From 8d5f4a8e9825345d05ebf13f6447f70718d3a7b2 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 19 Jun 2018 00:10:19 +0200 Subject: [PATCH] Switch to Little boxes, fixes #1 (#8) --- activitypub.py | 1423 +++++++++-------------------------- app.py | 1459 ++++++++++++++++++++---------------- config.py | 117 ++- dev-requirements.txt | 2 + requirements.txt | 6 +- tasks.py | 63 +- tests/federation_test.py | 401 +++++----- tests/integration_test.py | 9 +- utils/__init__.py | 6 +- utils/activitypub_utils.py | 65 -- utils/actor_service.py | 81 -- utils/content_helper.py | 58 -- utils/errors.py | 37 - utils/httpsig.py | 94 --- utils/key.py | 54 +- utils/linked_data_sig.py | 70 -- utils/object_service.py | 72 +- utils/opengraph.py | 30 +- utils/urlutils.py | 47 -- utils/webfinger.py | 75 -- 20 files changed, 1529 insertions(+), 2640 deletions(-) delete mode 100644 utils/activitypub_utils.py delete mode 100644 utils/actor_service.py delete mode 100644 utils/content_helper.py delete mode 100644 utils/errors.py delete mode 100644 utils/httpsig.py delete mode 100644 utils/linked_data_sig.py delete mode 100644 utils/urlutils.py delete mode 100644 utils/webfinger.py diff --git a/activitypub.py b/activitypub.py index cdb0bd1..21edb4c 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,77 +1,35 @@ import logging -import json -import binascii -import os from datetime import datetime -from enum import Enum +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Union from bson.objectid import ObjectId -from html2text import html2text from feedgen.feed import FeedGenerator +from html2text import html2text -from utils.actor_service import NotAnActorError -from utils.errors import BadActivityError -from utils.errors import UnexpectedActivityTypeError -from utils.errors import NotFromOutboxError -from utils import activitypub_utils -from config import USERNAME, BASE_URL, ID -from config import CTX_AS, CTX_SECURITY, AS_PUBLIC -from config import DB, ME, ACTOR_SERVICE -from config import OBJECT_SERVICE -from config import PUBLIC_INSTANCES import tasks - -from typing import List, Optional, Dict, Any, Union +from config import BASE_URL +from config import DB +from config import ID +from config import ME +from config import USER_AGENT +from config import USERNAME +from little_boxes import activitypub as ap +from little_boxes.backend import Backend +from little_boxes.collection import parse_collection as ap_parse_collection +from little_boxes.errors import Error logger = logging.getLogger(__name__) -# Helper/shortcut for typing -ObjectType = Dict[str, Any] -ObjectOrIDType = Union[str, ObjectType] - -COLLECTION_CTX = [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - { - "Hashtag": "as:Hashtag", - "sensitive": "as:sensitive", - } -] - - -class ActivityType(Enum): - """Supported activity `type`.""" - ANNOUNCE = 'Announce' - BLOCK = 'Block' - LIKE = 'Like' - CREATE = 'Create' - UPDATE = 'Update' - PERSON = 'Person' - ORDERED_COLLECTION = 'OrderedCollection' - ORDERED_COLLECTION_PAGE = 'OrderedCollectionPage' - COLLECTION_PAGE = 'CollectionPage' - COLLECTION = 'Collection' - NOTE = 'Note' - ACCEPT = 'Accept' - REJECT = 'Reject' - FOLLOW = 'Follow' - DELETE = 'Delete' - UNDO = 'Undo' - IMAGE = 'Image' - TOMBSTONE = 'Tombstone' - - -def random_object_id() -> str: - """Generates a random object ID.""" - return binascii.hexlify(os.urandom(8)).decode('utf-8') - - -def _remove_id(doc: ObjectType) -> ObjectType: +def _remove_id(doc: ap.ObjectType) -> ap.ObjectType: """Helper for removing MongoDB's `_id` field.""" doc = doc.copy() - if '_id' in doc: - del(doc['_id']) + if "_id" in doc: + del (doc["_id"]) return doc @@ -82,1125 +40,458 @@ def _to_list(data: Union[List[Any], Any]) -> List[Any]: return [data] -def clean_activity(activity: ObjectType) -> Dict[str, Any]: - """Clean the activity before rendering it. - - Remove the hidden bco and bcc field - """ - for field in ['bto', 'bcc']: - if field in activity: - del(activity[field]) - if activity['type'] == 'Create' and field in activity['object']: - del(activity['object'][field]) - return activity +def ensure_it_is_me(f): + """Method decorator used to track the events fired during tests.""" + def wrapper(*args, **kwargs): + if args[1].id != ME["id"]: + raise Error("unexpected actor") + return f(*args, **kwargs) -def _get_actor_id(actor: ObjectOrIDType) -> str: - """Helper for retrieving an actor `id`.""" - if isinstance(actor, dict): - return actor['id'] - return actor + return wrapper -class BaseActivity(object): - """Base class for ActivityPub activities.""" +class MicroblogPubBackend(Backend): + def user_agent(self) -> str: + return USER_AGENT - ACTIVITY_TYPE: Optional[ActivityType] = None - ALLOWED_OBJECT_TYPES: List[ActivityType] = [] - OBJECT_REQUIRED = False + def base_url(self) -> str: + return BASE_URL - def __init__(self, **kwargs) -> None: - # Ensure the class has an activity type defined - if not self.ACTIVITY_TYPE: - raise BadActivityError('Missing ACTIVITY_TYPE') + def activity_url(self, obj_id): + return f"{BASE_URL}/outbox/{obj_id}" - # XXX(tsileo): what to do about this check? - # Ensure the activity has a type and a valid one - # if kwargs.get('type') is None: - # raise BadActivityError('missing activity type') - - if kwargs.get('type') and kwargs.pop('type') != self.ACTIVITY_TYPE.value: - raise UnexpectedActivityTypeError(f'Expect the type to be {self.ACTIVITY_TYPE.value!r}') - - # Initialize the object - self._data: Dict[str, Any] = { - 'type': self.ACTIVITY_TYPE.value - } - logger.debug(f'initializing a {self.ACTIVITY_TYPE.value} activity: {kwargs}') - - if 'id' in kwargs: - self._data['id'] = kwargs.pop('id') - - if self.ACTIVITY_TYPE != ActivityType.PERSON: - actor = kwargs.get('actor') - if actor: - kwargs.pop('actor') - actor = self._validate_person(actor) - self._data['actor'] = actor - else: - # FIXME(tsileo): uses a special method to set the actor as "the instance" - if not self.NO_CONTEXT and self.ACTIVITY_TYPE != ActivityType.TOMBSTONE: - actor = ID - self._data['actor'] = actor - - if 'object' in kwargs: - obj = kwargs.pop('object') - if isinstance(obj, str): - self._data['object'] = obj - else: - if not self.ALLOWED_OBJECT_TYPES: - raise UnexpectedActivityTypeError('unexpected object') - if 'type' not in obj or (self.ACTIVITY_TYPE != ActivityType.CREATE and 'id' not in obj): - raise BadActivityError('invalid object, missing type') - if ActivityType(obj['type']) not in self.ALLOWED_OBJECT_TYPES: - raise UnexpectedActivityTypeError( - f'unexpected object type {obj["type"]} (allowed={self.ALLOWED_OBJECT_TYPES})' - ) - self._data['object'] = obj - - if '@context' not in kwargs: - if not self.NO_CONTEXT: - self._data['@context'] = CTX_AS - else: - self._data['@context'] = kwargs.pop('@context') - - # @context check - if not self.NO_CONTEXT: - if not isinstance(self._data['@context'], list): - self._data['@context'] = [self._data['@context']] - if CTX_SECURITY not in self._data['@context']: - self._data['@context'].append(CTX_SECURITY) - if isinstance(self._data['@context'][-1], dict): - self._data['@context'][-1]['Hashtag'] = 'as:Hashtag' - self._data['@context'][-1]['sensitive'] = 'as:sensitive' - else: - self._data['@context'].append({'Hashtag': 'as:Hashtag', 'sensitive': 'as:sensitive'}) - - allowed_keys = None - try: - allowed_keys = self._init(**kwargs) - logger.debug('calling custom init') - except NotImplementedError: - pass - - if allowed_keys: - # Allows an extra to (like for Accept and Follow) - kwargs.pop('to', None) - if len(set(kwargs.keys()) - set(allowed_keys)) > 0: - raise BadActivityError(f'extra data left: {kwargs!r}') - else: - # Remove keys with `None` value - valid_kwargs = {} - for k, v in kwargs.items(): - if v is None: - continue - valid_kwargs[k] = v - self._data.update(**valid_kwargs) - - def _init(self, **kwargs) -> Optional[List[str]]: - raise NotImplementedError - - def _verify(self) -> None: - raise NotImplementedError - - def verify(self) -> None: - """Verifies that the activity is valid.""" - if self.OBJECT_REQUIRED and 'object' not in self._data: - raise BadActivityError('activity must have an "object"') - - try: - self._verify() - except NotImplementedError: - pass - - def __repr__(self) -> str: - return '{}({!r})'.format(self.__class__.__qualname__, self._data.get('id')) - - def __str__(self) -> str: - return str(self._data.get('id', f'[new {self.ACTIVITY_TYPE} activity]')) - - def __getattr__(self, name: str) -> Any: - if self._data.get(name): - return self._data.get(name) - - @property - def type_enum(self) -> ActivityType: - return ActivityType(self.type) - - def _set_id(self, uri: str, obj_id: str) -> None: - raise NotImplementedError - - def set_id(self, uri: str, obj_id: str) -> None: - logger.debug(f'setting ID {uri} / {obj_id}') - self._data['id'] = uri - try: - self._set_id(uri, obj_id) - except NotImplementedError: - pass - - def _actor_id(self, obj: ObjectOrIDType) -> str: - if isinstance(obj, dict) and obj['type'] == ActivityType.PERSON.value: - obj_id = obj.get('id') - if not obj_id: - raise ValueError('missing object id') - return obj_id - else: - return str(obj) - - def _validate_person(self, obj: ObjectOrIDType) -> str: - obj_id = self._actor_id(obj) - try: - actor = ACTOR_SERVICE.get(obj_id) - except Exception: - return obj_id # FIXME(tsileo): handle this - if not actor: - raise ValueError('Invalid actor') - return actor['id'] - - def get_object(self) -> 'BaseActivity': - if self.__obj: - return self.__obj - if isinstance(self._data['object'], dict): - p = parse_activity(self._data['object']) - else: - if self.ACTIVITY_TYPE == ActivityType.FOLLOW: - p = Person(**ACTOR_SERVICE.get(self._data['object'])) - else: - obj = OBJECT_SERVICE.get(self._data['object']) - if ActivityType(obj.get('type')) not in self.ALLOWED_OBJECT_TYPES: - raise UnexpectedActivityTypeError(f'invalid object type {obj.get("type")}') - - p = parse_activity(obj) - - self.__obj: Optional[BaseActivity] = p - return p - - def reset_object_cache(self) -> None: - self.__obj = None - - def to_dict(self, embed: bool = False, embed_object_id_only: bool = False) -> ObjectType: - data = dict(self._data) - if embed: - for k in ['@context', 'signature']: - if k in data: - del(data[k]) - if data.get('object') and embed_object_id_only and isinstance(data['object'], dict): - try: - data['object'] = data['object']['id'] - except KeyError: - raise BadActivityError('embedded object does not have an id') - - return data - - def get_actor(self) -> 'BaseActivity': - actor = self._data.get('actor') - if not actor: - if self.type_enum == ActivityType.NOTE: - actor = str(self._data.get('attributedTo')) - else: - raise ValueError('failed to fetch actor') - - actor_id = self._actor_id(actor) - return Person(**ACTOR_SERVICE.get(actor_id)) - - def _pre_post_to_outbox(self) -> None: - raise NotImplementedError - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - raise NotImplementedError - - def _undo_outbox(self) -> None: - raise NotImplementedError - - def _pre_process_from_inbox(self) -> None: - raise NotImplementedError - - def _process_from_inbox(self) -> None: - raise NotImplementedError - - def _undo_inbox(self) -> None: - raise NotImplementedError - - def _undo_should_purge_cache(self) -> bool: - raise NotImplementedError - - def _should_purge_cache(self) -> bool: - raise NotImplementedError - - def process_from_inbox(self) -> None: - logger.debug(f'calling main process from inbox hook for {self}') - self.verify() - actor = self.get_actor() - - # Check for Block activity - if DB.outbox.find_one({'type': ActivityType.BLOCK.value, - 'activity.object': actor.id, - 'meta.undo': False}): - logger.info(f'actor {actor} is blocked, dropping the received activity {self}') - return - - if DB.inbox.find_one({'remote_id': self.id}): - # The activity is already in the inbox - logger.info(f'received duplicate activity {self}, dropping it') - return - - try: - self._pre_process_from_inbox() - logger.debug('called pre process from inbox hook') - except NotImplementedError: - logger.debug('pre process from inbox hook not implemented') - - activity = self.to_dict() - DB.inbox.insert_one({ - 'activity': activity, - 'type': self.type, - 'remote_id': self.id, - 'meta': {'undo': False, 'deleted': False}, - }) - logger.info('activity {self} saved') - - try: - self._process_from_inbox() - logger.debug('called process from inbox hook') - except NotImplementedError: - logger.debug('process from inbox hook not implemented') - - def post_to_outbox(self) -> None: - logger.debug(f'calling main post to outbox hook for {self}') - obj_id = random_object_id() - self.set_id(f'{ID}/outbox/{obj_id}', obj_id) - self.verify() - - try: - self._pre_post_to_outbox() - logger.debug(f'called pre post to outbox hook') - except NotImplementedError: - logger.debug('pre post to outbox hook not implemented') - - activity = self.to_dict() - DB.outbox.insert_one({ - 'id': obj_id, - 'activity': activity, - 'type': self.type, - 'remote_id': self.id, - 'meta': {'undo': False, 'deleted': False}, - }) - - recipients = self.recipients() - logger.info(f'recipients={recipients}') - activity = clean_activity(activity) - - try: - self._post_to_outbox(obj_id, activity, recipients) - logger.debug(f'called post to outbox hook') - except NotImplementedError: - logger.debug('post to outbox hook not implemented') - - payload = json.dumps(activity) - for recp in recipients: - logger.debug(f'posting to {recp}') - self._post_to_inbox(payload, recp) - - def _post_to_inbox(self, payload: str, to: str): - tasks.post_to_inbox.delay(payload, to) - - def _recipients(self) -> List[str]: - return [] - - def recipients(self) -> List[str]: - recipients = self._recipients() - - out: List[str] = [] - for recipient in recipients: - if recipient in PUBLIC_INSTANCES: - if recipient not in out: - out.append(str(recipient)) - continue - if recipient in [ME, AS_PUBLIC, None]: - continue - if isinstance(recipient, Person): - if recipient.id == ME: - continue - actor = recipient - else: - try: - actor = Person(**ACTOR_SERVICE.get(recipient)) - - if actor.endpoints: - shared_inbox = actor.endpoints.get('sharedInbox') - if shared_inbox not in out: - out.append(shared_inbox) - continue - - if actor.inbox and actor.inbox not in out: - out.append(actor.inbox) - - except NotAnActorError as error: - # Is the activity a `Collection`/`OrderedCollection`? - if error.activity and error.activity['type'] in [ActivityType.COLLECTION.value, - ActivityType.ORDERED_COLLECTION.value]: - for item in parse_collection(error.activity): - if item in [ME, AS_PUBLIC]: - continue - try: - col_actor = Person(**ACTOR_SERVICE.get(item)) - except NotAnActorError: - pass - - if col_actor.endpoints: - shared_inbox = col_actor.endpoints.get('sharedInbox') - if shared_inbox not in out: - out.append(shared_inbox) - continue - if col_actor.inbox and col_actor.inbox not in out: - out.append(col_actor.inbox) - - return out - - def build_undo(self) -> 'BaseActivity': - raise NotImplementedError - - def build_delete(self) -> 'BaseActivity': - raise NotImplementedError - - -class Person(BaseActivity): - ACTIVITY_TYPE = ActivityType.PERSON - - def _init(self, **kwargs): - # if 'icon' in kwargs: - # self._data['icon'] = Image(**kwargs.pop('icon')) - pass - - def _verify(self) -> None: - ACTOR_SERVICE.get(self._data['id']) - - -class Block(BaseActivity): - ACTIVITY_TYPE = ActivityType.BLOCK - OBJECT_REQUIRED = True - - -class Collection(BaseActivity): - ACTIVITY_TYPE = ActivityType.COLLECTION - - -class Image(BaseActivity): - ACTIVITY_TYPE = ActivityType.IMAGE - NO_CONTEXT = True - - def _init(self, **kwargs): - self._data.update( - url=kwargs.pop('url'), + @ensure_it_is_me + def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: + DB.outbox.insert_one( + { + "activity": activity.to_dict(), + "type": activity.type, + "remote_id": activity.id, + "meta": {"undo": False, "deleted": False}, + } ) - def __repr__(self): - return 'Image({!r})'.format(self._data.get('url')) - - -class Follow(BaseActivity): - ACTIVITY_TYPE = ActivityType.FOLLOW - ALLOWED_OBJECT_TYPES = [ActivityType.PERSON] - OBJECT_REQUIRED = True - - def _build_reply(self, reply_type: ActivityType) -> BaseActivity: - if reply_type == ActivityType.ACCEPT: - return Accept( - object=self.to_dict(embed=True), + @ensure_it_is_me + def outbox_is_blocked(self, as_actor: ap.Person, actor_id: str) -> bool: + return bool( + DB.outbox.find_one( + { + "type": ap.ActivityType.BLOCK.value, + "activity.object": actor_id, + "meta.undo": False, + } ) + ) - raise ValueError(f'type {reply_type} is invalid for building a reply') + def fetch_iri(self, iri: str) -> ap.ObjectType: + if iri == ME["id"]: + return ME - def _recipients(self) -> List[str]: - return [self.get_object().id] - - def _process_from_inbox(self) -> None: - accept = self.build_accept() - accept.post_to_outbox() - - remote_actor = self.get_actor().id - - if DB.followers.find({'remote_actor': remote_actor}).count() == 0: - DB.followers.insert_one({'remote_actor': remote_actor}) - - def _undo_inbox(self) -> None: - DB.followers.delete_one({'remote_actor': self.get_actor().id}) - - def _undo_outbox(self) -> None: - DB.following.delete_one({'remote_actor': self.get_object().id}) - - def build_accept(self) -> BaseActivity: - return self._build_reply(ActivityType.ACCEPT) - - def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True)) - - def _should_purge_cache(self) -> bool: - # Receiving a follow activity in the inbox should reset the application cache - return True - - -class Accept(BaseActivity): - ACTIVITY_TYPE = ActivityType.ACCEPT - ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW] - - def _recipients(self) -> List[str]: - return [self.get_object().get_actor().id] - - def _process_from_inbox(self) -> None: - remote_actor = self.get_actor().id - if DB.following.find({'remote_actor': remote_actor}).count() == 0: - DB.following.insert_one({'remote_actor': remote_actor}) - - def _should_purge_cache(self) -> bool: - # Receiving an accept activity in the inbox should reset the application cache - # (a follow request has been accepted) - return True - - -class Undo(BaseActivity): - ACTIVITY_TYPE = ActivityType.UNDO - ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW, ActivityType.LIKE, ActivityType.ANNOUNCE] - OBJECT_REQUIRED = True - - def _recipients(self) -> List[str]: - obj = self.get_object() - if obj.type_enum == ActivityType.FOLLOW: - return [obj.get_object().id] + # Check if the activity is owned by this server + if iri.startswith(BASE_URL): + is_a_note = False + if iri.endswith("/activity"): + iri = iri.replace("/activity", "") + is_a_note = True + data = DB.outbox.find_one({"remote_id": iri}) + if data: + if is_a_note: + return data["activity"]["object"] + return data["activity"] else: - return [obj.get_object().get_actor().id] - # TODO(tsileo): handle like and announce - raise Exception('TODO') + # Check if the activity is stored in the inbox + data = DB.inbox.find_one({"remote_id": iri}) + if data: + return data["activity"] - def _pre_process_from_inbox(self) -> None: - """Ensures an Undo activity comes from the same actor as the updated activity.""" - obj = self.get_object() - actor = self.get_actor() - if actor.id != obj.get_actor().id: - raise BadActivityError(f'{actor!r} cannot update {obj!r}') + # Fetch the URL via HTTP + return super().fetch_iri(iri) - def _process_from_inbox(self) -> None: - obj = self.get_object() - DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) + @ensure_it_is_me + def inbox_check_duplicate(self, as_actor: ap.Person, iri: str) -> bool: + return bool(DB.inbox.find_one({"remote_id": iri})) - try: - obj._undo_inbox() - except NotImplementedError: - pass + @ensure_it_is_me + def inbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: + DB.inbox.insert_one( + { + "activity": activity.to_dict(), + "type": activity.type, + "remote_id": activity.id, + "meta": {"undo": False, "deleted": False}, + } + ) - def _should_purge_cache(self) -> bool: - obj = self.get_object() - try: - # Receiving a undo activity regarding an activity that was mentioning a published activity - # should purge the cache - return obj._undo_should_purge_cache() - except NotImplementedError: - pass + @ensure_it_is_me + def post_to_remote_inbox(self, as_actor: ap.Person, payload: str, to: str) -> None: + tasks.post_to_inbox.delay(payload, to) - return False + @ensure_it_is_me + def new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: + remote_actor = follow.get_actor().id - def _pre_post_to_outbox(self) -> None: - """Ensures an Undo activity references an activity owned by the instance.""" - obj = self.get_object() - if not obj.id.startswith(ID): - raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + if DB.followers.find({"remote_actor": remote_actor}).count() == 0: + DB.followers.insert_one({"remote_actor": remote_actor}) - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - logger.debug('processing undo to outbox') - logger.debug('self={}'.format(self)) - obj = self.get_object() - logger.debug('obj={}'.format(obj)) - DB.outbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) + @ensure_it_is_me + def undo_new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: + # TODO(tsileo): update the follow to set undo + DB.followers.delete_one({"remote_actor": follow.get_actor().id}) - try: - obj._undo_outbox() - logger.debug(f'_undo_outbox called for {obj}') - except NotImplementedError: - logger.debug(f'_undo_outbox not implemented for {obj}') - pass + @ensure_it_is_me + def undo_new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: + # TODO(tsileo): update the follow to set undo + DB.following.delete_one({"remote_actor": follow.get_object().id}) + @ensure_it_is_me + def new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: + remote_actor = follow.get_object().id + if DB.following.find({"remote_actor": remote_actor}).count() == 0: + DB.following.insert_one({"remote_actor": remote_actor}) -class Like(BaseActivity): - ACTIVITY_TYPE = ActivityType.LIKE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] - OBJECT_REQUIRED = True - - def _recipients(self) -> List[str]: - return [self.get_object().get_actor().id] - - def _process_from_inbox(self): - obj = self.get_object() + @ensure_it_is_me + def inbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: + obj = like.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': 1}, - }) - # XXX(tsileo): notification?? + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}} + ) - def _undo_inbox(self) -> None: - obj = self.get_object() + @ensure_it_is_me + def inbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: + obj = like.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': -1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}} + ) - def _undo_should_purge_cache(self) -> bool: - # If a like coutn was decremented, we need to purge the application cache - return self.get_object().id.startswith(BASE_URL) - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]): - obj = self.get_object() + @ensure_it_is_me + def outbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: + obj = like.get_object() # Unlikely, but an actor can like it's own post - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': 1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}} + ) # Keep track of the like we just performed - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': obj_id}}) + DB.inbox.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.liked": like.id}} + ) - def _undo_outbox(self) -> None: - obj = self.get_object() + @ensure_it_is_me + def outbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: + obj = like.get_object() # Unlikely, but an actor can like it's own post - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': -1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}} + ) - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': False}}) + DB.inbox.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.liked": False}} + ) - def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True, embed_object_id_only=True)) - - -class Announce(BaseActivity): - ACTIVITY_TYPE = ActivityType.ANNOUNCE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] - - def _recipients(self) -> List[str]: - recipients = [] - - for field in ['to', 'cc']: - if field in self._data: - recipients.extend(_to_list(self._data[field])) - - return recipients - - def _process_from_inbox(self) -> None: - if isinstance(self._data['object'], str) and not self._data['object'].startswith('http'): + @ensure_it_is_me + def inbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: + if isinstance(announce._data["object"], str) and not announce._data[ + "object" + ].startswith("http"): # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else logger.warn( - f'received an Annouce referencing an OStatus notice ({self._data["object"]}), dropping the message' + f'received an Annouce referencing an OStatus notice ({announce._data["object"]}), dropping the message' ) return - # Save/cache the object, and make it part of the stream so we can fetch it - if isinstance(self._data['object'], str): - raw_obj = OBJECT_SERVICE.get( - self._data['object'], - reload_cache=True, - part_of_stream=True, - announce_published=self._data['published'], - ) - obj = parse_activity(raw_obj) + # FIXME(tsileo): Save/cache the object, and make it part of the stream so we can fetch it + if isinstance(announce._data["object"], str): + obj_iri = announce._data["object"] else: - obj = self.get_object() + obj_iri = self.get_object().id - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_boost': 1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj_iri}, {"$inc": {"meta.count_boost": 1}} + ) - def _undo_inbox(self) -> None: - obj = self.get_object() + @ensure_it_is_me + def inbox_undo_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: + obj = announce.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_boost': -1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_boost": -1}} + ) - def _undo_should_purge_cache(self) -> bool: - # If a like coutn was decremented, we need to purge the application cache - return self.get_object().id.startswith(BASE_URL) + @ensure_it_is_me + def outbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: + obj = announce.get_object() + DB.inbox.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.boosted": announce.id}} + ) - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - if isinstance(self._data['object'], str): - # Put the object in the cache - OBJECT_SERVICE.get( - self._data['object'], - reload_cache=True, - part_of_stream=True, - announce_published=self._data['published'], - ) + @ensure_it_is_me + def outbox_undo_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: + obj = announce.get_object() + DB.inbox.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.boosted": False}} + ) - obj = self.get_object() - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': obj_id}}) + @ensure_it_is_me + def inbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: + DB.inbox.update_one( + {"activity.object.id": delete.get_object().id}, + {"$set": {"meta.deleted": True}}, + ) + obj = delete.get_object() + if obj.ACTIVITY_TYPE != ap.ActivityType.NOTE: + obj = ap.parse_activity( + DB.inbox.find_one( + { + "activity.object.id": delete.get_object().id, + "type": ap.ActivityType.CREATE.value, + } + )["activity"] + ).get_object() - def _undo_outbox(self) -> None: - obj = self.get_object() - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': False}}) + logger.info(f"inbox_delete handle_replies obj={obj!r}") + if obj: + self._handle_replies_delete(as_actor, obj) - def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True)) - - -class Delete(BaseActivity): - ACTIVITY_TYPE = ActivityType.DELETE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.TOMBSTONE] - OBJECT_REQUIRED = True - - def _get_actual_object(self) -> BaseActivity: - obj = self.get_object() - if obj.type_enum == ActivityType.TOMBSTONE: - obj = parse_activity(OBJECT_SERVICE.get(obj.id)) - return obj - - def _recipients(self) -> List[str]: - obj = self._get_actual_object() - return obj._recipients() - - def _pre_process_from_inbox(self) -> None: - """Ensures a Delete activity comes from the same actor as the deleted activity.""" - obj = self._get_actual_object() - actor = self.get_actor() - if actor.id != obj.get_actor().id: - raise BadActivityError(f'{actor!r} cannot delete {obj!r}') - - def _process_from_inbox(self) -> None: - DB.inbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) - obj = self._get_actual_object() - if obj.type_enum == ActivityType.NOTE: - obj._delete_from_threads() + # FIXME(tsileo): handle threads + # obj = delete._get_actual_object() + # if obj.type_enum == ActivityType.NOTE: + # obj._delete_from_threads() # TODO(tsileo): also purge the cache if it's a reply of a published activity - def _pre_post_to_outbox(self) -> None: - """Ensures the Delete activity references a activity from the outbox (i.e. owned by the instance).""" - obj = self._get_actual_object() - if not obj.id.startswith(ID): - raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + @ensure_it_is_me + def outbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: + DB.outbox.update_one( + {"activity.object.id": delete.get_object().id}, + {"$set": {"meta.deleted": True}}, + ) + obj = delete.get_object() + if delete.get_object().ACTIVITY_TYPE != ap.ActivityType.NOTE: + obj = ap.parse_activity( + DB.outbox.find_one( + { + "activity.object.id": delete.get_object().id, + "type": ap.ActivityType.CREATE.value, + } + )["activity"] + ).get_object() - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - DB.outbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) + self._handle_replies_delete(as_actor, obj) - -class Update(BaseActivity): - ACTIVITY_TYPE = ActivityType.UPDATE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.PERSON] - OBJECT_REQUIRED = True - - def _pre_process_from_inbox(self) -> None: - """Ensures an Update activity comes from the same actor as the updated activity.""" - obj = self.get_object() - actor = self.get_actor() - if actor.id != obj.get_actor().id: - raise BadActivityError(f'{actor!r} cannot update {obj!r}') - - def _process_from_inbox(self): - obj = self.get_object() - if obj.type_enum == ActivityType.NOTE: - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'activity.object': obj.to_dict()}}) + @ensure_it_is_me + def inbox_update(self, as_actor: ap.Person, update: ap.Update) -> None: + obj = update.get_object() + if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE: + DB.inbox.update_one( + {"activity.object.id": obj.id}, + {"$set": {"activity.object": obj.to_dict()}}, + ) return - # If the object is a Person, it means the profile was updated, we just refresh our local cache - ACTOR_SERVICE.get(obj.id, reload_cache=True) + # FIXME(tsileo): handle update actor amd inbox_update_note/inbox_update_actor - # TODO(tsileo): implements _should_purge_cache if it's a reply of a published activity (i.e. in the outbox) + @ensure_it_is_me + def outbox_update(self, as_actor: ap.Person, _update: ap.Update) -> None: + obj = _update._data["object"] - def _pre_post_to_outbox(self) -> None: - obj = self.get_object() - if not obj.id.startswith(ID): - raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - obj = self._data['object'] - - update_prefix = 'activity.object.' - update: Dict[str, Any] = {'$set': dict(), '$unset': dict()} - update['$set'][f'{update_prefix}updated'] = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' + update_prefix = "activity.object." + update: Dict[str, Any] = {"$set": dict(), "$unset": dict()} + update["$set"][f"{update_prefix}updated"] = ( + datetime.utcnow().replace(microsecond=0).isoformat() + "Z" + ) for k, v in obj.items(): - if k in ['id', 'type']: + if k in ["id", "type"]: continue if v is None: - update['$unset'][f'{update_prefix}{k}'] = '' + update["$unset"][f"{update_prefix}{k}"] = "" else: - update['$set'][f'{update_prefix}{k}'] = v + update["$set"][f"{update_prefix}{k}"] = v - if len(update['$unset']) == 0: - del(update['$unset']) + if len(update["$unset"]) == 0: + del (update["$unset"]) - print(f'updating note from outbox {obj!r} {update}') - logger.info(f'updating note from outbox {obj!r} {update}') - DB.outbox.update_one({'activity.object.id': obj['id']}, update) + print(f"updating note from outbox {obj!r} {update}") + logger.info(f"updating note from outbox {obj!r} {update}") + DB.outbox.update_one({"activity.object.id": obj["id"]}, update) # FIXME(tsileo): should send an Update (but not a partial one, to all the note's recipients # (create a new Update with the result of the update, and send it without saving it?) + @ensure_it_is_me + def outbox_create(self, as_actor: ap.Person, create: ap.Create) -> None: + self._handle_replies(as_actor, create) -class Create(BaseActivity): - ACTIVITY_TYPE = ActivityType.CREATE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] - OBJECT_REQUIRED = True + @ensure_it_is_me + def inbox_create(self, as_actor: ap.Person, create: ap.Create) -> None: + self._handle_replies(as_actor, create) - def _set_id(self, uri: str, obj_id: str) -> None: - self._data['object']['id'] = uri + '/activity' - self._data['object']['url'] = ID + '/' + self.get_object().type.lower() + '/' + obj_id - self.reset_object_cache() - - def _init(self, **kwargs): - obj = self.get_object() - if not obj.attributedTo: - self._data['object']['attributedTo'] = self.get_actor().id - if not obj.published: - if self.published: - self._data['object']['published'] = self.published - else: - now = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' - self._data['published'] = now - self._data['object']['published'] = now - - def _recipients(self) -> List[str]: - # TODO(tsileo): audience support? - recipients = [] - for field in ['to', 'cc', 'bto', 'bcc']: - if field in self._data: - recipients.extend(_to_list(self._data[field])) - - recipients.extend(self.get_object()._recipients()) - - return recipients - - def _update_threads(self) -> None: - logger.debug('_update_threads hook') - obj = self.get_object() - - # TODO(tsileo): re-enable me - # tasks.fetch_og.delay('INBOX', self.id) - - threads = [] - reply = obj.get_local_reply() - print(f'initial_reply={reply}') - print(f'{obj}') - logger.debug(f'initial_reply={reply}') - reply_id = None - direct_reply = 1 - while reply is not None: - if not DB.inbox.find_one_and_update({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': 1, - 'meta.count_direct_reply': direct_reply, - }, - '$addToSet': {'meta.thread_children': obj.id}, - }): - DB.outbox.update_one({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': 1, - 'meta.count_direct_reply': direct_reply, - }, - '$addToSet': {'meta.thread_children': obj.id}, - }) - - direct_reply = 0 - reply_id = reply.id - reply = reply.get_local_reply() - logger.debug(f'next_reply={reply}') - threads.append(reply_id) - # FIXME(tsileo): obj.id is None!! - print(f'reply_id={reply_id} {obj.id} {obj._data} {self.id}') - - if reply_id: - if not DB.inbox.find_one_and_update({'activity.object.id': obj.id}, { - '$set': { - 'meta.thread_parents': threads, - 'meta.thread_root_parent': reply_id, - }, - }): - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$set': { - 'meta.thread_parents': threads, - 'meta.thread_root_parent': reply_id, - }, - }) - logger.debug('_update_threads done') - - def _process_from_inbox(self) -> None: - self._update_threads() - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - self._update_threads() - - def _should_purge_cache(self) -> bool: - # TODO(tsileo): handle reply of a reply... - obj = self.get_object() - in_reply_to = obj.inReplyTo - if in_reply_to: - local_activity = DB.outbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if local_activity: - return True - - return False - - -class Tombstone(BaseActivity): - ACTIVITY_TYPE = ActivityType.TOMBSTONE - - -class Note(BaseActivity): - ACTIVITY_TYPE = ActivityType.NOTE - - def _init(self, **kwargs): - print(self._data) - # Remove the `actor` field as `attributedTo` is used for `Note` instead - if 'actor' in self._data: - del(self._data['actor']) - if 'sensitive' not in kwargs: - self._data['sensitive'] = False - - def _recipients(self) -> List[str]: - # TODO(tsileo): audience support? - recipients: List[str] = [] - - # If the note is public, we publish it to the defined "public instances" - if AS_PUBLIC in self._data.get('to', []): - recipients.extend(PUBLIC_INSTANCES) - print('publishing to public instances') - print(recipients) - - for field in ['to', 'cc', 'bto', 'bcc']: - if field in self._data: - recipients.extend(_to_list(self._data[field])) - - return recipients - - def _delete_from_threads(self) -> None: - logger.debug('_delete_from_threads hook') - - reply = self.get_local_reply() - logger.debug(f'initial_reply={reply}') - direct_reply = -1 - while reply is not None: - if not DB.inbox.find_one_and_update({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': -1, - 'meta.count_direct_reply': direct_reply, - }, - '$pull': {'meta.thread_children': self.id}, - - }): - DB.outbox.update_one({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': 1, - 'meta.count_direct_reply': direct_reply, - }, - '$pull': {'meta.thread_children': self.id}, - }) - - direct_reply = 0 - reply = reply.get_local_reply() - logger.debug(f'next_reply={reply}') - - logger.debug('_delete_from_threads done') - return None - - def get_local_reply(self) -> Optional[BaseActivity]: - "Find the note reply if any.""" - in_reply_to = self.inReplyTo + @ensure_it_is_me + def _handle_replies_delete(self, as_actor: ap.Person, note: ap.Note) -> None: + in_reply_to = note.inReplyTo if not in_reply_to: - # This is the root comment - return None + pass - inbox_parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if inbox_parent: - return parse_activity(inbox_parent['activity']['object']) + if not DB.inbox.find_one_and_update( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, + ): + DB.outbox.update_one( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, + ) - outbox_parent = DB.outbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if outbox_parent: - return parse_activity(outbox_parent['activity']['object']) + @ensure_it_is_me + def _handle_replies(self, as_actor: ap.Person, create: ap.Create) -> None: + in_reply_to = create.get_object().inReplyTo + if not in_reply_to: + pass - # The parent is no stored on this instance - return None - - def build_create(self) -> BaseActivity: - """Wraps an activity in a Create activity.""" - create_payload = { - 'object': self.to_dict(embed=True), - 'actor': self.attributedTo or ME, - } - for field in ['published', 'to', 'bto', 'cc', 'bcc', 'audience']: - if field in self._data: - create_payload[field] = self._data[field] - - return Create(**create_payload) - - def build_like(self) -> BaseActivity: - return Like(object=self.id) - - def build_announce(self) -> BaseActivity: - return Announce( - object=self.id, - to=[AS_PUBLIC], - cc=[ID+'/followers', self.attributedTo], - published=datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', - ) - - def build_delete(self) -> BaseActivity: - return Delete(object=Tombstone(id=self.id).to_dict(embed=True)) - - def get_tombstone(self, deleted: Optional[str]) -> BaseActivity: - return Tombstone( - id=self.id, - published=self.published, - deleted=deleted, - updated=deleted, - ) - - -_ACTIVITY_TYPE_TO_CLS = { - ActivityType.IMAGE: Image, - ActivityType.PERSON: Person, - ActivityType.FOLLOW: Follow, - ActivityType.ACCEPT: Accept, - ActivityType.UNDO: Undo, - ActivityType.LIKE: Like, - ActivityType.ANNOUNCE: Announce, - ActivityType.UPDATE: Update, - ActivityType.DELETE: Delete, - ActivityType.CREATE: Create, - ActivityType.NOTE: Note, - ActivityType.BLOCK: Block, - ActivityType.COLLECTION: Collection, - ActivityType.TOMBSTONE: Tombstone, -} - - -def parse_activity(payload: ObjectType, expected: Optional[ActivityType] = None) -> BaseActivity: - t = ActivityType(payload['type']) - - if expected and t != expected: - raise UnexpectedActivityTypeError(f'expected a {expected.name} activity, got a {payload["type"]}') - - if t not in _ACTIVITY_TYPE_TO_CLS: - raise BadActivityError(f'unsupported activity type {payload["type"]}') - - activity = _ACTIVITY_TYPE_TO_CLS[t](**payload) - - return activity + if not DB.inbox.find_one_and_update( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, + ): + DB.outbox.update_one( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, + ) def gen_feed(): fg = FeedGenerator() - fg.id(f'{ID}') - fg.title(f'{USERNAME} notes') - fg.author({'name': USERNAME, 'email': 't@a4.io'}) - fg.link(href=ID, rel='alternate') - fg.description(f'{USERNAME} notes') - fg.logo(ME.get('icon', {}).get('url')) - fg.language('en') - for item in DB.outbox.find({'type': 'Create'}, limit=50): + fg.id(f"{ID}") + fg.title(f"{USERNAME} notes") + fg.author({"name": USERNAME, "email": "t@a4.io"}) + fg.link(href=ID, rel="alternate") + fg.description(f"{USERNAME} notes") + fg.logo(ME.get("icon", {}).get("url")) + fg.language("en") + for item in DB.outbox.find({"type": "Create"}, limit=50): fe = fg.add_entry() - fe.id(item['activity']['object'].get('url')) - fe.link(href=item['activity']['object'].get('url')) - fe.title(item['activity']['object']['content']) - fe.description(item['activity']['object']['content']) + fe.id(item["activity"]["object"].get("url")) + fe.link(href=item["activity"]["object"].get("url")) + fe.title(item["activity"]["object"]["content"]) + fe.description(item["activity"]["object"]["content"]) return fg def json_feed(path: str) -> Dict[str, Any]: """JSON Feed (https://jsonfeed.org/) document.""" data = [] - for item in DB.outbox.find({'type': 'Create'}, limit=50): - data.append({ - "id": item["id"], - "url": item['activity']['object'].get('url'), - "content_html": item['activity']['object']['content'], - "content_text": html2text(item['activity']['object']['content']), - "date_published": item['activity']['object'].get('published'), - }) + for item in DB.outbox.find({"type": "Create"}, limit=50): + data.append( + { + "id": item["id"], + "url": item["activity"]["object"].get("url"), + "content_html": item["activity"]["object"]["content"], + "content_text": html2text(item["activity"]["object"]["content"]), + "date_published": item["activity"]["object"].get("published"), + } + ) return { "version": "https://jsonfeed.org/version/1", - "user_comment": ("This is a microblog feed. You can add this to your feed reader using the following URL: " - + ID + path), + "user_comment": ( + "This is a microblog feed. You can add this to your feed reader using the following URL: " + + ID + + path + ), "title": USERNAME, "home_page_url": ID, "feed_url": ID + path, "author": { "name": USERNAME, "url": ID, - "avatar": ME.get('icon', {}).get('url'), + "avatar": ME.get("icon", {}).get("url"), }, "items": data, } -def build_inbox_json_feed(path: str, request_cursor: Optional[str] = None) -> Dict[str, Any]: +def build_inbox_json_feed( + path: str, request_cursor: Optional[str] = None +) -> Dict[str, Any]: data = [] cursor = None - q: Dict[str, Any] = {'type': 'Create', 'meta.deleted': False} + q: Dict[str, Any] = {"type": "Create", "meta.deleted": False} if request_cursor: - q['_id'] = {'$lt': request_cursor} + q["_id"] = {"$lt": request_cursor} - for item in DB.inbox.find(q, limit=50).sort('_id', -1): - actor = ACTOR_SERVICE.get(item['activity']['actor']) - data.append({ - "id": item["activity"]["id"], - "url": item['activity']['object'].get('url'), - "content_html": item['activity']['object']['content'], - "content_text": html2text(item['activity']['object']['content']), - "date_published": item['activity']['object'].get('published'), - "author": { - "name": actor.get('name', actor.get('preferredUsername')), - "url": actor.get('url'), - 'avatar': actor.get('icon', {}).get('url'), - }, - }) - cursor = str(item['_id']) + for item in DB.inbox.find(q, limit=50).sort("_id", -1): + actor = ap.get_backend().fetch_iri(item["activity"]["actor"]) + data.append( + { + "id": item["activity"]["id"], + "url": item["activity"]["object"].get("url"), + "content_html": item["activity"]["object"]["content"], + "content_text": html2text(item["activity"]["object"]["content"]), + "date_published": item["activity"]["object"].get("published"), + "author": { + "name": actor.get("name", actor.get("preferredUsername")), + "url": actor.get("url"), + "avatar": actor.get("icon", {}).get("url"), + }, + } + ) + cursor = str(item["_id"]) resp = { "version": "https://jsonfeed.org/version/1", - "title": f'{USERNAME}\'s stream', + "title": f"{USERNAME}'s stream", "home_page_url": ID, "feed_url": ID + path, "items": data, } if cursor and len(data) == 50: - resp['next_url'] = ID + path + '?cursor=' + cursor + resp["next_url"] = ID + path + "?cursor=" + cursor return resp -def parse_collection(payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None) -> List[str]: +def parse_collection( + payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None +) -> List[str]: """Resolve/fetch a `Collection`/`OrderedCollection`.""" # Resolve internal collections via MongoDB directly - if url == ID + '/followers': - return [doc['remote_actor'] for doc in DB.followers.find()] - elif url == ID + '/following': - return [doc['remote_actor'] for doc in DB.following.find()] + if url == ID + "/followers": + return [doc["remote_actor"] for doc in DB.followers.find()] + elif url == ID + "/following": + return [doc["remote_actor"] for doc in DB.following.find()] # Go through all the pages - return activitypub_utils.parse_collection(payload, url) + return ap_parse_collection(payload, url) def embed_collection(total_items, first_page_id): return { - "type": ActivityType.ORDERED_COLLECTION.value, + "type": ap.ActivityType.ORDERED_COLLECTION.value, "totalItems": total_items, - "first": f'{first_page_id}?page=first', + "first": f"{first_page_id}?page=first", "id": first_page_id, } -def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, col_name=None, first_page=False): +def build_ordered_collection( + col, q=None, cursor=None, map_func=None, limit=50, col_name=None, first_page=False +): col_name = col_name or col.name if q is None: q = {} if cursor: - q['_id'] = {'$lt': ObjectId(cursor)} - data = list(col.find(q, limit=limit).sort('_id', -1)) + q["_id"] = {"$lt": ObjectId(cursor)} + data = list(col.find(q, limit=limit).sort("_id", -1)) if not data: return { - 'id': BASE_URL + '/' + col_name, - 'totalItems': 0, - 'type': ActivityType.ORDERED_COLLECTION.value, - 'orederedItems': [], + "id": BASE_URL + "/" + col_name, + "totalItems": 0, + "type": ap.ActivityType.ORDERED_COLLECTION.value, + "orederedItems": [], } - start_cursor = str(data[0]['_id']) - next_page_cursor = str(data[-1]['_id']) + start_cursor = str(data[0]["_id"]) + next_page_cursor = str(data[-1]["_id"]) total_items = col.find(q).count() data = [_remove_id(doc) for doc in data] @@ -1210,41 +501,43 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, # No cursor, this is the first page and we return an OrderedCollection if not cursor: resp = { - '@context': COLLECTION_CTX, - 'id': f'{BASE_URL}/{col_name}', - 'totalItems': total_items, - 'type': ActivityType.ORDERED_COLLECTION.value, - 'first': { - 'id': f'{BASE_URL}/{col_name}?cursor={start_cursor}', - 'orderedItems': data, - 'partOf': f'{BASE_URL}/{col_name}', - 'totalItems': total_items, - 'type': ActivityType.ORDERED_COLLECTION_PAGE.value, + "@context": ap.COLLECTION_CTX, + "id": f"{BASE_URL}/{col_name}", + "totalItems": total_items, + "type": ap.ActivityType.ORDERED_COLLECTION.value, + "first": { + "id": f"{BASE_URL}/{col_name}?cursor={start_cursor}", + "orderedItems": data, + "partOf": f"{BASE_URL}/{col_name}", + "totalItems": total_items, + "type": ap.ActivityType.ORDERED_COLLECTION_PAGE.value, }, } if len(data) == limit: - resp['first']['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor + resp["first"]["next"] = ( + BASE_URL + "/" + col_name + "?cursor=" + next_page_cursor + ) if first_page: - return resp['first'] + return resp["first"] return resp # If there's a cursor, then we return an OrderedCollectionPage resp = { - '@context': COLLECTION_CTX, - 'type': ActivityType.ORDERED_COLLECTION_PAGE.value, - 'id': BASE_URL + '/' + col_name + '?cursor=' + start_cursor, - 'totalItems': total_items, - 'partOf': BASE_URL + '/' + col_name, - 'orderedItems': data, + "@context": ap.COLLECTION_CTX, + "type": ap.ActivityType.ORDERED_COLLECTION_PAGE.value, + "id": BASE_URL + "/" + col_name + "?cursor=" + start_cursor, + "totalItems": total_items, + "partOf": BASE_URL + "/" + col_name, + "orderedItems": data, } if len(data) == limit: - resp['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor + resp["next"] = BASE_URL + "/" + col_name + "?cursor=" + next_page_cursor if first_page: - return resp['first'] + return resp["first"] # XXX(tsileo): implements prev with prev=? diff --git a/app.py b/app.py index e3cd805..fd564c1 100644 --- a/app.py +++ b/app.py @@ -1,135 +1,145 @@ import binascii import hashlib import json -import urllib -import os -import mimetypes import logging -from functools import wraps +import mimetypes +import os +import urllib from datetime import datetime +from functools import wraps +from typing import Any +from typing import Dict +from urllib.parse import urlencode +from urllib.parse import urlparse -import timeago import bleach import mf2py -import pymongo import piexif +import pymongo +import timeago from bson.objectid import ObjectId from flask import Flask -from flask import abort -from flask import request -from flask import redirect from flask import Response -from flask import render_template -from flask import session +from flask import abort from flask import jsonify as flask_jsonify +from flask import redirect +from flask import render_template +from flask import request +from flask import session from flask import url_for +from flask_wtf.csrf import CSRFProtect from html2text import html2text -from itsdangerous import JSONWebSignatureSerializer from itsdangerous import BadSignature from passlib.hash import bcrypt from u2flib_server import u2f -from urllib.parse import urlparse, urlencode from werkzeug.utils import secure_filename -from flask_wtf.csrf import CSRFProtect import activitypub import config -from activitypub import ActivityType -from activitypub import clean_activity from activitypub import embed_collection -from utils.content_helper import parse_markdown -from config import KEY -from config import DB -from config import ME -from config import ID -from config import DOMAIN -from config import USERNAME -from config import BASE_URL -from config import ACTOR_SERVICE -from config import OBJECT_SERVICE -from config import PASS -from config import HEADERS -from config import VERSION -from config import DEBUG_MODE -from config import JWT from config import ADMIN_API_KEY +from config import BASE_URL +from config import DB +from config import DEBUG_MODE +from config import DOMAIN +from config import HEADERS +from config import ID +from config import JWT +from config import KEY +from config import ME +from config import PASS +from config import USERNAME +from config import VERSION from config import _drop_db from config import custom_cache_purge_hook -from utils.httpsig import HTTPSigAuth, verify_request +from little_boxes import activitypub as ap +from little_boxes.activitypub import ActivityType +from little_boxes.activitypub import clean_activity +from little_boxes.content_helper import parse_markdown +from little_boxes.errors import ActivityNotFoundError +from little_boxes.errors import Error +from little_boxes.errors import NotFromOutboxError +from little_boxes.httpsig import HTTPSigAuth + +# from little_boxes.httpsig import verify_request +from little_boxes.webfinger import get_actor_url +from little_boxes.webfinger import get_remote_follow_template from utils.key import get_secret_key -from utils.webfinger import get_remote_follow_template -from utils.webfinger import get_actor_url -from utils.errors import Error -from utils.errors import UnexpectedActivityTypeError -from utils.errors import BadActivityError -from utils.errors import NotFromOutboxError -from utils.errors import ActivityNotFoundError +from utils.object_service import ObjectService +OBJECT_SERVICE = ACTOR_SERVICE = ObjectService() + +back = activitypub.MicroblogPubBackend() +ap.use_backend(back) + +MY_PERSON = ap.Person(**ME) -from typing import Dict, Any - app = Flask(__name__) -app.secret_key = get_secret_key('flask') -app.config.update( - WTF_CSRF_CHECK_DEFAULT=False, -) +app.secret_key = get_secret_key("flask") +app.config.update(WTF_CSRF_CHECK_DEFAULT=False) csrf = CSRFProtect(app) logger = logging.getLogger(__name__) # Hook up Flask logging with gunicorn root_logger = logging.getLogger() -if os.getenv('FLASK_DEBUG'): +if os.getenv("FLASK_DEBUG"): logger.setLevel(logging.DEBUG) root_logger.setLevel(logging.DEBUG) else: - gunicorn_logger = logging.getLogger('gunicorn.error') + gunicorn_logger = logging.getLogger("gunicorn.error") root_logger.handlers = gunicorn_logger.handlers root_logger.setLevel(gunicorn_logger.level) -SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) +SIG_AUTH = HTTPSigAuth(KEY) + +OUTBOX = ap.Outbox(MY_PERSON) +INBOX = ap.Inbox(MY_PERSON) def verify_pass(pwd): - return bcrypt.verify(pwd, PASS) + return bcrypt.verify(pwd, PASS) + @app.context_processor def inject_config(): - return dict( - microblogpub_version=VERSION, - config=config, - logged_in=session.get('logged_in', False), - ) + return dict( + microblogpub_version=VERSION, + config=config, + logged_in=session.get("logged_in", False), + ) + @app.after_request def set_x_powered_by(response): - response.headers['X-Powered-By'] = 'microblog.pub' + response.headers["X-Powered-By"] = "microblog.pub" return response + # HTML/templates helper ALLOWED_TAGS = [ - 'a', - 'abbr', - 'acronym', - 'b', - 'blockquote', - 'code', - 'pre', - 'em', - 'i', - 'li', - 'ol', - 'strong', - 'ul', - 'span', - 'div', - 'p', - 'h1', - 'h2', - 'h3', - 'h4', - 'h5', - 'h6', + "a", + "abbr", + "acronym", + "b", + "blockquote", + "code", + "pre", + "em", + "i", + "li", + "ol", + "strong", + "ul", + "span", + "div", + "p", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", ] @@ -137,23 +147,23 @@ def clean_html(html): return bleach.clean(html, tags=ALLOWED_TAGS) -@app.template_filter() -def quote_plus(t): - return urllib.parse.quote_plus(t) +@app.template_filter() +def quote_plus(t): + return urllib.parse.quote_plus(t) -@app.template_filter() -def is_from_outbox(t): +@app.template_filter() +def is_from_outbox(t): return t.startswith(ID) -@app.template_filter() -def clean(html): - return clean_html(html) +@app.template_filter() +def clean(html): + return clean_html(html) -@app.template_filter() -def html2plaintext(body): +@app.template_filter() +def html2plaintext(body): return html2text(body) @@ -166,13 +176,16 @@ def domain(url): def get_actor(url): if not url: return None - print(f'GET_ACTOR {url}') + print(f"GET_ACTOR {url}") return ACTOR_SERVICE.get(url) + @app.template_filter() def format_time(val): if val: - return datetime.strftime(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), '%B %d, %Y, %H:%M %p') + return datetime.strftime( + datetime.strptime(val, "%Y-%m-%dT%H:%M:%SZ"), "%B %d, %Y, %H:%M %p" + ) return val @@ -180,26 +193,38 @@ def format_time(val): def format_timeago(val): if val: try: - return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), datetime.utcnow()) - except: - return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%S.%fZ'), datetime.utcnow()) - + return timeago.format( + datetime.strptime(val, "%Y-%m-%dT%H:%M:%SZ"), datetime.utcnow() + ) + except Exception: + return timeago.format( + datetime.strptime(val, "%Y-%m-%dT%H:%M:%S.%fZ"), datetime.utcnow() + ) + return val + def _is_img(filename): filename = filename.lower() - if (filename.endswith('.png') or filename.endswith('.jpg') or filename.endswith('.jpeg') or - filename.endswith('.gif') or filename.endswith('.svg')): + if ( + filename.endswith(".png") + or filename.endswith(".jpg") + or filename.endswith(".jpeg") + or filename.endswith(".gif") + or filename.endswith(".svg") + ): return True return False + @app.template_filter() def not_only_imgs(attachment): for a in attachment: - if not _is_img(a['url']): + if not _is_img(a["url"]): return True return False + @app.template_filter() def is_img(filename): return _is_img(filename) @@ -208,28 +233,29 @@ def is_img(filename): def login_required(f): @wraps(f) def decorated_function(*args, **kwargs): - if not session.get('logged_in'): - return redirect(url_for('login', next=request.url)) + if not session.get("logged_in"): + return redirect(url_for("login", next=request.url)) return f(*args, **kwargs) + return decorated_function def _api_required(): - if session.get('logged_in'): - if request.method not in ['GET', 'HEAD']: + if session.get("logged_in"): + if request.method not in ["GET", "HEAD"]: # If a standard API request is made with a "login session", it must havw a CSRF token csrf.protect() return # Token verification - token = request.headers.get('Authorization', '').replace('Bearer ', '') + token = request.headers.get("Authorization", "").replace("Bearer ", "") if not token: # IndieAuth token - token = request.form.get('access_token', '') + token = request.form.get("access_token", "") # Will raise a BadSignature on bad auth payload = JWT.loads(token) - logger.info(f'api call by {payload}') + logger.info(f"api call by {payload}") def api_required(f): @@ -241,31 +267,36 @@ def api_required(f): abort(401) return f(*args, **kwargs) + return decorated_function def jsonify(**data): - if '@context' not in data: - data['@context'] = config.CTX_AS + if "@context" not in data: + data["@context"] = config.CTX_AS return Response( response=json.dumps(data), - headers={'Content-Type': 'application/json' if app.debug else 'application/activity+json'}, + headers={ + "Content-Type": "application/json" + if app.debug + else "application/activity+json" + }, ) def is_api_request(): - h = request.headers.get('Accept') + h = request.headers.get("Accept") if h is None: return False - h = h.split(',')[0] - if h in HEADERS or h == 'application/json': + h = h.split(",")[0] + if h in HEADERS or h == "application/json": return True return False @app.errorhandler(ValueError) def handle_value_error(error): - logger.error(f'caught value error: {error!r}') + logger.error(f"caught value error: {error!r}") response = flask_jsonify(message=error.args[0]) response.status_code = 400 return response @@ -273,110 +304,114 @@ def handle_value_error(error): @app.errorhandler(Error) def handle_activitypub_error(error): - logger.error(f'caught activitypub error {error!r}') + logger.error(f"caught activitypub error {error!r}") response = flask_jsonify(error.to_dict()) response.status_code = error.status_code return response -# App routes +# App routes ####### # Login -@app.route('/logout') + +@app.route("/logout") @login_required def logout(): - session['logged_in'] = False - return redirect('/') + session["logged_in"] = False + return redirect("/") -@app.route('/login', methods=['POST', 'GET']) +@app.route("/login", methods=["POST", "GET"]) def login(): - devices = [doc['device'] for doc in DB.u2f.find()] + devices = [doc["device"] for doc in DB.u2f.find()] u2f_enabled = True if devices else False - if request.method == 'POST': + if request.method == "POST": csrf.protect() - pwd = request.form.get('pass') + pwd = request.form.get("pass") if pwd and verify_pass(pwd): if devices: - resp = json.loads(request.form.get('resp')) + resp = json.loads(request.form.get("resp")) print(resp) try: - u2f.complete_authentication(session['challenge'], resp) + u2f.complete_authentication(session["challenge"], resp) except ValueError as exc: - print('failed', exc) + print("failed", exc) abort(401) return finally: - session['challenge'] = None + session["challenge"] = None - session['logged_in'] = True - return redirect(request.args.get('redirect') or '/admin') + session["logged_in"] = True + return redirect(request.args.get("redirect") or "/admin") else: abort(401) payload = None if devices: payload = u2f.begin_authentication(ID, devices) - session['challenge'] = payload + session["challenge"] = payload return render_template( - 'login.html', - u2f_enabled=u2f_enabled, - me=ME, - payload=payload, + "login.html", u2f_enabled=u2f_enabled, me=ME, payload=payload ) -@app.route('/remote_follow', methods=['GET', 'POST']) +@app.route("/remote_follow", methods=["GET", "POST"]) def remote_follow(): - if request.method == 'GET': - return render_template('remote_follow.html') + if request.method == "GET": + return render_template("remote_follow.html") csrf.protect() - return redirect(get_remote_follow_template('@'+request.form.get('profile')).format(uri=f'{USERNAME}@{DOMAIN}')) + return redirect( + get_remote_follow_template("@" + request.form.get("profile")).format( + uri=f"{USERNAME}@{DOMAIN}" + ) + ) -@app.route('/authorize_follow', methods=['GET', 'POST']) +@app.route("/authorize_follow", methods=["GET", "POST"]) @login_required def authorize_follow(): - if request.method == 'GET': - return render_template('authorize_remote_follow.html', profile=request.args.get('profile')) + if request.method == "GET": + return render_template( + "authorize_remote_follow.html", profile=request.args.get("profile") + ) - actor = get_actor_url(request.form.get('profile')) + actor = get_actor_url(request.form.get("profile")) if not actor: abort(500) - if DB.following.find({'remote_actor': actor}).count() > 0: - return redirect('/following') + if DB.following.find({"remote_actor": actor}).count() > 0: + return redirect("/following") - follow = activitypub.Follow(object=actor) - follow.post_to_outbox() - return redirect('/following') + follow = activitypub.Follow(actor=MY_PERSON.id, object=actor) + OUTBOX.post(follow) + + return redirect("/following") -@app.route('/u2f/register', methods=['GET', 'POST']) +@app.route("/u2f/register", methods=["GET", "POST"]) @login_required def u2f_register(): # TODO(tsileo): ensure no duplicates - if request.method == 'GET': + if request.method == "GET": payload = u2f.begin_registration(ID) - session['challenge'] = payload - return render_template( - 'u2f.html', - payload=payload, - ) + session["challenge"] = payload + return render_template("u2f.html", payload=payload) else: - resp = json.loads(request.form.get('resp')) - device, device_cert = u2f.complete_registration(session['challenge'], resp) - session['challenge'] = None - DB.u2f.insert_one({'device': device, 'cert': device_cert}) - return '' + resp = json.loads(request.form.get("resp")) + device, device_cert = u2f.complete_registration(session["challenge"], resp) + session["challenge"] = None + DB.u2f.insert_one({"device": device, "cert": device_cert}) + return "" + ####### # Activity pub routes -@app.route('/') + +@app.route("/") def index(): if is_api_request(): return jsonify(**ME) @@ -384,31 +419,41 @@ def index(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 q = { - 'type': 'Create', - 'activity.object.type': 'Note', - 'activity.object.inReplyTo': None, - 'meta.deleted': False, + "type": "Create", + "activity.object.type": "Note", + "activity.object.inReplyTo": None, + "meta.deleted": False, } - c = request.args.get('cursor') + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.outbox.find({'$or': [q, {'type': 'Announce', 'meta.undo': False}]}, limit=limit).sort('_id', -1)) + outbox_data = list( + DB.outbox.find( + {"$or": [q, {"type": "Announce", "meta.undo": False}]}, limit=limit + ).sort("_id", -1) + ) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) for data in outbox_data: - if data['type'] == 'Announce': + if data["type"] == "Announce": print(data) - if data['activity']['object'].startswith('http'): - data['ref'] = {'activity': {'object': OBJECT_SERVICE.get(data['activity']['object'])}, 'meta': {}} - + if data["activity"]["object"].startswith("http"): + data["ref"] = { + "activity": { + "object": OBJECT_SERVICE.get(data["activity"]["object"]) + }, + "meta": {}, + } return render_template( - 'index.html', + "index.html", me=ME, - notes=DB.inbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False}).count(), + notes=DB.inbox.find( + {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + ).count(), followers=DB.followers.count(), following=DB.following.count(), outbox_data=outbox_data, @@ -416,34 +461,40 @@ def index(): ) -@app.route('/with_replies') +@app.route("/with_replies") def with_replies(): limit = 50 - q = { - 'type': 'Create', - 'activity.object.type': 'Note', - 'meta.deleted': False, - } - c = request.args.get('cursor') + q = {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.outbox.find({'$or': [q, {'type': 'Announce', 'meta.undo': False}]}, limit=limit).sort('_id', -1)) + outbox_data = list( + DB.outbox.find( + {"$or": [q, {"type": "Announce", "meta.undo": False}]}, limit=limit + ).sort("_id", -1) + ) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) for data in outbox_data: - if data['type'] == 'Announce': + if data["type"] == "Announce": print(data) - if data['activity']['object'].startswith('http'): - data['ref'] = {'activity': {'object': OBJECT_SERVICE.get(data['activity']['object'])}, 'meta': {}} - + if data["activity"]["object"].startswith("http"): + data["ref"] = { + "activity": { + "object": OBJECT_SERVICE.get(data["activity"]["object"]) + }, + "meta": {}, + } return render_template( - 'index.html', + "index.html", me=ME, - notes=DB.inbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False}).count(), + notes=DB.inbox.find( + {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + ).count(), followers=DB.followers.count(), following=DB.following.count(), outbox_data=outbox_data, @@ -452,17 +503,17 @@ def with_replies(): def _build_thread(data, include_children=True): - data['_requested'] = True - root_id = data['meta'].get('thread_root_parent', data['activity']['object']['id']) + data["_requested"] = True + root_id = data["meta"].get("thread_root_parent", data["activity"]["object"]["id"]) - thread_ids = data['meta'].get('thread_parents', []) + thread_ids = data["meta"].get("thread_parents", []) if include_children: - thread_ids.extend(data['meta'].get('thread_children', [])) + thread_ids.extend(data["meta"].get("thread_children", [])) query = { - 'activity.object.id': {'$in': thread_ids}, - 'type': 'Create', - 'meta.deleted': False, # TODO(tsileo): handle Tombstone instead of filtering them + "activity.object.id": {"$in": thread_ids}, + "type": "Create", + "meta.deleted": False, # TODO(tsileo): handle Tombstone instead of filtering them } # Fetch the root replies, and the children replies = [data] + list(DB.inbox.find(query)) + list(DB.outbox.find(query)) @@ -470,385 +521,427 @@ def _build_thread(data, include_children=True): # Index all the IDs in order to build a tree idx = {} for rep in replies: - rep_id = rep['activity']['object']['id'] + rep_id = rep["activity"]["object"]["id"] idx[rep_id] = rep.copy() - idx[rep_id]['_nodes'] = [] + idx[rep_id]["_nodes"] = [] # Build the tree for rep in replies: - rep_id = rep['activity']['object']['id'] + rep_id = rep["activity"]["object"]["id"] if rep_id == root_id: continue - reply_of = rep['activity']['object']['inReplyTo'] - idx[reply_of]['_nodes'].append(rep) + reply_of = rep["activity"]["object"]["inReplyTo"] + idx[reply_of]["_nodes"].append(rep) # Flatten the tree thread = [] + def _flatten(node, level=0): - node['_level'] = level + node["_level"] = level thread.append(node) - - for snode in sorted(idx[node['activity']['object']['id']]['_nodes'], key=lambda d: d['activity']['object']['published']): - _flatten(snode, level=level+1) + + for snode in sorted( + idx[node["activity"]["object"]["id"]]["_nodes"], + key=lambda d: d["activity"]["object"]["published"], + ): + _flatten(snode, level=level + 1) + _flatten(idx[root_id]) return thread -@app.route('/note/') -def note_by_id(note_id): - data = DB.outbox.find_one({'id': note_id}) - if not data: +@app.route("/note/") +def note_by_id(note_id): + data = DB.outbox.find_one({"remote_id": back.activity_url(note_id)}) + if not data: abort(404) - if data['meta'].get('deleted', False): + if data["meta"].get("deleted", False): abort(410) thread = _build_thread(data) + likes = list( + DB.inbox.find( + { + "meta.undo": False, + "type": ActivityType.LIKE.value, + "$or": [ + {"activity.object.id": data["activity"]["object"]["id"]}, + {"activity.object": data["activity"]["object"]["id"]}, + ], + } + ) + ) + likes = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in likes] - likes = list(DB.inbox.find({ - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - '$or': [{'activity.object.id': data['activity']['object']['id']}, - {'activity.object': data['activity']['object']['id']}], - })) - likes = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in likes] + shares = list( + DB.inbox.find( + { + "meta.undo": False, + "type": ActivityType.ANNOUNCE.value, + "$or": [ + {"activity.object.id": data["activity"]["object"]["id"]}, + {"activity.object": data["activity"]["object"]["id"]}, + ], + } + ) + ) + shares = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in shares] - shares = list(DB.inbox.find({ - 'meta.undo': False, - 'type': ActivityType.ANNOUNCE.value, - '$or': [{'activity.object.id': data['activity']['object']['id']}, - {'activity.object': data['activity']['object']['id']}], - })) - shares = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in shares] - - return render_template('note.html', likes=likes, shares=shares, me=ME, thread=thread, note=data) - - -@app.route('/nodeinfo') -def nodeinfo(): - return Response( - headers={'Content-Type': 'application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#'}, - response=json.dumps({ - 'version': '2.0', - 'software': {'name': 'microblogpub', 'version': f'Microblog.pub {VERSION}'}, - 'protocols': ['activitypub'], - 'services': {'inbound': [], 'outbound': []}, - 'openRegistrations': False, - 'usage': {'users': {'total': 1}, 'localPosts': DB.outbox.count()}, - 'metadata': { - 'sourceCode': 'https://github.com/tsileo/microblog.pub', - 'nodeName': f'@{USERNAME}@{DOMAIN}', - }, - }), + return render_template( + "note.html", likes=likes, shares=shares, me=ME, thread=thread, note=data ) -@app.route('/.well-known/nodeinfo') +@app.route("/nodeinfo") +def nodeinfo(): + return Response( + headers={ + "Content-Type": "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#" + }, + response=json.dumps( + { + "version": "2.0", + "software": { + "name": "microblogpub", + "version": f"Microblog.pub {VERSION}", + }, + "protocols": ["activitypub"], + "services": {"inbound": [], "outbound": []}, + "openRegistrations": False, + "usage": {"users": {"total": 1}, "localPosts": DB.outbox.count()}, + "metadata": { + "sourceCode": "https://github.com/tsileo/microblog.pub", + "nodeName": f"@{USERNAME}@{DOMAIN}", + }, + } + ), + ) + + +@app.route("/.well-known/nodeinfo") def wellknown_nodeinfo(): return flask_jsonify( links=[ { - 'rel': 'http://nodeinfo.diaspora.software/ns/schema/2.0', - 'href': f'{ID}/nodeinfo', + "rel": "http://nodeinfo.diaspora.software/ns/schema/2.0", + "href": f"{ID}/nodeinfo", } - - ], + ] ) -@app.route('/.well-known/webfinger') +@app.route("/.well-known/webfinger") def wellknown_webfinger(): """Enable WebFinger support, required for Mastodon interopability.""" - resource = request.args.get('resource') - if resource not in [f'acct:{USERNAME}@{DOMAIN}', ID]: + resource = request.args.get("resource") + if resource not in [f"acct:{USERNAME}@{DOMAIN}", ID]: abort(404) out = { - "subject": f'acct:{USERNAME}@{DOMAIN}', + "subject": f"acct:{USERNAME}@{DOMAIN}", "aliases": [ID], "links": [ - {"rel": "http://webfinger.net/rel/profile-page", "type": "text/html", "href": BASE_URL}, + { + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html", + "href": BASE_URL, + }, {"rel": "self", "type": "application/activity+json", "href": ID}, - {"rel":"http://ostatus.org/schema/1.0/subscribe", "template": BASE_URL+"/authorize_follow?profile={uri}"}, + { + "rel": "http://ostatus.org/schema/1.0/subscribe", + "template": BASE_URL + "/authorize_follow?profile={uri}", + }, ], } return Response( response=json.dumps(out), - headers={'Content-Type': 'application/jrd+json; charset=utf-8' if not app.debug else 'application/json'}, + headers={ + "Content-Type": "application/jrd+json; charset=utf-8" + if not app.debug + else "application/json" + }, ) def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: - if raw_doc['activity']['type'] != ActivityType.CREATE.value: + if raw_doc["activity"]["type"] != ActivityType.CREATE.value: return raw_doc - raw_doc['activity']['object']['replies'] = embed_collection( - raw_doc.get('meta', {}).get('count_direct_reply', 0), - f'{ID}/outbox/{raw_doc["id"]}/replies', + raw_doc["activity"]["object"]["replies"] = embed_collection( + raw_doc.get("meta", {}).get("count_direct_reply", 0), + f'{raw_doc["remote_id"]}/replies', ) - raw_doc['activity']['object']['likes'] = embed_collection( - raw_doc.get('meta', {}).get('count_like', 0), - f'{ID}/outbox/{raw_doc["id"]}/likes', + raw_doc["activity"]["object"]["likes"] = embed_collection( + raw_doc.get("meta", {}).get("count_like", 0), f'{raw_doc["remote_id"]}/likes' ) - raw_doc['activity']['object']['shares'] = embed_collection( - raw_doc.get('meta', {}).get('count_boost', 0), - f'{ID}/outbox/{raw_doc["id"]}/shares', + raw_doc["activity"]["object"]["shares"] = embed_collection( + raw_doc.get("meta", {}).get("count_boost", 0), f'{raw_doc["remote_id"]}/shares' ) return raw_doc def remove_context(activity: Dict[str, Any]) -> Dict[str, Any]: - if '@context' in activity: - del activity['@context'] + if "@context" in activity: + del activity["@context"] return activity def activity_from_doc(raw_doc: Dict[str, Any], embed: bool = False) -> Dict[str, Any]: raw_doc = add_extra_collection(raw_doc) - activity = clean_activity(raw_doc['activity']) + activity = clean_activity(raw_doc["activity"]) if embed: return remove_context(activity) return activity - -@app.route('/outbox', methods=['GET', 'POST']) -def outbox(): - if request.method == 'GET': - if not is_api_request(): - abort(404) +@app.route("/outbox", methods=["GET", "POST"]) +def outbox(): + if request.method == "GET": + if not is_api_request(): + abort(404) # TODO(tsileo): filter the outbox if not authenticated # FIXME(tsileo): filter deleted, add query support for build_ordered_collection q = { - 'meta.deleted': False, - #'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + "meta.deleted": False, + # 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } - return jsonify(**activitypub.build_ordered_collection( - DB.outbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: activity_from_doc(doc, embed=True), - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: activity_from_doc(doc, embed=True), + ) + ) # Handle POST request try: _api_required() except BadSignature: abort(401) - + data = request.get_json(force=True) print(data) - activity = activitypub.parse_activity(data) - - if activity.type_enum == ActivityType.NOTE: - activity = activity.build_create() - - activity.post_to_outbox() + activity = ap.parse_activity(data) + OUTBOX.post(activity) # Purge the cache if a custom hook is set, as new content was published custom_cache_purge_hook() - return Response(status=201, headers={'Location': activity.id}) + return Response(status=201, headers={"Location": activity.id}) -@app.route('/outbox/') +@app.route("/outbox/") def outbox_detail(item_id): - doc = DB.outbox.find_one({'id': item_id}) - if doc['meta'].get('deleted', False): - obj = activitypub.parse_activity(doc['activity']) + doc = DB.outbox.find_one({"remote_id": back.activity_url(item_id)}) + if doc["meta"].get("deleted", False): + obj = ap.parse_activity(doc["activity"]) resp = jsonify(**obj.get_object().get_tombstone()) resp.status_code = 410 return resp return jsonify(**activity_from_doc(doc)) -@app.route('/outbox//activity') +@app.route("/outbox//activity") def outbox_activity(item_id): # TODO(tsileo): handle Tombstone - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one( + {"remote_id": back.activity_url(item_id), "meta.deleted": False} + ) if not data: abort(404) obj = activity_from_doc(data) - if obj['type'] != ActivityType.CREATE.value: + if obj["type"] != ActivityType.CREATE.value: abort(404) - return jsonify(**obj['object']) + return jsonify(**obj["object"]) -@app.route('/outbox//replies') +@app.route("/outbox//replies") def outbox_activity_replies(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one( + {"remote_id": back.activity_url(item_id), "meta.deleted": False} + ) if not data: abort(404) - obj = activitypub.parse_activity(data['activity']) - if obj.type_enum != ActivityType.CREATE: + obj = ap.parse_activity(data["activity"]) + if obj.ACTIVITY_TYPE != ActivityType.CREATE: abort(404) q = { - 'meta.deleted': False, - 'type': ActivityType.CREATE.value, - 'activity.object.inReplyTo': obj.get_object().id, + "meta.deleted": False, + "type": ActivityType.CREATE.value, + "activity.object.inReplyTo": obj.get_object().id, } - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity']['object'], - col_name=f'outbox/{item_id}/replies', - first_page=request.args.get('page') == 'first', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["activity"]["object"], + col_name=f"outbox/{item_id}/replies", + first_page=request.args.get("page") == "first", + ) + ) -@app.route('/outbox//likes') +@app.route("/outbox//likes") def outbox_activity_likes(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one( + {"remote_id": back.activity_url(item_id), "meta.deleted": False} + ) if not data: abort(404) - obj = activitypub.parse_activity(data['activity']) - if obj.type_enum != ActivityType.CREATE: + obj = ap.parse_activity(data["activity"]) + if obj.ACTIVITY_TYPE != ActivityType.CREATE: abort(404) q = { - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - '$or': [{'activity.object.id': obj.get_object().id}, - {'activity.object': obj.get_object().id}], + "meta.undo": False, + "type": ActivityType.LIKE.value, + "$or": [ + {"activity.object.id": obj.get_object().id}, + {"activity.object": obj.get_object().id}, + ], } - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - col_name=f'outbox/{item_id}/likes', - first_page=request.args.get('page') == 'first', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: remove_context(doc["activity"]), + col_name=f"outbox/{item_id}/likes", + first_page=request.args.get("page") == "first", + ) + ) -@app.route('/outbox//shares') +@app.route("/outbox//shares") def outbox_activity_shares(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one( + {"remote_id": back.activity_url(item_id), "meta.deleted": False} + ) if not data: abort(404) - obj = activitypub.parse_activity(data['activity']) - if obj.type_enum != ActivityType.CREATE: + obj = ap.parse_activity(data["activity"]) + if obj.ACTIVITY_TYPE != ActivityType.CREATE: abort(404) q = { - 'meta.undo': False, - 'type': ActivityType.ANNOUNCE.value, - '$or': [{'activity.object.id': obj.get_object().id}, - {'activity.object': obj.get_object().id}], + "meta.undo": False, + "type": ActivityType.ANNOUNCE.value, + "$or": [ + {"activity.object.id": obj.get_object().id}, + {"activity.object": obj.get_object().id}, + ], } - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - col_name=f'outbox/{item_id}/shares', - first_page=request.args.get('page') == 'first', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: remove_context(doc["activity"]), + col_name=f"outbox/{item_id}/shares", + first_page=request.args.get("page") == "first", + ) + ) -@app.route('/admin', methods=['GET']) +@app.route("/admin", methods=["GET"]) @login_required def admin(): - q = { - 'meta.deleted': False, - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - } + q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} col_liked = DB.outbox.count(q) return render_template( - 'admin.html', - instances=list(DB.instances.find()), - inbox_size=DB.inbox.count(), - outbox_size=DB.outbox.count(), - object_cache_size=DB.objects_cache.count(), - actor_cache_size=DB.actors_cache.count(), - col_liked=col_liked, - col_followers=DB.followers.count(), - col_following=DB.following.count(), + "admin.html", + instances=list(DB.instances.find()), + inbox_size=DB.inbox.count(), + outbox_size=DB.outbox.count(), + object_cache_size=DB.objects_cache.count(), + actor_cache_size=DB.actors_cache.count(), + col_liked=col_liked, + col_followers=DB.followers.count(), + col_following=DB.following.count(), ) - -@app.route('/new', methods=['GET']) + +@app.route("/new", methods=["GET"]) @login_required def new(): reply_id = None - content = '' + content = "" thread = [] - if request.args.get('reply'): - data = DB.inbox.find_one({'activity.object.id': request.args.get('reply')}) + if request.args.get("reply"): + data = DB.inbox.find_one({"activity.object.id": request.args.get("reply")}) if not data: - data = DB.outbox.find_one({'activity.object.id': request.args.get('reply')}) + data = DB.outbox.find_one({"activity.object.id": request.args.get("reply")}) if not data: abort(400) - reply = activitypub.parse_activity(data['activity']) + reply = ap.parse_activity(data["activity"]) reply_id = reply.id - if reply.type_enum == ActivityType.CREATE: + if reply.ACTIVITY_TYPE == ActivityType.CREATE: reply_id = reply.get_object().id actor = reply.get_actor() domain = urlparse(actor.id).netloc # FIXME(tsileo): if reply of reply, fetch all participants - content = f'@{actor.preferredUsername}@{domain} ' - thread = _build_thread( - data, - include_children=False, - ) + content = f"@{actor.preferredUsername}@{domain} " + thread = _build_thread(data, include_children=False) - return render_template( - 'new.html', - reply=reply_id, - content=content, - thread=thread, - ) + return render_template("new.html", reply=reply_id, content=content, thread=thread) -@app.route('/notifications') +@app.route("/notifications") @login_required def notifications(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 q = { - 'type': 'Create', - 'activity.object.tag.type': 'Mention', - 'activity.object.tag.name': f'@{USERNAME}@{DOMAIN}', - 'meta.deleted': False, + "type": "Create", + "activity.object.tag.type": "Mention", + "activity.object.tag.name": f"@{USERNAME}@{DOMAIN}", + "meta.deleted": False, } # TODO(tsileo): also include replies via regex on Create replyTo - q = {'$or': [q, {'type': 'Follow'}, {'type': 'Accept'}, {'type': 'Undo', 'activity.object.type': 'Follow'}, - {'type': 'Announce', 'activity.object': {'$regex': f'^{BASE_URL}'}}, - {'type': 'Create', 'activity.object.inReplyTo': {'$regex': f'^{BASE_URL}'}}, - ]} + q = { + "$or": [ + q, + {"type": "Follow"}, + {"type": "Accept"}, + {"type": "Undo", "activity.object.type": "Follow"}, + {"type": "Announce", "activity.object": {"$regex": f"^{BASE_URL}"}}, + {"type": "Create", "activity.object.inReplyTo": {"$regex": f"^{BASE_URL}"}}, + ] + } print(q) - c = request.args.get('cursor') + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.inbox.find(q, limit=limit).sort('_id', -1)) + outbox_data = list(DB.inbox.find(q, limit=limit).sort("_id", -1)) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) # TODO(tsileo): fix the annonce handling, copy it from /stream - #for data in outbox_data: + # for data in outbox_data: # if data['type'] == 'Announce': # print(data) # if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: @@ -857,14 +950,10 @@ def notifications(): # else: # out.append(data) - return render_template( - 'stream.html', - inbox_data=outbox_data, - cursor=cursor, - ) + return render_template("stream.html", inbox_data=outbox_data, cursor=cursor) -@app.route('/api/key') +@app.route("/api/key") @login_required def api_user_key(): return flask_jsonify(api_key=ADMIN_API_KEY) @@ -878,25 +967,27 @@ def _user_api_arg(key: str, **kwargs): oid = request.args.get(key) or request.form.get(key) if not oid: - if 'default' in kwargs: - return kwargs.get('default') + if "default" in kwargs: + return kwargs.get("default") - raise ValueError(f'missing {key}') + raise ValueError(f"missing {key}") return oid def _user_api_get_note(from_outbox: bool = False): - oid = _user_api_arg('id') - note = activitypub.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE) + oid = _user_api_arg("id") + note = ap.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE) if from_outbox and not note.id.startswith(ID): - raise NotFromOutboxError(f'cannot delete {note.id}, id must be owned by the server') + raise NotFromOutboxError( + f"cannot load {note.id}, id must be owned by the server" + ) return note def _user_api_response(**kwargs): - _redirect = _user_api_arg('redirect', default=None) + _redirect = _user_api_arg("redirect", default=None) if _redirect: return redirect(_redirect) @@ -905,397 +996,430 @@ def _user_api_response(**kwargs): return resp -@app.route('/api/note/delete', methods=['POST']) +@app.route("/api/note/delete", methods=["POST"]) @api_required def api_delete(): """API endpoint to delete a Note activity.""" note = _user_api_get_note(from_outbox=True) delete = note.build_delete() - delete.post_to_outbox() + OUTBOX.post(delete) return _user_api_response(activity=delete.id) -@app.route('/api/boost', methods=['POST']) +@app.route("/api/boost", methods=["POST"]) @api_required def api_boost(): note = _user_api_get_note() - announce = note.build_announce() - announce.post_to_outbox() + announce = note.build_announce(MY_PERSON) + OUTBOX.post(announce) return _user_api_response(activity=announce.id) -@app.route('/api/like', methods=['POST']) +@app.route("/api/like", methods=["POST"]) @api_required def api_like(): note = _user_api_get_note() - like = note.build_like() - like.post_to_outbox() + like = note.build_like(MY_PERSON) + OUTBOX.post(like) return _user_api_response(activity=like.id) -@app.route('/api/undo', methods=['POST']) +@app.route("/api/undo", methods=["POST"]) @api_required def api_undo(): - oid = _user_api_arg('id') - doc = DB.outbox.find_one({'$or': [{'id': oid}, {'remote_id': oid}]}) + oid = _user_api_arg("id") + doc = DB.outbox.find_one( + {"$or": [{"remote_id": back.activity_url(oid)}, {"remote_id": oid}]} + ) if not doc: - raise ActivityNotFoundError(f'cannot found {oid}') + raise ActivityNotFoundError(f"cannot found {oid}") - obj = activitypub.parse_activity(doc.get('activity')) + obj = ap.parse_activity(doc.get("activity")) # FIXME(tsileo): detect already undo-ed and make this API call idempotent undo = obj.build_undo() - undo.post_to_outbox() + OUTBOX.post(undo) return _user_api_response(activity=undo.id) -@app.route('/stream') +@app.route("/stream") @login_required def stream(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 100 q = { - 'type': 'Create', - 'activity.object.type': 'Note', - 'activity.object.inReplyTo': None, - 'meta.deleted': False, + "type": "Create", + "activity.object.type": "Note", + "activity.object.inReplyTo": None, + "meta.deleted": False, } - c = request.args.get('cursor') + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.inbox.find( - { - '$or': [ - q, - { - 'type': 'Announce', - }, - ] - }, limit=limit).sort('activity.published', -1)) + outbox_data = list( + DB.inbox.find({"$or": [q, {"type": "Announce"}]}, limit=limit).sort( + "activity.published", -1 + ) + ) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) out = [] objcache = {} - cached = list(DB.objects_cache.find({'meta.part_of_stream': True}, limit=limit*3).sort('meta.announce_published', -1)) + cached = list( + DB.objects_cache.find({"meta.part_of_stream": True}, limit=limit * 3).sort( + "meta.announce_published", -1 + ) + ) for c in cached: - objcache[c['object_id']] = c['cached_object'] + objcache[c["object_id"]] = c["cached_object"] for data in outbox_data: - if data['type'] == 'Announce': - if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: - data['ref'] = {'activity': {'object': objcache[data['activity']['object']]}, 'meta': {}} + if data["type"] == "Announce": + if ( + data["activity"]["object"].startswith("http") + and data["activity"]["object"] in objcache + ): + data["ref"] = { + "activity": {"object": objcache[data["activity"]["object"]]}, + "meta": {}, + } out.append(data) else: - print('OMG', data) + print("OMG", data) else: out.append(data) - return render_template( - 'stream.html', - inbox_data=out, - cursor=cursor, - ) + return render_template("stream.html", inbox_data=out, cursor=cursor) -@app.route('/inbox', methods=['GET', 'POST']) -def inbox(): - if request.method == 'GET': - if not is_api_request(): - abort(404) +@app.route("/inbox", methods=["GET", "POST"]) +def inbox(): + if request.method == "GET": + if not is_api_request(): + abort(404) try: _api_required() except BadSignature: abort(404) - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q={'meta.deleted': False}, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q={"meta.deleted": False}, + cursor=request.args.get("cursor"), + map_func=lambda doc: remove_context(doc["activity"]), + ) + ) - data = request.get_json(force=True) - logger.debug(f'req_headers={request.headers}') - logger.debug(f'raw_data={data}') - try: + data = request.get_json(force=True) + logger.debug(f"req_headers={request.headers}") + logger.debug(f"raw_data={data}") + """try: if not verify_request(ACTOR_SERVICE): - raise Exception('failed to verify request') + raise Exception("failed to verify request") except Exception: - logger.exception('failed to verify request, trying to verify the payload by fetching the remote') + logger.exception( + "failed to verify request, trying to verify the payload by fetching the remote" + ) try: - data = OBJECT_SERVICE.get(data['id']) + data = OBJECT_SERVICE.get(data["id"]) except Exception: logger.exception(f'failed to fetch remote id at {data["id"]}') return Response( status=422, - headers={'Content-Type': 'application/json'}, - response=json.dumps({'error': 'failed to verify request (using HTTP signatures or fetching the IRI)'}), + headers={"Content-Type": "application/json"}, + response=json.dumps( + { + "error": "failed to verify request (using HTTP signatures or fetching the IRI)" + } + ), ) + """ + activity = ap.parse_activity(data) + logger.debug(f"inbox activity={activity}/{data}") + INBOX.post(activity) - activity = activitypub.parse_activity(data) - logger.debug(f'inbox activity={activity}/{data}') - activity.process_from_inbox() - - return Response( - status=201, - ) + return Response(status=201) -@app.route('/api/debug', methods=['GET', 'DELETE']) +def without_id(l): + out = [] + for d in l: + if "_id" in d: + del d["_id"] + out.append(d) + return out + + +@app.route("/api/debug", methods=["GET", "DELETE"]) @api_required def api_debug(): """Endpoint used/needed for testing, only works in DEBUG_MODE.""" if not DEBUG_MODE: - return flask_jsonify(message='DEBUG_MODE is off') + return flask_jsonify(message="DEBUG_MODE is off") - if request.method == 'DELETE': + if request.method == "DELETE": _drop_db() - return flask_jsonify(message='DB dropped') + return flask_jsonify(message="DB dropped") return flask_jsonify( inbox=DB.inbox.count(), outbox=DB.outbox.count(), + outbox_data=without_id(DB.outbox.find()), ) -@app.route('/api/upload', methods=['POST']) +@app.route("/api/upload", methods=["POST"]) @api_required def api_upload(): - file = request.files['file'] + file = request.files["file"] rfilename = secure_filename(file.filename) prefix = hashlib.sha256(os.urandom(32)).hexdigest()[:6] mtype = mimetypes.guess_type(rfilename)[0] - filename = f'{prefix}_{rfilename}' - file.save(os.path.join('static', 'media', filename)) + filename = f"{prefix}_{rfilename}" + file.save(os.path.join("static", "media", filename)) # Remove EXIF metadata - if filename.lower().endswith('.jpg') or filename.lower().endswith('.jpeg'): - piexif.remove(os.path.join('static', 'media', filename)) + if filename.lower().endswith(".jpg") or filename.lower().endswith(".jpeg"): + piexif.remove(os.path.join("static", "media", filename)) - print('upload OK') + print("upload OK") print(filename) attachment = [ - {'mediaType': mtype, - 'name': rfilename, - 'type': 'Document', - 'url': BASE_URL + f'/static/media/{filename}' - }, + { + "mediaType": mtype, + "name": rfilename, + "type": "Document", + "url": BASE_URL + f"/static/media/{filename}", + } ] print(attachment) - content = request.args.get('content') - to = request.args.get('to') - note = activitypub.Note( - cc=[ID+'/followers'], - to=[to if to else config.AS_PUBLIC], + content = request.args.get("content") + to = request.args.get("to") + note = ap.Note( + attributedTo=MY_PERSON.id, + cc=[ID + "/followers"], + to=[to if to else ap.AS_PUBLIC], content=content, # TODO(tsileo): handle markdown attachment=attachment, ) - print('post_note_init') + print("post_note_init") print(note) create = note.build_create() print(create) print(create.to_dict()) - create.post_to_outbox() - print('posted') - - return Response( - status=201, - response='OK', - ) + OUTBOX.post(create) + print("posted") + + return Response(status=201, response="OK") -@app.route('/api/new_note', methods=['POST']) -@api_required -def api_new_note(): - source = _user_api_arg('content') +@app.route("/api/new_note", methods=["POST"]) +@api_required +def api_new_note(): + source = _user_api_arg("content") if not source: - raise ValueError('missing content') - + raise ValueError("missing content") + _reply, reply = None, None try: - _reply = _user_api_arg('reply') + _reply = _user_api_arg("reply") except ValueError: pass - content, tags = parse_markdown(source) - to = request.args.get('to') - cc = [ID+'/followers'] - + content, tags = parse_markdown(source) + to = request.args.get("to") + cc = [ID + "/followers"] + if _reply: - reply = activitypub.parse_activity(OBJECT_SERVICE.get(_reply)) + reply = ap.parse_activity(OBJECT_SERVICE.get(_reply)) cc.append(reply.attributedTo) for tag in tags: - if tag['type'] == 'Mention': - cc.append(tag['href']) + if tag["type"] == "Mention": + cc.append(tag["href"]) - note = activitypub.Note( - cc=list(set(cc)), - to=[to if to else config.AS_PUBLIC], + note = ap.Note( + attributedTo=MY_PERSON.id, + cc=list(set(cc)), + to=[to if to else ap.AS_PUBLIC], content=content, tag=tags, - source={'mediaType': 'text/markdown', 'content': source}, - inReplyTo=reply.id if reply else None + source={"mediaType": "text/markdown", "content": source}, + inReplyTo=reply.id if reply else None, ) create = note.build_create() - create.post_to_outbox() + OUTBOX.post(create) return _user_api_response(activity=create.id) -@app.route('/api/stream') +@app.route("/api/stream") @api_required def api_stream(): return Response( - response=json.dumps(activitypub.build_inbox_json_feed('/api/stream', request.args.get('cursor'))), - headers={'Content-Type': 'application/json'}, + response=json.dumps( + activitypub.build_inbox_json_feed("/api/stream", request.args.get("cursor")) + ), + headers={"Content-Type": "application/json"}, ) -@app.route('/api/block', methods=['POST']) +@app.route("/api/block", methods=["POST"]) @api_required def api_block(): - actor = _user_api_arg('actor') + actor = _user_api_arg("actor") - existing = DB.outbox.find_one({ - 'type': ActivityType.BLOCK.value, - 'activity.object': actor, - 'meta.undo': False, - }) + existing = DB.outbox.find_one( + {"type": ActivityType.BLOCK.value, "activity.object": actor, "meta.undo": False} + ) if existing: - return _user_api_response(activity=existing['activity']['id']) + return _user_api_response(activity=existing["activity"]["id"]) - block = activitypub.Block(object=actor) - block.post_to_outbox() + block = ap.Block(actor=MY_PERSON.id, object=actor) + OUTBOX.post(block) return _user_api_response(activity=block.id) -@app.route('/api/follow', methods=['POST']) +@app.route("/api/follow", methods=["POST"]) @api_required def api_follow(): - actor = _user_api_arg('actor') + actor = _user_api_arg("actor") - existing = DB.following.find_one({'remote_actor': actor}) + existing = DB.following.find_one({"remote_actor": actor}) if existing: - return _user_api_response(activity=existing['activity']['id']) + return _user_api_response(activity=existing["activity"]["id"]) - follow = activitypub.Follow(object=actor) - follow.post_to_outbox() + follow = ap.Follow(actor=MY_PERSON.id, object=actor) + OUTBOX.post(follow) return _user_api_response(activity=follow.id) -@app.route('/followers') -def followers(): - if is_api_request(): +@app.route("/followers") +def followers(): + if is_api_request(): return jsonify( **activitypub.build_ordered_collection( DB.followers, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['remote_actor'], + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["remote_actor"], ) - ) + ) - followers = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.followers.find(limit=50)] - return render_template( - 'followers.html', + followers = [ + ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.followers.find(limit=50) + ] + return render_template( + "followers.html", me=ME, - notes=DB.inbox.find({'object.object.type': 'Note'}).count(), + notes=DB.inbox.find({"object.object.type": "Note"}).count(), followers=DB.followers.count(), following=DB.following.count(), followers_data=followers, ) -@app.route('/following') +@app.route("/following") def following(): if is_api_request(): return jsonify( **activitypub.build_ordered_collection( DB.following, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['remote_actor'], - ), + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["remote_actor"], + ) ) - - following = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.following.find(limit=50)] + + following = [ + ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.following.find(limit=50) + ] return render_template( - 'following.html', + "following.html", me=ME, - notes=DB.inbox.find({'object.object.type': 'Note'}).count(), + notes=DB.inbox.find({"object.object.type": "Note"}).count(), followers=DB.followers.count(), following=DB.following.count(), following_data=following, ) -@app.route('/tags/') +@app.route("/tags/") def tags(tag): - if not DB.outbox.count({'activity.object.tag.type': 'Hashtag', 'activity.object.tag.name': '#'+tag}): + if not DB.outbox.count( + {"activity.object.tag.type": "Hashtag", "activity.object.tag.name": "#" + tag} + ): abort(404) if not is_api_request(): return render_template( - 'tags.html', + "tags.html", tag=tag, - outbox_data=DB.outbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False, - 'activity.object.tag.type': 'Hashtag', - 'activity.object.tag.name': '#'+tag}), + outbox_data=DB.outbox.find( + { + "type": "Create", + "activity.object.type": "Note", + "meta.deleted": False, + "activity.object.tag.type": "Hashtag", + "activity.object.tag.name": "#" + tag, + } + ), ) q = { - 'meta.deleted': False, - 'meta.undo': False, - 'type': ActivityType.CREATE.value, - 'activity.object.tag.type': 'Hashtag', - 'activity.object.tag.name': '#'+tag, + "meta.deleted": False, + "meta.undo": False, + "type": ActivityType.CREATE.value, + "activity.object.tag.type": "Hashtag", + "activity.object.tag.name": "#" + tag, } - return jsonify(**activitypub.build_ordered_collection( - DB.outbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity']['object']['id'], - col_name=f'tags/{tag}', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["activity"]["object"]["id"], + col_name=f"tags/{tag}", + ) + ) -@app.route('/liked') +@app.route("/liked") def liked(): if not is_api_request(): abort(404) - q = { - 'meta.deleted': False, - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - } - return jsonify(**activitypub.build_ordered_collection( - DB.outbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity']['object'], - col_name='liked', - )) + q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} + return jsonify( + **activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["activity"]["object"], + col_name="liked", + ) + ) + ####### # IndieAuth def build_auth_resp(payload): - if request.headers.get('Accept') == 'application/json': + if request.headers.get("Accept") == "application/json": return Response( status=200, - headers={'Content-Type': 'application/json'}, + headers={"Content-Type": "application/json"}, response=json.dumps(payload), ) return Response( status=200, - headers={'Content-Type': 'application/x-www-form-urlencoded'}, + headers={"Content-Type": "application/x-www-form-urlencoded"}, response=urlencode(payload), ) @@ -1308,43 +1432,37 @@ def _get_prop(props, name, default=None): return items return default + def get_client_id_data(url): data = mf2py.parse(url=url) - for item in data['items']: - if 'h-x-app' in item['type'] or 'h-app' in item['type']: - props = item.get('properties', {}) + for item in data["items"]: + if "h-x-app" in item["type"] or "h-app" in item["type"]: + props = item.get("properties", {}) print(props) return dict( - logo=_get_prop(props, 'logo'), - name=_get_prop(props, 'name'), - url=_get_prop(props, 'url'), + logo=_get_prop(props, "logo"), + name=_get_prop(props, "name"), + url=_get_prop(props, "url"), ) - return dict( - logo=None, - name=url, - url=url, - ) + return dict(logo=None, name=url, url=url) -@app.route('/indieauth/flow', methods=['POST']) -@login_required -def indieauth_flow(): - auth = dict( - scope=' '.join(request.form.getlist('scopes')), - me=request.form.get('me'), - client_id=request.form.get('client_id'), - state=request.form.get('state'), - redirect_uri=request.form.get('redirect_uri'), - response_type=request.form.get('response_type'), +@app.route("/indieauth/flow", methods=["POST"]) +@login_required +def indieauth_flow(): + auth = dict( + scope=" ".join(request.form.getlist("scopes")), + me=request.form.get("me"), + client_id=request.form.get("client_id"), + state=request.form.get("state"), + redirect_uri=request.form.get("redirect_uri"), + response_type=request.form.get("response_type"), ) - code = binascii.hexlify(os.urandom(8)).decode('utf-8') - auth.update( - code=code, - verified=False, - ) + code = binascii.hexlify(os.urandom(8)).decode("utf-8") + auth.update(code=code, verified=False) print(auth) - if not auth['redirect_uri']: + if not auth["redirect_uri"]: abort(500) DB.indieauth.insert_one(auth) @@ -1354,23 +1472,23 @@ def indieauth_flow(): return redirect(red) -# @app.route('/indieauth', methods=['GET', 'POST']) -def indieauth_endpoint(): - if request.method == 'GET': - if not session.get('logged_in'): - return redirect(url_for('login', next=request.url)) +# @app.route('/indieauth', methods=['GET', 'POST']) +def indieauth_endpoint(): + if request.method == "GET": + if not session.get("logged_in"): + return redirect(url_for("login", next=request.url)) - me = request.args.get('me') - # FIXME(tsileo): ensure me == ID - client_id = request.args.get('client_id') - redirect_uri = request.args.get('redirect_uri') - state = request.args.get('state', '') - response_type = request.args.get('response_type', 'id') - scope = request.args.get('scope', '').split() + me = request.args.get("me") + # FIXME(tsileo): ensure me == ID + client_id = request.args.get("client_id") + redirect_uri = request.args.get("redirect_uri") + state = request.args.get("state", "") + response_type = request.args.get("response_type", "id") + scope = request.args.get("scope", "").split() - print('STATE', state) + print("STATE", state) return render_template( - 'indieauth_flow.html', + "indieauth_flow.html", client=get_client_id_data(client_id), scopes=scope, redirect_uri=redirect_uri, @@ -1381,14 +1499,18 @@ def indieauth_endpoint(): ) # Auth verification via POST - code = request.form.get('code') - redirect_uri = request.form.get('redirect_uri') - client_id = request.form.get('client_id') + code = request.form.get("code") + redirect_uri = request.form.get("redirect_uri") + client_id = request.form.get("client_id") auth = DB.indieauth.find_one_and_update( - {'code': code, 'redirect_uri': redirect_uri, 'client_id': client_id}, #}, # , 'verified': False}, - {'$set': {'verified': True}}, - sort=[('_id', pymongo.DESCENDING)], + { + "code": code, + "redirect_uri": redirect_uri, + "client_id": client_id, + }, # }, # , 'verified': False}, + {"$set": {"verified": True}}, + sort=[("_id", pymongo.DESCENDING)], ) print(auth) print(code, redirect_uri, client_id) @@ -1397,33 +1519,42 @@ def indieauth_endpoint(): abort(403) return - session['logged_in'] = True - me = auth['me'] - state = auth['state'] - scope = ' '.join(auth['scope']) - print('STATE', state) - return build_auth_resp({'me': me, 'state': state, 'scope': scope}) + session["logged_in"] = True + me = auth["me"] + state = auth["state"] + scope = " ".join(auth["scope"]) + print("STATE", state) + return build_auth_resp({"me": me, "state": state, "scope": scope}) -@app.route('/token', methods=['GET', 'POST']) +@app.route("/token", methods=["GET", "POST"]) def token_endpoint(): - if request.method == 'POST': - code = request.form.get('code') - me = request.form.get('me') - redirect_uri = request.form.get('redirect_uri') - client_id = request.form.get('client_id') + if request.method == "POST": + code = request.form.get("code") + me = request.form.get("me") + redirect_uri = request.form.get("redirect_uri") + client_id = request.form.get("client_id") - auth = DB.indieauth.find_one({'code': code, 'me': me, 'redirect_uri': redirect_uri, 'client_id': client_id}) + auth = DB.indieauth.find_one( + { + "code": code, + "me": me, + "redirect_uri": redirect_uri, + "client_id": client_id, + } + ) if not auth: abort(403) - scope = ' '.join(auth['scope']) - payload = dict(me=me, client_id=client_id, scope=scope, ts=datetime.now().timestamp()) - token = JWT.dumps(payload).decode('utf-8') + scope = " ".join(auth["scope"]) + payload = dict( + me=me, client_id=client_id, scope=scope, ts=datetime.now().timestamp() + ) + token = JWT.dumps(payload).decode("utf-8") - return build_auth_resp({'me': me, 'scope': scope, 'access_token': token}) + return build_auth_resp({"me": me, "scope": scope, "access_token": token}) # Token verification - token = request.headers.get('Authorization').replace('Bearer ', '') + token = request.headers.get("Authorization").replace("Bearer ", "") try: payload = JWT.loads(token) except BadSignature: @@ -1431,8 +1562,10 @@ def token_endpoint(): # TODO(tsileo): handle expiration - return build_auth_resp({ - 'me': payload['me'], - 'scope': payload['scope'], - 'client_id': payload['client_id'], - }) + return build_auth_resp( + { + "me": payload["me"], + "scope": payload["scope"], + "client_id": payload["client_id"], + } + ) diff --git a/config.py b/config.py index 506a4d3..44659c1 100644 --- a/config.py +++ b/config.py @@ -1,15 +1,17 @@ -import subprocess import os -import yaml -from pymongo import MongoClient -import requests -from itsdangerous import JSONWebSignatureSerializer +import subprocess from datetime import datetime -from utils import strtobool -from utils.key import Key, KEY_DIR, get_secret_key -from utils.actor_service import ActorService -from utils.object_service import ObjectService +import requests +import yaml +from itsdangerous import JSONWebSignatureSerializer +from pymongo import MongoClient + +from little_boxes import strtobool +from utils.key import KEY_DIR +from utils.key import get_key +from utils.key import get_secret_key + def noop(): pass @@ -17,103 +19,94 @@ def noop(): CUSTOM_CACHE_HOOKS = False try: - from cache_hooks import purge as custom_cache_purge_hook + from cache_hooks import purge as custom_cache_purge_hook except ModuleNotFoundError: custom_cache_purge_hook = noop -VERSION = subprocess.check_output(['git', 'describe', '--always']).split()[0].decode('utf-8') +VERSION = ( + subprocess.check_output(["git", "describe", "--always"]).split()[0].decode("utf-8") +) -DEBUG_MODE = strtobool(os.getenv('MICROBLOGPUB_DEBUG', 'false')) +DEBUG_MODE = strtobool(os.getenv("MICROBLOGPUB_DEBUG", "false")) -CTX_AS = 'https://www.w3.org/ns/activitystreams' -CTX_SECURITY = 'https://w3id.org/security/v1' -AS_PUBLIC = 'https://www.w3.org/ns/activitystreams#Public' +CTX_AS = "https://www.w3.org/ns/activitystreams" +CTX_SECURITY = "https://w3id.org/security/v1" +AS_PUBLIC = "https://www.w3.org/ns/activitystreams#Public" HEADERS = [ - 'application/activity+json', - 'application/ld+json;profile=https://www.w3.org/ns/activitystreams', + "application/activity+json", + "application/ld+json;profile=https://www.w3.org/ns/activitystreams", 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - 'application/ld+json', + "application/ld+json", ] -with open(os.path.join(KEY_DIR, 'me.yml')) as f: +with open(os.path.join(KEY_DIR, "me.yml")) as f: conf = yaml.load(f) - USERNAME = conf['username'] - NAME = conf['name'] - DOMAIN = conf['domain'] - SCHEME = 'https' if conf.get('https', True) else 'http' - BASE_URL = SCHEME + '://' + DOMAIN + USERNAME = conf["username"] + NAME = conf["name"] + DOMAIN = conf["domain"] + SCHEME = "https" if conf.get("https", True) else "http" + BASE_URL = SCHEME + "://" + DOMAIN ID = BASE_URL - SUMMARY = conf['summary'] - ICON_URL = conf['icon_url'] - PASS = conf['pass'] - PUBLIC_INSTANCES = conf.get('public_instances', []) + SUMMARY = conf["summary"] + ICON_URL = conf["icon_url"] + PASS = conf["pass"] + PUBLIC_INSTANCES = conf.get("public_instances", []) # TODO(tsileo): choose dark/light style - THEME_COLOR = conf.get('theme_color') + THEME_COLOR = conf.get("theme_color") USER_AGENT = ( - f'{requests.utils.default_user_agent()} ' - f'(microblog.pub/{VERSION}; +{BASE_URL})' + f"{requests.utils.default_user_agent()} " f"(microblog.pub/{VERSION}; +{BASE_URL})" ) -# TODO(tsileo): use 'mongo:27017; -# mongo_client = MongoClient(host=['mongo:27017']) mongo_client = MongoClient( - host=[os.getenv('MICROBLOGPUB_MONGODB_HOST', 'localhost:27017')], + host=[os.getenv("MICROBLOGPUB_MONGODB_HOST", "localhost:27017")] ) -DB_NAME = '{}_{}'.format(USERNAME, DOMAIN.replace('.', '_')) +DB_NAME = "{}_{}".format(USERNAME, DOMAIN.replace(".", "_")) DB = mongo_client[DB_NAME] + def _drop_db(): if not DEBUG_MODE: return mongo_client.drop_database(DB_NAME) -KEY = Key(USERNAME, DOMAIN, create=True) + +KEY = get_key(ID, USERNAME, DOMAIN) -JWT_SECRET = get_secret_key('jwt') +JWT_SECRET = get_secret_key("jwt") JWT = JSONWebSignatureSerializer(JWT_SECRET) + def _admin_jwt_token() -> str: - return JWT.dumps({'me': 'ADMIN', 'ts': datetime.now().timestamp()}).decode('utf-8') # type: ignore + return JWT.dumps( # type: ignore + {"me": "ADMIN", "ts": datetime.now().timestamp()} + ).decode( # type: ignore + "utf-8" + ) -ADMIN_API_KEY = get_secret_key('admin_api_key', _admin_jwt_token) +ADMIN_API_KEY = get_secret_key("admin_api_key", _admin_jwt_token) ME = { - "@context": [ - CTX_AS, - CTX_SECURITY, - ], + "@context": [CTX_AS, CTX_SECURITY], "type": "Person", "id": ID, - "following": ID+"/following", - "followers": ID+"/followers", - "liked": ID+"/liked", - "inbox": ID+"/inbox", - "outbox": ID+"/outbox", + "following": ID + "/following", + "followers": ID + "/followers", + "liked": ID + "/liked", + "inbox": ID + "/inbox", + "outbox": ID + "/outbox", "preferredUsername": USERNAME, "name": NAME, "summary": SUMMARY, "endpoints": {}, "url": ID, - "icon": { - "mediaType": "image/png", - "type": "Image", - "url": ICON_URL, - }, - "publicKey": { - "id": ID+"#main-key", - "owner": ID, - "publicKeyPem": KEY.pubkey_pem, - }, + "icon": {"mediaType": "image/png", "type": "Image", "url": ICON_URL}, + "publicKey": KEY.to_dict(), } -print(ME) - -ACTOR_SERVICE = ActorService(USER_AGENT, DB.actors_cache, ID, ME, DB.instances) -OBJECT_SERVICE = ObjectService(USER_AGENT, DB.objects_cache, DB.inbox, DB.outbox, DB.instances) diff --git a/dev-requirements.txt b/dev-requirements.txt index 62e71f2..7db7fab 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,8 @@ +git+https://github.com/tsileo/little-boxes.git pytest requests html2text pyyaml flake8 mypy +black diff --git a/requirements.txt b/requirements.txt index 425405f..eb16141 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,21 +2,19 @@ libsass gunicorn piexif requests -markdown python-u2flib-server Flask Flask-WTF Celery pymongo -pyld timeago bleach -pycryptodome html2text feedgen itsdangerous bcrypt mf2py passlib -pyyaml git+https://github.com/erikriver/opengraph.git +git+https://github.com/tsileo/little-boxes.git +pyyaml diff --git a/tasks.py b/tasks.py index a30854a..a5c85db 100644 --- a/tasks.py +++ b/tasks.py @@ -1,47 +1,52 @@ -import os import json import logging +import os import random import requests from celery import Celery from requests.exceptions import HTTPError -from config import HEADERS -from config import ID from config import DB +from config import HEADERS from config import KEY from config import USER_AGENT -from utils.httpsig import HTTPSigAuth +from little_boxes.httpsig import HTTPSigAuth +from little_boxes.linked_data_sig import generate_signature from utils.opengraph import fetch_og_metadata -from utils.linked_data_sig import generate_signature - log = logging.getLogger(__name__) -app = Celery('tasks', broker=os.getenv('MICROBLOGPUB_AMQP_BROKER', 'pyamqp://guest@localhost//')) -SigAuth = HTTPSigAuth(ID+'#main-key', KEY.privkey) +app = Celery( + "tasks", broker=os.getenv("MICROBLOGPUB_AMQP_BROKER", "pyamqp://guest@localhost//") +) +SigAuth = HTTPSigAuth(KEY) @app.task(bind=True, max_retries=12) def post_to_inbox(self, payload: str, to: str) -> None: try: - log.info('payload=%s', payload) - log.info('generating sig') + log.info("payload=%s", payload) + log.info("generating sig") signed_payload = json.loads(payload) - generate_signature(signed_payload, KEY.privkey) - log.info('to=%s', to) - resp = requests.post(to, data=json.dumps(signed_payload), auth=SigAuth, headers={ - 'Content-Type': HEADERS[1], - 'Accept': HEADERS[1], - 'User-Agent': USER_AGENT, - }) - log.info('resp=%s', resp) - log.info('resp_body=%s', resp.text) + generate_signature(signed_payload, KEY) + log.info("to=%s", to) + resp = requests.post( + to, + data=json.dumps(signed_payload), + auth=SigAuth, + headers={ + "Content-Type": HEADERS[1], + "Accept": HEADERS[1], + "User-Agent": USER_AGENT, + }, + ) + log.info("resp=%s", resp) + log.info("resp_body=%s", resp.text) resp.raise_for_status() except HTTPError as err: - log.exception('request failed') + log.exception("request failed") if 400 >= err.response.status_code >= 499: - log.info('client error, no retry') + log.info("client error, no retry") return self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) @@ -49,11 +54,15 @@ def post_to_inbox(self, payload: str, to: str) -> None: @app.task(bind=True, max_retries=12) def fetch_og(self, col, remote_id): try: - log.info('fetch_og_meta remote_id=%s col=%s', remote_id, col) - if col == 'INBOX': - log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.inbox, remote_id)) - elif col == 'OUTBOX': - log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.outbox, remote_id)) + log.info("fetch_og_meta remote_id=%s col=%s", remote_id, col) + if col == "INBOX": + log.info( + "%d links saved", fetch_og_metadata(USER_AGENT, DB.inbox, remote_id) + ) + elif col == "OUTBOX": + log.info( + "%d links saved", fetch_og_metadata(USER_AGENT, DB.outbox, remote_id) + ) except Exception as err: - self.log.exception('failed') + self.log.exception("failed") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) diff --git a/tests/federation_test.py b/tests/federation_test.py index e050afc..6e0a7ea 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -1,12 +1,12 @@ -import time import os +import time +from typing import List +from typing import Tuple import requests from html2text import html2text -from utils import activitypub_utils -from typing import Tuple -from typing import List +from little_boxes.collection import parse_collection def resp2plaintext(resp): @@ -22,33 +22,38 @@ class Instance(object): self.docker_url = docker_url or host_url self._create_delay = 10 with open( - os.path.join(os.path.dirname(os.path.abspath(__file__)), f'fixtures/{name}/config/admin_api_key.key') + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + f"fixtures/{name}/config/admin_api_key.key", + ) ) as f: api_key = f.read() - self._auth_headers = {'Authorization': f'Bearer {api_key}'} + self._auth_headers = {"Authorization": f"Bearer {api_key}"} - def _do_req(self, url, headers): + def _do_req(self, url): """Used to parse collection.""" url = url.replace(self.docker_url, self.host_url) - resp = requests.get(url, headers=headers) + resp = requests.get(url, headers={'Accept': 'application/activity+json'}) resp.raise_for_status() return resp.json() def _parse_collection(self, payload=None, url=None): """Parses a collection (go through all the pages).""" - return activitypub_utils.parse_collection(url=url, payload=payload, do_req=self._do_req) + return parse_collection( + url=url, payload=payload, fetcher=self._do_req, + ) def ping(self): """Ensures the homepage is reachable.""" - resp = requests.get(f'{self.host_url}/') + resp = requests.get(f"{self.host_url}/") resp.raise_for_status() assert resp.status_code == 200 def debug(self): """Returns the debug infos (number of items in the inbox/outbox.""" resp = requests.get( - f'{self.host_url}/api/debug', - headers={**self._auth_headers, 'Accept': 'application/json'}, + f"{self.host_url}/api/debug", + headers={**self._auth_headers, "Accept": "application/json"}, ) resp.raise_for_status() @@ -57,8 +62,8 @@ class Instance(object): def drop_db(self): """Drops the MongoDB DB.""" resp = requests.delete( - f'{self.host_url}/api/debug', - headers={**self._auth_headers, 'Accept': 'application/json'}, + f"{self.host_url}/api/debug", + headers={**self._auth_headers, "Accept": "application/json"}, ) resp.raise_for_status() @@ -68,100 +73,92 @@ class Instance(object): """Blocks an actor.""" # Instance1 follows instance2 resp = requests.post( - f'{self.host_url}/api/block', - params={'actor': actor_url}, + f"{self.host_url}/api/block", + params={"actor": actor_url}, headers=self._auth_headers, ) assert resp.status_code == 201 # We need to wait for the Follow/Accept dance - time.sleep(self._create_delay/2) - return resp.json().get('activity') + time.sleep(self._create_delay / 2) + return resp.json().get("activity") - def follow(self, instance: 'Instance') -> str: + def follow(self, instance: "Instance") -> str: """Follows another instance.""" # Instance1 follows instance2 resp = requests.post( - f'{self.host_url}/api/follow', - json={'actor': instance.docker_url}, + f"{self.host_url}/api/follow", + json={"actor": instance.docker_url}, headers=self._auth_headers, ) assert resp.status_code == 201 # We need to wait for the Follow/Accept dance time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def new_note(self, content, reply=None) -> str: """Creates a new note.""" - params = {'content': content} + params = {"content": content} if reply: - params['reply'] = reply + params["reply"] = reply resp = requests.post( - f'{self.host_url}/api/new_note', - json=params, - headers=self._auth_headers, + f"{self.host_url}/api/new_note", json=params, headers=self._auth_headers ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def boost(self, oid: str) -> str: """Creates an Announce activity.""" resp = requests.post( - f'{self.host_url}/api/boost', - json={'id': oid}, - headers=self._auth_headers, + f"{self.host_url}/api/boost", json={"id": oid}, headers=self._auth_headers ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def like(self, oid: str) -> str: """Creates a Like activity.""" resp = requests.post( - f'{self.host_url}/api/like', - json={'id': oid}, - headers=self._auth_headers, + f"{self.host_url}/api/like", json={"id": oid}, headers=self._auth_headers ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def delete(self, oid: str) -> str: """Creates a Delete activity.""" resp = requests.post( - f'{self.host_url}/api/note/delete', - json={'id': oid}, + f"{self.host_url}/api/note/delete", + json={"id": oid}, headers=self._auth_headers, ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def undo(self, oid: str) -> str: """Creates a Undo activity.""" resp = requests.post( - f'{self.host_url}/api/undo', - json={'id': oid}, - headers=self._auth_headers, + f"{self.host_url}/api/undo", json={"id": oid}, headers=self._auth_headers ) assert resp.status_code == 201 # We need to wait for the Follow/Accept dance time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def followers(self) -> List[str]: """Parses the followers collection.""" resp = requests.get( - f'{self.host_url}/followers', - headers={'Accept': 'application/activity+json'}, + f"{self.host_url}/followers", + headers={"Accept": "application/activity+json"}, ) resp.raise_for_status() @@ -172,8 +169,8 @@ class Instance(object): def following(self): """Parses the following collection.""" resp = requests.get( - f'{self.host_url}/following', - headers={'Accept': 'application/activity+json'}, + f"{self.host_url}/following", + headers={"Accept": "application/activity+json"}, ) resp.raise_for_status() @@ -184,8 +181,8 @@ class Instance(object): def outbox(self): """Returns the instance outbox.""" resp = requests.get( - f'{self.host_url}/following', - headers={'Accept': 'application/activity+json'}, + f"{self.host_url}/following", + headers={"Accept": "application/activity+json"}, ) resp.raise_for_status() return resp.json() @@ -194,7 +191,7 @@ class Instance(object): """Fetches a specific item from the instance outbox.""" resp = requests.get( aid.replace(self.docker_url, self.host_url), - headers={'Accept': 'application/activity+json'}, + headers={"Accept": "application/activity+json"}, ) resp.raise_for_status() return resp.json() @@ -202,8 +199,8 @@ class Instance(object): def stream_jsonfeed(self): """Returns the "stream"'s JSON feed.""" resp = requests.get( - f'{self.host_url}/api/stream', - headers={**self._auth_headers, 'Accept': 'application/json'}, + f"{self.host_url}/api/stream", + headers={**self._auth_headers, "Accept": "application/json"}, ) resp.raise_for_status() return resp.json() @@ -211,10 +208,14 @@ class Instance(object): def _instances() -> Tuple[Instance, Instance]: """Initializes the client for the two test instances.""" - instance1 = Instance('instance1', 'http://localhost:5006', 'http://instance1_web_1:5005') + instance1 = Instance( + "instance1", "http://localhost:5006", "http://instance1_web_1:5005" + ) instance1.ping() - instance2 = Instance('instance2', 'http://localhost:5007', 'http://instance2_web_1:5005') + instance2 = Instance( + "instance2", "http://localhost:5007", "http://instance2_web_1:5005" + ) instance2.ping() # Return the DB @@ -230,12 +231,12 @@ def test_follow() -> None: # Instance1 follows instance2 instance1.follow(instance2) instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 1 # An Accept activity should be there - assert instance1_debug['outbox'] == 1 # We've sent a Follow activity + assert instance1_debug["inbox"] == 1 # An Accept activity should be there + assert instance1_debug["outbox"] == 1 # We've sent a Follow activity instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 1 # An Follow activity should be there - assert instance2_debug['outbox'] == 1 # We've sent a Accept activity + assert instance2_debug["inbox"] == 1 # An Follow activity should be there + assert instance2_debug["outbox"] == 1 # We've sent a Accept activity assert instance2.followers() == [instance1.docker_url] assert instance1.following() == [instance2.docker_url] @@ -247,12 +248,12 @@ def test_follow_unfollow(): # Instance1 follows instance2 follow_id = instance1.follow(instance2) instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 1 # An Accept activity should be there - assert instance1_debug['outbox'] == 1 # We've sent a Follow activity + assert instance1_debug["inbox"] == 1 # An Accept activity should be there + assert instance1_debug["outbox"] == 1 # We've sent a Follow activity instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 1 # An Follow activity should be there - assert instance2_debug['outbox'] == 1 # We've sent a Accept activity + assert instance2_debug["inbox"] == 1 # An Follow activity should be there + assert instance2_debug["outbox"] == 1 # We've sent a Accept activity assert instance2.followers() == [instance1.docker_url] assert instance1.following() == [instance2.docker_url] @@ -263,12 +264,12 @@ def test_follow_unfollow(): assert instance1.following() == [] instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 1 # An Accept activity should be there - assert instance1_debug['outbox'] == 2 # We've sent a Follow and a Undo activity + assert instance1_debug["inbox"] == 1 # An Accept activity should be there + assert instance1_debug["outbox"] == 2 # We've sent a Follow and a Undo activity instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 2 # An Follow and Undo activity should be there - assert instance2_debug['outbox'] == 1 # We've sent a Accept activity + assert instance2_debug["inbox"] == 2 # An Follow and Undo activity should be there + assert instance2_debug["outbox"] == 1 # We've sent a Accept activity def test_post_content(): @@ -279,17 +280,19 @@ def test_post_content(): instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id def test_block_and_post_content(): @@ -300,18 +303,22 @@ def test_block_and_post_content(): instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 instance2.block(instance1.docker_url) - instance1.new_note('hello') + instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 2 # An Follow, Accept activity should be there, Create should have been dropped - assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow activity + the Block activity + assert ( + instance2_debug["inbox"] == 2 + ) # An Follow, Accept activity should be there, Create should have been dropped + assert ( + instance2_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow activity + the Block activity # Ensure the post is not visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 def test_post_content_and_delete(): @@ -322,26 +329,30 @@ def test_post_content_and_delete(): instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id - instance1.delete(f'{create_id}/activity') + instance1.delete(f"{create_id}/activity") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 4 # An Follow, Accept and Create and Delete activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 4 + ) # An Follow, Accept and Create and Delete activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post has been delete from instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 def test_post_content_and_like(): @@ -351,26 +362,26 @@ def test_post_content_and_like(): instance1.follow(instance2) instance2.follow(instance1) - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id # Now, instance2 like the note - like_id = instance2.like(f'{create_id}/activity') + like_id = instance2.like(f"{create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # Follow, Accept and Like - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 3 # Follow, Accept and Like + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'likes' in note - assert note['likes']['totalItems'] == 1 - likes = instance1._parse_collection(url=note['likes']['first']) + note = instance1.outbox_get(f"{create_id}/activity") + assert "likes" in note + assert note["likes"]["totalItems"] == 1 + likes = instance1._parse_collection(url=note["likes"]["first"]) assert len(likes) == 1 - assert likes[0]['id'] == like_id + assert likes[0]["id"] == like_id def test_post_content_and_like_unlike() -> None: @@ -380,36 +391,36 @@ def test_post_content_and_like_unlike() -> None: instance1.follow(instance2) instance2.follow(instance1) - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id # Now, instance2 like the note - like_id = instance2.like(f'{create_id}/activity') + like_id = instance2.like(f"{create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # Follow, Accept and Like - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 3 # Follow, Accept and Like + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'likes' in note - assert note['likes']['totalItems'] == 1 - likes = instance1._parse_collection(url=note['likes']['first']) + note = instance1.outbox_get(f"{create_id}/activity") + assert "likes" in note + assert note["likes"]["totalItems"] == 1 + likes = instance1._parse_collection(url=note["likes"]["first"]) assert len(likes) == 1 - assert likes[0]['id'] == like_id + assert likes[0]["id"] == like_id instance2.undo(like_id) instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 4 # Follow, Accept and Like and Undo - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 4 # Follow, Accept and Like and Undo + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'likes' in note - assert note['likes']['totalItems'] == 0 + note = instance1.outbox_get(f"{create_id}/activity") + assert "likes" in note + assert note["likes"]["totalItems"] == 0 def test_post_content_and_boost() -> None: @@ -419,26 +430,26 @@ def test_post_content_and_boost() -> None: instance1.follow(instance2) instance2.follow(instance1) - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id # Now, instance2 like the note - boost_id = instance2.boost(f'{create_id}/activity') + boost_id = instance2.boost(f"{create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # Follow, Accept and Announce - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 3 # Follow, Accept and Announce + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'shares' in note - assert note['shares']['totalItems'] == 1 - shares = instance1._parse_collection(url=note['shares']['first']) + note = instance1.outbox_get(f"{create_id}/activity") + assert "shares" in note + assert note["shares"]["totalItems"] == 1 + shares = instance1._parse_collection(url=note["shares"]["first"]) assert len(shares) == 1 - assert shares[0]['id'] == boost_id + assert shares[0]["id"] == boost_id def test_post_content_and_boost_unboost() -> None: @@ -448,36 +459,36 @@ def test_post_content_and_boost_unboost() -> None: instance1.follow(instance2) instance2.follow(instance1) - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id # Now, instance2 like the note - boost_id = instance2.boost(f'{create_id}/activity') + boost_id = instance2.boost(f"{create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # Follow, Accept and Announce - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 3 # Follow, Accept and Announce + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'shares' in note - assert note['shares']['totalItems'] == 1 - shares = instance1._parse_collection(url=note['shares']['first']) + note = instance1.outbox_get(f"{create_id}/activity") + assert "shares" in note + assert note["shares"]["totalItems"] == 1 + shares = instance1._parse_collection(url=note["shares"]["first"]) assert len(shares) == 1 - assert shares[0]['id'] == boost_id + assert shares[0]["id"] == boost_id instance2.undo(boost_id) instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 4 # Follow, Accept and Announce and Undo - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 4 # Follow, Accept and Announce and Undo + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'shares' in note - assert note['shares']['totalItems'] == 0 + note = instance1.outbox_get(f"{create_id}/activity") + assert "shares" in note + assert note["shares"]["totalItems"] == 0 def test_post_content_and_post_reply() -> None: @@ -488,40 +499,50 @@ def test_post_content_and_post_reply() -> None: instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 - instance1_create_id = instance1.new_note('hello') + instance1_create_id = instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream instance2_inbox_stream = instance2.stream_jsonfeed() - assert len(instance2_inbox_stream['items']) == 1 - assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id + assert len(instance2_inbox_stream["items"]) == 1 + assert instance2_inbox_stream["items"][0]["id"] == instance1_create_id instance2_create_id = instance2.new_note( - f'hey @instance1@{instance1.docker_url}', - reply=f'{instance1_create_id}/activity', + f"hey @instance1@{instance1.docker_url}", + reply=f"{instance1_create_id}/activity", ) instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert ( + instance2_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance1_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert ( + instance1_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity instance1_inbox_stream = instance1.stream_jsonfeed() - assert len(instance1_inbox_stream['items']) == 1 - assert instance1_inbox_stream['items'][0]['id'] == instance2_create_id + assert len(instance1_inbox_stream["items"]) == 1 + assert instance1_inbox_stream["items"][0]["id"] == instance2_create_id - instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') - assert 'replies' in instance1_note - assert instance1_note['replies']['totalItems'] == 1 - replies = instance1._parse_collection(url=instance1_note['replies']['first']) + instance1_note = instance1.outbox_get(f"{instance1_create_id}/activity") + assert "replies" in instance1_note + assert instance1_note["replies"]["totalItems"] == 1 + replies = instance1._parse_collection(url=instance1_note["replies"]["first"]) assert len(replies) == 1 - assert replies[0]['id'] == f'{instance2_create_id}/activity' + assert replies[0]["id"] == f"{instance2_create_id}/activity" def test_post_content_and_post_reply_and_delete() -> None: @@ -532,44 +553,58 @@ def test_post_content_and_post_reply_and_delete() -> None: instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 - instance1_create_id = instance1.new_note('hello') + instance1_create_id = instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream instance2_inbox_stream = instance2.stream_jsonfeed() - assert len(instance2_inbox_stream['items']) == 1 - assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id + assert len(instance2_inbox_stream["items"]) == 1 + assert instance2_inbox_stream["items"][0]["id"] == instance1_create_id instance2_create_id = instance2.new_note( - f'hey @instance1@{instance1.docker_url}', - reply=f'{instance1_create_id}/activity', + f"hey @instance1@{instance1.docker_url}", + reply=f"{instance1_create_id}/activity", ) instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert ( + instance2_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance1_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert ( + instance1_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity instance1_inbox_stream = instance1.stream_jsonfeed() - assert len(instance1_inbox_stream['items']) == 1 - assert instance1_inbox_stream['items'][0]['id'] == instance2_create_id + assert len(instance1_inbox_stream["items"]) == 1 + assert instance1_inbox_stream["items"][0]["id"] == instance2_create_id - instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') - assert 'replies' in instance1_note - assert instance1_note['replies']['totalItems'] == 1 + instance1_note = instance1.outbox_get(f"{instance1_create_id}/activity") + assert "replies" in instance1_note + assert instance1_note["replies"]["totalItems"] == 1 - instance2.delete(f'{instance2_create_id}/activity') + instance2.delete(f"{instance2_create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 4 # An Follow, Accept and Create and Delete activity should be there - assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance1_debug["inbox"] == 4 + ) # An Follow, Accept and Create and Delete activity should be there + assert ( + instance1_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity - instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') - assert 'replies' in instance1_note - assert instance1_note['replies']['totalItems'] == 0 + instance1_note = instance1.outbox_get(f"{instance1_create_id}/activity") + assert "replies" in instance1_note + assert instance1_note["replies"]["totalItems"] == 0 diff --git a/tests/integration_test.py b/tests/integration_test.py index 4270b4b..dbfe19b 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -9,7 +9,10 @@ from html2text import html2text def config(): """Return the current config as a dict.""" import yaml - with open(os.path.join(os.path.dirname(__file__), '..', 'config/me.yml'), 'rb') as f: + + with open( + os.path.join(os.path.dirname(__file__), "..", "config/me.yml"), "rb" + ) as f: yield yaml.load(f) @@ -20,9 +23,9 @@ def resp2plaintext(resp): def test_ping_homepage(config): """Ensure the homepage is accessible.""" - resp = requests.get('http://localhost:5005') + resp = requests.get("http://localhost:5005") resp.raise_for_status() assert resp.status_code == 200 body = resp2plaintext(resp) - assert config['name'] in body + assert config["name"] in body assert f"@{config['username']}@{config['domain']}" in body diff --git a/utils/__init__.py b/utils/__init__.py index c30c37d..cdf368d 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -4,9 +4,9 @@ logger = logging.getLogger(__name__) def strtobool(s: str) -> bool: - if s in ['y', 'yes', 'true', 'on', '1']: + if s in ["y", "yes", "true", "on", "1"]: return True - if s in ['n', 'no', 'false', 'off', '0']: + if s in ["n", "no", "false", "off", "0"]: return False - raise ValueError(f'cannot convert {s} to bool') + raise ValueError(f"cannot convert {s} to bool") diff --git a/utils/activitypub_utils.py b/utils/activitypub_utils.py deleted file mode 100644 index 0275f54..0000000 --- a/utils/activitypub_utils.py +++ /dev/null @@ -1,65 +0,0 @@ -from typing import Optional, Dict, List, Any - -import requests - -from .errors import RecursionLimitExceededError -from .errors import UnexpectedActivityTypeError - - -def _do_req(url: str, headers: Dict[str, str]) -> Dict[str, Any]: - resp = requests.get(url, headers=headers) - resp.raise_for_status() - return resp.json() - - -def parse_collection( - payload: Optional[Dict[str, Any]] = None, - url: Optional[str] = None, - user_agent: Optional[str] = None, - level: int = 0, - do_req: Any = _do_req, -) -> List[str]: - """Resolve/fetch a `Collection`/`OrderedCollection`.""" - if level > 3: - raise RecursionLimitExceededError('recursion limit exceeded') - - # Go through all the pages - headers = {'Accept': 'application/activity+json'} - if user_agent: - headers['User-Agent'] = user_agent - - out: List[str] = [] - if url: - payload = do_req(url, headers) - if not payload: - raise ValueError('must at least prove a payload or an URL') - - if payload['type'] in ['Collection', 'OrderedCollection']: - if 'orderedItems' in payload: - return payload['orderedItems'] - if 'items' in payload: - return payload['items'] - if 'first' in payload: - if 'orderedItems' in payload['first']: - out.extend(payload['first']['orderedItems']) - if 'items' in payload['first']: - out.extend(payload['first']['items']) - n = payload['first'].get('next') - if n: - out.extend(parse_collection(url=n, user_agent=user_agent, level=level+1, do_req=do_req)) - return out - - while payload: - if payload['type'] in ['CollectionPage', 'OrderedCollectionPage']: - if 'orderedItems' in payload: - out.extend(payload['orderedItems']) - if 'items' in payload: - out.extend(payload['items']) - n = payload.get('next') - if n is None: - break - payload = do_req(n, headers) - else: - raise UnexpectedActivityTypeError('unexpected activity type {}'.format(payload['type'])) - - return out diff --git a/utils/actor_service.py b/utils/actor_service.py deleted file mode 100644 index 9982235..0000000 --- a/utils/actor_service.py +++ /dev/null @@ -1,81 +0,0 @@ -import logging - -import requests -from urllib.parse import urlparse -from Crypto.PublicKey import RSA - -from .urlutils import check_url -from .errors import ActivityNotFoundError - -logger = logging.getLogger(__name__) - - -class NotAnActorError(Exception): - def __init__(self, activity): - self.activity = activity - - -class ActorService(object): - def __init__(self, user_agent, col, actor_id, actor_data, instances): - logger.debug(f'Initializing ActorService user_agent={user_agent}') - self._user_agent = user_agent - self._col = col - self._in_mem = {actor_id: actor_data} - self._instances = instances - self._known_instances = set() - - def _fetch(self, actor_url): - logger.debug(f'fetching remote object {actor_url}') - - check_url(actor_url) - - resp = requests.get(actor_url, headers={ - 'Accept': 'application/activity+json', - 'User-Agent': self._user_agent, - }) - if resp.status_code == 404: - raise ActivityNotFoundError(f'{actor_url} cannot be fetched, 404 not found error') - - resp.raise_for_status() - return resp.json() - - def get(self, actor_url, reload_cache=False): - logger.info(f'get actor {actor_url} (reload_cache={reload_cache})') - - if actor_url in self._in_mem: - return self._in_mem[actor_url] - - instance = urlparse(actor_url)._replace(path='', query='', fragment='').geturl() - if instance not in self._known_instances: - self._known_instances.add(instance) - if not self._instances.find_one({'instance': instance}): - self._instances.insert({'instance': instance, 'first_object': actor_url}) - - if reload_cache: - actor = self._fetch(actor_url) - self._in_mem[actor_url] = actor - self._col.update({'actor_id': actor_url}, {'$set': {'cached_response': actor}}, upsert=True) - return actor - - cached_actor = self._col.find_one({'actor_id': actor_url}) - if cached_actor: - return cached_actor['cached_response'] - - actor = self._fetch(actor_url) - if not 'type' in actor: - raise NotAnActorError(None) - if actor['type'] != 'Person': - raise NotAnActorError(actor) - - self._col.update({'actor_id': actor_url}, {'$set': {'cached_response': actor}}, upsert=True) - self._in_mem[actor_url] = actor - return actor - - def get_public_key(self, actor_url, reload_cache=False): - profile = self.get(actor_url, reload_cache=reload_cache) - pub = profile['publicKey'] - return pub['id'], RSA.importKey(pub['publicKeyPem']) - - def get_inbox_url(self, actor_url, reload_cache=False): - profile = self.get(actor_url, reload_cache=reload_cache) - return profile.get('inbox') diff --git a/utils/content_helper.py b/utils/content_helper.py deleted file mode 100644 index b254e2b..0000000 --- a/utils/content_helper.py +++ /dev/null @@ -1,58 +0,0 @@ -import typing -import re - -from bleach.linkifier import Linker -from markdown import markdown - -from utils.webfinger import get_actor_url -from config import USERNAME, BASE_URL, ID -from config import ACTOR_SERVICE - -from typing import List, Optional, Tuple, Dict, Any, Union, Type - - -def set_attrs(attrs, new=False): - attrs[(None, u'target')] = u'_blank' - attrs[(None, u'class')] = u'external' - attrs[(None, u'rel')] = u'noopener' - attrs[(None, u'title')] = attrs[(None, u'href')] - return attrs - - -LINKER = Linker(callbacks=[set_attrs]) -HASHTAG_REGEX = re.compile(r"(#[\d\w\.]+)") -MENTION_REGEX = re.compile(r"@[\d\w_.+-]+@[\d\w-]+\.[\d\w\-.]+") - - -def hashtagify(content: str) -> Tuple[str, List[Dict[str, str]]]: - tags = [] - for hashtag in re.findall(HASHTAG_REGEX, content): - tag = hashtag[1:] - link = f'' - tags.append(dict(href=f'{BASE_URL}/tags/{tag}', name=hashtag, type='Hashtag')) - content = content.replace(hashtag, link) - return content, tags - - -def mentionify(content: str) -> Tuple[str, List[Dict[str, str]]]: - tags = [] - for mention in re.findall(MENTION_REGEX, content): - _, username, domain = mention.split('@') - actor_url = get_actor_url(mention) - p = ACTOR_SERVICE.get(actor_url) - print(p) - tags.append(dict(type='Mention', href=p['id'], name=mention)) - link = f'@{username}' - content = content.replace(mention, link) - return content, tags - - -def parse_markdown(content: str) -> Tuple[str, List[Dict[str, str]]]: - tags = [] - content = LINKER.linkify(content) - content, hashtag_tags = hashtagify(content) - tags.extend(hashtag_tags) - content, mention_tags = mentionify(content) - tags.extend(mention_tags) - content = markdown(content) - return content, tags diff --git a/utils/errors.py b/utils/errors.py deleted file mode 100644 index 7ffe744..0000000 --- a/utils/errors.py +++ /dev/null @@ -1,37 +0,0 @@ - -class Error(Exception): - status_code = 400 - - def __init__(self, message, status_code=None, payload=None): - Exception.__init__(self) - self.message = message - if status_code is not None: - self.status_code = status_code - self.payload = payload - - def to_dict(self): - rv = dict(self.payload or ()) - rv['message'] = self.message - return rv - - def __repr__(self): - return f'{self.__class__.__qualname__}({self.message!r}, payload={self.payload!r}, status_code={self.status_code})' - - -class NotFromOutboxError(Error): - pass - -class ActivityNotFoundError(Error): - status_code = 404 - - -class BadActivityError(Error): - pass - - -class RecursionLimitExceededError(BadActivityError): - pass - - -class UnexpectedActivityTypeError(BadActivityError): - pass diff --git a/utils/httpsig.py b/utils/httpsig.py deleted file mode 100644 index 8437784..0000000 --- a/utils/httpsig.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Implements HTTP signature for Flask requests. - -Mastodon instances won't accept requests that are not signed using this scheme. - -""" -from datetime import datetime -from urllib.parse import urlparse -from typing import Any, Dict, Optional -import base64 -import hashlib -import logging - -from flask import request -from requests.auth import AuthBase - -from Crypto.Signature import PKCS1_v1_5 -from Crypto.Hash import SHA256 - -logger = logging.getLogger(__name__) - - -def _build_signed_string(signed_headers: str, method: str, path: str, headers: Any, body_digest: str) -> str: - out = [] - for signed_header in signed_headers.split(' '): - if signed_header == '(request-target)': - out.append('(request-target): '+method.lower()+' '+path) - elif signed_header == 'digest': - out.append('digest: '+body_digest) - else: - out.append(signed_header+': '+headers[signed_header]) - return '\n'.join(out) - - -def _parse_sig_header(val: Optional[str]) -> Optional[Dict[str, str]]: - if not val: - return None - out = {} - for data in val.split(','): - k, v = data.split('=', 1) - out[k] = v[1:len(v)-1] - return out - - -def _verify_h(signed_string, signature, pubkey): - signer = PKCS1_v1_5.new(pubkey) - digest = SHA256.new() - digest.update(signed_string.encode('utf-8')) - return signer.verify(digest, signature) - - -def _body_digest() -> str: - h = hashlib.new('sha256') - h.update(request.data) - return 'SHA-256='+base64.b64encode(h.digest()).decode('utf-8') - - -def verify_request(actor_service) -> bool: - hsig = _parse_sig_header(request.headers.get('Signature')) - if not hsig: - logger.debug('no signature in header') - return False - logger.debug(f'hsig={hsig}') - signed_string = _build_signed_string(hsig['headers'], request.method, request.path, request.headers, _body_digest()) - _, rk = actor_service.get_public_key(hsig['keyId']) - return _verify_h(signed_string, base64.b64decode(hsig['signature']), rk) - - -class HTTPSigAuth(AuthBase): - def __init__(self, keyid, privkey): - self.keyid = keyid - self.privkey = privkey - - def __call__(self, r): - logger.info(f'keyid={self.keyid}') - host = urlparse(r.url).netloc - bh = hashlib.new('sha256') - bh.update(r.body.encode('utf-8')) - bodydigest = 'SHA-256='+base64.b64encode(bh.digest()).decode('utf-8') - date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') - r.headers.update({'Digest': bodydigest, 'Date': date}) - r.headers.update({'Host': host}) - sigheaders = '(request-target) user-agent host date digest content-type' - to_be_signed = _build_signed_string(sigheaders, r.method, r.path_url, r.headers, bodydigest) - signer = PKCS1_v1_5.new(self.privkey) - digest = SHA256.new() - digest.update(to_be_signed.encode('utf-8')) - sig = base64.b64encode(signer.sign(digest)) - sig = sig.decode('utf-8') - headers = { - 'Signature': f'keyId="{self.keyid}",algorithm="rsa-sha256",headers="{sigheaders}",signature="{sig}"' - } - logger.info(f'signed request headers={headers}') - r.headers.update(headers) - return r diff --git a/utils/key.py b/utils/key.py index f5a2455..e7012ae 100644 --- a/utils/key.py +++ b/utils/key.py @@ -1,22 +1,22 @@ -import os import binascii - -from Crypto.PublicKey import RSA +import os from typing import Callable -KEY_DIR = os.path.join( - os.path.dirname(os.path.abspath(__file__)), '..', 'config' -) +from little_boxes.key import Key + +KEY_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "config") def _new_key() -> str: - return binascii.hexlify(os.urandom(32)).decode('utf-8') + return binascii.hexlify(os.urandom(32)).decode("utf-8") + def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str: - key_path = os.path.join(KEY_DIR, f'{name}.key') + """Loads or generates a cryptographic key.""" + key_path = os.path.join(KEY_DIR, f"{name}.key") if not os.path.exists(key_path): k = new_key() - with open(key_path, 'w+') as f: + with open(key_path, "w+") as f: f.write(k) return k @@ -24,23 +24,19 @@ def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str: return f.read() -class Key(object): - DEFAULT_KEY_SIZE = 2048 - def __init__(self, user: str, domain: str, create: bool = True) -> None: - user = user.replace('.', '_') - domain = domain.replace('.', '_') - key_path = os.path.join(KEY_DIR, f'key_{user}_{domain}.pem') - if os.path.isfile(key_path): - with open(key_path) as f: - self.privkey_pem = f.read() - self.privkey = RSA.importKey(self.privkey_pem) - self.pubkey_pem = self.privkey.publickey().exportKey('PEM').decode('utf-8') - else: - if not create: - raise Exception('must init private key first') - k = RSA.generate(self.DEFAULT_KEY_SIZE) - self.privkey_pem = k.exportKey('PEM').decode('utf-8') - self.pubkey_pem = k.publickey().exportKey('PEM').decode('utf-8') - with open(key_path, 'w') as f: - f.write(self.privkey_pem) - self.privkey = k +def get_key(owner: str, user: str, domain: str) -> Key: + """"Loads or generates an RSA key.""" + k = Key(owner) + user = user.replace(".", "_") + domain = domain.replace(".", "_") + key_path = os.path.join(KEY_DIR, f"key_{user}_{domain}.pem") + if os.path.isfile(key_path): + with open(key_path) as f: + privkey_pem = f.read() + k.load(privkey_pem) + else: + k.new() + with open(key_path, "w") as f: + f.write(k.privkey_pem) + + return k diff --git a/utils/linked_data_sig.py b/utils/linked_data_sig.py deleted file mode 100644 index 834c9bd..0000000 --- a/utils/linked_data_sig.py +++ /dev/null @@ -1,70 +0,0 @@ -from pyld import jsonld -import hashlib -from datetime import datetime - -from Crypto.Signature import PKCS1_v1_5 -from Crypto.Hash import SHA256 -import base64 - -from typing import Any, Dict - - -# cache the downloaded "schemas", otherwise the library is super slow -# (https://github.com/digitalbazaar/pyld/issues/70) -_CACHE: Dict[str, Any] = {} -LOADER = jsonld.requests_document_loader() - -def _caching_document_loader(url: str) -> Any: - if url in _CACHE: - return _CACHE[url] - resp = LOADER(url) - _CACHE[url] = resp - return resp - -jsonld.set_document_loader(_caching_document_loader) - - -def options_hash(doc): - doc = dict(doc['signature']) - for k in ['type', 'id', 'signatureValue']: - if k in doc: - del doc[k] - doc['@context'] = 'https://w3id.org/identity/v1' - normalized = jsonld.normalize(doc, {'algorithm': 'URDNA2015', 'format': 'application/nquads'}) - h = hashlib.new('sha256') - h.update(normalized.encode('utf-8')) - return h.hexdigest() - - -def doc_hash(doc): - doc = dict(doc) - if 'signature' in doc: - del doc['signature'] - normalized = jsonld.normalize(doc, {'algorithm': 'URDNA2015', 'format': 'application/nquads'}) - h = hashlib.new('sha256') - h.update(normalized.encode('utf-8')) - return h.hexdigest() - - -def verify_signature(doc, pubkey): - to_be_signed = options_hash(doc) + doc_hash(doc) - signature = doc['signature']['signatureValue'] - signer = PKCS1_v1_5.new(pubkey) - digest = SHA256.new() - digest.update(to_be_signed.encode('utf-8')) - return signer.verify(digest, base64.b64decode(signature)) - - -def generate_signature(doc, privkey): - options = { - 'type': 'RsaSignature2017', - 'creator': doc['actor'] + '#main-key', - 'created': datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', - } - doc['signature'] = options - to_be_signed = options_hash(doc) + doc_hash(doc) - signer = PKCS1_v1_5.new(privkey) - digest = SHA256.new() - digest.update(to_be_signed.encode('utf-8')) - sig = base64.b64encode(signer.sign(digest)) - options['signatureValue'] = sig.decode('utf-8') diff --git a/utils/object_service.py b/utils/object_service.py index 1ebc0ce..e46f9b1 100644 --- a/utils/object_service.py +++ b/utils/object_service.py @@ -1,67 +1,21 @@ -import requests -from urllib.parse import urlparse +import logging -from .urlutils import check_url -from .errors import ActivityNotFoundError +from little_boxes.activitypub import get_backend + +logger = logging.getLogger(__name__) class ObjectService(object): - def __init__(self, user_agent, col, inbox, outbox, instances): - self._user_agent = user_agent - self._col = col - self._inbox = inbox - self._outbox = outbox - self._instances = instances - self._known_instances = set() + def __init__(self): + logger.debug("Initializing ObjectService") + self._cache = {} - def _fetch_remote(self, object_id): - print(f'fetch remote {object_id}') - check_url(object_id) - resp = requests.get(object_id, headers={ - 'Accept': 'application/activity+json', - 'User-Agent': self._user_agent, - }) - if resp.status_code == 404: - raise ActivityNotFoundError(f'{object_id} cannot be fetched, 404 error not found') + def get(self, iri, reload_cache=False): + logger.info(f"get actor {iri} (reload_cache={reload_cache})") - resp.raise_for_status() - return resp.json() - - def _fetch(self, object_id): - instance = urlparse(object_id)._replace(path='', query='', fragment='').geturl() - if instance not in self._known_instances: - self._known_instances.add(instance) - if not self._instances.find_one({'instance': instance}): - self._instances.insert({'instance': instance, 'first_object': object_id}) - - obj = self._inbox.find_one({'$or': [{'remote_id': object_id}, {'type': 'Create', 'activity.object.id': object_id}]}) - if obj: - if obj['remote_id'] == object_id: - return obj['activity'] - return obj['activity']['object'] - - obj = self._outbox.find_one({'$or': [{'remote_id': object_id}, {'type': 'Create', 'activity.object.id': object_id}]}) - if obj: - if obj['remote_id'] == object_id: - return obj['activity'] - return obj['activity']['object'] - - return self._fetch_remote(object_id) - - def get(self, object_id, reload_cache=False, part_of_stream=False, announce_published=None): - if reload_cache: - obj = self._fetch(object_id) - self._col.update({'object_id': object_id}, {'$set': {'cached_object': obj, 'meta.part_of_stream': part_of_stream, 'meta.announce_published': announce_published}}, upsert=True) - return obj - - cached_object = self._col.find_one({'object_id': object_id}) - if cached_object: - print(f'ObjectService: {cached_object}') - return cached_object['cached_object'] - - obj = self._fetch(object_id) - - self._col.update({'object_id': object_id}, {'$set': {'cached_object': obj, 'meta.part_of_stream': part_of_stream, 'meta.announce_published': announce_published}}, upsert=True) - # print(f'ObjectService: {obj}') + if not reload_cache and iri in self._cache: + return self._cache[iri] + obj = get_backend().fetch_iri(iri) + self._cache[iri] = obj return obj diff --git a/utils/opengraph.py b/utils/opengraph.py index a53c07b..597ad3c 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -1,36 +1,34 @@ -from urllib.parse import urlparse - -import ipaddress import opengraph import requests from bs4 import BeautifulSoup -from .urlutils import is_url_valid, check_url +from little_boxes.urlutils import check_url +from little_boxes.urlutils import is_url_valid def links_from_note(note): - tags_href= set() - for t in note.get('tag', []): - h = t.get('href') + tags_href = set() + for t in note.get("tag", []): + h = t.get("href") if h: # TODO(tsileo): fetch the URL for Actor profile, type=mention tags_href.add(h) links = set() - soup = BeautifulSoup(note['content']) - for link in soup.find_all('a'): - h = link.get('href') - if h.startswith('http') and h not in tags_href and is_url_valid(h): + soup = BeautifulSoup(note["content"]) + for link in soup.find_all("a"): + h = link.get("href") + if h.startswith("http") and h not in tags_href and is_url_valid(h): links.add(h) return links def fetch_og_metadata(user_agent, col, remote_id): - doc = col.find_one({'remote_id': remote_id}) + doc = col.find_one({"remote_id": remote_id}) if not doc: raise ValueError - note = doc['activity']['object'] + note = doc["activity"]["object"] print(note) links = links_from_note(note) if not links: @@ -39,9 +37,11 @@ def fetch_og_metadata(user_agent, col, remote_id): htmls = [] for l in links: check_url(l) - r = requests.get(l, headers={'User-Agent': user_agent}) + r = requests.get(l, headers={"User-Agent": user_agent}) r.raise_for_status() htmls.append(r.text) links_og_metadata = [dict(opengraph.OpenGraph(html=html)) for html in htmls] - col.update_one({'remote_id': remote_id}, {'$set': {'meta.og_metadata': links_og_metadata}}) + col.update_one( + {"remote_id": remote_id}, {"$set": {"meta.og_metadata": links_og_metadata}} + ) return len(links) diff --git a/utils/urlutils.py b/utils/urlutils.py deleted file mode 100644 index 99f900d..0000000 --- a/utils/urlutils.py +++ /dev/null @@ -1,47 +0,0 @@ -import logging -import os -import socket -import ipaddress -from urllib.parse import urlparse - -from . import strtobool -from .errors import Error - -logger = logging.getLogger(__name__) - - -class InvalidURLError(Error): - pass - - -def is_url_valid(url: str) -> bool: - parsed = urlparse(url) - if parsed.scheme not in ['http', 'https']: - return False - - # XXX in debug mode, we want to allow requests to localhost to test the federation with local instances - debug_mode = strtobool(os.getenv('MICROBLOGPUB_DEBUG', 'false')) - if debug_mode: - return True - - if parsed.hostname in ['localhost']: - return False - - try: - ip_address = socket.getaddrinfo(parsed.hostname, parsed.port or 80)[0][4][0] - except socket.gaierror: - logger.exception(f'failed to lookup url {url}') - return False - - if ipaddress.ip_address(ip_address).is_private: - logger.info(f'rejecting private URL {url}') - return False - - return True - - -def check_url(url: str) -> None: - if not is_url_valid(url): - raise InvalidURLError(f'"{url}" is invalid') - - return None diff --git a/utils/webfinger.py b/utils/webfinger.py deleted file mode 100644 index 8e6fdc7..0000000 --- a/utils/webfinger.py +++ /dev/null @@ -1,75 +0,0 @@ -from urllib.parse import urlparse -from typing import Dict, Any -from typing import Optional -import logging - -import requests - -from .urlutils import check_url - - -logger = logging.getLogger(__name__) - - -def webfinger(resource: str) -> Optional[Dict[str, Any]]: - """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL. - """ - logger.info(f'performing webfinger resolution for {resource}') - protos = ['https', 'http'] - if resource.startswith('http://'): - protos.reverse() - host = urlparse(resource).netloc - elif resource.startswith('https://'): - host = urlparse(resource).netloc - else: - if resource.startswith('acct:'): - resource = resource[5:] - if resource.startswith('@'): - resource = resource[1:] - _, host = resource.split('@', 1) - resource='acct:'+resource - - # Security check on the url (like not calling localhost) - check_url(f'https://{host}') - - for i, proto in enumerate(protos): - try: - url = f'{proto}://{host}/.well-known/webfinger' - resp = requests.get( - url, - {'resource': resource} - ) - except requests.ConnectionError: - # If we tried https first and the domain is "http only" - if i == 0: - continue - break - if resp.status_code == 404: - return None - resp.raise_for_status() - return resp.json() - - -def get_remote_follow_template(resource: str) -> Optional[str]: - data = webfinger(resource) - if data is None: - return None - for link in data['links']: - if link.get('rel') == 'http://ostatus.org/schema/1.0/subscribe': - return link.get('template') - return None - - -def get_actor_url(resource: str) -> Optional[str]: - """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL. - - Returns: - the Actor URL or None if the resolution failed. - """ - data = webfinger(resource) - if data is None: - return None - for link in data['links']: - if link.get('rel') == 'self' and link.get('type') == 'application/activity+json': - return link.get('href') - return None