From d9362adb25cd988962269632cd6de8e963a82944 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 16 Jun 2018 22:02:10 +0200 Subject: [PATCH] Black/isort --- activitypub.py | 26 +- app.py | 1295 +++++++++++++++++++----------------- config.py | 17 +- dev-requirements.txt | 1 + tasks.py | 58 +- tests/federation_test.py | 8 +- utils/activitypub_utils.py | 5 +- utils/actor_service.py | 4 +- utils/content_helper.py | 25 +- utils/httpsig.py | 13 +- utils/key.py | 3 +- utils/linked_data_sig.py | 11 +- utils/object_service.py | 5 +- utils/opengraph.py | 5 +- utils/urlutils.py | 2 +- utils/webfinger.py | 8 +- 16 files changed, 814 insertions(+), 672 deletions(-) diff --git a/activitypub.py b/activitypub.py index 697ec59..9097d05 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,23 +1,30 @@ import logging - from datetime import datetime +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Union from bson.objectid import ObjectId -from html2text import html2text from feedgen.feed import FeedGenerator +from html2text import html2text +import tasks +from config import BASE_URL +from config import DB +from config import ID +from config import ME +from config import USER_AGENT +from config import USERNAME from little_boxes import activitypub as ap from little_boxes.backend import Backend from little_boxes.collection import parse_collection as ap_parse_collection -from config import USERNAME, BASE_URL, ID -from config import DB, ME -import tasks - -from typing import List, Optional, Dict, Any, Union - logger = logging.getLogger(__name__) +MY_PERSON = ap.Person(**ME) + def _remove_id(doc: ap.ObjectType) -> ap.ObjectType: """Helper for removing MongoDB's `_id` field.""" @@ -35,6 +42,9 @@ def _to_list(data: Union[List[Any], Any]) -> List[Any]: class MicroblogPubBackend(Backend): + def user_agent(self) -> str: + return USER_AGENT + def base_url(self) -> str: return BASE_URL diff --git a/app.py b/app.py index 6b104d2..e337204 100644 --- a/app.py +++ b/app.py @@ -1,135 +1,137 @@ import binascii import hashlib import json -import urllib -import os -import mimetypes import logging -from functools import wraps +import mimetypes +import os +import urllib from datetime import datetime +from functools import wraps +from typing import Any +from typing import Dict +from urllib.parse import urlencode +from urllib.parse import urlparse -import timeago import bleach import mf2py -import pymongo import piexif +import pymongo +import timeago from bson.objectid import ObjectId from flask import Flask -from flask import abort -from flask import request -from flask import redirect from flask import Response -from flask import render_template -from flask import session +from flask import abort from flask import jsonify as flask_jsonify +from flask import redirect +from flask import render_template +from flask import request +from flask import session from flask import url_for +from flask_wtf.csrf import CSRFProtect from html2text import html2text -from itsdangerous import JSONWebSignatureSerializer from itsdangerous import BadSignature +from itsdangerous import JSONWebSignatureSerializer from passlib.hash import bcrypt from u2flib_server import u2f -from urllib.parse import urlparse, urlencode from werkzeug.utils import secure_filename -from flask_wtf.csrf import CSRFProtect import activitypub import config -from activitypub import ActivityType -from activitypub import clean_activity from activitypub import embed_collection -from utils.content_helper import parse_markdown -from config import KEY -from config import DB -from config import ME -from config import ID -from config import DOMAIN -from config import USERNAME -from config import BASE_URL from config import ACTOR_SERVICE +from config import ADMIN_API_KEY +from config import BASE_URL +from config import DB +from config import DEBUG_MODE +from config import DOMAIN +from config import HEADERS +from config import ID +from config import JWT +from config import KEY +from config import ME from config import OBJECT_SERVICE from config import PASS -from config import HEADERS +from config import USERNAME from config import VERSION -from config import DEBUG_MODE -from config import JWT -from config import ADMIN_API_KEY from config import _drop_db from config import custom_cache_purge_hook -from utils.httpsig import HTTPSigAuth, verify_request -from utils.key import get_secret_key -from utils.webfinger import get_remote_follow_template -from utils.webfinger import get_actor_url -from utils.errors import Error -from utils.errors import UnexpectedActivityTypeError -from utils.errors import BadActivityError -from utils.errors import NotFromOutboxError +from little_boxes.activitypub import ActivityType +from little_boxes.activitypub import clean_activity +from little_boxes.errors import BadActivityError +from little_boxes.errors import Error +from little_boxes.errors import UnexpectedActivityTypeError +from little_boxes.httpsig import HTTPSigAuth +from little_boxes.httpsig import verify_request +from little_boxes.webfinger import get_actor_url +from little_boxes.webfinger import get_remote_follow_template +from utils.content_helper import parse_markdown from utils.errors import ActivityNotFoundError - - -from typing import Dict, Any +from utils.errors import NotFromOutboxError +from utils.key import get_secret_key app = Flask(__name__) -app.secret_key = get_secret_key('flask') -app.config.update( - WTF_CSRF_CHECK_DEFAULT=False, -) +app.secret_key = get_secret_key("flask") +app.config.update(WTF_CSRF_CHECK_DEFAULT=False) csrf = CSRFProtect(app) logger = logging.getLogger(__name__) # Hook up Flask logging with gunicorn root_logger = logging.getLogger() -if os.getenv('FLASK_DEBUG'): +if os.getenv("FLASK_DEBUG"): logger.setLevel(logging.DEBUG) root_logger.setLevel(logging.DEBUG) else: - gunicorn_logger = logging.getLogger('gunicorn.error') + gunicorn_logger = logging.getLogger("gunicorn.error") root_logger.handlers = gunicorn_logger.handlers root_logger.setLevel(gunicorn_logger.level) -SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) +SIG_AUTH = HTTPSigAuth(KEY) def verify_pass(pwd): - return bcrypt.verify(pwd, PASS) + return bcrypt.verify(pwd, PASS) + @app.context_processor def inject_config(): - return dict( - microblogpub_version=VERSION, - config=config, - logged_in=session.get('logged_in', False), - ) + return dict( + microblogpub_version=VERSION, + config=config, + logged_in=session.get("logged_in", False), + ) + @app.after_request def set_x_powered_by(response): - response.headers['X-Powered-By'] = 'microblog.pub' + response.headers["X-Powered-By"] = "microblog.pub" return response + # HTML/templates helper ALLOWED_TAGS = [ - 'a', - 'abbr', - 'acronym', - 'b', - 'blockquote', - 'code', - 'pre', - 'em', - 'i', - 'li', - 'ol', - 'strong', - 'ul', - 'span', - 'div', - 'p', - 'h1', - 'h2', - 'h3', - 'h4', - 'h5', - 'h6', + "a", + "abbr", + "acronym", + "b", + "blockquote", + "code", + "pre", + "em", + "i", + "li", + "ol", + "strong", + "ul", + "span", + "div", + "p", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", ] @@ -166,13 +168,16 @@ def domain(url): def get_actor(url): if not url: return None - print(f'GET_ACTOR {url}') + print(f"GET_ACTOR {url}") return ACTOR_SERVICE.get(url) + @app.template_filter() def format_time(val): if val: - return datetime.strftime(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), '%B %d, %Y, %H:%M %p') + return datetime.strftime( + datetime.strptime(val, "%Y-%m-%dT%H:%M:%SZ"), "%B %d, %Y, %H:%M %p" + ) return val @@ -180,26 +185,38 @@ def format_time(val): def format_timeago(val): if val: try: - return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), datetime.utcnow()) - except: - return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%S.%fZ'), datetime.utcnow()) + return timeago.format( + datetime.strptime(val, "%Y-%m-%dT%H:%M:%SZ"), datetime.utcnow() + ) + except Exception: + return timeago.format( + datetime.strptime(val, "%Y-%m-%dT%H:%M:%S.%fZ"), datetime.utcnow() + ) return val + def _is_img(filename): filename = filename.lower() - if (filename.endswith('.png') or filename.endswith('.jpg') or filename.endswith('.jpeg') or - filename.endswith('.gif') or filename.endswith('.svg')): + if ( + filename.endswith(".png") + or filename.endswith(".jpg") + or filename.endswith(".jpeg") + or filename.endswith(".gif") + or filename.endswith(".svg") + ): return True return False + @app.template_filter() def not_only_imgs(attachment): for a in attachment: - if not _is_img(a['url']): + if not _is_img(a["url"]): return True return False + @app.template_filter() def is_img(filename): return _is_img(filename) @@ -208,28 +225,29 @@ def is_img(filename): def login_required(f): @wraps(f) def decorated_function(*args, **kwargs): - if not session.get('logged_in'): - return redirect(url_for('login', next=request.url)) + if not session.get("logged_in"): + return redirect(url_for("login", next=request.url)) return f(*args, **kwargs) + return decorated_function def _api_required(): - if session.get('logged_in'): - if request.method not in ['GET', 'HEAD']: + if session.get("logged_in"): + if request.method not in ["GET", "HEAD"]: # If a standard API request is made with a "login session", it must havw a CSRF token csrf.protect() return # Token verification - token = request.headers.get('Authorization', '').replace('Bearer ', '') + token = request.headers.get("Authorization", "").replace("Bearer ", "") if not token: # IndieAuth token - token = request.form.get('access_token', '') + token = request.form.get("access_token", "") # Will raise a BadSignature on bad auth payload = JWT.loads(token) - logger.info(f'api call by {payload}') + logger.info(f"api call by {payload}") def api_required(f): @@ -241,31 +259,36 @@ def api_required(f): abort(401) return f(*args, **kwargs) + return decorated_function def jsonify(**data): - if '@context' not in data: - data['@context'] = config.CTX_AS + if "@context" not in data: + data["@context"] = config.CTX_AS return Response( response=json.dumps(data), - headers={'Content-Type': 'application/json' if app.debug else 'application/activity+json'}, + headers={ + "Content-Type": "application/json" + if app.debug + else "application/activity+json" + }, ) def is_api_request(): - h = request.headers.get('Accept') + h = request.headers.get("Accept") if h is None: return False - h = h.split(',')[0] - if h in HEADERS or h == 'application/json': + h = h.split(",")[0] + if h in HEADERS or h == "application/json": return True return False @app.errorhandler(ValueError) def handle_value_error(error): - logger.error(f'caught value error: {error!r}') + logger.error(f"caught value error: {error!r}") response = flask_jsonify(message=error.args[0]) response.status_code = 400 return response @@ -273,7 +296,7 @@ def handle_value_error(error): @app.errorhandler(Error) def handle_activitypub_error(error): - logger.error(f'caught activitypub error {error!r}') + logger.error(f"caught activitypub error {error!r}") response = flask_jsonify(error.to_dict()) response.status_code = error.status_code return response @@ -284,99 +307,102 @@ def handle_activitypub_error(error): ####### # Login -@app.route('/logout') + +@app.route("/logout") @login_required def logout(): - session['logged_in'] = False - return redirect('/') + session["logged_in"] = False + return redirect("/") -@app.route('/login', methods=['POST', 'GET']) +@app.route("/login", methods=["POST", "GET"]) def login(): - devices = [doc['device'] for doc in DB.u2f.find()] + devices = [doc["device"] for doc in DB.u2f.find()] u2f_enabled = True if devices else False - if request.method == 'POST': + if request.method == "POST": csrf.protect() - pwd = request.form.get('pass') + pwd = request.form.get("pass") if pwd and verify_pass(pwd): if devices: - resp = json.loads(request.form.get('resp')) + resp = json.loads(request.form.get("resp")) print(resp) try: - u2f.complete_authentication(session['challenge'], resp) + u2f.complete_authentication(session["challenge"], resp) except ValueError as exc: - print('failed', exc) + print("failed", exc) abort(401) return finally: - session['challenge'] = None + session["challenge"] = None - session['logged_in'] = True - return redirect(request.args.get('redirect') or '/admin') + session["logged_in"] = True + return redirect(request.args.get("redirect") or "/admin") else: abort(401) payload = None if devices: payload = u2f.begin_authentication(ID, devices) - session['challenge'] = payload + session["challenge"] = payload return render_template( - 'login.html', - u2f_enabled=u2f_enabled, - me=ME, - payload=payload, + "login.html", u2f_enabled=u2f_enabled, me=ME, payload=payload ) -@app.route('/remote_follow', methods=['GET', 'POST']) +@app.route("/remote_follow", methods=["GET", "POST"]) def remote_follow(): - if request.method == 'GET': - return render_template('remote_follow.html') + if request.method == "GET": + return render_template("remote_follow.html") csrf.protect() - return redirect(get_remote_follow_template('@'+request.form.get('profile')).format(uri=f'{USERNAME}@{DOMAIN}')) + return redirect( + get_remote_follow_template("@" + request.form.get("profile")).format( + uri=f"{USERNAME}@{DOMAIN}" + ) + ) -@app.route('/authorize_follow', methods=['GET', 'POST']) +@app.route("/authorize_follow", methods=["GET", "POST"]) @login_required def authorize_follow(): - if request.method == 'GET': - return render_template('authorize_remote_follow.html', profile=request.args.get('profile')) + if request.method == "GET": + return render_template( + "authorize_remote_follow.html", profile=request.args.get("profile") + ) - actor = get_actor_url(request.form.get('profile')) + actor = get_actor_url(request.form.get("profile")) if not actor: abort(500) - if DB.following.find({'remote_actor': actor}).count() > 0: - return redirect('/following') + if DB.following.find({"remote_actor": actor}).count() > 0: + return redirect("/following") follow = activitypub.Follow(object=actor) follow.post_to_outbox() - return redirect('/following') + return redirect("/following") -@app.route('/u2f/register', methods=['GET', 'POST']) +@app.route("/u2f/register", methods=["GET", "POST"]) @login_required def u2f_register(): # TODO(tsileo): ensure no duplicates - if request.method == 'GET': + if request.method == "GET": payload = u2f.begin_registration(ID) - session['challenge'] = payload - return render_template( - 'u2f.html', - payload=payload, - ) + session["challenge"] = payload + return render_template("u2f.html", payload=payload) else: - resp = json.loads(request.form.get('resp')) - device, device_cert = u2f.complete_registration(session['challenge'], resp) - session['challenge'] = None - DB.u2f.insert_one({'device': device, 'cert': device_cert}) - return '' + resp = json.loads(request.form.get("resp")) + device, device_cert = u2f.complete_registration(session["challenge"], resp) + session["challenge"] = None + DB.u2f.insert_one({"device": device, "cert": device_cert}) + return "" + ####### # Activity pub routes -@app.route('/') + +@app.route("/") def index(): if is_api_request(): return jsonify(**ME) @@ -384,31 +410,41 @@ def index(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 q = { - 'type': 'Create', - 'activity.object.type': 'Note', - 'activity.object.inReplyTo': None, - 'meta.deleted': False, + "type": "Create", + "activity.object.type": "Note", + "activity.object.inReplyTo": None, + "meta.deleted": False, } - c = request.args.get('cursor') + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.outbox.find({'$or': [q, {'type': 'Announce', 'meta.undo': False}]}, limit=limit).sort('_id', -1)) + outbox_data = list( + DB.outbox.find( + {"$or": [q, {"type": "Announce", "meta.undo": False}]}, limit=limit + ).sort("_id", -1) + ) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) for data in outbox_data: - if data['type'] == 'Announce': + if data["type"] == "Announce": print(data) - if data['activity']['object'].startswith('http'): - data['ref'] = {'activity': {'object': OBJECT_SERVICE.get(data['activity']['object'])}, 'meta': {}} - + if data["activity"]["object"].startswith("http"): + data["ref"] = { + "activity": { + "object": OBJECT_SERVICE.get(data["activity"]["object"]) + }, + "meta": {}, + } return render_template( - 'index.html', + "index.html", me=ME, - notes=DB.inbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False}).count(), + notes=DB.inbox.find( + {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + ).count(), followers=DB.followers.count(), following=DB.following.count(), outbox_data=outbox_data, @@ -416,34 +452,40 @@ def index(): ) -@app.route('/with_replies') +@app.route("/with_replies") def with_replies(): limit = 50 - q = { - 'type': 'Create', - 'activity.object.type': 'Note', - 'meta.deleted': False, - } - c = request.args.get('cursor') + q = {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.outbox.find({'$or': [q, {'type': 'Announce', 'meta.undo': False}]}, limit=limit).sort('_id', -1)) + outbox_data = list( + DB.outbox.find( + {"$or": [q, {"type": "Announce", "meta.undo": False}]}, limit=limit + ).sort("_id", -1) + ) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) for data in outbox_data: - if data['type'] == 'Announce': + if data["type"] == "Announce": print(data) - if data['activity']['object'].startswith('http'): - data['ref'] = {'activity': {'object': OBJECT_SERVICE.get(data['activity']['object'])}, 'meta': {}} - + if data["activity"]["object"].startswith("http"): + data["ref"] = { + "activity": { + "object": OBJECT_SERVICE.get(data["activity"]["object"]) + }, + "meta": {}, + } return render_template( - 'index.html', + "index.html", me=ME, - notes=DB.inbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False}).count(), + notes=DB.inbox.find( + {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + ).count(), followers=DB.followers.count(), following=DB.following.count(), outbox_data=outbox_data, @@ -452,17 +494,17 @@ def with_replies(): def _build_thread(data, include_children=True): - data['_requested'] = True - root_id = data['meta'].get('thread_root_parent', data['activity']['object']['id']) + data["_requested"] = True + root_id = data["meta"].get("thread_root_parent", data["activity"]["object"]["id"]) - thread_ids = data['meta'].get('thread_parents', []) + thread_ids = data["meta"].get("thread_parents", []) if include_children: - thread_ids.extend(data['meta'].get('thread_children', [])) + thread_ids.extend(data["meta"].get("thread_children", [])) query = { - 'activity.object.id': {'$in': thread_ids}, - 'type': 'Create', - 'meta.deleted': False, # TODO(tsileo): handle Tombstone instead of filtering them + "activity.object.id": {"$in": thread_ids}, + "type": "Create", + "meta.deleted": False, # TODO(tsileo): handle Tombstone instead of filtering them } # Fetch the root replies, and the children replies = [data] + list(DB.inbox.find(query)) + list(DB.outbox.find(query)) @@ -470,131 +512,166 @@ def _build_thread(data, include_children=True): # Index all the IDs in order to build a tree idx = {} for rep in replies: - rep_id = rep['activity']['object']['id'] + rep_id = rep["activity"]["object"]["id"] idx[rep_id] = rep.copy() - idx[rep_id]['_nodes'] = [] + idx[rep_id]["_nodes"] = [] # Build the tree for rep in replies: - rep_id = rep['activity']['object']['id'] + rep_id = rep["activity"]["object"]["id"] if rep_id == root_id: continue - reply_of = rep['activity']['object']['inReplyTo'] - idx[reply_of]['_nodes'].append(rep) + reply_of = rep["activity"]["object"]["inReplyTo"] + idx[reply_of]["_nodes"].append(rep) # Flatten the tree thread = [] + def _flatten(node, level=0): - node['_level'] = level + node["_level"] = level thread.append(node) - for snode in sorted(idx[node['activity']['object']['id']]['_nodes'], key=lambda d: d['activity']['object']['published']): - _flatten(snode, level=level+1) + for snode in sorted( + idx[node["activity"]["object"]["id"]]["_nodes"], + key=lambda d: d["activity"]["object"]["published"], + ): + _flatten(snode, level=level + 1) + _flatten(idx[root_id]) return thread -@app.route('/note/') +@app.route("/note/") def note_by_id(note_id): - data = DB.outbox.find_one({'id': note_id}) + data = DB.outbox.find_one({"id": note_id}) if not data: abort(404) - if data['meta'].get('deleted', False): + if data["meta"].get("deleted", False): abort(410) thread = _build_thread(data) + likes = list( + DB.inbox.find( + { + "meta.undo": False, + "type": ActivityType.LIKE.value, + "$or": [ + {"activity.object.id": data["activity"]["object"]["id"]}, + {"activity.object": data["activity"]["object"]["id"]}, + ], + } + ) + ) + likes = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in likes] - likes = list(DB.inbox.find({ - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - '$or': [{'activity.object.id': data['activity']['object']['id']}, - {'activity.object': data['activity']['object']['id']}], - })) - likes = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in likes] + shares = list( + DB.inbox.find( + { + "meta.undo": False, + "type": ActivityType.ANNOUNCE.value, + "$or": [ + {"activity.object.id": data["activity"]["object"]["id"]}, + {"activity.object": data["activity"]["object"]["id"]}, + ], + } + ) + ) + shares = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in shares] - shares = list(DB.inbox.find({ - 'meta.undo': False, - 'type': ActivityType.ANNOUNCE.value, - '$or': [{'activity.object.id': data['activity']['object']['id']}, - {'activity.object': data['activity']['object']['id']}], - })) - shares = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in shares] - - return render_template('note.html', likes=likes, shares=shares, me=ME, thread=thread, note=data) - - -@app.route('/nodeinfo') -def nodeinfo(): - return Response( - headers={'Content-Type': 'application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#'}, - response=json.dumps({ - 'version': '2.0', - 'software': {'name': 'microblogpub', 'version': f'Microblog.pub {VERSION}'}, - 'protocols': ['activitypub'], - 'services': {'inbound': [], 'outbound': []}, - 'openRegistrations': False, - 'usage': {'users': {'total': 1}, 'localPosts': DB.outbox.count()}, - 'metadata': { - 'sourceCode': 'https://github.com/tsileo/microblog.pub', - 'nodeName': f'@{USERNAME}@{DOMAIN}', - }, - }), + return render_template( + "note.html", likes=likes, shares=shares, me=ME, thread=thread, note=data ) -@app.route('/.well-known/nodeinfo') +@app.route("/nodeinfo") +def nodeinfo(): + return Response( + headers={ + "Content-Type": "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#" + }, + response=json.dumps( + { + "version": "2.0", + "software": { + "name": "microblogpub", + "version": f"Microblog.pub {VERSION}", + }, + "protocols": ["activitypub"], + "services": {"inbound": [], "outbound": []}, + "openRegistrations": False, + "usage": {"users": {"total": 1}, "localPosts": DB.outbox.count()}, + "metadata": { + "sourceCode": "https://github.com/tsileo/microblog.pub", + "nodeName": f"@{USERNAME}@{DOMAIN}", + }, + } + ), + ) + + +@app.route("/.well-known/nodeinfo") def wellknown_nodeinfo(): return flask_jsonify( links=[ { - 'rel': 'http://nodeinfo.diaspora.software/ns/schema/2.0', - 'href': f'{ID}/nodeinfo', + "rel": "http://nodeinfo.diaspora.software/ns/schema/2.0", + "href": f"{ID}/nodeinfo", } - - ], + ] ) -@app.route('/.well-known/webfinger') +@app.route("/.well-known/webfinger") def wellknown_webfinger(): """Enable WebFinger support, required for Mastodon interopability.""" - resource = request.args.get('resource') - if resource not in [f'acct:{USERNAME}@{DOMAIN}', ID]: + resource = request.args.get("resource") + if resource not in [f"acct:{USERNAME}@{DOMAIN}", ID]: abort(404) out = { - "subject": f'acct:{USERNAME}@{DOMAIN}', + "subject": f"acct:{USERNAME}@{DOMAIN}", "aliases": [ID], "links": [ - {"rel": "http://webfinger.net/rel/profile-page", "type": "text/html", "href": BASE_URL}, + { + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html", + "href": BASE_URL, + }, {"rel": "self", "type": "application/activity+json", "href": ID}, - {"rel":"http://ostatus.org/schema/1.0/subscribe", "template": BASE_URL+"/authorize_follow?profile={uri}"}, + { + "rel": "http://ostatus.org/schema/1.0/subscribe", + "template": BASE_URL + "/authorize_follow?profile={uri}", + }, ], } return Response( response=json.dumps(out), - headers={'Content-Type': 'application/jrd+json; charset=utf-8' if not app.debug else 'application/json'}, + headers={ + "Content-Type": "application/jrd+json; charset=utf-8" + if not app.debug + else "application/json" + }, ) def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: - if raw_doc['activity']['type'] != ActivityType.CREATE.value: + if raw_doc["activity"]["type"] != ActivityType.CREATE.value: return raw_doc - raw_doc['activity']['object']['replies'] = embed_collection( - raw_doc.get('meta', {}).get('count_direct_reply', 0), + raw_doc["activity"]["object"]["replies"] = embed_collection( + raw_doc.get("meta", {}).get("count_direct_reply", 0), f'{ID}/outbox/{raw_doc["id"]}/replies', ) - raw_doc['activity']['object']['likes'] = embed_collection( - raw_doc.get('meta', {}).get('count_like', 0), + raw_doc["activity"]["object"]["likes"] = embed_collection( + raw_doc.get("meta", {}).get("count_like", 0), f'{ID}/outbox/{raw_doc["id"]}/likes', ) - raw_doc['activity']['object']['shares'] = embed_collection( - raw_doc.get('meta', {}).get('count_boost', 0), + raw_doc["activity"]["object"]["shares"] = embed_collection( + raw_doc.get("meta", {}).get("count_boost", 0), f'{ID}/outbox/{raw_doc["id"]}/shares', ) @@ -602,37 +679,38 @@ def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: def remove_context(activity: Dict[str, Any]) -> Dict[str, Any]: - if '@context' in activity: - del activity['@context'] + if "@context" in activity: + del activity["@context"] return activity def activity_from_doc(raw_doc: Dict[str, Any], embed: bool = False) -> Dict[str, Any]: raw_doc = add_extra_collection(raw_doc) - activity = clean_activity(raw_doc['activity']) + activity = clean_activity(raw_doc["activity"]) if embed: return remove_context(activity) return activity - -@app.route('/outbox', methods=['GET', 'POST']) +@app.route("/outbox", methods=["GET", "POST"]) def outbox(): - if request.method == 'GET': + if request.method == "GET": if not is_api_request(): abort(404) # TODO(tsileo): filter the outbox if not authenticated # FIXME(tsileo): filter deleted, add query support for build_ordered_collection q = { - 'meta.deleted': False, - #'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + "meta.deleted": False, + # 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } - return jsonify(**activitypub.build_ordered_collection( - DB.outbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: activity_from_doc(doc, embed=True), - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: activity_from_doc(doc, embed=True), + ) + ) # Handle POST request try: @@ -652,203 +730,207 @@ def outbox(): # Purge the cache if a custom hook is set, as new content was published custom_cache_purge_hook() - return Response(status=201, headers={'Location': activity.id}) + return Response(status=201, headers={"Location": activity.id}) -@app.route('/outbox/') +@app.route("/outbox/") def outbox_detail(item_id): - doc = DB.outbox.find_one({'id': item_id}) - if doc['meta'].get('deleted', False): - obj = activitypub.parse_activity(doc['activity']) + doc = DB.outbox.find_one({"id": item_id}) + if doc["meta"].get("deleted", False): + obj = activitypub.parse_activity(doc["activity"]) resp = jsonify(**obj.get_object().get_tombstone()) resp.status_code = 410 return resp return jsonify(**activity_from_doc(doc)) -@app.route('/outbox//activity') +@app.route("/outbox//activity") def outbox_activity(item_id): # TODO(tsileo): handle Tombstone - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) if not data: abort(404) obj = activity_from_doc(data) - if obj['type'] != ActivityType.CREATE.value: + if obj["type"] != ActivityType.CREATE.value: abort(404) - return jsonify(**obj['object']) + return jsonify(**obj["object"]) -@app.route('/outbox//replies') +@app.route("/outbox//replies") def outbox_activity_replies(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) if not data: abort(404) - obj = activitypub.parse_activity(data['activity']) + obj = activitypub.parse_activity(data["activity"]) if obj.type_enum != ActivityType.CREATE: abort(404) q = { - 'meta.deleted': False, - 'type': ActivityType.CREATE.value, - 'activity.object.inReplyTo': obj.get_object().id, + "meta.deleted": False, + "type": ActivityType.CREATE.value, + "activity.object.inReplyTo": obj.get_object().id, } - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity']['object'], - col_name=f'outbox/{item_id}/replies', - first_page=request.args.get('page') == 'first', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["activity"]["object"], + col_name=f"outbox/{item_id}/replies", + first_page=request.args.get("page") == "first", + ) + ) -@app.route('/outbox//likes') +@app.route("/outbox//likes") def outbox_activity_likes(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) if not data: abort(404) - obj = activitypub.parse_activity(data['activity']) + obj = activitypub.parse_activity(data["activity"]) if obj.type_enum != ActivityType.CREATE: abort(404) q = { - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - '$or': [{'activity.object.id': obj.get_object().id}, - {'activity.object': obj.get_object().id}], + "meta.undo": False, + "type": ActivityType.LIKE.value, + "$or": [ + {"activity.object.id": obj.get_object().id}, + {"activity.object": obj.get_object().id}, + ], } - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - col_name=f'outbox/{item_id}/likes', - first_page=request.args.get('page') == 'first', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: remove_context(doc["activity"]), + col_name=f"outbox/{item_id}/likes", + first_page=request.args.get("page") == "first", + ) + ) -@app.route('/outbox//shares') +@app.route("/outbox//shares") def outbox_activity_shares(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) if not data: abort(404) - obj = activitypub.parse_activity(data['activity']) + obj = activitypub.parse_activity(data["activity"]) if obj.type_enum != ActivityType.CREATE: abort(404) q = { - 'meta.undo': False, - 'type': ActivityType.ANNOUNCE.value, - '$or': [{'activity.object.id': obj.get_object().id}, - {'activity.object': obj.get_object().id}], + "meta.undo": False, + "type": ActivityType.ANNOUNCE.value, + "$or": [ + {"activity.object.id": obj.get_object().id}, + {"activity.object": obj.get_object().id}, + ], } - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - col_name=f'outbox/{item_id}/shares', - first_page=request.args.get('page') == 'first', - )) - - -@app.route('/admin', methods=['GET']) -@login_required -def admin(): - q = { - 'meta.deleted': False, - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - } - col_liked = DB.outbox.count(q) - - return render_template( - 'admin.html', - instances=list(DB.instances.find()), - inbox_size=DB.inbox.count(), - outbox_size=DB.outbox.count(), - object_cache_size=DB.objects_cache.count(), - actor_cache_size=DB.actors_cache.count(), - col_liked=col_liked, - col_followers=DB.followers.count(), - col_following=DB.following.count(), + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: remove_context(doc["activity"]), + col_name=f"outbox/{item_id}/shares", + first_page=request.args.get("page") == "first", + ) ) -@app.route('/new', methods=['GET']) +@app.route("/admin", methods=["GET"]) +@login_required +def admin(): + q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} + col_liked = DB.outbox.count(q) + + return render_template( + "admin.html", + instances=list(DB.instances.find()), + inbox_size=DB.inbox.count(), + outbox_size=DB.outbox.count(), + object_cache_size=DB.objects_cache.count(), + actor_cache_size=DB.actors_cache.count(), + col_liked=col_liked, + col_followers=DB.followers.count(), + col_following=DB.following.count(), + ) + + +@app.route("/new", methods=["GET"]) @login_required def new(): reply_id = None - content = '' + content = "" thread = [] - if request.args.get('reply'): - data = DB.inbox.find_one({'activity.object.id': request.args.get('reply')}) + if request.args.get("reply"): + data = DB.inbox.find_one({"activity.object.id": request.args.get("reply")}) if not data: - data = DB.outbox.find_one({'activity.object.id': request.args.get('reply')}) + data = DB.outbox.find_one({"activity.object.id": request.args.get("reply")}) if not data: abort(400) - reply = activitypub.parse_activity(data['activity']) + reply = activitypub.parse_activity(data["activity"]) reply_id = reply.id if reply.type_enum == ActivityType.CREATE: reply_id = reply.get_object().id actor = reply.get_actor() domain = urlparse(actor.id).netloc # FIXME(tsileo): if reply of reply, fetch all participants - content = f'@{actor.preferredUsername}@{domain} ' - thread = _build_thread( - data, - include_children=False, - ) + content = f"@{actor.preferredUsername}@{domain} " + thread = _build_thread(data, include_children=False) - return render_template( - 'new.html', - reply=reply_id, - content=content, - thread=thread, - ) + return render_template("new.html", reply=reply_id, content=content, thread=thread) -@app.route('/notifications') +@app.route("/notifications") @login_required def notifications(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 q = { - 'type': 'Create', - 'activity.object.tag.type': 'Mention', - 'activity.object.tag.name': f'@{USERNAME}@{DOMAIN}', - 'meta.deleted': False, + "type": "Create", + "activity.object.tag.type": "Mention", + "activity.object.tag.name": f"@{USERNAME}@{DOMAIN}", + "meta.deleted": False, } # TODO(tsileo): also include replies via regex on Create replyTo - q = {'$or': [q, {'type': 'Follow'}, {'type': 'Accept'}, {'type': 'Undo', 'activity.object.type': 'Follow'}, - {'type': 'Announce', 'activity.object': {'$regex': f'^{BASE_URL}'}}, - {'type': 'Create', 'activity.object.inReplyTo': {'$regex': f'^{BASE_URL}'}}, - ]} + q = { + "$or": [ + q, + {"type": "Follow"}, + {"type": "Accept"}, + {"type": "Undo", "activity.object.type": "Follow"}, + {"type": "Announce", "activity.object": {"$regex": f"^{BASE_URL}"}}, + {"type": "Create", "activity.object.inReplyTo": {"$regex": f"^{BASE_URL}"}}, + ] + } print(q) - c = request.args.get('cursor') + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.inbox.find(q, limit=limit).sort('_id', -1)) + outbox_data = list(DB.inbox.find(q, limit=limit).sort("_id", -1)) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) # TODO(tsileo): fix the annonce handling, copy it from /stream - #for data in outbox_data: + # for data in outbox_data: # if data['type'] == 'Announce': # print(data) # if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: @@ -857,14 +939,10 @@ def notifications(): # else: # out.append(data) - return render_template( - 'stream.html', - inbox_data=outbox_data, - cursor=cursor, - ) + return render_template("stream.html", inbox_data=outbox_data, cursor=cursor) -@app.route('/api/key') +@app.route("/api/key") @login_required def api_user_key(): return flask_jsonify(api_key=ADMIN_API_KEY) @@ -878,25 +956,29 @@ def _user_api_arg(key: str, **kwargs): oid = request.args.get(key) or request.form.get(key) if not oid: - if 'default' in kwargs: - return kwargs.get('default') + if "default" in kwargs: + return kwargs.get("default") - raise ValueError(f'missing {key}') + raise ValueError(f"missing {key}") return oid def _user_api_get_note(from_outbox: bool = False): - oid = _user_api_arg('id') - note = activitypub.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE) + oid = _user_api_arg("id") + note = activitypub.parse_activity( + OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE + ) if from_outbox and not note.id.startswith(ID): - raise NotFromOutboxError(f'cannot delete {note.id}, id must be owned by the server') + raise NotFromOutboxError( + f"cannot delete {note.id}, id must be owned by the server" + ) return note def _user_api_response(**kwargs): - _redirect = _user_api_arg('redirect', default=None) + _redirect = _user_api_arg("redirect", default=None) if _redirect: return redirect(_redirect) @@ -905,7 +987,7 @@ def _user_api_response(**kwargs): return resp -@app.route('/api/note/delete', methods=['POST']) +@app.route("/api/note/delete", methods=["POST"]) @api_required def api_delete(): """API endpoint to delete a Note activity.""" @@ -917,7 +999,7 @@ def api_delete(): return _user_api_response(activity=delete.id) -@app.route('/api/boost', methods=['POST']) +@app.route("/api/boost", methods=["POST"]) @api_required def api_boost(): note = _user_api_get_note() @@ -928,7 +1010,7 @@ def api_boost(): return _user_api_response(activity=announce.id) -@app.route('/api/like', methods=['POST']) +@app.route("/api/like", methods=["POST"]) @api_required def api_like(): note = _user_api_get_note() @@ -939,15 +1021,15 @@ def api_like(): return _user_api_response(activity=like.id) -@app.route('/api/undo', methods=['POST']) +@app.route("/api/undo", methods=["POST"]) @api_required def api_undo(): - oid = _user_api_arg('id') - doc = DB.outbox.find_one({'$or': [{'id': oid}, {'remote_id': oid}]}) + oid = _user_api_arg("id") + doc = DB.outbox.find_one({"$or": [{"id": oid}, {"remote_id": oid}]}) if not doc: - raise ActivityNotFoundError(f'cannot found {oid}') + raise ActivityNotFoundError(f"cannot found {oid}") - obj = activitypub.parse_activity(doc.get('activity')) + obj = activitypub.parse_activity(doc.get("activity")) # FIXME(tsileo): detect already undo-ed and make this API call idempotent undo = obj.build_undo() undo.post_to_outbox() @@ -955,58 +1037,60 @@ def api_undo(): return _user_api_response(activity=undo.id) -@app.route('/stream') +@app.route("/stream") @login_required def stream(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 100 q = { - 'type': 'Create', - 'activity.object.type': 'Note', - 'activity.object.inReplyTo': None, - 'meta.deleted': False, + "type": "Create", + "activity.object.type": "Note", + "activity.object.inReplyTo": None, + "meta.deleted": False, } - c = request.args.get('cursor') + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.inbox.find( - { - '$or': [ - q, - { - 'type': 'Announce', - }, - ] - }, limit=limit).sort('activity.published', -1)) + outbox_data = list( + DB.inbox.find({"$or": [q, {"type": "Announce"}]}, limit=limit).sort( + "activity.published", -1 + ) + ) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) out = [] objcache = {} - cached = list(DB.objects_cache.find({'meta.part_of_stream': True}, limit=limit*3).sort('meta.announce_published', -1)) + cached = list( + DB.objects_cache.find({"meta.part_of_stream": True}, limit=limit * 3).sort( + "meta.announce_published", -1 + ) + ) for c in cached: - objcache[c['object_id']] = c['cached_object'] + objcache[c["object_id"]] = c["cached_object"] for data in outbox_data: - if data['type'] == 'Announce': - if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: - data['ref'] = {'activity': {'object': objcache[data['activity']['object']]}, 'meta': {}} + if data["type"] == "Announce": + if ( + data["activity"]["object"].startswith("http") + and data["activity"]["object"] in objcache + ): + data["ref"] = { + "activity": {"object": objcache[data["activity"]["object"]]}, + "meta": {}, + } out.append(data) else: - print('OMG', data) + print("OMG", data) else: out.append(data) - return render_template( - 'stream.html', - inbox_data=out, - cursor=cursor, - ) + return render_template("stream.html", inbox_data=out, cursor=cursor) -@app.route('/inbox', methods=['GET', 'POST']) +@app.route("/inbox", methods=["GET", "POST"]) def inbox(): - if request.method == 'GET': + if request.method == "GET": if not is_api_request(): abort(404) try: @@ -1014,135 +1098,136 @@ def inbox(): except BadSignature: abort(404) - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q={'meta.deleted': False}, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q={"meta.deleted": False}, + cursor=request.args.get("cursor"), + map_func=lambda doc: remove_context(doc["activity"]), + ) + ) data = request.get_json(force=True) - logger.debug(f'req_headers={request.headers}') - logger.debug(f'raw_data={data}') + logger.debug(f"req_headers={request.headers}") + logger.debug(f"raw_data={data}") try: if not verify_request(ACTOR_SERVICE): - raise Exception('failed to verify request') + raise Exception("failed to verify request") except Exception: - logger.exception('failed to verify request, trying to verify the payload by fetching the remote') + logger.exception( + "failed to verify request, trying to verify the payload by fetching the remote" + ) try: - data = OBJECT_SERVICE.get(data['id']) + data = OBJECT_SERVICE.get(data["id"]) except Exception: logger.exception(f'failed to fetch remote id at {data["id"]}') return Response( status=422, - headers={'Content-Type': 'application/json'}, - response=json.dumps({'error': 'failed to verify request (using HTTP signatures or fetching the IRI)'}), + headers={"Content-Type": "application/json"}, + response=json.dumps( + { + "error": "failed to verify request (using HTTP signatures or fetching the IRI)" + } + ), ) activity = activitypub.parse_activity(data) - logger.debug(f'inbox activity={activity}/{data}') + logger.debug(f"inbox activity={activity}/{data}") activity.process_from_inbox() - return Response( - status=201, - ) + return Response(status=201) -@app.route('/api/debug', methods=['GET', 'DELETE']) +@app.route("/api/debug", methods=["GET", "DELETE"]) @api_required def api_debug(): """Endpoint used/needed for testing, only works in DEBUG_MODE.""" if not DEBUG_MODE: - return flask_jsonify(message='DEBUG_MODE is off') + return flask_jsonify(message="DEBUG_MODE is off") - if request.method == 'DELETE': + if request.method == "DELETE": _drop_db() - return flask_jsonify(message='DB dropped') + return flask_jsonify(message="DB dropped") - return flask_jsonify( - inbox=DB.inbox.count(), - outbox=DB.outbox.count(), - ) + return flask_jsonify(inbox=DB.inbox.count(), outbox=DB.outbox.count()) -@app.route('/api/upload', methods=['POST']) +@app.route("/api/upload", methods=["POST"]) @api_required def api_upload(): - file = request.files['file'] + file = request.files["file"] rfilename = secure_filename(file.filename) prefix = hashlib.sha256(os.urandom(32)).hexdigest()[:6] mtype = mimetypes.guess_type(rfilename)[0] - filename = f'{prefix}_{rfilename}' - file.save(os.path.join('static', 'media', filename)) + filename = f"{prefix}_{rfilename}" + file.save(os.path.join("static", "media", filename)) # Remove EXIF metadata - if filename.lower().endswith('.jpg') or filename.lower().endswith('.jpeg'): - piexif.remove(os.path.join('static', 'media', filename)) + if filename.lower().endswith(".jpg") or filename.lower().endswith(".jpeg"): + piexif.remove(os.path.join("static", "media", filename)) - print('upload OK') + print("upload OK") print(filename) attachment = [ - {'mediaType': mtype, - 'name': rfilename, - 'type': 'Document', - 'url': BASE_URL + f'/static/media/{filename}' - }, + { + "mediaType": mtype, + "name": rfilename, + "type": "Document", + "url": BASE_URL + f"/static/media/{filename}", + } ] print(attachment) - content = request.args.get('content') - to = request.args.get('to') + content = request.args.get("content") + to = request.args.get("to") note = activitypub.Note( - cc=[ID+'/followers'], + cc=[ID + "/followers"], to=[to if to else config.AS_PUBLIC], content=content, # TODO(tsileo): handle markdown attachment=attachment, ) - print('post_note_init') + print("post_note_init") print(note) create = note.build_create() print(create) print(create.to_dict()) create.post_to_outbox() - print('posted') + print("posted") - return Response( - status=201, - response='OK', - ) + return Response(status=201, response="OK") -@app.route('/api/new_note', methods=['POST']) +@app.route("/api/new_note", methods=["POST"]) @api_required def api_new_note(): - source = _user_api_arg('content') + source = _user_api_arg("content") if not source: - raise ValueError('missing content') + raise ValueError("missing content") _reply, reply = None, None try: - _reply = _user_api_arg('reply') + _reply = _user_api_arg("reply") except ValueError: pass content, tags = parse_markdown(source) - to = request.args.get('to') - cc = [ID+'/followers'] + to = request.args.get("to") + cc = [ID + "/followers"] if _reply: reply = activitypub.parse_activity(OBJECT_SERVICE.get(_reply)) cc.append(reply.attributedTo) for tag in tags: - if tag['type'] == 'Mention': - cc.append(tag['href']) + if tag["type"] == "Mention": + cc.append(tag["href"]) note = activitypub.Note( cc=list(set(cc)), to=[to if to else config.AS_PUBLIC], content=content, tag=tags, - source={'mediaType': 'text/markdown', 'content': source}, - inReplyTo=reply.id if reply else None + source={"mediaType": "text/markdown", "content": source}, + inReplyTo=reply.id if reply else None, ) create = note.build_create() create.post_to_outbox() @@ -1150,27 +1235,27 @@ def api_new_note(): return _user_api_response(activity=create.id) -@app.route('/api/stream') +@app.route("/api/stream") @api_required def api_stream(): return Response( - response=json.dumps(activitypub.build_inbox_json_feed('/api/stream', request.args.get('cursor'))), - headers={'Content-Type': 'application/json'}, + response=json.dumps( + activitypub.build_inbox_json_feed("/api/stream", request.args.get("cursor")) + ), + headers={"Content-Type": "application/json"}, ) -@app.route('/api/block', methods=['POST']) +@app.route("/api/block", methods=["POST"]) @api_required def api_block(): - actor = _user_api_arg('actor') + actor = _user_api_arg("actor") - existing = DB.outbox.find_one({ - 'type': ActivityType.BLOCK.value, - 'activity.object': actor, - 'meta.undo': False, - }) + existing = DB.outbox.find_one( + {"type": ActivityType.BLOCK.value, "activity.object": actor, "meta.undo": False} + ) if existing: - return _user_api_response(activity=existing['activity']['id']) + return _user_api_response(activity=existing["activity"]["id"]) block = activitypub.Block(object=actor) block.post_to_outbox() @@ -1178,14 +1263,14 @@ def api_block(): return _user_api_response(activity=block.id) -@app.route('/api/follow', methods=['POST']) +@app.route("/api/follow", methods=["POST"]) @api_required def api_follow(): - actor = _user_api_arg('actor') + actor = _user_api_arg("actor") - existing = DB.following.find_one({'remote_actor': actor}) + existing = DB.following.find_one({"remote_actor": actor}) if existing: - return _user_api_response(activity=existing['activity']['id']) + return _user_api_response(activity=existing["activity"]["id"]) follow = activitypub.Follow(object=actor) follow.post_to_outbox() @@ -1193,109 +1278,122 @@ def api_follow(): return _user_api_response(activity=follow.id) -@app.route('/followers') +@app.route("/followers") def followers(): if is_api_request(): return jsonify( **activitypub.build_ordered_collection( DB.followers, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['remote_actor'], + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["remote_actor"], ) ) - followers = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.followers.find(limit=50)] + followers = [ + ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.followers.find(limit=50) + ] return render_template( - 'followers.html', + "followers.html", me=ME, - notes=DB.inbox.find({'object.object.type': 'Note'}).count(), + notes=DB.inbox.find({"object.object.type": "Note"}).count(), followers=DB.followers.count(), following=DB.following.count(), followers_data=followers, ) -@app.route('/following') +@app.route("/following") def following(): if is_api_request(): return jsonify( **activitypub.build_ordered_collection( DB.following, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['remote_actor'], - ), + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["remote_actor"], + ) ) - following = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.following.find(limit=50)] + following = [ + ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.following.find(limit=50) + ] return render_template( - 'following.html', + "following.html", me=ME, - notes=DB.inbox.find({'object.object.type': 'Note'}).count(), + notes=DB.inbox.find({"object.object.type": "Note"}).count(), followers=DB.followers.count(), following=DB.following.count(), following_data=following, ) -@app.route('/tags/') +@app.route("/tags/") def tags(tag): - if not DB.outbox.count({'activity.object.tag.type': 'Hashtag', 'activity.object.tag.name': '#'+tag}): + if not DB.outbox.count( + {"activity.object.tag.type": "Hashtag", "activity.object.tag.name": "#" + tag} + ): abort(404) if not is_api_request(): return render_template( - 'tags.html', + "tags.html", tag=tag, - outbox_data=DB.outbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False, - 'activity.object.tag.type': 'Hashtag', - 'activity.object.tag.name': '#'+tag}), + outbox_data=DB.outbox.find( + { + "type": "Create", + "activity.object.type": "Note", + "meta.deleted": False, + "activity.object.tag.type": "Hashtag", + "activity.object.tag.name": "#" + tag, + } + ), ) q = { - 'meta.deleted': False, - 'meta.undo': False, - 'type': ActivityType.CREATE.value, - 'activity.object.tag.type': 'Hashtag', - 'activity.object.tag.name': '#'+tag, + "meta.deleted": False, + "meta.undo": False, + "type": ActivityType.CREATE.value, + "activity.object.tag.type": "Hashtag", + "activity.object.tag.name": "#" + tag, } - return jsonify(**activitypub.build_ordered_collection( - DB.outbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity']['object']['id'], - col_name=f'tags/{tag}', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["activity"]["object"]["id"], + col_name=f"tags/{tag}", + ) + ) -@app.route('/liked') +@app.route("/liked") def liked(): if not is_api_request(): abort(404) - q = { - 'meta.deleted': False, - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - } - return jsonify(**activitypub.build_ordered_collection( - DB.outbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity']['object'], - col_name='liked', - )) + q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} + return jsonify( + **activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["activity"]["object"], + col_name="liked", + ) + ) + ####### # IndieAuth def build_auth_resp(payload): - if request.headers.get('Accept') == 'application/json': + if request.headers.get("Accept") == "application/json": return Response( status=200, - headers={'Content-Type': 'application/json'}, + headers={"Content-Type": "application/json"}, response=json.dumps(payload), ) return Response( status=200, - headers={'Content-Type': 'application/x-www-form-urlencoded'}, + headers={"Content-Type": "application/x-www-form-urlencoded"}, response=urlencode(payload), ) @@ -1308,43 +1406,37 @@ def _get_prop(props, name, default=None): return items return default + def get_client_id_data(url): data = mf2py.parse(url=url) - for item in data['items']: - if 'h-x-app' in item['type'] or 'h-app' in item['type']: - props = item.get('properties', {}) + for item in data["items"]: + if "h-x-app" in item["type"] or "h-app" in item["type"]: + props = item.get("properties", {}) print(props) return dict( - logo=_get_prop(props, 'logo'), - name=_get_prop(props, 'name'), - url=_get_prop(props, 'url'), + logo=_get_prop(props, "logo"), + name=_get_prop(props, "name"), + url=_get_prop(props, "url"), ) - return dict( - logo=None, - name=url, - url=url, - ) + return dict(logo=None, name=url, url=url) -@app.route('/indieauth/flow', methods=['POST']) +@app.route("/indieauth/flow", methods=["POST"]) @login_required def indieauth_flow(): auth = dict( - scope=' '.join(request.form.getlist('scopes')), - me=request.form.get('me'), - client_id=request.form.get('client_id'), - state=request.form.get('state'), - redirect_uri=request.form.get('redirect_uri'), - response_type=request.form.get('response_type'), + scope=" ".join(request.form.getlist("scopes")), + me=request.form.get("me"), + client_id=request.form.get("client_id"), + state=request.form.get("state"), + redirect_uri=request.form.get("redirect_uri"), + response_type=request.form.get("response_type"), ) - code = binascii.hexlify(os.urandom(8)).decode('utf-8') - auth.update( - code=code, - verified=False, - ) + code = binascii.hexlify(os.urandom(8)).decode("utf-8") + auth.update(code=code, verified=False) print(auth) - if not auth['redirect_uri']: + if not auth["redirect_uri"]: abort(500) DB.indieauth.insert_one(auth) @@ -1356,21 +1448,21 @@ def indieauth_flow(): # @app.route('/indieauth', methods=['GET', 'POST']) def indieauth_endpoint(): - if request.method == 'GET': - if not session.get('logged_in'): - return redirect(url_for('login', next=request.url)) + if request.method == "GET": + if not session.get("logged_in"): + return redirect(url_for("login", next=request.url)) - me = request.args.get('me') + me = request.args.get("me") # FIXME(tsileo): ensure me == ID - client_id = request.args.get('client_id') - redirect_uri = request.args.get('redirect_uri') - state = request.args.get('state', '') - response_type = request.args.get('response_type', 'id') - scope = request.args.get('scope', '').split() + client_id = request.args.get("client_id") + redirect_uri = request.args.get("redirect_uri") + state = request.args.get("state", "") + response_type = request.args.get("response_type", "id") + scope = request.args.get("scope", "").split() - print('STATE', state) + print("STATE", state) return render_template( - 'indieauth_flow.html', + "indieauth_flow.html", client=get_client_id_data(client_id), scopes=scope, redirect_uri=redirect_uri, @@ -1381,14 +1473,18 @@ def indieauth_endpoint(): ) # Auth verification via POST - code = request.form.get('code') - redirect_uri = request.form.get('redirect_uri') - client_id = request.form.get('client_id') + code = request.form.get("code") + redirect_uri = request.form.get("redirect_uri") + client_id = request.form.get("client_id") auth = DB.indieauth.find_one_and_update( - {'code': code, 'redirect_uri': redirect_uri, 'client_id': client_id}, #}, # , 'verified': False}, - {'$set': {'verified': True}}, - sort=[('_id', pymongo.DESCENDING)], + { + "code": code, + "redirect_uri": redirect_uri, + "client_id": client_id, + }, # }, # , 'verified': False}, + {"$set": {"verified": True}}, + sort=[("_id", pymongo.DESCENDING)], ) print(auth) print(code, redirect_uri, client_id) @@ -1397,33 +1493,42 @@ def indieauth_endpoint(): abort(403) return - session['logged_in'] = True - me = auth['me'] - state = auth['state'] - scope = ' '.join(auth['scope']) - print('STATE', state) - return build_auth_resp({'me': me, 'state': state, 'scope': scope}) + session["logged_in"] = True + me = auth["me"] + state = auth["state"] + scope = " ".join(auth["scope"]) + print("STATE", state) + return build_auth_resp({"me": me, "state": state, "scope": scope}) -@app.route('/token', methods=['GET', 'POST']) +@app.route("/token", methods=["GET", "POST"]) def token_endpoint(): - if request.method == 'POST': - code = request.form.get('code') - me = request.form.get('me') - redirect_uri = request.form.get('redirect_uri') - client_id = request.form.get('client_id') + if request.method == "POST": + code = request.form.get("code") + me = request.form.get("me") + redirect_uri = request.form.get("redirect_uri") + client_id = request.form.get("client_id") - auth = DB.indieauth.find_one({'code': code, 'me': me, 'redirect_uri': redirect_uri, 'client_id': client_id}) + auth = DB.indieauth.find_one( + { + "code": code, + "me": me, + "redirect_uri": redirect_uri, + "client_id": client_id, + } + ) if not auth: abort(403) - scope = ' '.join(auth['scope']) - payload = dict(me=me, client_id=client_id, scope=scope, ts=datetime.now().timestamp()) - token = JWT.dumps(payload).decode('utf-8') + scope = " ".join(auth["scope"]) + payload = dict( + me=me, client_id=client_id, scope=scope, ts=datetime.now().timestamp() + ) + token = JWT.dumps(payload).decode("utf-8") - return build_auth_resp({'me': me, 'scope': scope, 'access_token': token}) + return build_auth_resp({"me": me, "scope": scope, "access_token": token}) # Token verification - token = request.headers.get('Authorization').replace('Bearer ', '') + token = request.headers.get("Authorization").replace("Bearer ", "") try: payload = JWT.loads(token) except BadSignature: @@ -1431,8 +1536,10 @@ def token_endpoint(): # TODO(tsileo): handle expiration - return build_auth_resp({ - 'me': payload['me'], - 'scope': payload['scope'], - 'client_id': payload['client_id'], - }) + return build_auth_resp( + { + "me": payload["me"], + "scope": payload["scope"], + "client_id": payload["client_id"], + } + ) diff --git a/config.py b/config.py index 9adbff9..c0e1858 100644 --- a/config.py +++ b/config.py @@ -1,13 +1,16 @@ -import subprocess import os -import yaml -from pymongo import MongoClient -import requests -from itsdangerous import JSONWebSignatureSerializer +import subprocess from datetime import datetime -from little_boxes.utils import strtobool -from utils.key import KEY_DIR, get_key, get_secret_key +import requests +import yaml +from itsdangerous import JSONWebSignatureSerializer +from pymongo import MongoClient + +from little_boxes import strtobool +from utils.key import KEY_DIR +from utils.key import get_key +from utils.key import get_secret_key def noop(): diff --git a/dev-requirements.txt b/dev-requirements.txt index 62e71f2..a4ab4e5 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,3 +4,4 @@ html2text pyyaml flake8 mypy +black diff --git a/tasks.py b/tasks.py index a30854a..37b0257 100644 --- a/tasks.py +++ b/tasks.py @@ -1,47 +1,53 @@ -import os import json import logging +import os import random import requests from celery import Celery from requests.exceptions import HTTPError +from config import DB from config import HEADERS from config import ID -from config import DB from config import KEY from config import USER_AGENT from utils.httpsig import HTTPSigAuth -from utils.opengraph import fetch_og_metadata from utils.linked_data_sig import generate_signature - +from utils.opengraph import fetch_og_metadata log = logging.getLogger(__name__) -app = Celery('tasks', broker=os.getenv('MICROBLOGPUB_AMQP_BROKER', 'pyamqp://guest@localhost//')) -SigAuth = HTTPSigAuth(ID+'#main-key', KEY.privkey) +app = Celery( + "tasks", broker=os.getenv("MICROBLOGPUB_AMQP_BROKER", "pyamqp://guest@localhost//") +) +SigAuth = HTTPSigAuth(ID + "#main-key", KEY.privkey) @app.task(bind=True, max_retries=12) def post_to_inbox(self, payload: str, to: str) -> None: try: - log.info('payload=%s', payload) - log.info('generating sig') + log.info("payload=%s", payload) + log.info("generating sig") signed_payload = json.loads(payload) generate_signature(signed_payload, KEY.privkey) - log.info('to=%s', to) - resp = requests.post(to, data=json.dumps(signed_payload), auth=SigAuth, headers={ - 'Content-Type': HEADERS[1], - 'Accept': HEADERS[1], - 'User-Agent': USER_AGENT, - }) - log.info('resp=%s', resp) - log.info('resp_body=%s', resp.text) + log.info("to=%s", to) + resp = requests.post( + to, + data=json.dumps(signed_payload), + auth=SigAuth, + headers={ + "Content-Type": HEADERS[1], + "Accept": HEADERS[1], + "User-Agent": USER_AGENT, + }, + ) + log.info("resp=%s", resp) + log.info("resp_body=%s", resp.text) resp.raise_for_status() except HTTPError as err: - log.exception('request failed') + log.exception("request failed") if 400 >= err.response.status_code >= 499: - log.info('client error, no retry') + log.info("client error, no retry") return self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) @@ -49,11 +55,15 @@ def post_to_inbox(self, payload: str, to: str) -> None: @app.task(bind=True, max_retries=12) def fetch_og(self, col, remote_id): try: - log.info('fetch_og_meta remote_id=%s col=%s', remote_id, col) - if col == 'INBOX': - log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.inbox, remote_id)) - elif col == 'OUTBOX': - log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.outbox, remote_id)) + log.info("fetch_og_meta remote_id=%s col=%s", remote_id, col) + if col == "INBOX": + log.info( + "%d links saved", fetch_og_metadata(USER_AGENT, DB.inbox, remote_id) + ) + elif col == "OUTBOX": + log.info( + "%d links saved", fetch_og_metadata(USER_AGENT, DB.outbox, remote_id) + ) except Exception as err: - self.log.exception('failed') + self.log.exception("failed") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) diff --git a/tests/federation_test.py b/tests/federation_test.py index e050afc..a2dba10 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -1,12 +1,12 @@ -import time import os +import time +from typing import List +from typing import Tuple import requests from html2text import html2text -from utils import activitypub_utils -from typing import Tuple -from typing import List +from utils import activitypub_utils def resp2plaintext(resp): diff --git a/utils/activitypub_utils.py b/utils/activitypub_utils.py index 0275f54..3204237 100644 --- a/utils/activitypub_utils.py +++ b/utils/activitypub_utils.py @@ -1,4 +1,7 @@ -from typing import Optional, Dict, List, Any +from typing import Any +from typing import Dict +from typing import List +from typing import Optional import requests diff --git a/utils/actor_service.py b/utils/actor_service.py index 9982235..f13a6cf 100644 --- a/utils/actor_service.py +++ b/utils/actor_service.py @@ -1,11 +1,11 @@ import logging +from urllib.parse import urlparse import requests -from urllib.parse import urlparse from Crypto.PublicKey import RSA -from .urlutils import check_url from .errors import ActivityNotFoundError +from .urlutils import check_url logger = logging.getLogger(__name__) diff --git a/utils/content_helper.py b/utils/content_helper.py index b254e2b..8ea8cf2 100644 --- a/utils/content_helper.py +++ b/utils/content_helper.py @@ -1,14 +1,21 @@ -import typing -import re +import re +import typing +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple +from typing import Type +from typing import Union -from bleach.linkifier import Linker -from markdown import markdown +from bleach.linkifier import Linker +from markdown import markdown -from utils.webfinger import get_actor_url -from config import USERNAME, BASE_URL, ID -from config import ACTOR_SERVICE - -from typing import List, Optional, Tuple, Dict, Any, Union, Type +from config import ACTOR_SERVICE +from config import BASE_URL +from config import ID +from config import USERNAME +from utils.webfinger import get_actor_url def set_attrs(attrs, new=False): diff --git a/utils/httpsig.py b/utils/httpsig.py index 8437784..609ec3d 100644 --- a/utils/httpsig.py +++ b/utils/httpsig.py @@ -3,19 +3,20 @@ Mastodon instances won't accept requests that are not signed using this scheme. """ -from datetime import datetime -from urllib.parse import urlparse -from typing import Any, Dict, Optional import base64 import hashlib import logging +from datetime import datetime +from typing import Any +from typing import Dict +from typing import Optional +from urllib.parse import urlparse +from Crypto.Hash import SHA256 +from Crypto.Signature import PKCS1_v1_5 from flask import request from requests.auth import AuthBase -from Crypto.Signature import PKCS1_v1_5 -from Crypto.Hash import SHA256 - logger = logging.getLogger(__name__) diff --git a/utils/key.py b/utils/key.py index 18162a5..e7012ae 100644 --- a/utils/key.py +++ b/utils/key.py @@ -1,6 +1,5 @@ -import os import binascii - +import os from typing import Callable from little_boxes.key import Key diff --git a/utils/linked_data_sig.py b/utils/linked_data_sig.py index 834c9bd..b75166e 100644 --- a/utils/linked_data_sig.py +++ b/utils/linked_data_sig.py @@ -1,13 +1,12 @@ -from pyld import jsonld +import base64 import hashlib from datetime import datetime +from typing import Any +from typing import Dict -from Crypto.Signature import PKCS1_v1_5 from Crypto.Hash import SHA256 -import base64 - -from typing import Any, Dict - +from Crypto.Signature import PKCS1_v1_5 +from pyld import jsonld # cache the downloaded "schemas", otherwise the library is super slow # (https://github.com/digitalbazaar/pyld/issues/70) diff --git a/utils/object_service.py b/utils/object_service.py index 1ebc0ce..594fa10 100644 --- a/utils/object_service.py +++ b/utils/object_service.py @@ -1,8 +1,9 @@ -import requests from urllib.parse import urlparse -from .urlutils import check_url +import requests + from .errors import ActivityNotFoundError +from .urlutils import check_url class ObjectService(object): diff --git a/utils/opengraph.py b/utils/opengraph.py index a53c07b..8bafece 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -1,11 +1,12 @@ +import ipaddress from urllib.parse import urlparse -import ipaddress import opengraph import requests from bs4 import BeautifulSoup -from .urlutils import is_url_valid, check_url +from .urlutils import check_url +from .urlutils import is_url_valid def links_from_note(note): diff --git a/utils/urlutils.py b/utils/urlutils.py index 99f900d..360d209 100644 --- a/utils/urlutils.py +++ b/utils/urlutils.py @@ -1,7 +1,7 @@ +import ipaddress import logging import os import socket -import ipaddress from urllib.parse import urlparse from . import strtobool diff --git a/utils/webfinger.py b/utils/webfinger.py index 8e6fdc7..344dc01 100644 --- a/utils/webfinger.py +++ b/utils/webfinger.py @@ -1,13 +1,13 @@ -from urllib.parse import urlparse -from typing import Dict, Any -from typing import Optional import logging +from typing import Any +from typing import Dict +from typing import Optional +from urllib.parse import urlparse import requests from .urlutils import check_url - logger = logging.getLogger(__name__)