From 63ca0b68e1823f35d5ec6b925f630308bd8df56c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 11 Jul 2018 23:22:47 +0200 Subject: [PATCH] Fix the tasks and formatting --- activitypub.py | 27 ++++++++++++----------- app.py | 46 +++++++++++++++++++++++++++------------- config.py | 2 +- tasks.py | 41 +++++++++++++---------------------- tests/federation_test.py | 1 - utils/opengraph.py | 1 - 6 files changed, 62 insertions(+), 56 deletions(-) diff --git a/activitypub.py b/activitypub.py index 8deed50..2e748fa 100644 --- a/activitypub.py +++ b/activitypub.py @@ -8,11 +8,16 @@ from typing import List from typing import Optional from bson.objectid import ObjectId +from cachetools import LRUCache from feedgen.feed import FeedGenerator from html2text import html2text +from little_boxes import activitypub as ap +from little_boxes import strtobool +from little_boxes.activitypub import _to_list +from little_boxes.backend import Backend +from little_boxes.errors import ActivityGoneError +from little_boxes.errors import Error -import tasks -from cachetools import LRUCache from config import BASE_URL from config import DB from config import EXTRA_INBOXES @@ -20,12 +25,6 @@ from config import ID from config import ME from config import USER_AGENT from config import USERNAME -from little_boxes import activitypub as ap -from little_boxes import strtobool -from little_boxes.activitypub import _to_list -from little_boxes.backend import Backend -from little_boxes.errors import ActivityGoneError -from little_boxes.errors import Error logger = logging.getLogger(__name__) @@ -95,9 +94,10 @@ class MicroblogPubBackend(Backend): } ) - tasks.cache_attachments.delay(activity.id) - if box == Box.INBOX: - tasks.process_new_activity.delay(activity.id) + self.save_cb(box, activity.id) + + def set_save_cb(self, cb): + self.save_cb = cb @ensure_it_is_me def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: @@ -198,9 +198,12 @@ class MicroblogPubBackend(Backend): def inbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: self.save(Box.INBOX, activity) + def set_post_to_remote_inbox(self, cb): + self.post_to_remote_inbox_cb = cb + @ensure_it_is_me def post_to_remote_inbox(self, as_actor: ap.Person, payload: str, to: str) -> None: - tasks.post_to_inbox.delay(payload, to) + self.post_to_remote_inbox_cb(payload, to) @ensure_it_is_me def new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: diff --git a/app.py b/app.py index 0287efc..425b5b6 100644 --- a/app.py +++ b/app.py @@ -33,12 +33,27 @@ from flask import url_for from flask_wtf.csrf import CSRFProtect from html2text import html2text from itsdangerous import BadSignature +from little_boxes import activitypub as ap +from little_boxes.activitypub import ActivityType +from little_boxes.activitypub import _to_list +from little_boxes.activitypub import clean_activity +from little_boxes.activitypub import get_backend +from little_boxes.content_helper import parse_markdown +from little_boxes.errors import ActivityGoneError +from little_boxes.errors import ActivityNotFoundError +from little_boxes.errors import Error +from little_boxes.errors import NotFromOutboxError +from little_boxes.httpsig import HTTPSigAuth +from little_boxes.httpsig import verify_request +from little_boxes.webfinger import get_actor_url +from little_boxes.webfinger import get_remote_follow_template from passlib.hash import bcrypt from u2flib_server import u2f from werkzeug.utils import secure_filename import activitypub import config +import tasks from activitypub import Box from activitypub import embed_collection from config import ADMIN_API_KEY @@ -58,24 +73,23 @@ from config import USERNAME from config import VERSION from config import _drop_db from config import custom_cache_purge_hook -from little_boxes import activitypub as ap -from little_boxes.activitypub import ActivityType -from little_boxes.activitypub import _to_list -from little_boxes.activitypub import clean_activity -from little_boxes.activitypub import get_backend -from little_boxes.content_helper import parse_markdown -from little_boxes.errors import ActivityGoneError -from little_boxes.errors import ActivityNotFoundError -from little_boxes.errors import Error -from little_boxes.errors import NotFromOutboxError -from little_boxes.httpsig import HTTPSigAuth -from little_boxes.httpsig import verify_request -from little_boxes.webfinger import get_actor_url -from little_boxes.webfinger import get_remote_follow_template from utils.key import get_secret_key from utils.media import Kind back = activitypub.MicroblogPubBackend() + + +def save_cb(box: Box, iri: str) -> None: + tasks.cache_attachments.delay(iri) + if box == Box.INBOX: + tasks.process_new_activity.delay(iri) + + +back.set_save_cb(save_cb) + + +back.set_post_to_remote_inbox(tasks.post_to_inbox.delay) + ap.use_backend(back) MY_PERSON = ap.Person(**ME) @@ -1569,7 +1583,9 @@ def following(): ) following, older_than, newer_than = paginated_query(DB.activities, q) - following = [get_backend().fetch_iri(doc["activity"]["object"]) for doc in following] + following = [ + get_backend().fetch_iri(doc["activity"]["object"]) for doc in following + ] return render_template( "following.html", following_data=following, diff --git a/config.py b/config.py index 9e7b844..8c78fb6 100644 --- a/config.py +++ b/config.py @@ -8,9 +8,9 @@ import requests import sass import yaml from itsdangerous import JSONWebSignatureSerializer +from little_boxes import strtobool from pymongo import MongoClient -from little_boxes import strtobool from utils.key import KEY_DIR from utils.key import get_key from utils.key import get_secret_key diff --git a/tasks.py b/tasks.py index 2787a0b..7f3a3a8 100644 --- a/tasks.py +++ b/tasks.py @@ -5,18 +5,18 @@ import random import requests from celery import Celery +from little_boxes import activitypub as ap +from little_boxes.httpsig import HTTPSigAuth +from little_boxes.linked_data_sig import generate_signature from requests.exceptions import HTTPError -from little_boxes import activitypub as ap +import activitypub from config import DB from config import HEADERS from config import KEY -from config import USER_AGENT -from little_boxes.httpsig import HTTPSigAuth -from little_boxes.linked_data_sig import generate_signature -from utils.opengraph import fetch_og_metadata -from utils.media import Kind from config import MEDIA_CACHE +from config import USER_AGENT +from utils.media import Kind log = logging.getLogger(__name__) app = Celery( @@ -25,6 +25,10 @@ app = Celery( SigAuth = HTTPSigAuth(KEY) +back = activitypub.MicroblogPubBackend() +ap.use_backend(back) + + @app.task(bind=True, max_retries=12) def process_new_activity(self, iri: str) -> None: try: @@ -32,7 +36,7 @@ def process_new_activity(self, iri: str) -> None: log.info(f"activity={activity!r}") tag_stream = False - if activity.has_type(ap.ActivityType.ANNOUCE): + if activity.has_type(ap.ActivityType.ANNOUNCE): tag_stream = True elif activity.has_type(ap.ActivityType.CREATE): note = activity.get_object() @@ -40,7 +44,9 @@ def process_new_activity(self, iri: str) -> None: tag_stream = True log.info(f"{iri} tag_stream={tag_stream}") - DB.update_one({"remote_id": activity.id}, {"$set": {"meta.stream": tag_stream}}) + DB.activities.update_one( + {"remote_id": activity.id}, {"$set": {"meta.stream": tag_stream}} + ) log.info(f"new activity {iri} processed") except Exception as err: @@ -71,7 +77,7 @@ def cache_attachments(self, iri: str) -> None: for attachment in activity.get_object()._data.get("attachment", []): MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) - log.info(f"attachmwents cached for {iri}") + log.info(f"attachments cached for {iri}") except Exception as err: log.exception(f"failed to process new activity {iri}") @@ -105,20 +111,3 @@ def post_to_inbox(self, payload: str, to: str) -> None: log.info("client error, no retry") return self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) - - -@app.task(bind=True, max_retries=12) -def fetch_og(self, col, remote_id): - try: - log.info("fetch_og_meta remote_id=%s col=%s", remote_id, col) - if col == "INBOX": - log.info( - "%d links saved", fetch_og_metadata(USER_AGENT, DB.inbox, remote_id) - ) - elif col == "OUTBOX": - log.info( - "%d links saved", fetch_og_metadata(USER_AGENT, DB.outbox, remote_id) - ) - except Exception as err: - self.log.exception("failed") - self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) diff --git a/tests/federation_test.py b/tests/federation_test.py index f926812..20569bb 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -5,7 +5,6 @@ from typing import Tuple import requests from html2text import html2text - from little_boxes.collection import parse_collection diff --git a/utils/opengraph.py b/utils/opengraph.py index 597ad3c..b543269 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -1,7 +1,6 @@ import opengraph import requests from bs4 import BeautifulSoup - from little_boxes.urlutils import check_url from little_boxes.urlutils import is_url_valid