Fix the tasks and formatting

activitypub.py

@@ -8,11 +8,16 @@ from typing import List
 from typing import Optional
 
 from bson.objectid import ObjectId
+from cachetools import LRUCache
 from feedgen.feed import FeedGenerator
 from html2text import html2text
+from little_boxes import activitypub as ap
+from little_boxes import strtobool
+from little_boxes.activitypub import _to_list
+from little_boxes.backend import Backend
+from little_boxes.errors import ActivityGoneError
+from little_boxes.errors import Error
 
-import tasks
-from cachetools import LRUCache
 from config import BASE_URL
 from config import DB
 from config import EXTRA_INBOXES
@@ -20,12 +25,6 @@ from config import ID
 from config import ME
 from config import USER_AGENT
 from config import USERNAME
-from little_boxes import activitypub as ap
-from little_boxes import strtobool
-from little_boxes.activitypub import _to_list
-from little_boxes.backend import Backend
-from little_boxes.errors import ActivityGoneError
-from little_boxes.errors import Error
 
 logger = logging.getLogger(__name__)
 
@@ -95,9 +94,10 @@ class MicroblogPubBackend(Backend):
             }
         )
 
-        tasks.cache_attachments.delay(activity.id)
-        if box == Box.INBOX:
-            tasks.process_new_activity.delay(activity.id)
+        self.save_cb(box, activity.id)
+
+    def set_save_cb(self, cb):
+        self.save_cb = cb
 
     @ensure_it_is_me
     def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None:
@@ -198,9 +198,12 @@ class MicroblogPubBackend(Backend):
     def inbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None:
         self.save(Box.INBOX, activity)
 
+    def set_post_to_remote_inbox(self, cb):
+        self.post_to_remote_inbox_cb = cb
+
     @ensure_it_is_me
     def post_to_remote_inbox(self, as_actor: ap.Person, payload: str, to: str) -> None:
-        tasks.post_to_inbox.delay(payload, to)
+        self.post_to_remote_inbox_cb(payload, to)
 
     @ensure_it_is_me
     def new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None:

app.py

@@ -33,12 +33,27 @@ from flask import url_for
 from flask_wtf.csrf import CSRFProtect
 from html2text import html2text
 from itsdangerous import BadSignature
+from little_boxes import activitypub as ap
+from little_boxes.activitypub import ActivityType
+from little_boxes.activitypub import _to_list
+from little_boxes.activitypub import clean_activity
+from little_boxes.activitypub import get_backend
+from little_boxes.content_helper import parse_markdown
+from little_boxes.errors import ActivityGoneError
+from little_boxes.errors import ActivityNotFoundError
+from little_boxes.errors import Error
+from little_boxes.errors import NotFromOutboxError
+from little_boxes.httpsig import HTTPSigAuth
+from little_boxes.httpsig import verify_request
+from little_boxes.webfinger import get_actor_url
+from little_boxes.webfinger import get_remote_follow_template
 from passlib.hash import bcrypt
 from u2flib_server import u2f
 from werkzeug.utils import secure_filename
 
 import activitypub
 import config
+import tasks
 from activitypub import Box
 from activitypub import embed_collection
 from config import ADMIN_API_KEY
@@ -58,24 +73,23 @@ from config import USERNAME
 from config import VERSION
 from config import _drop_db
 from config import custom_cache_purge_hook
-from little_boxes import activitypub as ap
-from little_boxes.activitypub import ActivityType
-from little_boxes.activitypub import _to_list
-from little_boxes.activitypub import clean_activity
-from little_boxes.activitypub import get_backend
-from little_boxes.content_helper import parse_markdown
-from little_boxes.errors import ActivityGoneError
-from little_boxes.errors import ActivityNotFoundError
-from little_boxes.errors import Error
-from little_boxes.errors import NotFromOutboxError
-from little_boxes.httpsig import HTTPSigAuth
-from little_boxes.httpsig import verify_request
-from little_boxes.webfinger import get_actor_url
-from little_boxes.webfinger import get_remote_follow_template
 from utils.key import get_secret_key
 from utils.media import Kind
 
 back = activitypub.MicroblogPubBackend()
+
+
+def save_cb(box: Box, iri: str) -> None:
+    tasks.cache_attachments.delay(iri)
+    if box == Box.INBOX:
+        tasks.process_new_activity.delay(iri)
+
+
+back.set_save_cb(save_cb)
+
+
+back.set_post_to_remote_inbox(tasks.post_to_inbox.delay)
+
 ap.use_backend(back)
 
 MY_PERSON = ap.Person(**ME)
@@ -1569,7 +1583,9 @@ def following():
         )
 
     following, older_than, newer_than = paginated_query(DB.activities, q)
-    following = [get_backend().fetch_iri(doc["activity"]["object"]) for doc in following]
+    following = [
+        get_backend().fetch_iri(doc["activity"]["object"]) for doc in following
+    ]
     return render_template(
         "following.html",
         following_data=following,

config.py

@@ -8,9 +8,9 @@ import requests
 import sass
 import yaml
 from itsdangerous import JSONWebSignatureSerializer
+from little_boxes import strtobool
 from pymongo import MongoClient
 
-from little_boxes import strtobool
 from utils.key import KEY_DIR
 from utils.key import get_key
 from utils.key import get_secret_key

tasks.py

@@ -5,18 +5,18 @@ import random
 
 import requests
 from celery import Celery
+from little_boxes import activitypub as ap
+from little_boxes.httpsig import HTTPSigAuth
+from little_boxes.linked_data_sig import generate_signature
 from requests.exceptions import HTTPError
 
-from little_boxes import activitypub as ap
+import activitypub
 from config import DB
 from config import HEADERS
 from config import KEY
-from config import USER_AGENT
-from little_boxes.httpsig import HTTPSigAuth
-from little_boxes.linked_data_sig import generate_signature
-from utils.opengraph import fetch_og_metadata
-from utils.media import Kind
 from config import MEDIA_CACHE
+from config import USER_AGENT
+from utils.media import Kind
 
 log = logging.getLogger(__name__)
 app = Celery(
@@ -25,6 +25,10 @@ app = Celery(
 SigAuth = HTTPSigAuth(KEY)
 
 
+back = activitypub.MicroblogPubBackend()
+ap.use_backend(back)
+
+
 @app.task(bind=True, max_retries=12)
 def process_new_activity(self, iri: str) -> None:
     try:
@@ -32,7 +36,7 @@ def process_new_activity(self, iri: str) -> None:
         log.info(f"activity={activity!r}")
 
         tag_stream = False
-        if activity.has_type(ap.ActivityType.ANNOUCE):
+        if activity.has_type(ap.ActivityType.ANNOUNCE):
             tag_stream = True
         elif activity.has_type(ap.ActivityType.CREATE):
             note = activity.get_object()
@@ -40,7 +44,9 @@ def process_new_activity(self, iri: str) -> None:
                 tag_stream = True
 
         log.info(f"{iri} tag_stream={tag_stream}")
-        DB.update_one({"remote_id": activity.id}, {"$set": {"meta.stream": tag_stream}})
+        DB.activities.update_one(
+            {"remote_id": activity.id}, {"$set": {"meta.stream": tag_stream}}
+        )
 
         log.info(f"new activity {iri} processed")
     except Exception as err:
@@ -71,7 +77,7 @@ def cache_attachments(self, iri: str) -> None:
             for attachment in activity.get_object()._data.get("attachment", []):
                 MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT)
 
-        log.info(f"attachmwents cached for {iri}")
+        log.info(f"attachments cached for {iri}")
 
     except Exception as err:
         log.exception(f"failed to process new activity {iri}")
@@ -105,20 +111,3 @@ def post_to_inbox(self, payload: str, to: str) -> None:
             log.info("client error, no retry")
             return
         self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries))
-
-
-@app.task(bind=True, max_retries=12)
-def fetch_og(self, col, remote_id):
-    try:
-        log.info("fetch_og_meta remote_id=%s col=%s", remote_id, col)
-        if col == "INBOX":
-            log.info(
-                "%d links saved", fetch_og_metadata(USER_AGENT, DB.inbox, remote_id)
-            )
-        elif col == "OUTBOX":
-            log.info(
-                "%d links saved", fetch_og_metadata(USER_AGENT, DB.outbox, remote_id)
-            )
-    except Exception as err:
-        self.log.exception("failed")
-        self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries))

tests/federation_test.py

@@ -5,7 +5,6 @@ from typing import Tuple
 
 import requests
 from html2text import html2text
-
 from little_boxes.collection import parse_collection
 
 

utils/opengraph.py

@@ -1,7 +1,6 @@
 import opengraph
 import requests
 from bs4 import BeautifulSoup
-
 from little_boxes.urlutils import check_url
 from little_boxes.urlutils import is_url_valid