Make most of the HTTP requests async

app/activitypub.py

@@ -103,16 +103,17 @@ class NotAnObjectError(Exception):
         self.resp = resp
 
 
-def fetch(url: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
+async def fetch(url: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
-    resp = httpx.get(
+    async with httpx.AsyncClient() as client:
-        url,
+        resp = await client.get(
-        headers={
+            url,
-            "User-Agent": config.USER_AGENT,
+            headers={
-            "Accept": config.AP_CONTENT_TYPE,
+                "User-Agent": config.USER_AGENT,
-        },
+                "Accept": config.AP_CONTENT_TYPE,
-        params=params,
+            },
-        follow_redirects=True,
+            params=params,
-    )
+            follow_redirects=True,
+        )
 
     # Special handling for deleted object
     if resp.status_code == 410:
@@ -125,7 +126,7 @@ def fetch(url: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
         raise NotAnObjectError(url, resp)
 
 
-def parse_collection(  # noqa: C901
+async def parse_collection(  # noqa: C901
     url: str | None = None,
     payload: RawObject | None = None,
     level: int = 0,
@@ -137,7 +138,7 @@ def parse_collection(  # noqa: C901
     # Go through all the pages
     out: list[RawObject] = []
     if url:
-        payload = fetch(url)
+        payload = await fetch(url)
     if not payload:
         raise ValueError("must at least prove a payload or an URL")
 
@@ -155,7 +156,9 @@ def parse_collection(  # noqa: C901
             return payload["items"]
         if "first" in payload:
             if isinstance(payload["first"], str):
-                out.extend(parse_collection(url=payload["first"], level=level + 1))
+                out.extend(
+                    await parse_collection(url=payload["first"], level=level + 1)
+                )
             else:
                 if "orderedItems" in payload["first"]:
                     out.extend(payload["first"]["orderedItems"])
@@ -163,7 +166,7 @@ def parse_collection(  # noqa: C901
                     out.extend(payload["first"]["items"])
                 n = payload["first"].get("next")
                 if n:
-                    out.extend(parse_collection(url=n, level=level + 1))
+                    out.extend(await parse_collection(url=n, level=level + 1))
         return out
 
     while payload:
@@ -175,7 +178,7 @@ def parse_collection(  # noqa: C901
             n = payload.get("next")
             if n is None:
                 break
-            payload = fetch(n)
+            payload = await fetch(n)
         else:
             raise ValueError("unexpected activity type {}".format(payload["type"]))
 
@@ -263,18 +266,6 @@ def remove_context(raw_object: RawObject) -> RawObject:
     return a
 
 
-def get(url: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
-    resp = httpx.get(
-        url,
-        headers={"User-Agent": config.USER_AGENT, "Accept": config.AP_CONTENT_TYPE},
-        params=params,
-        follow_redirects=True,
-        auth=auth,
-    )
-    resp.raise_for_status()
-    return resp.json()
-
-
 def post(url: str, payload: dict[str, Any]) -> httpx.Response:
     resp = httpx.post(
         url,

app/actor.py

@@ -160,7 +160,7 @@ async def fetch_actor(db_session: AsyncSession, actor_id: str) -> "ActorModel":
     if existing_actor:
         return existing_actor
 
-    ap_actor = ap.get(actor_id)
+    ap_actor = await ap.fetch(actor_id)
     return await save_actor(db_session, ap_actor)
 
 

app/ap_object.py

@@ -178,26 +178,35 @@ class Attachment(BaseModel):
 
 
 class RemoteObject(Object):
-    def __init__(self, raw_object: ap.RawObject, actor: Actor | None = None):
+    def __init__(self, raw_object: ap.RawObject, actor: Actor):
         self._raw_object = raw_object
-        self._actor: Actor
+        self._actor = actor
 
+        if self._actor.ap_id != ap.get_actor_id(self._raw_object):
+            raise ValueError(f"Invalid actor {self._actor.ap_id}")
+
+    @classmethod
+    async def from_raw_object(
+        cls,
+        raw_object: ap.RawObject,
+        actor: Actor | None = None,
+    ):
         # Pre-fetch the actor
         actor_id = ap.get_actor_id(raw_object)
         if actor_id == LOCAL_ACTOR.ap_id:
-            self._actor = LOCAL_ACTOR
+            _actor = LOCAL_ACTOR
         elif actor:
             if actor.ap_id != actor_id:
                 raise ValueError(
                     f"Invalid actor, got {actor.ap_id}, " f"expected {actor_id}"
                 )
-            self._actor = actor
+            _actor = actor  # type: ignore
         else:
-            self._actor = RemoteActor(
+            _actor = RemoteActor(
-                ap_actor=ap.fetch(ap.get_actor_id(raw_object)),
+                ap_actor=await ap.fetch(ap.get_actor_id(raw_object)),
             )
 
-        self._og_meta = None
+        return cls(raw_object, _actor)
 
     @property
     def og_meta(self) -> list[dict[str, Any]] | None:

app/boxes.py

@@ -52,7 +52,7 @@ async def save_outbox_object(
     relates_to_actor_id: int | None = None,
     source: str | None = None,
 ) -> models.OutboxObject:
-    ra = RemoteObject(raw_object)
+    ra = await RemoteObject.from_raw_object(raw_object)
 
     outbox_object = models.OutboxObject(
         public_id=public_id,
@@ -368,13 +368,13 @@ async def _compute_recipients(
             continue
 
         # Fetch the object
-        raw_object = ap.fetch(r)
+        raw_object = await ap.fetch(r)
         if raw_object.get("type") in ap.ACTOR_TYPES:
             saved_actor = await save_actor(db_session, raw_object)
             recipients.add(saved_actor.shared_inbox_url or saved_actor.inbox_url)
         else:
             # Assume it's a collection of actors
-            for raw_actor in ap.parse_collection(payload=raw_object):
+            for raw_actor in await ap.parse_collection(payload=raw_object):
                 actor = RemoteActor(raw_actor)
                 recipients.add(actor.shared_inbox_url or actor.inbox_url)
 
@@ -741,7 +741,7 @@ async def save_to_inbox(db_session: AsyncSession, raw_object: ap.RawObject) -> N
                 # Save it as an inbox object
                 if not ra.activity_object_ap_id:
                     raise ValueError("Should never happen")
-                announced_raw_object = ap.fetch(ra.activity_object_ap_id)
+                announced_raw_object = await ap.fetch(ra.activity_object_ap_id)
                 announced_actor = await fetch_actor(
                     db_session, ap.get_actor_id(announced_raw_object)
                 )
@@ -830,7 +830,7 @@ async def fetch_actor_collection(db_session: AsyncSession, url: str) -> list[Act
         else:
             raise ValueError(f"internal collection for {url}) not supported")
 
-    return [RemoteActor(actor) for actor in ap.parse_collection(url)]
+    return [RemoteActor(actor) for actor in await ap.parse_collection(url)]
 
 
 @dataclass

app/httpsig.py

@@ -63,11 +63,11 @@ def _body_digest(body: bytes) -> str:
 
 
 @lru_cache(32)
-def _get_public_key(key_id: str) -> Key:
+async def _get_public_key(key_id: str) -> Key:
     # TODO: use DB to use cache actor
     from app import activitypub as ap
 
-    actor = ap.fetch(key_id)
+    actor = await ap.fetch(key_id)
     if actor["type"] == "Key":
         # The Key is not embedded in the Person
         k = Key(actor["owner"], actor["id"])
@@ -111,7 +111,7 @@ async def httpsig_checker(
     )
 
     try:
-        k = _get_public_key(hsig["keyId"])
+        k = await _get_public_key(hsig["keyId"])
     except ap.ObjectIsGoneError:
         logger.info("Actor is gone")
         return HTTPSigInfo(has_valid_signature=False)

app/lookup.py

@@ -10,13 +10,13 @@ from app.database import AsyncSession
 
 async def lookup(db_session: AsyncSession, query: str) -> Actor | RemoteObject:
     if query.startswith("@"):
-        query = webfinger.get_actor_url(query)  # type: ignore  # None check below
+        query = await webfinger.get_actor_url(query)  # type: ignore  # None check below
 
         if not query:
             raise ap.NotAnObjectError(query)
 
     try:
-        ap_obj = ap.fetch(query)
+        ap_obj = await ap.fetch(query)
     except ap.NotAnObjectError as not_an_object_error:
         resp = not_an_object_error.resp
         if not resp:
@@ -26,7 +26,7 @@ async def lookup(db_session: AsyncSession, query: str) -> Actor | RemoteObject:
         if resp.headers.get("content-type", "").startswith("text/html"):
             for alternate in mf2py.parse(doc=resp.text).get("alternates", []):
                 if alternate.get("type") == "application/activity+json":
-                    alternate_obj = ap.fetch(alternate["url"])
+                    alternate_obj = await ap.fetch(alternate["url"])
 
         if alternate_obj:
             ap_obj = alternate_obj
@@ -37,4 +37,4 @@ async def lookup(db_session: AsyncSession, query: str) -> Actor | RemoteObject:
         actor = await fetch_actor(db_session, ap_obj["id"])
         return actor
     else:
-        return RemoteObject(ap_obj)
+        return await RemoteObject.from_raw_object(ap_obj)

app/main.py

@@ -604,7 +604,7 @@ async def post_remote_follow(
     if not profile.startswith("@"):
         profile = f"@{profile}"
 
-    remote_follow_template = get_remote_follow_template(profile)
+    remote_follow_template = await get_remote_follow_template(profile)
     if not remote_follow_template:
         raise HTTPException(status_code=404)
 

app/source.py

@@ -52,7 +52,7 @@ async def _mentionify(
             )
         ).scalar_one_or_none()
         if not actor:
-            actor_url = webfinger.get_actor_url(mention)
+            actor_url = await webfinger.get_actor_url(mention)
             if not actor_url:
                 # FIXME(ts): raise an error?
                 continue

app/webfinger.py

@@ -7,7 +7,7 @@ from loguru import logger
 from app import config
 
 
-def webfinger(
+async def webfinger(
     resource: str,
 ) -> dict[str, Any] | None:  # noqa: C901
     """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL."""
@@ -28,37 +28,38 @@ def webfinger(
 
     is_404 = False
 
-    for i, proto in enumerate(protos):
+    async with httpx.AsyncClient() as client:
-        try:
+        for i, proto in enumerate(protos):
-            url = f"{proto}://{host}/.well-known/webfinger"
+            try:
-            resp = httpx.get(
+                url = f"{proto}://{host}/.well-known/webfinger"
-                url,
+                resp = await client.get(
-                params={"resource": resource},
+                    url,
-                headers={
+                    params={"resource": resource},
-                    "User-Agent": config.USER_AGENT,
+                    headers={
-                },
+                        "User-Agent": config.USER_AGENT,
-            )
+                    },
-            break
+                )
-        except httpx.HTTPStatusError as http_error:
+                break
-            logger.exception("HTTP error")
+            except httpx.HTTPStatusError as http_error:
-            if http_error.response.status_code in [403, 404, 410]:
+                logger.exception("HTTP error")
-                is_404 = True
+                if http_error.response.status_code in [403, 404, 410]:
-                continue
+                    is_404 = True
-            raise
+                    continue
-        except httpx.HTTPError:
+                raise
-            logger.exception("req failed")
+            except httpx.HTTPError:
-            # If we tried https first and the domain is "http only"
+                logger.exception("req failed")
-            if i == 0:
+                # If we tried https first and the domain is "http only"
-                continue
+                if i == 0:
-            break
+                    continue
+                break
     if is_404:
         return None
 
     return resp.json()
 
 
-def get_remote_follow_template(resource: str) -> str | None:
+async def get_remote_follow_template(resource: str) -> str | None:
-    data = webfinger(resource)
+    data = await webfinger(resource)
     if data is None:
         return None
     for link in data["links"]:
@@ -67,13 +68,13 @@ def get_remote_follow_template(resource: str) -> str | None:
     return None
 
 
-def get_actor_url(resource: str) -> str | None:
+async def get_actor_url(resource: str) -> str | None:
     """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL.
 
     Returns:
         the Actor URL or None if the resolution failed.
     """
-    data = webfinger(resource)
+    data = await webfinger(resource)
     if data is None:
         return None
     for link in data["links"]:

tests/test_inbox.py

@@ -43,7 +43,8 @@ def test_inbox_follow_request(
         factories.build_follow_activity(
             from_remote_actor=ra,
             for_remote_actor=LOCAL_ACTOR,
-        )
+        ),
+        ra,
     )
     with mock_httpsig_checker(ra):
         response = client.post(
@@ -100,7 +101,8 @@ def test_inbox_accept_follow_request(
             from_remote_actor=LOCAL_ACTOR,
             for_remote_actor=ra,
             outbox_public_id=follow_id,
-        )
+        ),
+        LOCAL_ACTOR,
     )
     outbox_object = factories.OutboxObjectFactory.from_remote_object(
         follow_id, follow_from_outbox
@@ -111,7 +113,8 @@ def test_inbox_accept_follow_request(
         factories.build_accept_activity(
             from_remote_actor=ra,
             for_remote_object=follow_from_outbox,
-        )
+        ),
+        ra,
     )
     with mock_httpsig_checker(ra):
         response = client.post(

tests/test_outbox.py

@@ -112,7 +112,8 @@ def test_send_create_activity__with_followers(
             from_remote_actor=ra,
             for_remote_actor=LOCAL_ACTOR,
             outbox_public_id=follow_id,
-        )
+        ),
+        ra,
     )
     inbox_object = factories.InboxObjectFactory.from_remote_object(
         follow_from_inbox, actor

tests/test_process_outgoing_activities.py

@@ -31,7 +31,8 @@ def _setup_outbox_object() -> models.OutboxObject:
             from_remote_actor=LOCAL_ACTOR,
             for_remote_actor=ra,
             outbox_public_id=follow_id,
-        )
+        ),
+        LOCAL_ACTOR,
     )
     outbox_object = factories.OutboxObjectFactory.from_remote_object(
         follow_id, follow_from_outbox