From 88b57f29afb3aff233dc789ca73d52026303905d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 13 Jul 2022 20:05:15 +0200 Subject: [PATCH] HTTP clients tweaks --- app/activitypub.py | 10 +++++++++- app/httpsig.py | 4 ++-- app/main.py | 5 ++++- poetry.lock | 34 +++++++++++++++++++++++++++++++++- pyproject.toml | 2 +- 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/app/activitypub.py b/app/activitypub.py index c82a027..09cead3 100644 --- a/app/activitypub.py +++ b/app/activitypub.py @@ -14,6 +14,8 @@ from app.key import get_pubkey_as_pem if TYPE_CHECKING: from app.actor import Actor +_HTTPX_TRANSPORT = httpx.AsyncHTTPTransport(retries=1) + RawObject = dict[str, Any] AS_CTX = "https://www.w3.org/ns/activitystreams" AS_PUBLIC = "https://www.w3.org/ns/activitystreams#Public" @@ -49,6 +51,10 @@ class ObjectIsGoneError(Exception): pass +class ObjectNotFoundError(Exception): + pass + + class VisibilityEnum(str, enum.Enum): PUBLIC = "public" UNLISTED = "unlisted" @@ -104,7 +110,7 @@ class NotAnObjectError(Exception): async def fetch(url: str, params: dict[str, Any] | None = None) -> RawObject: - async with httpx.AsyncClient() as client: + async with httpx.AsyncClient(transport=_HTTPX_TRANSPORT) as client: resp = await client.get( url, headers={ @@ -119,6 +125,8 @@ async def fetch(url: str, params: dict[str, Any] | None = None) -> RawObject: # Special handling for deleted object if resp.status_code == 410: raise ObjectIsGoneError(f"{url} is gone") + elif resp.status_code == 404: + raise ObjectNotFoundError(f"{url} not found") resp.raise_for_status() try: diff --git a/app/httpsig.py b/app/httpsig.py index bce718b..8ec267e 100644 --- a/app/httpsig.py +++ b/app/httpsig.py @@ -138,8 +138,8 @@ async def httpsig_checker( try: k = await _get_public_key(db_session, hsig["keyId"]) - except ap.ObjectIsGoneError: - logger.info("Actor is gone") + except (ap.ObjectIsGoneError, ap.ObjectNotFoundError): + logger.info("Actor is gone or not found") return HTTPSigInfo(has_valid_signature=False, is_ap_actor_gone=True) except Exception: logger.exception(f'Failed to fetch HTTP sig key {hsig["keyId"]}') diff --git a/app/main.py b/app/main.py index 90ef66f..2ce4dd1 100644 --- a/app/main.py +++ b/app/main.py @@ -66,6 +66,9 @@ _RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCac # TODO(ts): # # Next: +# - incoming activity worker +# - handle remove activity +# - retries httpx? # - DB models for webmentions # - allow to undo follow requests # - indieauth tweaks @@ -760,7 +763,7 @@ async def nodeinfo( ) -proxy_client = httpx.AsyncClient(follow_redirects=True) +proxy_client = httpx.AsyncClient(follow_redirects=True, http2=True) @app.get("/proxy/media/{encoded_url}") diff --git a/poetry.lock b/poetry.lock index 5ddc072..04a5747 100644 --- a/poetry.lock +++ b/poetry.lock @@ -365,6 +365,26 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "h2" +version = "4.1.0" +description = "HTTP/2 State-Machine based protocol implementation" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +hpack = ">=4.0,<5" +hyperframe = ">=6.0,<7" + +[[package]] +name = "hpack" +version = "4.0.0" +description = "Pure-Python HPACK header compression" +category = "main" +optional = false +python-versions = ">=3.6.1" + [[package]] name = "html2text" version = "2020.1.16" @@ -419,6 +439,7 @@ python-versions = ">=3.7" [package.dependencies] certifi = "*" +h2 = {version = ">=3,<5", optional = true, markers = "extra == \"http2\""} httpcore = ">=0.15.0,<0.16.0" rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]} sniffio = "*" @@ -440,6 +461,14 @@ python-versions = ">=3.7" [package.extras] tests = ["freezegun", "pytest", "pytest-cov"] +[[package]] +name = "hyperframe" +version = "6.0.1" +description = "HTTP/2 framing layer for Python" +category = "main" +optional = false +python-versions = ">=3.6.1" + [[package]] name = "idna" version = "3.3" @@ -1173,7 +1202,7 @@ dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"] [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "fd741c6c1c1e85cb1b39150df503bc64b28244b65222180c6768409fcfd1d70a" +content-hash = "7bc5ba65a004438ac015dcd01c27e1d327dbf491f9f881a48a2a790bb0bbf710" [metadata.files] aiosqlite = [ @@ -1454,6 +1483,8 @@ h11 = [ {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"}, ] +h2 = [] +hpack = [] html2text = [ {file = "html2text-2020.1.16-py3-none-any.whl", hash = "sha256:c7c629882da0cf377d66f073329ccf34a12ed2adf0169b9285ae4e63ef54c82b"}, {file = "html2text-2020.1.16.tar.gz", hash = "sha256:e296318e16b059ddb97f7a8a1d6a5c1d7af4544049a01e261731d2d5cc277bbb"}, @@ -1474,6 +1505,7 @@ humanize = [ {file = "humanize-4.2.3-py3-none-any.whl", hash = "sha256:bed628920d45cd5018abb095710f0c03a8336d6ac0790e7647c6a328f3880b81"}, {file = "humanize-4.2.3.tar.gz", hash = "sha256:2bc1fdd831cd00557d3010abdd84d3e41b4a96703a3eaf6c24ee290b26b75a44"}, ] +hyperframe = [] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, diff --git a/pyproject.toml b/pyproject.toml index 7a543af..9e284c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ bcrypt = "^3.2.2" itsdangerous = "^2.1.2" python-multipart = "^0.0.5" tomli = "^2.0.1" -httpx = "^0.23.0" +httpx = {extras = ["http2"], version = "^0.23.0"} SQLAlchemy = {extras = ["asyncio"], version = "^1.4.39"} alembic = "^1.8.0" bleach = "^5.0.0"