jonny/microblog.pub
1
0
Fork 0
forked from forks/microblog.pub
Code Pull requests Activity

Tweak OG error handling

Browse source
This commit is contained in:
Thomas Sileo 2019-08-06 21:51:58 +02:00
parent fa3d2ca288
commit f01015c73b
1 changed files with 10 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
utils/opengraph.py
View file

@ -57,10 +57,12 @@ def fetch_og_metadata(user_agent, links):
h = requests.head(l, headers={"User-Agent": user_agent}, timeout=3) h = requests.head(l, headers={"User-Agent": user_agent}, timeout=3)
h.raise_for_status() h.raise_for_status()
except requests.HTTPError as http_err: except requests.HTTPError as http_err:
logger.debug(f"failed to HEAD {l}, got a {http_err.response.status_code}") logger.debug(
f"failed to HEAD {l}, got a {http_err.response.status_code}: {http_err.response.text}"
)
continue continue
except requests.Timeout: except requests.RequestException as err:
logger.debug(f"HEAD {l} timed out") logger.debug(f"failed to HEAD {l}: {err!r}")
continue continue
if not h.headers.get("content-type").startswith("text/html"): if not h.headers.get("content-type").startswith("text/html"):
@ -71,10 +73,12 @@ def fetch_og_metadata(user_agent, links):
r = requests.get(l, headers={"User-Agent": user_agent}, timeout=5) r = requests.get(l, headers={"User-Agent": user_agent}, timeout=5)
r.raise_for_status() r.raise_for_status()
except requests.HTTPError as http_err: except requests.HTTPError as http_err:
logger.debug(f"failed to GET {l}, got a {http_err.response.status_code}") logger.debug(
f"failed to GET {l}, got a {http_err.response.status_code}: {http_err.response.text}"
)
continue continue
except requests.Timeout: except requests.RequestException as err:
logger.debug(f"GET {l} timed out") logger.debug(f"failed to GET {l}: {err!r}")
continue continue
r.encoding = "UTF-8" r.encoding = "UTF-8"