forks/microblog.pub
1
0
Fork 1
mirror of https://git.sr.ht/~tsileo/microblog.pub synced 2024-11-15 03:04:28 +00:00
Code Issues Projects Releases Packages Wiki Activity

More Open Graph tweaks

Browse source
This commit is contained in:
Thomas Sileo 2018-08-05 14:45:44 +02:00
parent c585f07857
commit 3d8d3efd25
1 changed files with 4 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
utils/opengraph.py
View file

@ -44,12 +44,15 @@ def fetch_og_metadata(user_agent, links):
r = requests.get(l, headers={"User-Agent": user_agent}, timeout=15) r = requests.get(l, headers={"User-Agent": user_agent}, timeout=15)
r.raise_for_status() r.raise_for_status()
if not r.headers.get("content-type").startswith("text/html"):
logger.debug(f"skipping {l}")
continue
html = r.text html = r.text
try: try:
data = dict(opengraph.OpenGraph(html=html)) data = dict(opengraph.OpenGraph(html=html))
except Exception: except Exception:
logger.exception("failed to parse {l}") logger.exception(f"failed to parse {l}")
continue continue
if data.get("url"): if data.get("url"):
res.append(data) res.append(data)