Custom emoji support

This commit is contained in:
Thomas Sileo 2022-06-27 20:55:44 +02:00
parent 5b025a8e45
commit 09ce33579a
17 changed files with 357 additions and 70 deletions

View file

@ -18,6 +18,7 @@ from app.actor import get_actors_metadata
from app.boxes import get_inbox_object_by_ap_id from app.boxes import get_inbox_object_by_ap_id
from app.boxes import get_outbox_object_by_ap_id from app.boxes import get_outbox_object_by_ap_id
from app.boxes import send_follow from app.boxes import send_follow
from app.config import EMOJIS
from app.config import generate_csrf_token from app.config import generate_csrf_token
from app.config import session_serializer from app.config import session_serializer
from app.config import verify_csrf_token from app.config import verify_csrf_token
@ -25,6 +26,7 @@ from app.config import verify_password
from app.database import get_db from app.database import get_db
from app.lookup import lookup from app.lookup import lookup
from app.uploads import save_upload from app.uploads import save_upload
from app.utils.emoji import EMOJIS_BY_NAME
def user_session_or_redirect( def user_session_or_redirect(
@ -123,36 +125,11 @@ def admin_new(
(v.name, ap.VisibilityEnum.get_display_name(v)) (v.name, ap.VisibilityEnum.get_display_name(v))
for v in ap.VisibilityEnum for v in ap.VisibilityEnum
], ],
}, "emojis": EMOJIS.split(" "),
) "custom_emojis": sorted(
[dat for name, dat in EMOJIS_BY_NAME.items()],
key=lambda obj: obj["name"],
@router.get("/stream") ),
def stream(
request: Request,
db: Session = Depends(get_db),
) -> templates.TemplateResponse:
stream = (
db.query(models.InboxObject)
.filter(
models.InboxObject.ap_type.in_(["Note", "Article", "Video", "Announce"]),
models.InboxObject.is_hidden_from_stream.is_(False),
models.InboxObject.undone_by_inbox_object_id.is_(None),
)
.options(
# joinedload(models.InboxObject.relates_to_inbox_object),
joinedload(models.InboxObject.relates_to_outbox_object),
)
.order_by(models.InboxObject.ap_published_at.desc())
.limit(20)
.all()
)
return templates.render_template(
db,
request,
"admin_stream.html",
{
"stream": stream,
}, },
) )
@ -452,7 +429,7 @@ def admin_actions_unpin(
@router.post("/actions/new") @router.post("/actions/new")
def admin_actions_new( def admin_actions_new(
request: Request, request: Request,
files: list[UploadFile], files: list[UploadFile] = [],
content: str = Form(), content: str = Form(),
redirect_url: str = Form(), redirect_url: str = Form(),
in_reply_to: str | None = Form(None), in_reply_to: str | None = Form(None),
@ -501,7 +478,7 @@ def login_validation(
if not verify_password(password): if not verify_password(password):
raise HTTPException(status_code=401) raise HTTPException(status_code=401)
resp = RedirectResponse("/admin", status_code=302) resp = RedirectResponse("/admin/inbox", status_code=302)
resp.set_cookie("session", session_serializer.dumps({"is_logged_in": True})) # type: ignore # noqa: E501 resp.set_cookie("session", session_serializer.dumps({"is_logged_in": True})) # type: ignore # noqa: E501
return resp return resp

View file

@ -341,7 +341,7 @@ def _compute_recipients(db: Session, ap_object: ap.RawObject) -> set[str]:
db.query(models.Actor).filter(models.Actor.ap_id == r).one_or_none() db.query(models.Actor).filter(models.Actor.ap_id == r).one_or_none()
) )
if known_actor: if known_actor:
recipients.add(known_actor.shared_inbox_url or actor.inbox_url) recipients.add(known_actor.shared_inbox_url or known_actor.inbox_url)
continue continue
# Fetch the object # Fetch the object

View file

@ -10,6 +10,8 @@ from fastapi import Request
from itsdangerous import TimedSerializer from itsdangerous import TimedSerializer
from itsdangerous import TimestampSigner from itsdangerous import TimestampSigner
from app.utils.emoji import _load_emojis
ROOT_DIR = Path().parent.resolve() ROOT_DIR = Path().parent.resolve()
_CONFIG_FILE = os.getenv("MICROBLOGPUB_CONFIG_FILE", "me.toml") _CONFIG_FILE = os.getenv("MICROBLOGPUB_CONFIG_FILE", "me.toml")
@ -76,6 +78,11 @@ SQLALCHEMY_DATABASE_URL = CONFIG.sqlalchemy_database_url or f"sqlite:///{DB_PATH
KEY_PATH = ( KEY_PATH = (
(ROOT_DIR / CONFIG.key_path) if CONFIG.key_path else ROOT_DIR / "data" / "key.pem" (ROOT_DIR / CONFIG.key_path) if CONFIG.key_path else ROOT_DIR / "data" / "key.pem"
) )
EMOJIS = "😺 😸 😹 😻 😼 😽 🙀 😿 😾"
# Emoji template for the FE
EMOJI_TPL = '<img src="/static/twemoji/{filename}.svg" alt="{raw}" class="emoji">'
_load_emojis(ROOT_DIR, BASE_URL)
session_serializer = TimedSerializer(CONFIG.secret, salt="microblogpub.login") session_serializer = TimedSerializer(CONFIG.secret, salt="microblogpub.login")

View file

@ -52,6 +52,7 @@ from app.config import verify_csrf_token
from app.database import get_db from app.database import get_db
from app.templates import is_current_user_admin from app.templates import is_current_user_admin
from app.uploads import UPLOAD_DIR from app.uploads import UPLOAD_DIR
from app.utils.emoji import EMOJIS_BY_NAME
from app.webfinger import get_remote_follow_template from app.webfinger import get_remote_follow_template
# TODO(ts): # TODO(ts):
@ -520,6 +521,16 @@ def tag_by_name(
) )
@app.get("/e/{name}")
def emoji_by_name(name: str) -> ActivityPubResponse:
try:
emoji = EMOJIS_BY_NAME[f":{name}:"]
except KeyError:
raise HTTPException(status_code=404)
return ActivityPubResponse({"@context": ap.AS_CTX, **emoji})
@app.post("/inbox") @app.post("/inbox")
async def inbox( async def inbox(
request: Request, request: Request,

View file

@ -140,6 +140,6 @@ nav.flexbox {
float: right; float: right;
} }
} }
.custom-emoji { .emoji, .custom-emoji {
max-width: 25px; max-width: 25px;
} }

View file

@ -8,6 +8,7 @@ from app import webfinger
from app.actor import Actor from app.actor import Actor
from app.actor import fetch_actor from app.actor import fetch_actor
from app.config import BASE_URL from app.config import BASE_URL
from app.utils import emoji
def _set_a_attrs(attrs, new=False): def _set_a_attrs(attrs, new=False):
@ -78,5 +79,10 @@ def markdownify(
if mentionify: if mentionify:
content, mention_tags, mentioned_actors = _mentionify(db, content) content, mention_tags, mentioned_actors = _mentionify(db, content)
tags.extend(mention_tags) tags.extend(mention_tags)
# Handle custom emoji
tags.extend(emoji.tags(content))
content = markdown(content, extensions=["mdx_linkify"]) content = markdown(content, extensions=["mdx_linkify"])
return content, tags, mentioned_actors return content, tags, mentioned_actors

2
app/static/emoji/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
*
!.gitignore

2
app/static/twemoji/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
*
!.gitignore

View file

@ -6,6 +6,7 @@ from typing import Any
from urllib.parse import urlparse from urllib.parse import urlparse
import bleach import bleach
import emoji
import html2text import html2text
import timeago # type: ignore import timeago # type: ignore
from bs4 import BeautifulSoup # type: ignore from bs4 import BeautifulSoup # type: ignore
@ -16,6 +17,7 @@ from sqlalchemy.orm import Session
from starlette.templating import _TemplateResponse as TemplateResponse from starlette.templating import _TemplateResponse as TemplateResponse
from app import activitypub as ap from app import activitypub as ap
from app import config
from app import models from app import models
from app.actor import LOCAL_ACTOR from app.actor import LOCAL_ACTOR
from app.ap_object import Attachment from app.ap_object import Attachment
@ -171,7 +173,8 @@ def _update_inline_imgs(content):
def _clean_html(html: str, note: Object) -> str: def _clean_html(html: str, note: Object) -> str:
try: try:
return _replace_custom_emojis( return _emojify(
_replace_custom_emojis(
bleach.clean( bleach.clean(
_update_inline_imgs(highlight(html)), _update_inline_imgs(highlight(html)),
tags=ALLOWED_TAGS, tags=ALLOWED_TAGS,
@ -180,6 +183,7 @@ def _clean_html(html: str, note: Object) -> str:
), ),
note, note,
) )
)
except Exception: except Exception:
raise raise
@ -229,11 +233,24 @@ def _html2text(content: str) -> str:
return H2T.handle(content) return H2T.handle(content)
def _replace_emoji(u, data):
filename = hex(ord(u))[2:]
return config.EMOJI_TPL.format(filename=filename, raw=u)
def _emojify(text: str):
return emoji.replace_emoji(
text,
replace=_replace_emoji,
)
_templates.env.filters["domain"] = _filter_domain _templates.env.filters["domain"] = _filter_domain
_templates.env.filters["media_proxy_url"] = _media_proxy_url _templates.env.filters["media_proxy_url"] = _media_proxy_url
_templates.env.filters["clean_html"] = _clean_html _templates.env.filters["clean_html"] = _clean_html
_templates.env.filters["timeago"] = _timeago _templates.env.filters["timeago"] = _timeago
_templates.env.filters["format_date"] = _format_date _templates.env.filters["format_date"] = _format_date
_templates.env.filters["has_media_type"] = _has_media_type _templates.env.filters["has_media_type"] = _has_media_type
_templates.env.filters["pluralize"] = _pluralize
_templates.env.filters["html2text"] = _html2text _templates.env.filters["html2text"] = _html2text
_templates.env.filters["emojify"] = _emojify
_templates.env.filters["pluralize"] = _pluralize

View file

@ -17,6 +17,13 @@
{% endfor %} {% endfor %}
</select> </select>
</p> </p>
{% for emoji in emojis %}
<span class="ji">{{ emoji | emojify | safe }}</span>
{% endfor %}
{% for emoji in custom_emojis %}
<span class="ji"><img src="{{ emoji.icon.url }}" alt="{{ emoji.name }}" title="{{ emoji.name }}" class="custom-emoji"></span>
{% endfor %}
<textarea name="content" rows="10" cols="50" autofocus="autofocus" designMode="on" style="font-size:1.2em;width:95%;">{{ content }}</textarea> <textarea name="content" rows="10" cols="50" autofocus="autofocus" designMode="on" style="font-size:1.2em;width:95%;">{{ content }}</textarea>
<input type="hidden" name="in_reply_to" value="{{ request.query_params.in_reply_to }}"> <input type="hidden" name="in_reply_to" value="{{ request.query_params.in_reply_to }}">
<p> <p>
@ -26,5 +33,38 @@
<input type="submit" value="Publish"> <input type="submit" value="Publish">
</p> </p>
</form> </form>
<script>
// The new post textarea
var ta = document.getElementsByTagName("textarea")[0];
// Helper for inserting text (emojis) in the textarea
function insertAtCursor (textToInsert) {
ta.focus();
const isSuccess = document.execCommand("insertText", false, textToInsert);
// Firefox (non-standard method)
if (!isSuccess) {
// Credits to https://www.everythingfrontend.com/posts/insert-text-into-textarea-at-cursor-position.html
// get current text of the input
const value = ta.value;
// save selection start and end position
const start = ta.selectionStart;
const end = ta.selectionEnd;
// update the value with our text inserted
ta.value = value.slice(0, start) + textToInsert + value.slice(end);
// update cursor to be at the end of insertion
ta.selectionStart = ta.selectionEnd = start + textToInsert.length;
}
}
// Emoji click callback func
var ji = function (ev) {
insertAtCursor(ev.target.attributes.alt.value + " ");
ta.focus()
//console.log(document.execCommand('insertText', false /*no UI*/, ev.target.attributes.alt.value));
}
// Enable the click for each emojis
var items = document.getElementsByClassName("ji")
for (var i = 0; i < items.length; i++) {
items[i].addEventListener('click', ji);
}
</script>
{% endblock %} {% endblock %}

45
app/utils/emoji.py Normal file
View file

@ -0,0 +1,45 @@
import mimetypes
import re
import typing
from pathlib import Path
if typing.TYPE_CHECKING:
from app.activitypub import RawObject
EMOJI_REGEX = re.compile(r"(:[\d\w]+:)")
EMOJIS: dict[str, "RawObject"] = {}
EMOJIS_BY_NAME: dict[str, "RawObject"] = {}
def _load_emojis(root_dir: Path, base_url: str) -> None:
if EMOJIS:
return
for emoji in (root_dir / "app" / "static" / "emoji").iterdir():
mt = mimetypes.guess_type(emoji.name)[0]
if mt and mt.startswith("image/"):
name = emoji.name.split(".")[0]
ap_emoji: "RawObject" = {
"type": "Emoji",
"name": f":{name}:",
"updated": "1970-01-01T00:00:00Z", # XXX: we don't track date
"id": f"{base_url}/e/{name}",
"icon": {
"mediaType": mt,
"type": "Image",
"url": f"{base_url}/static/emoji/{emoji.name}",
},
}
EMOJIS[emoji.name] = ap_emoji
EMOJIS_BY_NAME[ap_emoji["name"]] = ap_emoji
def tags(content: str) -> list["RawObject"]:
tags = []
added = set()
for e in re.findall(EMOJI_REGEX, content):
if e not in added and e in EMOJIS_BY_NAME:
tags.append(EMOJIS_BY_NAME[e])
added.add(e)
return tags

89
poetry.lock generated
View file

@ -176,6 +176,14 @@ python-versions = "*"
[package.dependencies] [package.dependencies]
beautifulsoup4 = "*" beautifulsoup4 = "*"
[[package]]
name = "cachetools"
version = "5.2.0"
description = "Extensible memoizing collections and decorators"
category = "main"
optional = false
python-versions = "~=3.7"
[[package]] [[package]]
name = "certifi" name = "certifi"
version = "2022.5.18.1" version = "2022.5.18.1"
@ -239,6 +247,17 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""}
[package.extras] [package.extras]
development = ["black", "flake8", "mypy", "pytest", "types-colorama"] development = ["black", "flake8", "mypy", "pytest", "types-colorama"]
[[package]]
name = "emoji"
version = "1.7.0"
description = "Emoji for Python"
category = "main"
optional = false
python-versions = "*"
[package.extras]
dev = ["pytest", "coverage", "coveralls"]
[[package]] [[package]]
name = "factory-boy" name = "factory-boy"
version = "3.2.1" version = "3.2.1"
@ -308,6 +327,14 @@ mccabe = ">=0.6.0,<0.7.0"
pycodestyle = ">=2.8.0,<2.9.0" pycodestyle = ">=2.8.0,<2.9.0"
pyflakes = ">=2.4.0,<2.5.0" pyflakes = ">=2.4.0,<2.5.0"
[[package]]
name = "frozendict"
version = "2.3.2"
description = "A simple immutable dictionary"
category = "main"
optional = false
python-versions = ">=3.6"
[[package]] [[package]]
name = "greenlet" name = "greenlet"
version = "1.1.2" version = "1.1.2"
@ -725,6 +752,25 @@ category = "main"
optional = false optional = false
python-versions = ">=3.6" python-versions = ">=3.6"
[[package]]
name = "pyld"
version = "2.0.3"
description = "Python implementation of the JSON-LD API"
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
cachetools = "*"
frozendict = "*"
lxml = "*"
[package.extras]
aiohttp = ["aiohttp"]
cachetools = ["cachetools"]
frozendict = ["frozendict"]
requests = ["requests"]
[[package]] [[package]]
name = "pyparsing" name = "pyparsing"
version = "3.0.9" version = "3.0.9"
@ -959,6 +1005,14 @@ category = "dev"
optional = false optional = false
python-versions = "*" python-versions = "*"
[[package]]
name = "types-emoji"
version = "1.7.2"
description = "Typing stubs for emoji"
category = "dev"
optional = false
python-versions = "*"
[[package]] [[package]]
name = "types-markdown" name = "types-markdown"
version = "3.3.28" version = "3.3.28"
@ -1080,7 +1134,7 @@ dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"]
[metadata] [metadata]
lock-version = "1.1" lock-version = "1.1"
python-versions = "^3.10" python-versions = "^3.10"
content-hash = "2bb47cf626fbd4c87803f8b0362470892a04239b2530676c861401e4352f7483" content-hash = "e8f20d21a8c7822fbc3c183376d694fc0109e90851377bc6b7316c5c72e880b0"
[metadata.files] [metadata.files]
alembic = [ alembic = [
@ -1168,6 +1222,10 @@ boussole = [
bs4 = [ bs4 = [
{file = "bs4-0.0.1.tar.gz", hash = "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"}, {file = "bs4-0.0.1.tar.gz", hash = "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"},
] ]
cachetools = [
{file = "cachetools-5.2.0-py3-none-any.whl", hash = "sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db"},
{file = "cachetools-5.2.0.tar.gz", hash = "sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757"},
]
certifi = [ certifi = [
{file = "certifi-2022.5.18.1-py3-none-any.whl", hash = "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"}, {file = "certifi-2022.5.18.1-py3-none-any.whl", hash = "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"},
{file = "certifi-2022.5.18.1.tar.gz", hash = "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7"}, {file = "certifi-2022.5.18.1.tar.gz", hash = "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7"},
@ -1240,6 +1298,9 @@ colorlog = [
{file = "colorlog-6.6.0-py2.py3-none-any.whl", hash = "sha256:351c51e866c86c3217f08e4b067a7974a678be78f07f85fc2d55b8babde6d94e"}, {file = "colorlog-6.6.0-py2.py3-none-any.whl", hash = "sha256:351c51e866c86c3217f08e4b067a7974a678be78f07f85fc2d55b8babde6d94e"},
{file = "colorlog-6.6.0.tar.gz", hash = "sha256:344f73204009e4c83c5b6beb00b3c45dc70fcdae3c80db919e0a4171d006fde8"}, {file = "colorlog-6.6.0.tar.gz", hash = "sha256:344f73204009e4c83c5b6beb00b3c45dc70fcdae3c80db919e0a4171d006fde8"},
] ]
emoji = [
{file = "emoji-1.7.0.tar.gz", hash = "sha256:65c54533ea3c78f30d0729288998715f418d7467de89ec258a31c0ce8660a1d1"},
]
factory-boy = [ factory-boy = [
{file = "factory_boy-3.2.1-py2.py3-none-any.whl", hash = "sha256:eb02a7dd1b577ef606b75a253b9818e6f9eaf996d94449c9d5ebb124f90dc795"}, {file = "factory_boy-3.2.1-py2.py3-none-any.whl", hash = "sha256:eb02a7dd1b577ef606b75a253b9818e6f9eaf996d94449c9d5ebb124f90dc795"},
{file = "factory_boy-3.2.1.tar.gz", hash = "sha256:a98d277b0c047c75eb6e4ab8508a7f81fb03d2cb21986f627913546ef7a2a55e"}, {file = "factory_boy-3.2.1.tar.gz", hash = "sha256:a98d277b0c047c75eb6e4ab8508a7f81fb03d2cb21986f627913546ef7a2a55e"},
@ -1259,6 +1320,25 @@ flake8 = [
{file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"},
{file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"}, {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"},
] ]
frozendict = [
{file = "frozendict-2.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4fb171d1e84d17335365877e19d17440373b47ca74a73c06f65ac0b16d01e87f"},
{file = "frozendict-2.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0a3640e9d7533d164160b758351aa49d9e85bbe0bd76d219d4021e90ffa6a52"},
{file = "frozendict-2.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:87cfd00fafbc147d8cd2590d1109b7db8ac8d7d5bdaa708ba46caee132b55d4d"},
{file = "frozendict-2.3.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:fb09761e093cfabb2f179dbfdb2521e1ec5701df714d1eb5c51fa7849027be19"},
{file = "frozendict-2.3.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82176dc7adf01cf8f0193e909401939415a230a1853f4a672ec1629a06ceae18"},
{file = "frozendict-2.3.2-cp36-cp36m-win_amd64.whl", hash = "sha256:c1c70826aa4a50fa283fe161834ac4a3ac7c753902c980bb8b595b0998a38ddb"},
{file = "frozendict-2.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1db5035ddbed995badd1a62c4102b5e207b5aeb24472df2c60aba79639d7996b"},
{file = "frozendict-2.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4246fc4cb1413645ba4d3513939b90d979a5bae724be605a10b2b26ee12f839c"},
{file = "frozendict-2.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:680cd42fb0a255da1ce45678ccbd7f69da750d5243809524ebe8f45b2eda6e6b"},
{file = "frozendict-2.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a7f3a181d6722c92a9fab12d0c5c2b006a18ca5666098531f316d1e1c8984e3"},
{file = "frozendict-2.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1cb866eabb3c1384a7fe88e1e1033e2b6623073589012ab637c552bf03f6364"},
{file = "frozendict-2.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:952c5e5e664578c5c2ce8489ee0ab6a1855da02b58ef593ee728fc10d672641a"},
{file = "frozendict-2.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:608b77904cd0117cd816df605a80d0043a5326ee62529327d2136c792165a823"},
{file = "frozendict-2.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0eed41fd326f0bcc779837d8d9e1374da1bc9857fe3b9f2910195bbd5fff3aeb"},
{file = "frozendict-2.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:bde28db6b5868dd3c45b3555f9d1dc5a1cca6d93591502fa5dcecce0dde6a335"},
{file = "frozendict-2.3.2-py3-none-any.whl", hash = "sha256:6882a9bbe08ab9b5ff96ce11bdff3fe40b114b9813bc6801261e2a7b45e20012"},
{file = "frozendict-2.3.2.tar.gz", hash = "sha256:7fac4542f0a13fbe704db4942f41ba3abffec5af8b100025973e59dff6a09d0d"},
]
greenlet = [ greenlet = [
{file = "greenlet-1.1.2-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:58df5c2a0e293bf665a51f8a100d3e9956febfbf1d9aaf8c0677cf70218910c6"}, {file = "greenlet-1.1.2-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:58df5c2a0e293bf665a51f8a100d3e9956febfbf1d9aaf8c0677cf70218910c6"},
{file = "greenlet-1.1.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:aec52725173bd3a7b56fe91bc56eccb26fbdff1386ef123abb63c84c5b43b63a"}, {file = "greenlet-1.1.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:aec52725173bd3a7b56fe91bc56eccb26fbdff1386ef123abb63c84c5b43b63a"},
@ -1683,6 +1763,9 @@ pygments = [
{file = "Pygments-2.12.0-py3-none-any.whl", hash = "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"}, {file = "Pygments-2.12.0-py3-none-any.whl", hash = "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"},
{file = "Pygments-2.12.0.tar.gz", hash = "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb"}, {file = "Pygments-2.12.0.tar.gz", hash = "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb"},
] ]
pyld = [
{file = "PyLD-2.0.3.tar.gz", hash = "sha256:287445f888c3a332ccbd20a14844c66c2fcbaeab3c99acd506a0788e2ebb2f82"},
]
pyparsing = [ pyparsing = [
{file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"},
{file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"},
@ -1823,6 +1906,10 @@ types-bleach = [
{file = "types-bleach-5.0.2.tar.gz", hash = "sha256:e1498c512a62117496cf82be3d129972bb89fd1d6482b001cdeb2759ab3c82f5"}, {file = "types-bleach-5.0.2.tar.gz", hash = "sha256:e1498c512a62117496cf82be3d129972bb89fd1d6482b001cdeb2759ab3c82f5"},
{file = "types_bleach-5.0.2-py3-none-any.whl", hash = "sha256:6fcb75ee4b69190fe60340147b66442cecddaefe3c0629433a4240da1ec2dcf6"}, {file = "types_bleach-5.0.2-py3-none-any.whl", hash = "sha256:6fcb75ee4b69190fe60340147b66442cecddaefe3c0629433a4240da1ec2dcf6"},
] ]
types-emoji = [
{file = "types-emoji-1.7.2.tar.gz", hash = "sha256:a7660fb507b30cb80bcec2d01417d828f1258b9b2cd9fa80918e8e5470c5e037"},
{file = "types_emoji-1.7.2-py3-none-any.whl", hash = "sha256:f4c18bb43e33dc267c650b73d7ae0cd71708c75c79063706d0b91fa9416190c8"},
]
types-markdown = [ types-markdown = [
{file = "types-Markdown-3.3.28.tar.gz", hash = "sha256:733ba19dad58d5dca1206390f55fa285573535b7c369b94dd367bbc34bf7e4de"}, {file = "types-Markdown-3.3.28.tar.gz", hash = "sha256:733ba19dad58d5dca1206390f55fa285573535b7c369b94dd367bbc34bf7e4de"},
{file = "types_Markdown-3.3.28-py3-none-any.whl", hash = "sha256:7868cfa3f8a2304d9ecea2ca9b02c14fcb2e34bd26fdbaf01d8c4d362a85d345"}, {file = "types_Markdown-3.3.28-py3-none-any.whl", hash = "sha256:7868cfa3f8a2304d9ecea2ca9b02c14fcb2e34bd26fdbaf01d8c4d362a85d345"},

View file

@ -36,6 +36,8 @@ Pillow = "^9.1.1"
blurhash-python = "^1.1.3" blurhash-python = "^1.1.3"
html2text = "^2020.1.16" html2text = "^2020.1.16"
feedgen = "^0.9.0" feedgen = "^0.9.0"
emoji = "^1.7.0"
PyLD = "^2.0.3"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
black = "^22.3.0" black = "^22.3.0"
@ -53,6 +55,7 @@ types-Markdown = "^3.3.28"
factory-boy = "^3.2.1" factory-boy = "^3.2.1"
pytest-asyncio = "^0.18.3" pytest-asyncio = "^0.18.3"
types-Pillow = "^9.0.20" types-Pillow = "^9.0.20"
types-emoji = "^1.7.2"
[build-system] [build-system]
requires = ["poetry-core>=1.0.0"] requires = ["poetry-core>=1.0.0"]

View file

@ -1,5 +1,9 @@
import io
import tarfile
from pathlib import Path
from typing import Optional from typing import Optional
import httpx
from invoke import Context # type: ignore from invoke import Context # type: ignore
from invoke import run # type: ignore from invoke import run # type: ignore
from invoke import task # type: ignore from invoke import task # type: ignore
@ -67,3 +71,23 @@ def tests(ctx, k=None):
pty=True, pty=True,
echo=True, echo=True,
) )
@task
def download_twemoji(ctx):
# type: (Context) -> None
resp = httpx.get(
"https://github.com/twitter/twemoji/archive/refs/tags/v14.0.2.tar.gz",
follow_redirects=True,
)
resp.raise_for_status()
tf = tarfile.open(fileobj=io.BytesIO(resp.content))
members = [
member
for member in tf.getmembers()
if member.name.startswith("twemoji-14.0.2/assets/svg/")
]
for member in members:
emoji_name = Path(member.name).name
with open(f"app/static/twemoji/{emoji_name}", "wb") as f:
f.write(tf.extractfile(member).read()) # type: ignore

View file

@ -204,3 +204,8 @@ class InboxObjectFactory(factory.alchemy.SQLAlchemyModelFactory):
# Hide replies from the stream # Hide replies from the stream
is_hidden_from_stream=True if ro.in_reply_to else False, is_hidden_from_stream=True if ro.in_reply_to else False,
) )
class FollowerFactory(factory.alchemy.SQLAlchemyModelFactory):
class Meta(BaseModelMeta):
model = models.Follower

61
tests/test_emoji.py Normal file
View file

@ -0,0 +1,61 @@
from fastapi.testclient import TestClient
from app import activitypub as ap
from app import models
from app.config import generate_csrf_token
from app.database import Session
from app.utils.emoji import EMOJIS_BY_NAME
from tests.utils import generate_admin_session_cookies
def test_emoji_are_loaded() -> None:
assert len(EMOJIS_BY_NAME) >= 1
def test_emoji_ap_endpoint(db: Session, client: TestClient) -> None:
response = client.get("/e/goose_hacker", headers={"Accept": ap.AP_CONTENT_TYPE})
assert response.status_code == 200
assert response.headers["content-type"] == ap.AP_CONTENT_TYPE
emoji_resp = response.json()
assert emoji_resp["type"] == "Emoji"
def test_emoji_ap_endpoint__not_found(db: Session, client: TestClient) -> None:
response = client.get("/e/goose_hacker2", headers={"Accept": ap.AP_CONTENT_TYPE})
assert response.status_code == 404
def test_emoji_note_with_emoji(db: Session, client: TestClient) -> None:
# Call admin endpoint to create a note with
note_content = "😺 :goose_hacker:"
response = client.post(
"/admin/actions/new",
data={
"redirect_url": "http://testserver/",
"content": note_content,
"visibility": ap.VisibilityEnum.PUBLIC.name,
"csrf_token": generate_csrf_token(),
},
cookies=generate_admin_session_cookies(),
)
# Then the server returns a 302
assert response.status_code == 302
# And the Follow activity was created in the outbox
outbox_object = db.query(models.OutboxObject).one()
assert outbox_object.ap_type == "Note"
assert len(outbox_object.tags) == 1
emoji_tag = outbox_object.tags[0]
assert emoji_tag["type"] == "Emoji"
assert emoji_tag["name"] == ":goose_hacker:"
url = emoji_tag["icon"]["url"]
# And the custom emoji is rendered in the HTML version
html_resp = client.get("/o/" + outbox_object.public_id)
html_resp.raise_for_status()
assert html_resp.status_code == 200
assert url in html_resp.text
# And the unicode emoji is rendered with twemoji
assert f'/static/twemoji/{hex(ord("😺"))[2:]}.svg' in html_resp.text

View file

@ -120,32 +120,6 @@ def test_process_next_outgoing_activity__error_500(
assert outgoing_activity.tries == 1 assert outgoing_activity.tries == 1
def test_process_next_outgoing_activity__connect_error(
db: Session,
respx_mock: respx.MockRouter,
) -> None:
outbox_object = _setup_outbox_object()
recipient_inbox_url = "https://example.com/inbox"
respx_mock.post(recipient_inbox_url).mock(side_effect=httpx.ConnectError)
# And an outgoing activity
outgoing_activity = factories.OutgoingActivityFactory(
recipient=recipient_inbox_url,
outbox_object_id=outbox_object.id,
)
# When processing the next outgoing activity
# Then it is processed
assert process_next_outgoing_activity(db) is True
assert respx_mock.calls.call_count == 1
outgoing_activity = db.query(models.OutgoingActivity).one()
assert outgoing_activity.is_sent is False
assert outgoing_activity.error is not None
assert outgoing_activity.tries == 1
def test_process_next_outgoing_activity__errored( def test_process_next_outgoing_activity__errored(
db: Session, db: Session,
respx_mock: respx.MockRouter, respx_mock: respx.MockRouter,
@ -179,5 +153,31 @@ def test_process_next_outgoing_activity__errored(
assert process_next_outgoing_activity(db) is False assert process_next_outgoing_activity(db) is False
def test_process_next_outgoing_activity__connect_error(
db: Session,
respx_mock: respx.MockRouter,
) -> None:
outbox_object = _setup_outbox_object()
recipient_inbox_url = "https://example.com/inbox"
respx_mock.post(recipient_inbox_url).mock(side_effect=httpx.ConnectError)
# And an outgoing activity
outgoing_activity = factories.OutgoingActivityFactory(
recipient=recipient_inbox_url,
outbox_object_id=outbox_object.id,
)
# When processing the next outgoing activity
# Then it is processed
assert process_next_outgoing_activity(db) is True
assert respx_mock.calls.call_count == 1
outgoing_activity = db.query(models.OutgoingActivity).one()
assert outgoing_activity.is_sent is False
assert outgoing_activity.error is not None
assert outgoing_activity.tries == 1
# TODO(ts): # TODO(ts):
# - parse retry after # - parse retry after