Custom emoji support

app/admin.py

@@ -18,6 +18,7 @@ from app.actor import get_actors_metadata
 from app.boxes import get_inbox_object_by_ap_id
 from app.boxes import get_outbox_object_by_ap_id
 from app.boxes import send_follow
+from app.config import EMOJIS
 from app.config import generate_csrf_token
 from app.config import session_serializer
 from app.config import verify_csrf_token
@@ -25,6 +26,7 @@ from app.config import verify_password
 from app.database import get_db
 from app.lookup import lookup
 from app.uploads import save_upload
+from app.utils.emoji import EMOJIS_BY_NAME
 
 
 def user_session_or_redirect(
@@ -123,36 +125,11 @@ def admin_new(
                 (v.name, ap.VisibilityEnum.get_display_name(v))
                 for v in ap.VisibilityEnum
             ],
-        },
+            "emojis": EMOJIS.split(" "),
-    )
+            "custom_emojis": sorted(
-
+                [dat for name, dat in EMOJIS_BY_NAME.items()],
-
+                key=lambda obj: obj["name"],
-@router.get("/stream")
+            ),
-def stream(
-    request: Request,
-    db: Session = Depends(get_db),
-) -> templates.TemplateResponse:
-    stream = (
-        db.query(models.InboxObject)
-        .filter(
-            models.InboxObject.ap_type.in_(["Note", "Article", "Video", "Announce"]),
-            models.InboxObject.is_hidden_from_stream.is_(False),
-            models.InboxObject.undone_by_inbox_object_id.is_(None),
-        )
-        .options(
-            # joinedload(models.InboxObject.relates_to_inbox_object),
-            joinedload(models.InboxObject.relates_to_outbox_object),
-        )
-        .order_by(models.InboxObject.ap_published_at.desc())
-        .limit(20)
-        .all()
-    )
-    return templates.render_template(
-        db,
-        request,
-        "admin_stream.html",
-        {
-            "stream": stream,
         },
     )
 
@@ -452,7 +429,7 @@ def admin_actions_unpin(
 @router.post("/actions/new")
 def admin_actions_new(
     request: Request,
-    files: list[UploadFile],
+    files: list[UploadFile] = [],
     content: str = Form(),
     redirect_url: str = Form(),
     in_reply_to: str | None = Form(None),
@@ -501,7 +478,7 @@ def login_validation(
     if not verify_password(password):
         raise HTTPException(status_code=401)
 
-    resp = RedirectResponse("/admin", status_code=302)
+    resp = RedirectResponse("/admin/inbox", status_code=302)
     resp.set_cookie("session", session_serializer.dumps({"is_logged_in": True}))  # type: ignore  # noqa: E501
 
     return resp

app/boxes.py

@@ -341,7 +341,7 @@ def _compute_recipients(db: Session, ap_object: ap.RawObject) -> set[str]:
             db.query(models.Actor).filter(models.Actor.ap_id == r).one_or_none()
         )
         if known_actor:
-            recipients.add(known_actor.shared_inbox_url or actor.inbox_url)
+            recipients.add(known_actor.shared_inbox_url or known_actor.inbox_url)
             continue
 
         # Fetch the object

app/config.py

@@ -10,6 +10,8 @@ from fastapi import Request
 from itsdangerous import TimedSerializer
 from itsdangerous import TimestampSigner
 
+from app.utils.emoji import _load_emojis
+
 ROOT_DIR = Path().parent.resolve()
 
 _CONFIG_FILE = os.getenv("MICROBLOGPUB_CONFIG_FILE", "me.toml")
@@ -76,6 +78,11 @@ SQLALCHEMY_DATABASE_URL = CONFIG.sqlalchemy_database_url or f"sqlite:///{DB_PATH
 KEY_PATH = (
     (ROOT_DIR / CONFIG.key_path) if CONFIG.key_path else ROOT_DIR / "data" / "key.pem"
 )
+EMOJIS = "😺 😸 😹 😻 😼 😽 🙀 😿 😾"
+# Emoji template for the FE
+EMOJI_TPL = '<img src="/static/twemoji/{filename}.svg" alt="{raw}" class="emoji">'
+
+_load_emojis(ROOT_DIR, BASE_URL)
 
 
 session_serializer = TimedSerializer(CONFIG.secret, salt="microblogpub.login")

app/main.py

@@ -52,6 +52,7 @@ from app.config import verify_csrf_token
 from app.database import get_db
 from app.templates import is_current_user_admin
 from app.uploads import UPLOAD_DIR
+from app.utils.emoji import EMOJIS_BY_NAME
 from app.webfinger import get_remote_follow_template
 
 # TODO(ts):
@@ -520,6 +521,16 @@ def tag_by_name(
     )
 
 
+@app.get("/e/{name}")
+def emoji_by_name(name: str) -> ActivityPubResponse:
+    try:
+        emoji = EMOJIS_BY_NAME[f":{name}:"]
+    except KeyError:
+        raise HTTPException(status_code=404)
+
+    return ActivityPubResponse({"@context": ap.AS_CTX, **emoji})
+
+
 @app.post("/inbox")
 async def inbox(
     request: Request,

app/scss/main.scss

@@ -140,6 +140,6 @@ nav.flexbox {
     float: right;
   }
 }
-.custom-emoji {
+.emoji, .custom-emoji {
   max-width: 25px;
 }

app/source.py

@@ -8,6 +8,7 @@ from app import webfinger
 from app.actor import Actor
 from app.actor import fetch_actor
 from app.config import BASE_URL
+from app.utils import emoji
 
 
 def _set_a_attrs(attrs, new=False):
@@ -78,5 +79,10 @@ def markdownify(
     if mentionify:
         content, mention_tags, mentioned_actors = _mentionify(db, content)
         tags.extend(mention_tags)
+
+    # Handle custom emoji
+    tags.extend(emoji.tags(content))
+
     content = markdown(content, extensions=["mdx_linkify"])
+
     return content, tags, mentioned_actors

app/static/emoji/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

app/static/twemoji/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

app/templates.py

@@ -6,6 +6,7 @@ from typing import Any
 from urllib.parse import urlparse
 
 import bleach
+import emoji
 import html2text
 import timeago  # type: ignore
 from bs4 import BeautifulSoup  # type: ignore
@@ -16,6 +17,7 @@ from sqlalchemy.orm import Session
 from starlette.templating import _TemplateResponse as TemplateResponse
 
 from app import activitypub as ap
+from app import config
 from app import models
 from app.actor import LOCAL_ACTOR
 from app.ap_object import Attachment
@@ -171,7 +173,8 @@ def _update_inline_imgs(content):
 
 def _clean_html(html: str, note: Object) -> str:
     try:
-        return _replace_custom_emojis(
+        return _emojify(
+            _replace_custom_emojis(
                 bleach.clean(
                     _update_inline_imgs(highlight(html)),
                     tags=ALLOWED_TAGS,
@@ -180,6 +183,7 @@ def _clean_html(html: str, note: Object) -> str:
                 ),
                 note,
             )
+        )
     except Exception:
         raise
 
@@ -229,11 +233,24 @@ def _html2text(content: str) -> str:
     return H2T.handle(content)
 
 
+def _replace_emoji(u, data):
+    filename = hex(ord(u))[2:]
+    return config.EMOJI_TPL.format(filename=filename, raw=u)
+
+
+def _emojify(text: str):
+    return emoji.replace_emoji(
+        text,
+        replace=_replace_emoji,
+    )
+
+
 _templates.env.filters["domain"] = _filter_domain
 _templates.env.filters["media_proxy_url"] = _media_proxy_url
 _templates.env.filters["clean_html"] = _clean_html
 _templates.env.filters["timeago"] = _timeago
 _templates.env.filters["format_date"] = _format_date
 _templates.env.filters["has_media_type"] = _has_media_type
-_templates.env.filters["pluralize"] = _pluralize
 _templates.env.filters["html2text"] = _html2text
+_templates.env.filters["emojify"] = _emojify
+_templates.env.filters["pluralize"] = _pluralize

app/templates/admin_new.html

@@ -17,6 +17,13 @@
         {% endfor %}
     </select>
     </p>
+    {% for emoji in emojis %}
+    <span class="ji">{{ emoji | emojify | safe }}</span>
+    {% endfor %}
+    {% for emoji in custom_emojis %}
+    <span class="ji"><img src="{{ emoji.icon.url }}" alt="{{ emoji.name }}" title="{{ emoji.name }}" class="custom-emoji"></span>
+    {% endfor %}
+
     <textarea name="content" rows="10" cols="50" autofocus="autofocus" designMode="on" style="font-size:1.2em;width:95%;">{{ content }}</textarea>
     <input type="hidden" name="in_reply_to" value="{{ request.query_params.in_reply_to }}">
     <p>
@@ -26,5 +33,38 @@
         <input type="submit" value="Publish">
     </p>
 </form>
+<script>
+// The new post textarea
+var ta = document.getElementsByTagName("textarea")[0];
+// Helper for inserting text (emojis) in the textarea
+function insertAtCursor (textToInsert) {
+    ta.focus();
+    const isSuccess = document.execCommand("insertText", false, textToInsert);
 
+    // Firefox (non-standard method)
+    if (!isSuccess) {
+        // Credits to https://www.everythingfrontend.com/posts/insert-text-into-textarea-at-cursor-position.html
+        // get current text of the input
+        const value = ta.value;
+        // save selection start and end position
+        const start = ta.selectionStart;
+        const end = ta.selectionEnd;
+        // update the value with our text inserted
+        ta.value = value.slice(0, start) + textToInsert + value.slice(end);
+        // update cursor to be at the end of insertion
+        ta.selectionStart = ta.selectionEnd = start + textToInsert.length;
+    }
+}
+// Emoji click callback func
+var ji = function (ev) {
+    insertAtCursor(ev.target.attributes.alt.value + " ");
+    ta.focus()
+    //console.log(document.execCommand('insertText', false /*no UI*/, ev.target.attributes.alt.value));
+}
+// Enable the click for each emojis
+var items = document.getElementsByClassName("ji")
+for (var i = 0; i < items.length; i++) {
+    items[i].addEventListener('click', ji);
+}
+</script>
 {% endblock %}

app/utils/emoji.py

@@ -0,0 +1,45 @@
+import mimetypes
+import re
+import typing
+from pathlib import Path
+
+if typing.TYPE_CHECKING:
+    from app.activitypub import RawObject
+
+EMOJI_REGEX = re.compile(r"(:[\d\w]+:)")
+
+EMOJIS: dict[str, "RawObject"] = {}
+EMOJIS_BY_NAME: dict[str, "RawObject"] = {}
+
+
+def _load_emojis(root_dir: Path, base_url: str) -> None:
+    if EMOJIS:
+        return
+    for emoji in (root_dir / "app" / "static" / "emoji").iterdir():
+        mt = mimetypes.guess_type(emoji.name)[0]
+        if mt and mt.startswith("image/"):
+            name = emoji.name.split(".")[0]
+            ap_emoji: "RawObject" = {
+                "type": "Emoji",
+                "name": f":{name}:",
+                "updated": "1970-01-01T00:00:00Z",  # XXX: we don't track date
+                "id": f"{base_url}/e/{name}",
+                "icon": {
+                    "mediaType": mt,
+                    "type": "Image",
+                    "url": f"{base_url}/static/emoji/{emoji.name}",
+                },
+            }
+            EMOJIS[emoji.name] = ap_emoji
+            EMOJIS_BY_NAME[ap_emoji["name"]] = ap_emoji
+
+
+def tags(content: str) -> list["RawObject"]:
+    tags = []
+    added = set()
+    for e in re.findall(EMOJI_REGEX, content):
+        if e not in added and e in EMOJIS_BY_NAME:
+            tags.append(EMOJIS_BY_NAME[e])
+            added.add(e)
+
+    return tags

poetry.lock

@@ -176,6 +176,14 @@ python-versions = "*"
 [package.dependencies]
 beautifulsoup4 = "*"
 
+[[package]]
+name = "cachetools"
+version = "5.2.0"
+description = "Extensible memoizing collections and decorators"
+category = "main"
+optional = false
+python-versions = "~=3.7"
+
 [[package]]
 name = "certifi"
 version = "2022.5.18.1"
@@ -239,6 +247,17 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""}
 [package.extras]
 development = ["black", "flake8", "mypy", "pytest", "types-colorama"]
 
+[[package]]
+name = "emoji"
+version = "1.7.0"
+description = "Emoji for Python"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.extras]
+dev = ["pytest", "coverage", "coveralls"]
+
 [[package]]
 name = "factory-boy"
 version = "3.2.1"
@@ -308,6 +327,14 @@ mccabe = ">=0.6.0,<0.7.0"
 pycodestyle = ">=2.8.0,<2.9.0"
 pyflakes = ">=2.4.0,<2.5.0"
 
+[[package]]
+name = "frozendict"
+version = "2.3.2"
+description = "A simple immutable dictionary"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
 [[package]]
 name = "greenlet"
 version = "1.1.2"
@@ -725,6 +752,25 @@ category = "main"
 optional = false
 python-versions = ">=3.6"
 
+[[package]]
+name = "pyld"
+version = "2.0.3"
+description = "Python implementation of the JSON-LD API"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+cachetools = "*"
+frozendict = "*"
+lxml = "*"
+
+[package.extras]
+aiohttp = ["aiohttp"]
+cachetools = ["cachetools"]
+frozendict = ["frozendict"]
+requests = ["requests"]
+
 [[package]]
 name = "pyparsing"
 version = "3.0.9"
@@ -959,6 +1005,14 @@ category = "dev"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "types-emoji"
+version = "1.7.2"
+description = "Typing stubs for emoji"
+category = "dev"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "types-markdown"
 version = "3.3.28"
@@ -1080,7 +1134,7 @@ dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.10"
-content-hash = "2bb47cf626fbd4c87803f8b0362470892a04239b2530676c861401e4352f7483"
+content-hash = "e8f20d21a8c7822fbc3c183376d694fc0109e90851377bc6b7316c5c72e880b0"
 
 [metadata.files]
 alembic = [
@@ -1168,6 +1222,10 @@ boussole = [
 bs4 = [
     {file = "bs4-0.0.1.tar.gz", hash = "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"},
 ]
+cachetools = [
+    {file = "cachetools-5.2.0-py3-none-any.whl", hash = "sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db"},
+    {file = "cachetools-5.2.0.tar.gz", hash = "sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757"},
+]
 certifi = [
     {file = "certifi-2022.5.18.1-py3-none-any.whl", hash = "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"},
     {file = "certifi-2022.5.18.1.tar.gz", hash = "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7"},
@@ -1240,6 +1298,9 @@ colorlog = [
     {file = "colorlog-6.6.0-py2.py3-none-any.whl", hash = "sha256:351c51e866c86c3217f08e4b067a7974a678be78f07f85fc2d55b8babde6d94e"},
     {file = "colorlog-6.6.0.tar.gz", hash = "sha256:344f73204009e4c83c5b6beb00b3c45dc70fcdae3c80db919e0a4171d006fde8"},
 ]
+emoji = [
+    {file = "emoji-1.7.0.tar.gz", hash = "sha256:65c54533ea3c78f30d0729288998715f418d7467de89ec258a31c0ce8660a1d1"},
+]
 factory-boy = [
     {file = "factory_boy-3.2.1-py2.py3-none-any.whl", hash = "sha256:eb02a7dd1b577ef606b75a253b9818e6f9eaf996d94449c9d5ebb124f90dc795"},
     {file = "factory_boy-3.2.1.tar.gz", hash = "sha256:a98d277b0c047c75eb6e4ab8508a7f81fb03d2cb21986f627913546ef7a2a55e"},
@@ -1259,6 +1320,25 @@ flake8 = [
     {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"},
     {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"},
 ]
+frozendict = [
+    {file = "frozendict-2.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4fb171d1e84d17335365877e19d17440373b47ca74a73c06f65ac0b16d01e87f"},
+    {file = "frozendict-2.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0a3640e9d7533d164160b758351aa49d9e85bbe0bd76d219d4021e90ffa6a52"},
+    {file = "frozendict-2.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:87cfd00fafbc147d8cd2590d1109b7db8ac8d7d5bdaa708ba46caee132b55d4d"},
+    {file = "frozendict-2.3.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:fb09761e093cfabb2f179dbfdb2521e1ec5701df714d1eb5c51fa7849027be19"},
+    {file = "frozendict-2.3.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82176dc7adf01cf8f0193e909401939415a230a1853f4a672ec1629a06ceae18"},
+    {file = "frozendict-2.3.2-cp36-cp36m-win_amd64.whl", hash = "sha256:c1c70826aa4a50fa283fe161834ac4a3ac7c753902c980bb8b595b0998a38ddb"},
+    {file = "frozendict-2.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1db5035ddbed995badd1a62c4102b5e207b5aeb24472df2c60aba79639d7996b"},
+    {file = "frozendict-2.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4246fc4cb1413645ba4d3513939b90d979a5bae724be605a10b2b26ee12f839c"},
+    {file = "frozendict-2.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:680cd42fb0a255da1ce45678ccbd7f69da750d5243809524ebe8f45b2eda6e6b"},
+    {file = "frozendict-2.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a7f3a181d6722c92a9fab12d0c5c2b006a18ca5666098531f316d1e1c8984e3"},
+    {file = "frozendict-2.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1cb866eabb3c1384a7fe88e1e1033e2b6623073589012ab637c552bf03f6364"},
+    {file = "frozendict-2.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:952c5e5e664578c5c2ce8489ee0ab6a1855da02b58ef593ee728fc10d672641a"},
+    {file = "frozendict-2.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:608b77904cd0117cd816df605a80d0043a5326ee62529327d2136c792165a823"},
+    {file = "frozendict-2.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0eed41fd326f0bcc779837d8d9e1374da1bc9857fe3b9f2910195bbd5fff3aeb"},
+    {file = "frozendict-2.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:bde28db6b5868dd3c45b3555f9d1dc5a1cca6d93591502fa5dcecce0dde6a335"},
+    {file = "frozendict-2.3.2-py3-none-any.whl", hash = "sha256:6882a9bbe08ab9b5ff96ce11bdff3fe40b114b9813bc6801261e2a7b45e20012"},
+    {file = "frozendict-2.3.2.tar.gz", hash = "sha256:7fac4542f0a13fbe704db4942f41ba3abffec5af8b100025973e59dff6a09d0d"},
+]
 greenlet = [
     {file = "greenlet-1.1.2-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:58df5c2a0e293bf665a51f8a100d3e9956febfbf1d9aaf8c0677cf70218910c6"},
     {file = "greenlet-1.1.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:aec52725173bd3a7b56fe91bc56eccb26fbdff1386ef123abb63c84c5b43b63a"},
@@ -1683,6 +1763,9 @@ pygments = [
     {file = "Pygments-2.12.0-py3-none-any.whl", hash = "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"},
     {file = "Pygments-2.12.0.tar.gz", hash = "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb"},
 ]
+pyld = [
+    {file = "PyLD-2.0.3.tar.gz", hash = "sha256:287445f888c3a332ccbd20a14844c66c2fcbaeab3c99acd506a0788e2ebb2f82"},
+]
 pyparsing = [
     {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"},
     {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"},
@@ -1823,6 +1906,10 @@ types-bleach = [
     {file = "types-bleach-5.0.2.tar.gz", hash = "sha256:e1498c512a62117496cf82be3d129972bb89fd1d6482b001cdeb2759ab3c82f5"},
     {file = "types_bleach-5.0.2-py3-none-any.whl", hash = "sha256:6fcb75ee4b69190fe60340147b66442cecddaefe3c0629433a4240da1ec2dcf6"},
 ]
+types-emoji = [
+    {file = "types-emoji-1.7.2.tar.gz", hash = "sha256:a7660fb507b30cb80bcec2d01417d828f1258b9b2cd9fa80918e8e5470c5e037"},
+    {file = "types_emoji-1.7.2-py3-none-any.whl", hash = "sha256:f4c18bb43e33dc267c650b73d7ae0cd71708c75c79063706d0b91fa9416190c8"},
+]
 types-markdown = [
     {file = "types-Markdown-3.3.28.tar.gz", hash = "sha256:733ba19dad58d5dca1206390f55fa285573535b7c369b94dd367bbc34bf7e4de"},
     {file = "types_Markdown-3.3.28-py3-none-any.whl", hash = "sha256:7868cfa3f8a2304d9ecea2ca9b02c14fcb2e34bd26fdbaf01d8c4d362a85d345"},

pyproject.toml

@@ -36,6 +36,8 @@ Pillow = "^9.1.1"
 blurhash-python = "^1.1.3"
 html2text = "^2020.1.16"
 feedgen = "^0.9.0"
+emoji = "^1.7.0"
+PyLD = "^2.0.3"
 
 [tool.poetry.dev-dependencies]
 black = "^22.3.0"
@@ -53,6 +55,7 @@ types-Markdown = "^3.3.28"
 factory-boy = "^3.2.1"
 pytest-asyncio = "^0.18.3"
 types-Pillow = "^9.0.20"
+types-emoji = "^1.7.2"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

tasks.py

@@ -1,5 +1,9 @@
+import io
+import tarfile
+from pathlib import Path
 from typing import Optional
 
+import httpx
 from invoke import Context  # type: ignore
 from invoke import run  # type: ignore
 from invoke import task  # type: ignore
@@ -67,3 +71,23 @@ def tests(ctx, k=None):
         pty=True,
         echo=True,
     )
+
+
+@task
+def download_twemoji(ctx):
+    # type: (Context) -> None
+    resp = httpx.get(
+        "https://github.com/twitter/twemoji/archive/refs/tags/v14.0.2.tar.gz",
+        follow_redirects=True,
+    )
+    resp.raise_for_status()
+    tf = tarfile.open(fileobj=io.BytesIO(resp.content))
+    members = [
+        member
+        for member in tf.getmembers()
+        if member.name.startswith("twemoji-14.0.2/assets/svg/")
+    ]
+    for member in members:
+        emoji_name = Path(member.name).name
+        with open(f"app/static/twemoji/{emoji_name}", "wb") as f:
+            f.write(tf.extractfile(member).read())  # type: ignore

tests/factories.py

@@ -204,3 +204,8 @@ class InboxObjectFactory(factory.alchemy.SQLAlchemyModelFactory):
             # Hide replies from the stream
             is_hidden_from_stream=True if ro.in_reply_to else False,
         )
+
+
+class FollowerFactory(factory.alchemy.SQLAlchemyModelFactory):
+    class Meta(BaseModelMeta):
+        model = models.Follower

tests/test_emoji.py

@@ -0,0 +1,61 @@
+from fastapi.testclient import TestClient
+
+from app import activitypub as ap
+from app import models
+from app.config import generate_csrf_token
+from app.database import Session
+from app.utils.emoji import EMOJIS_BY_NAME
+from tests.utils import generate_admin_session_cookies
+
+
+def test_emoji_are_loaded() -> None:
+    assert len(EMOJIS_BY_NAME) >= 1
+
+
+def test_emoji_ap_endpoint(db: Session, client: TestClient) -> None:
+    response = client.get("/e/goose_hacker", headers={"Accept": ap.AP_CONTENT_TYPE})
+    assert response.status_code == 200
+    assert response.headers["content-type"] == ap.AP_CONTENT_TYPE
+    emoji_resp = response.json()
+    assert emoji_resp["type"] == "Emoji"
+
+
+def test_emoji_ap_endpoint__not_found(db: Session, client: TestClient) -> None:
+    response = client.get("/e/goose_hacker2", headers={"Accept": ap.AP_CONTENT_TYPE})
+    assert response.status_code == 404
+
+
+def test_emoji_note_with_emoji(db: Session, client: TestClient) -> None:
+    # Call admin endpoint to create a note with
+    note_content = "😺 :goose_hacker:"
+
+    response = client.post(
+        "/admin/actions/new",
+        data={
+            "redirect_url": "http://testserver/",
+            "content": note_content,
+            "visibility": ap.VisibilityEnum.PUBLIC.name,
+            "csrf_token": generate_csrf_token(),
+        },
+        cookies=generate_admin_session_cookies(),
+    )
+
+    # Then the server returns a 302
+    assert response.status_code == 302
+
+    # And the Follow activity was created in the outbox
+    outbox_object = db.query(models.OutboxObject).one()
+    assert outbox_object.ap_type == "Note"
+    assert len(outbox_object.tags) == 1
+    emoji_tag = outbox_object.tags[0]
+    assert emoji_tag["type"] == "Emoji"
+    assert emoji_tag["name"] == ":goose_hacker:"
+    url = emoji_tag["icon"]["url"]
+
+    # And the custom emoji is rendered in the HTML version
+    html_resp = client.get("/o/" + outbox_object.public_id)
+    html_resp.raise_for_status()
+    assert html_resp.status_code == 200
+    assert url in html_resp.text
+    # And the unicode emoji is rendered with twemoji
+    assert f'/static/twemoji/{hex(ord("😺"))[2:]}.svg' in html_resp.text

tests/test_process_outgoing_activities.py

@@ -120,32 +120,6 @@ def test_process_next_outgoing_activity__error_500(
     assert outgoing_activity.tries == 1
 
 
-def test_process_next_outgoing_activity__connect_error(
-    db: Session,
-    respx_mock: respx.MockRouter,
-) -> None:
-    outbox_object = _setup_outbox_object()
-    recipient_inbox_url = "https://example.com/inbox"
-    respx_mock.post(recipient_inbox_url).mock(side_effect=httpx.ConnectError)
-
-    # And an outgoing activity
-    outgoing_activity = factories.OutgoingActivityFactory(
-        recipient=recipient_inbox_url,
-        outbox_object_id=outbox_object.id,
-    )
-
-    # When processing the next outgoing activity
-    # Then it is processed
-    assert process_next_outgoing_activity(db) is True
-
-    assert respx_mock.calls.call_count == 1
-
-    outgoing_activity = db.query(models.OutgoingActivity).one()
-    assert outgoing_activity.is_sent is False
-    assert outgoing_activity.error is not None
-    assert outgoing_activity.tries == 1
-
-
 def test_process_next_outgoing_activity__errored(
     db: Session,
     respx_mock: respx.MockRouter,
@@ -179,5 +153,31 @@ def test_process_next_outgoing_activity__errored(
     assert process_next_outgoing_activity(db) is False
 
 
+def test_process_next_outgoing_activity__connect_error(
+    db: Session,
+    respx_mock: respx.MockRouter,
+) -> None:
+    outbox_object = _setup_outbox_object()
+    recipient_inbox_url = "https://example.com/inbox"
+    respx_mock.post(recipient_inbox_url).mock(side_effect=httpx.ConnectError)
+
+    # And an outgoing activity
+    outgoing_activity = factories.OutgoingActivityFactory(
+        recipient=recipient_inbox_url,
+        outbox_object_id=outbox_object.id,
+    )
+
+    # When processing the next outgoing activity
+    # Then it is processed
+    assert process_next_outgoing_activity(db) is True
+
+    assert respx_mock.calls.call_count == 1
+
+    outgoing_activity = db.query(models.OutgoingActivity).one()
+    assert outgoing_activity.is_sent is False
+    assert outgoing_activity.error is not None
+    assert outgoing_activity.tries == 1
+
+
 # TODO(ts):
 # - parse retry after