diff --git a/blueprints/tasks.py b/blueprints/tasks.py
index 950e4c8..d6c463b 100644
--- a/blueprints/tasks.py
+++ b/blueprints/tasks.py
@@ -2,6 +2,8 @@ import json
 import traceback
 from datetime import datetime
 from datetime import timezone
+from typing import Any
+from typing import Dict
 
 import flask
 import requests
@@ -559,9 +561,6 @@ def task_process_reply() -> _Response:
         if reply.has_type(ap.ActivityType.CREATE):
             reply = reply.get_object()
 
-        # Store some metadata for the UI
-        # FIXME(tsileo): be able to display: "In reply to @user@domain.tld"?
-
         new_replies = [activity, reply]
 
         while 1:
@@ -579,29 +578,36 @@ def task_process_reply() -> _Response:
 
         app.logger.info(f"root_reply={reply!r} for activity={activity!r}")
 
-        # Ensure the "root reply" is present in the inbox/outbox
-        if not find_one_activity(
-            {**by_object_id(root_reply), **by_type(ap.ActivityType.CREATE)}
-        ):
-            return ""
-
         # In case the activity was from the inbox
         update_one_activity(
             {**by_object_id(activity.id), **by_type(ap.ActivityType.CREATE)},
             upsert({MetaKey.THREAD_ROOT_PARENT: root_reply}),
         )
 
-        for new_reply in new_replies:
+        for (new_reply_idx, new_reply) in enumerate(new_replies):
             if find_one_activity(
                 {**by_object_id(new_reply.id), **by_type(ap.ActivityType.CREATE)}
             ) or DB.replies.find_one(by_remote_id(new_reply.id)):
                 continue
 
             actor = new_reply.get_actor()
+            is_root_reply = new_reply_idx == len(new_replies) - 1
+            if is_root_reply:
+                reply_flags: Dict[str, Any] = {}
+            else:
+                reply_actor = new_replies[new_reply_idx + 1].get_actor()
+                is_in_reply_to_self = actor.id == reply_actor.id
+                reply_flags = {MetaKey.IN_REPLY_TO_SELF.value: is_in_reply_to_self}
+                if not is_in_reply_to_self:
+                    reply_flags[MetaKey.IN_REPLY_TO_ACTOR.value] = reply_actor.to_dict(
+                        embed=True
+                    )
+
             # Save the reply with the cached actor and the thread flag/ID
             save_reply(
                 new_reply,
                 {
+                    **reply_flags,
                     MetaKey.THREAD_ROOT_PARENT.value: root_reply,
                     MetaKey.ACTOR.value: actor.to_dict(embed=True),
                     MetaKey.ACTOR_HASH.value: _actor_hash(actor),
diff --git a/core/activitypub.py b/core/activitypub.py
index 2f6f693..2f064f2 100644
--- a/core/activitypub.py
+++ b/core/activitypub.py
@@ -735,6 +735,15 @@ def handle_replies(create: ap.Create) -> None:
         )
         return None
 
+    # Update the activity to save some data about the reply
+    if reply.get_actor().id == create.get_actor().id:
+        in_reply_to_data = {MetaKey.IN_REPLY_TO_SELF: True}
+    else:
+        in_reply_to_data = {
+            MetaKey.IN_REPLY_TO_ACTOR: reply.get_actor().to_dict(embed=True)
+        }
+    update_one_activity(by_remote_id(create.id), upsert(in_reply_to_data))
+
     # It's a regular reply, try to increment the reply counter
     creply = DB.activities.find_one_and_update(
         {**by_object_id(in_reply_to), **by_type(ap.ActivityType.CREATE)},
@@ -747,4 +756,4 @@ def handle_replies(create: ap.Create) -> None:
         )
 
     # Spawn a task to process it (and determine if it needs to be saved)
-    Tasks.process_reply(create.get_object().id)
+    Tasks.process_reply(create.get_object_id())
diff --git a/core/meta.py b/core/meta.py
index 422437c..f5dde02 100644
--- a/core/meta.py
+++ b/core/meta.py
@@ -40,6 +40,9 @@ class MetaKey(Enum):
     PUBLIC = "public"
     THREAD_ROOT_PARENT = "thread_root_parent"
 
+    IN_REPLY_TO_SELF = "in_reply_to_self"
+    IN_REPLY_TO_ACTOR = "in_reply_to_actor"
+
     SERVER = "server"
     VISIBILITY = "visibility"
     OBJECT_VISIBILITY = "object_visibility"