Add stats CLI command

app/utils/stats.py

@@ -0,0 +1,157 @@
+import asyncio
+from dataclasses import dataclass
+
+import humanize
+from sqlalchemy import case
+from sqlalchemy import func
+from sqlalchemy import or_
+from sqlalchemy import select
+from tabulate import tabulate
+
+from app import models
+from app.config import ROOT_DIR
+from app.database import AsyncSession
+from app.database import async_session
+from app.database import now
+
+_DATA_DIR = ROOT_DIR / "data"
+
+
+@dataclass
+class DiskUsageStats:
+    data_dir_size: int
+    upload_dir_size: int
+
+
+def get_disk_usage_stats() -> DiskUsageStats:
+    du_stats = DiskUsageStats(
+        data_dir_size=0,
+        upload_dir_size=0,
+    )
+    for f in _DATA_DIR.glob("**/*"):
+        if f.is_file():
+            stat = f.stat()
+            du_stats.data_dir_size += stat.st_size
+            if str(f.parent).endswith("/data/uploads"):
+                du_stats.upload_dir_size += stat.st_size
+
+    return du_stats
+
+
+@dataclass
+class OutgoingActivityStatsItem:
+    total_count: int
+    waiting_count: int
+    sent_count: int
+    errored_count: int
+
+
+@dataclass
+class OutgoingActivityStats:
+    total: OutgoingActivityStatsItem
+    from_inbox: OutgoingActivityStatsItem
+    from_outbox: OutgoingActivityStatsItem
+
+
+async def get_outgoing_activity_stats(
+    db_session: AsyncSession,
+) -> OutgoingActivityStats:
+    async def _get_stats(f) -> OutgoingActivityStatsItem:
+        row = (
+            await db_session.execute(
+                select(
+                    func.count(models.OutgoingActivity.id).label("total_count"),
+                    func.sum(
+                        case(
+                            [
+                                (
+                                    or_(
+                                        models.OutgoingActivity.next_try > now(),
+                                        models.OutgoingActivity.tries == 0,
+                                    ),
+                                    1,
+                                ),
+                            ],
+                            else_=0,
+                        )
+                    ).label("waiting_count"),
+                    func.sum(
+                        case(
+                            [
+                                (models.OutgoingActivity.is_sent.is_(True), 1),
+                            ],
+                            else_=0,
+                        )
+                    ).label("sent_count"),
+                    func.sum(
+                        case(
+                            [
+                                (models.OutgoingActivity.is_errored.is_(True), 1),
+                            ],
+                            else_=0,
+                        )
+                    ).label("errored_count"),
+                ).where(f)
+            )
+        ).one()
+        return OutgoingActivityStatsItem(**dict(row))
+
+    from_inbox = await _get_stats(models.OutgoingActivity.inbox_object_id.is_not(None))
+    from_outbox = await _get_stats(
+        models.OutgoingActivity.outbox_object_id.is_not(None)
+    )
+
+    return OutgoingActivityStats(
+        from_inbox=from_inbox,
+        from_outbox=from_outbox,
+        total=OutgoingActivityStatsItem(
+            total_count=from_inbox.total_count + from_outbox.total_count,
+            waiting_count=from_inbox.waiting_count + from_outbox.waiting_count,
+            sent_count=from_inbox.sent_count + from_outbox.sent_count,
+            errored_count=from_inbox.errored_count + from_outbox.errored_count,
+        ),
+    )
+
+
+def print_stats() -> None:
+    async def _get_stats():
+        async with async_session() as session:
+            dat = await get_outgoing_activity_stats(session)
+
+        return dat
+
+    outgoing_activity_stats = asyncio.run(_get_stats())
+    disk_usage_stats = get_disk_usage_stats()
+
+    print()
+    print(
+        tabulate(
+            [
+                (
+                    "data/",
+                    humanize.naturalsize(disk_usage_stats.data_dir_size),
+                ),
+                (
+                    "data/uploads/",
+                    humanize.naturalsize(disk_usage_stats.upload_dir_size),
+                ),
+            ],
+            headers=["Disk usage", "size"],
+        )
+    )
+
+    print()
+    print(
+        tabulate(
+            [
+                (name, s.total_count, s.waiting_count, s.sent_count, s.errored_count)
+                for (name, s) in [
+                    ("total", outgoing_activity_stats.total),
+                    ("outbox", outgoing_activity_stats.from_outbox),
+                    ("forwarded", outgoing_activity_stats.from_inbox),
+                ]
+            ],
+            headers=["Outgoing activities", "total", "waiting", "sent", "errored"],
+        )
+    )
+    print()

poetry.lock

@@ -993,6 +993,17 @@ anyio = ">=3.4.0,<5"
 [package.extras]
 full = ["itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"]
 
+[[package]]
+name = "tabulate"
+version = "0.8.10"
+description = "Pretty-print tabular data"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+
+[package.extras]
+widechars = ["wcwidth"]
+
 [[package]]
 name = "tomli"
 version = "2.0.1"
@@ -1068,6 +1079,14 @@ python-versions = "*"
 [package.dependencies]
 types-urllib3 = "<1.27"
 
+[[package]]
+name = "types-tabulate"
+version = "0.8.11"
+description = "Typing stubs for tabulate"
+category = "dev"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "types-urllib3"
 version = "1.26.16"
@@ -1154,7 +1173,7 @@ dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.10"
-content-hash = "ae7b5b5dfd9a30bc585c27be3d79e48c13b5cbb60b917034bc93e8038c4d3d8f"
+content-hash = "fd741c6c1c1e85cb1b39150df503bc64b28244b65222180c6768409fcfd1d70a"
 
 [metadata.files]
 aiosqlite = [
@@ -1960,6 +1979,7 @@ starlette = [
     {file = "starlette-0.19.1-py3-none-any.whl", hash = "sha256:5a60c5c2d051f3a8eb546136aa0c9399773a689595e099e0877704d5888279bf"},
     {file = "starlette-0.19.1.tar.gz", hash = "sha256:c6d21096774ecb9639acad41b86b7706e52ba3bf1dc13ea4ed9ad593d47e24c7"},
 ]
+tabulate = []
 tomli = [
     {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
     {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
@@ -1996,6 +2016,7 @@ types-requests = [
     {file = "types-requests-2.28.0.tar.gz", hash = "sha256:9863d16dfbb3fa55dcda64fa3b989e76e8859033b26c1e1623e30465cfe294d3"},
     {file = "types_requests-2.28.0-py3-none-any.whl", hash = "sha256:85383b4ef0535f639c3f06c5bbb6494bbf59570c4cd88bbcf540f0b2ac1b49ab"},
 ]
+types-tabulate = []
 types-urllib3 = [
     {file = "types-urllib3-1.26.16.tar.gz", hash = "sha256:8bb3832c684c30cbed40b96e28bc04703becb2b97d82ac65ba4b968783453b0e"},
     {file = "types_urllib3-1.26.16-py3-none-any.whl", hash = "sha256:20588c285e5ca336d908d2705994830a83cfb6bda40fc356bbafaf430a262013"},

pyproject.toml

@@ -39,6 +39,7 @@ PyLD = "^2.0.3"
 aiosqlite = "^0.17.0"
 cachetools = "^5.2.0"
 humanize = "^4.2.3"
+tabulate = "^0.8.10"
 
 [tool.poetry.dev-dependencies]
 black = "^22.3.0"
@@ -60,6 +61,7 @@ types-emoji = "^1.7.2"
 types-cachetools = "^5.2.1"
 sqlalchemy2-stubs = "^0.0.2-alpha.24"
 types-python-dateutil = "^2.8.18"
+types-tabulate = "^0.8.11"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

tasks.py

@@ -135,3 +135,11 @@ def install_deps(ctx):
 def update(ctx):
     # type: (Context) -> None
     print("Done")
+
+
+@task
+def stats(ctx):
+    # type: (Context) -> None
+    from app.utils.stats import print_stats
+
+    print_stats()

tests/test_process_outgoing_activities.py

@@ -84,6 +84,7 @@ def test_process_next_outgoing_activity__server_200(
     outgoing_activity = factories.OutgoingActivityFactory(
         recipient=recipient_inbox_url,
         outbox_object_id=outbox_object.id,
+        inbox_object_id=None,
     )
 
     # When processing the next outgoing activity
@@ -174,6 +175,7 @@ def test_process_next_outgoing_activity__connect_error(
     outgoing_activity = factories.OutgoingActivityFactory(
         recipient=recipient_inbox_url,
         outbox_object_id=outbox_object.id,
+        inbox_object_id=None,
     )
 
     # When processing the next outgoing activity