From 758a28d0e1d8a80230edd048208468254bf02126 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 8 Jul 2022 21:17:08 +0200 Subject: [PATCH] Add stats CLI command --- app/utils/stats.py | 157 ++++++++++++++++++++++ poetry.lock | 23 +++- pyproject.toml | 2 + tasks.py | 8 ++ tests/test_process_outgoing_activities.py | 2 + 5 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 app/utils/stats.py diff --git a/app/utils/stats.py b/app/utils/stats.py new file mode 100644 index 0000000..8bf2efb --- /dev/null +++ b/app/utils/stats.py @@ -0,0 +1,157 @@ +import asyncio +from dataclasses import dataclass + +import humanize +from sqlalchemy import case +from sqlalchemy import func +from sqlalchemy import or_ +from sqlalchemy import select +from tabulate import tabulate + +from app import models +from app.config import ROOT_DIR +from app.database import AsyncSession +from app.database import async_session +from app.database import now + +_DATA_DIR = ROOT_DIR / "data" + + +@dataclass +class DiskUsageStats: + data_dir_size: int + upload_dir_size: int + + +def get_disk_usage_stats() -> DiskUsageStats: + du_stats = DiskUsageStats( + data_dir_size=0, + upload_dir_size=0, + ) + for f in _DATA_DIR.glob("**/*"): + if f.is_file(): + stat = f.stat() + du_stats.data_dir_size += stat.st_size + if str(f.parent).endswith("/data/uploads"): + du_stats.upload_dir_size += stat.st_size + + return du_stats + + +@dataclass +class OutgoingActivityStatsItem: + total_count: int + waiting_count: int + sent_count: int + errored_count: int + + +@dataclass +class OutgoingActivityStats: + total: OutgoingActivityStatsItem + from_inbox: OutgoingActivityStatsItem + from_outbox: OutgoingActivityStatsItem + + +async def get_outgoing_activity_stats( + db_session: AsyncSession, +) -> OutgoingActivityStats: + async def _get_stats(f) -> OutgoingActivityStatsItem: + row = ( + await db_session.execute( + select( + func.count(models.OutgoingActivity.id).label("total_count"), + func.sum( + case( + [ + ( + or_( + models.OutgoingActivity.next_try > now(), + models.OutgoingActivity.tries == 0, + ), + 1, + ), + ], + else_=0, + ) + ).label("waiting_count"), + func.sum( + case( + [ + (models.OutgoingActivity.is_sent.is_(True), 1), + ], + else_=0, + ) + ).label("sent_count"), + func.sum( + case( + [ + (models.OutgoingActivity.is_errored.is_(True), 1), + ], + else_=0, + ) + ).label("errored_count"), + ).where(f) + ) + ).one() + return OutgoingActivityStatsItem(**dict(row)) + + from_inbox = await _get_stats(models.OutgoingActivity.inbox_object_id.is_not(None)) + from_outbox = await _get_stats( + models.OutgoingActivity.outbox_object_id.is_not(None) + ) + + return OutgoingActivityStats( + from_inbox=from_inbox, + from_outbox=from_outbox, + total=OutgoingActivityStatsItem( + total_count=from_inbox.total_count + from_outbox.total_count, + waiting_count=from_inbox.waiting_count + from_outbox.waiting_count, + sent_count=from_inbox.sent_count + from_outbox.sent_count, + errored_count=from_inbox.errored_count + from_outbox.errored_count, + ), + ) + + +def print_stats() -> None: + async def _get_stats(): + async with async_session() as session: + dat = await get_outgoing_activity_stats(session) + + return dat + + outgoing_activity_stats = asyncio.run(_get_stats()) + disk_usage_stats = get_disk_usage_stats() + + print() + print( + tabulate( + [ + ( + "data/", + humanize.naturalsize(disk_usage_stats.data_dir_size), + ), + ( + "data/uploads/", + humanize.naturalsize(disk_usage_stats.upload_dir_size), + ), + ], + headers=["Disk usage", "size"], + ) + ) + + print() + print( + tabulate( + [ + (name, s.total_count, s.waiting_count, s.sent_count, s.errored_count) + for (name, s) in [ + ("total", outgoing_activity_stats.total), + ("outbox", outgoing_activity_stats.from_outbox), + ("forwarded", outgoing_activity_stats.from_inbox), + ] + ], + headers=["Outgoing activities", "total", "waiting", "sent", "errored"], + ) + ) + print() diff --git a/poetry.lock b/poetry.lock index 04d7cde..6cb05d4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -993,6 +993,17 @@ anyio = ">=3.4.0,<5" [package.extras] full = ["itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"] +[[package]] +name = "tabulate" +version = "0.8.10" +description = "Pretty-print tabular data" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +widechars = ["wcwidth"] + [[package]] name = "tomli" version = "2.0.1" @@ -1068,6 +1079,14 @@ python-versions = "*" [package.dependencies] types-urllib3 = "<1.27" +[[package]] +name = "types-tabulate" +version = "0.8.11" +description = "Typing stubs for tabulate" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "types-urllib3" version = "1.26.16" @@ -1154,7 +1173,7 @@ dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"] [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "ae7b5b5dfd9a30bc585c27be3d79e48c13b5cbb60b917034bc93e8038c4d3d8f" +content-hash = "fd741c6c1c1e85cb1b39150df503bc64b28244b65222180c6768409fcfd1d70a" [metadata.files] aiosqlite = [ @@ -1960,6 +1979,7 @@ starlette = [ {file = "starlette-0.19.1-py3-none-any.whl", hash = "sha256:5a60c5c2d051f3a8eb546136aa0c9399773a689595e099e0877704d5888279bf"}, {file = "starlette-0.19.1.tar.gz", hash = "sha256:c6d21096774ecb9639acad41b86b7706e52ba3bf1dc13ea4ed9ad593d47e24c7"}, ] +tabulate = [] tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, @@ -1996,6 +2016,7 @@ types-requests = [ {file = "types-requests-2.28.0.tar.gz", hash = "sha256:9863d16dfbb3fa55dcda64fa3b989e76e8859033b26c1e1623e30465cfe294d3"}, {file = "types_requests-2.28.0-py3-none-any.whl", hash = "sha256:85383b4ef0535f639c3f06c5bbb6494bbf59570c4cd88bbcf540f0b2ac1b49ab"}, ] +types-tabulate = [] types-urllib3 = [ {file = "types-urllib3-1.26.16.tar.gz", hash = "sha256:8bb3832c684c30cbed40b96e28bc04703becb2b97d82ac65ba4b968783453b0e"}, {file = "types_urllib3-1.26.16-py3-none-any.whl", hash = "sha256:20588c285e5ca336d908d2705994830a83cfb6bda40fc356bbafaf430a262013"}, diff --git a/pyproject.toml b/pyproject.toml index 2379695..7a543af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ PyLD = "^2.0.3" aiosqlite = "^0.17.0" cachetools = "^5.2.0" humanize = "^4.2.3" +tabulate = "^0.8.10" [tool.poetry.dev-dependencies] black = "^22.3.0" @@ -60,6 +61,7 @@ types-emoji = "^1.7.2" types-cachetools = "^5.2.1" sqlalchemy2-stubs = "^0.0.2-alpha.24" types-python-dateutil = "^2.8.18" +types-tabulate = "^0.8.11" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tasks.py b/tasks.py index 9955183..6dde133 100644 --- a/tasks.py +++ b/tasks.py @@ -135,3 +135,11 @@ def install_deps(ctx): def update(ctx): # type: (Context) -> None print("Done") + + +@task +def stats(ctx): + # type: (Context) -> None + from app.utils.stats import print_stats + + print_stats() diff --git a/tests/test_process_outgoing_activities.py b/tests/test_process_outgoing_activities.py index a65e6db..b2b3eaa 100644 --- a/tests/test_process_outgoing_activities.py +++ b/tests/test_process_outgoing_activities.py @@ -84,6 +84,7 @@ def test_process_next_outgoing_activity__server_200( outgoing_activity = factories.OutgoingActivityFactory( recipient=recipient_inbox_url, outbox_object_id=outbox_object.id, + inbox_object_id=None, ) # When processing the next outgoing activity @@ -174,6 +175,7 @@ def test_process_next_outgoing_activity__connect_error( outgoing_activity = factories.OutgoingActivityFactory( recipient=recipient_inbox_url, outbox_object_id=outbox_object.id, + inbox_object_id=None, ) # When processing the next outgoing activity