Add stats CLI command

This commit is contained in:
Thomas Sileo 2022-07-08 21:17:08 +02:00
parent 592de1b22d
commit 758a28d0e1
5 changed files with 191 additions and 1 deletions

157
app/utils/stats.py Normal file
View file

@ -0,0 +1,157 @@
import asyncio
from dataclasses import dataclass
import humanize
from sqlalchemy import case
from sqlalchemy import func
from sqlalchemy import or_
from sqlalchemy import select
from tabulate import tabulate
from app import models
from app.config import ROOT_DIR
from app.database import AsyncSession
from app.database import async_session
from app.database import now
_DATA_DIR = ROOT_DIR / "data"
@dataclass
class DiskUsageStats:
data_dir_size: int
upload_dir_size: int
def get_disk_usage_stats() -> DiskUsageStats:
du_stats = DiskUsageStats(
data_dir_size=0,
upload_dir_size=0,
)
for f in _DATA_DIR.glob("**/*"):
if f.is_file():
stat = f.stat()
du_stats.data_dir_size += stat.st_size
if str(f.parent).endswith("/data/uploads"):
du_stats.upload_dir_size += stat.st_size
return du_stats
@dataclass
class OutgoingActivityStatsItem:
total_count: int
waiting_count: int
sent_count: int
errored_count: int
@dataclass
class OutgoingActivityStats:
total: OutgoingActivityStatsItem
from_inbox: OutgoingActivityStatsItem
from_outbox: OutgoingActivityStatsItem
async def get_outgoing_activity_stats(
db_session: AsyncSession,
) -> OutgoingActivityStats:
async def _get_stats(f) -> OutgoingActivityStatsItem:
row = (
await db_session.execute(
select(
func.count(models.OutgoingActivity.id).label("total_count"),
func.sum(
case(
[
(
or_(
models.OutgoingActivity.next_try > now(),
models.OutgoingActivity.tries == 0,
),
1,
),
],
else_=0,
)
).label("waiting_count"),
func.sum(
case(
[
(models.OutgoingActivity.is_sent.is_(True), 1),
],
else_=0,
)
).label("sent_count"),
func.sum(
case(
[
(models.OutgoingActivity.is_errored.is_(True), 1),
],
else_=0,
)
).label("errored_count"),
).where(f)
)
).one()
return OutgoingActivityStatsItem(**dict(row))
from_inbox = await _get_stats(models.OutgoingActivity.inbox_object_id.is_not(None))
from_outbox = await _get_stats(
models.OutgoingActivity.outbox_object_id.is_not(None)
)
return OutgoingActivityStats(
from_inbox=from_inbox,
from_outbox=from_outbox,
total=OutgoingActivityStatsItem(
total_count=from_inbox.total_count + from_outbox.total_count,
waiting_count=from_inbox.waiting_count + from_outbox.waiting_count,
sent_count=from_inbox.sent_count + from_outbox.sent_count,
errored_count=from_inbox.errored_count + from_outbox.errored_count,
),
)
def print_stats() -> None:
async def _get_stats():
async with async_session() as session:
dat = await get_outgoing_activity_stats(session)
return dat
outgoing_activity_stats = asyncio.run(_get_stats())
disk_usage_stats = get_disk_usage_stats()
print()
print(
tabulate(
[
(
"data/",
humanize.naturalsize(disk_usage_stats.data_dir_size),
),
(
"data/uploads/",
humanize.naturalsize(disk_usage_stats.upload_dir_size),
),
],
headers=["Disk usage", "size"],
)
)
print()
print(
tabulate(
[
(name, s.total_count, s.waiting_count, s.sent_count, s.errored_count)
for (name, s) in [
("total", outgoing_activity_stats.total),
("outbox", outgoing_activity_stats.from_outbox),
("forwarded", outgoing_activity_stats.from_inbox),
]
],
headers=["Outgoing activities", "total", "waiting", "sent", "errored"],
)
)
print()

23
poetry.lock generated
View file

@ -993,6 +993,17 @@ anyio = ">=3.4.0,<5"
[package.extras] [package.extras]
full = ["itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"] full = ["itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"]
[[package]]
name = "tabulate"
version = "0.8.10"
description = "Pretty-print tabular data"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[package.extras]
widechars = ["wcwidth"]
[[package]] [[package]]
name = "tomli" name = "tomli"
version = "2.0.1" version = "2.0.1"
@ -1068,6 +1079,14 @@ python-versions = "*"
[package.dependencies] [package.dependencies]
types-urllib3 = "<1.27" types-urllib3 = "<1.27"
[[package]]
name = "types-tabulate"
version = "0.8.11"
description = "Typing stubs for tabulate"
category = "dev"
optional = false
python-versions = "*"
[[package]] [[package]]
name = "types-urllib3" name = "types-urllib3"
version = "1.26.16" version = "1.26.16"
@ -1154,7 +1173,7 @@ dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"]
[metadata] [metadata]
lock-version = "1.1" lock-version = "1.1"
python-versions = "^3.10" python-versions = "^3.10"
content-hash = "ae7b5b5dfd9a30bc585c27be3d79e48c13b5cbb60b917034bc93e8038c4d3d8f" content-hash = "fd741c6c1c1e85cb1b39150df503bc64b28244b65222180c6768409fcfd1d70a"
[metadata.files] [metadata.files]
aiosqlite = [ aiosqlite = [
@ -1960,6 +1979,7 @@ starlette = [
{file = "starlette-0.19.1-py3-none-any.whl", hash = "sha256:5a60c5c2d051f3a8eb546136aa0c9399773a689595e099e0877704d5888279bf"}, {file = "starlette-0.19.1-py3-none-any.whl", hash = "sha256:5a60c5c2d051f3a8eb546136aa0c9399773a689595e099e0877704d5888279bf"},
{file = "starlette-0.19.1.tar.gz", hash = "sha256:c6d21096774ecb9639acad41b86b7706e52ba3bf1dc13ea4ed9ad593d47e24c7"}, {file = "starlette-0.19.1.tar.gz", hash = "sha256:c6d21096774ecb9639acad41b86b7706e52ba3bf1dc13ea4ed9ad593d47e24c7"},
] ]
tabulate = []
tomli = [ tomli = [
{file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
{file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
@ -1996,6 +2016,7 @@ types-requests = [
{file = "types-requests-2.28.0.tar.gz", hash = "sha256:9863d16dfbb3fa55dcda64fa3b989e76e8859033b26c1e1623e30465cfe294d3"}, {file = "types-requests-2.28.0.tar.gz", hash = "sha256:9863d16dfbb3fa55dcda64fa3b989e76e8859033b26c1e1623e30465cfe294d3"},
{file = "types_requests-2.28.0-py3-none-any.whl", hash = "sha256:85383b4ef0535f639c3f06c5bbb6494bbf59570c4cd88bbcf540f0b2ac1b49ab"}, {file = "types_requests-2.28.0-py3-none-any.whl", hash = "sha256:85383b4ef0535f639c3f06c5bbb6494bbf59570c4cd88bbcf540f0b2ac1b49ab"},
] ]
types-tabulate = []
types-urllib3 = [ types-urllib3 = [
{file = "types-urllib3-1.26.16.tar.gz", hash = "sha256:8bb3832c684c30cbed40b96e28bc04703becb2b97d82ac65ba4b968783453b0e"}, {file = "types-urllib3-1.26.16.tar.gz", hash = "sha256:8bb3832c684c30cbed40b96e28bc04703becb2b97d82ac65ba4b968783453b0e"},
{file = "types_urllib3-1.26.16-py3-none-any.whl", hash = "sha256:20588c285e5ca336d908d2705994830a83cfb6bda40fc356bbafaf430a262013"}, {file = "types_urllib3-1.26.16-py3-none-any.whl", hash = "sha256:20588c285e5ca336d908d2705994830a83cfb6bda40fc356bbafaf430a262013"},

View file

@ -39,6 +39,7 @@ PyLD = "^2.0.3"
aiosqlite = "^0.17.0" aiosqlite = "^0.17.0"
cachetools = "^5.2.0" cachetools = "^5.2.0"
humanize = "^4.2.3" humanize = "^4.2.3"
tabulate = "^0.8.10"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
black = "^22.3.0" black = "^22.3.0"
@ -60,6 +61,7 @@ types-emoji = "^1.7.2"
types-cachetools = "^5.2.1" types-cachetools = "^5.2.1"
sqlalchemy2-stubs = "^0.0.2-alpha.24" sqlalchemy2-stubs = "^0.0.2-alpha.24"
types-python-dateutil = "^2.8.18" types-python-dateutil = "^2.8.18"
types-tabulate = "^0.8.11"
[build-system] [build-system]
requires = ["poetry-core>=1.0.0"] requires = ["poetry-core>=1.0.0"]

View file

@ -135,3 +135,11 @@ def install_deps(ctx):
def update(ctx): def update(ctx):
# type: (Context) -> None # type: (Context) -> None
print("Done") print("Done")
@task
def stats(ctx):
# type: (Context) -> None
from app.utils.stats import print_stats
print_stats()

View file

@ -84,6 +84,7 @@ def test_process_next_outgoing_activity__server_200(
outgoing_activity = factories.OutgoingActivityFactory( outgoing_activity = factories.OutgoingActivityFactory(
recipient=recipient_inbox_url, recipient=recipient_inbox_url,
outbox_object_id=outbox_object.id, outbox_object_id=outbox_object.id,
inbox_object_id=None,
) )
# When processing the next outgoing activity # When processing the next outgoing activity
@ -174,6 +175,7 @@ def test_process_next_outgoing_activity__connect_error(
outgoing_activity = factories.OutgoingActivityFactory( outgoing_activity = factories.OutgoingActivityFactory(
recipient=recipient_inbox_url, recipient=recipient_inbox_url,
outbox_object_id=outbox_object.id, outbox_object_id=outbox_object.id,
inbox_object_id=None,
) )
# When processing the next outgoing activity # When processing the next outgoing activity