From b156c57ff18f1bc3a178e2ea40a4068b69bc16a3 Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Mon, 2 Jan 2023 23:54:57 -0800 Subject: [PATCH] ok that's enough debugging for tonight. getting the SQL models to work by using some simple local data and it's choking on the relationships. by jove we've done it again and made it too complicated --- .env.sample | 3 ++ README.md | 7 +++- diyalgo/__init__.py | 2 +- diyalgo/client/init.py | 15 ++++++++ diyalgo/config.py | 12 +++++++ diyalgo/expansions/timeline.py | 19 ++++++++++ diyalgo/models/account.py | 64 ++++++++++++++++++++-------------- diyalgo/models/attachment.py | 16 ++++++--- diyalgo/models/emojis.py | 17 +++++++-- diyalgo/models/links.py | 28 +++++++++++++++ diyalgo/models/poll.py | 16 ++++++--- diyalgo/models/status.py | 35 ++++++++++--------- diyalgo/models/tag.py | 13 +++++-- poetry.lock | 17 ++++++++- pyproject.toml | 4 +++ 15 files changed, 208 insertions(+), 60 deletions(-) create mode 100644 .env.sample create mode 100644 diyalgo/client/init.py create mode 100644 diyalgo/config.py create mode 100644 diyalgo/expansions/timeline.py create mode 100644 diyalgo/models/links.py diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..313c984 --- /dev/null +++ b/.env.sample @@ -0,0 +1,3 @@ +MASTO_URL= +MASTO_TOKEN= +LOGDIR= \ No newline at end of file diff --git a/README.md b/README.md index 8f201d3..9c67b42 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,8 @@ # diyalgo -DIY Algoritms for mastodon \ No newline at end of file +![PyPI](https://img.shields.io/pypi/v/diyalgo) + +DIY Algoritms for mastodon + +Just uploading to PyPI for now to squat on the package name. Will release +ah um a version of this soon :) \ No newline at end of file diff --git a/diyalgo/__init__.py b/diyalgo/__init__.py index 8d1c8b6..3f7333e 100644 --- a/diyalgo/__init__.py +++ b/diyalgo/__init__.py @@ -1 +1 @@ - +from diyalgo.config import Config diff --git a/diyalgo/client/init.py b/diyalgo/client/init.py new file mode 100644 index 0000000..4b70750 --- /dev/null +++ b/diyalgo/client/init.py @@ -0,0 +1,15 @@ +from typing import Optional +from diyalgo import Config + +from mastodon import Mastodon + +def log_in(config:Optional[Config]=None) -> Mastodon: + if config is None: + config = Config() + + client = Mastodon( + access_token=config.MASTO_TOKEN, + api_base_url=config.MASTO_URL + ) + + return client \ No newline at end of file diff --git a/diyalgo/config.py b/diyalgo/config.py new file mode 100644 index 0000000..985bad2 --- /dev/null +++ b/diyalgo/config.py @@ -0,0 +1,12 @@ +from pathlib import Path +from typing import Optional +from pydantic import BaseSettings, AnyHttpUrl, EmailStr + +class Config(BaseSettings): + MASTO_URL:AnyHttpUrl + MASTO_TOKEN: Optional[str] = None + LOGDIR:Path = Path().home() / '.mastotools' + + class Config: + env_file = '.env' + env_file_encoding = 'utf-8' \ No newline at end of file diff --git a/diyalgo/expansions/timeline.py b/diyalgo/expansions/timeline.py new file mode 100644 index 0000000..8e4242e --- /dev/null +++ b/diyalgo/expansions/timeline.py @@ -0,0 +1,19 @@ +from typing import List, Literal +import pdb + +from mastodon import Mastodon + +from diyalgo.models import Status + +TIMELINES = Literal['home', 'local', 'public', 'tag', 'hashtag', 'list', 'id'] + +def fetch_timeline( + client:Mastodon, + timeline:TIMELINES="public", + **kwargs + ) -> List[Status]: + tl = client.timeline(timeline=timeline, **kwargs) + tl = client.fetch_remaining(tl) + pdb.set_trace() + tl = [Status(**status) for status in tl] + return tl diff --git a/diyalgo/models/account.py b/diyalgo/models/account.py index 0e0fa25..f7b9958 100644 --- a/diyalgo/models/account.py +++ b/diyalgo/models/account.py @@ -1,28 +1,14 @@ from pydantic import BaseModel, AnyHttpUrl -from sqlmodel import Field, SQLModel +from sqlmodel import Field, SQLModel, Relationship from datetime import datetime -from typing import Optional, List +from typing import Optional, List, TYPE_CHECKING from bs4 import BeautifulSoup -from diyalgo.models import CustomEmoji +from diyalgo.models.links import EmojiAccountLink, EmojiStatusLink -class AccountField(BaseModel): - name: str - value: str - verified_at: Optional[datetime] = None - url: Optional[AnyHttpUrl] = None - - def __init__(self, name:str, value:str, verified_at:Optional[datetime] = None): - soup = BeautifulSoup(value, 'lxml') - a = soup.find('a') - if a is not None: - url = a.get('href') - else: - url = None - super().__init__(name=name, value=value, url=url, verified_at=verified_at) - - class Config: - extra = "ignore" +if TYPE_CHECKING: + from diyalgo.models import Status + from diyalgo.models import CustomEmoji class Account(SQLModel, table=True): @@ -32,26 +18,50 @@ class Account(SQLModel, table=True): avatar: str avatar_static: str bot: bool - created_at:datetime + # created_at:datetime discoverable:bool display_name:str - emojis: List[CustomEmoji] = Field(default_factory=list) - fields: List[AccountField] = Field(default_factory=list) + emojis: List['CustomEmoji'] = Relationship(back_populates='accounts', link_model=EmojiAccountLink) + # fields: List["AccountField"] = Relationship(back_populates='account') followers_count:int following_count:int group: bool header: str - last_status_at: Optional[datetime] = None + # last_status_at: Optional[datetime] = None limited: Optional[bool] = None locked: bool - moved: Optional['Account'] = None + # moved: Optional['Account'] = Relationship() noindex: Optional[bool] = None header_static: str note: str + statuses: List['Status'] = Relationship(back_populates='account') statuses_count: int suspended: Optional[bool] = None url: AnyHttpUrl username: str - class Config: - extra = 'ignore' \ No newline at end of file + # class Config: + # extra = 'ignore' + +class AccountField(SQLModel): + id: Optional[int] = Field(primary_key=True, default=None) + name: str + value: str + # verified_at: Optional[datetime] = None + # url: Optional[AnyHttpUrl] = None + + account_id: Optional[int] = Field(default=None, foreign_key='account.id') + account: Account = Relationship(back_populates='fields') + + # def __init__(self, value:str, **kwargs): + # soup = BeautifulSoup(value, 'lxml') + # a = soup.find('a') + # if a is not None: + # url = a.get('href') + # else: + # url = None + # super().__init__(value=value, url=url, **kwargs) + # + # class Config: + # extra = "ignore" + diff --git a/diyalgo/models/attachment.py b/diyalgo/models/attachment.py index 5074ff2..84ef492 100644 --- a/diyalgo/models/attachment.py +++ b/diyalgo/models/attachment.py @@ -1,12 +1,20 @@ -from typing import Literal, Optional -from sqlmodel import Field, SQLModel +from typing import Literal, Optional, TYPE_CHECKING +from sqlmodel import Field, SQLModel, Relationship + +if TYPE_CHECKING: + from diyalgo.models import Status class MediaAttachment(SQLModel, table=True): id: int = Field(primary_key=True) blurhash: str description: str - meta: dict + # meta: dict preview_url: str remote_url: str - type: Literal['unknown', 'image', 'gifv', 'video', 'audio'] + type: str #Literal['unknown', 'image', 'gifv', 'video', 'audio'] url: str + status_id: Optional[int] = Field(default=None, foreign_key='status.id') + status: 'Status' = Relationship(back_populates='media_attachments') + + # class Config: + # extra = 'ignore' diff --git a/diyalgo/models/emojis.py b/diyalgo/models/emojis.py index 14384a3..24745b0 100644 --- a/diyalgo/models/emojis.py +++ b/diyalgo/models/emojis.py @@ -1,8 +1,19 @@ -from sqlmodel import Field, SQLModel +from typing import Optional, List, TYPE_CHECKING +from sqlmodel import Field, SQLModel, Relationship + +from diyalgo.models.links import EmojiAccountLink, EmojiStatusLink + +if TYPE_CHECKING: + from diyalgo.models import Account, Status + class CustomEmoji(SQLModel, table=True): - shortcode: str = Field(primary_key=True) + id: Optional[int] = Field(primary_key=True, default=None) + shortcode: str url: str static_url: str visible_in_picker: bool - category: str \ No newline at end of file + category: str + + accounts: List['Account'] = Relationship(back_populates='emojis', link_model=EmojiAccountLink) + statuses: List['Status'] = Relationship(back_populates='emojis', link_model=EmojiStatusLink) \ No newline at end of file diff --git a/diyalgo/models/links.py b/diyalgo/models/links.py new file mode 100644 index 0000000..c815d38 --- /dev/null +++ b/diyalgo/models/links.py @@ -0,0 +1,28 @@ +from typing import Optional + +from sqlmodel import SQLModel, Field + + +class EmojiAccountLink(SQLModel, table=True): + emoji_id: Optional[int] = Field( + default=None, foreign_key='customemoji.id', primary_key=True + ) + account_id: Optional[int] = Field( + default=None, foreign_key='account.id', primary_key=True + ) + +class EmojiStatusLink(SQLModel, table=True): + emoji_id: Optional[int] = Field( + default=None, foreign_key='customemoji.id', primary_key=True + ) + status_id: Optional[int] = Field( + default=None, foreign_key='status.id', primary_key=True + ) + +class TagStatusLink(SQLModel, table=True): + tag_id: Optional[int] = Field( + default=None, foreign_key='tag.id', primary_key=True + ) + status_id: Optional[int] = Field( + default=None, foreign_key='status.id', primary_key=True + ) \ No newline at end of file diff --git a/diyalgo/models/poll.py b/diyalgo/models/poll.py index 0d89296..ffb3af7 100644 --- a/diyalgo/models/poll.py +++ b/diyalgo/models/poll.py @@ -1,22 +1,28 @@ from datetime import datetime from typing import Optional, List, TYPE_CHECKING -from sqlmodel import Field, SQLModel +from sqlmodel import Field, SQLModel, Relationship if TYPE_CHECKING: - from diyalgo.models import CustomEmoji + from diyalgo.models import CustomEmoji, Status -class PollOption(SQLModel): +class PollOption(SQLModel, table=True): + id: Optional[int] = Field(primary_key=True, default=None) + poll_id: Optional[int] = Field(default=None, foreign_key='poll.id') + poll: 'Poll' = Relationship(back_populates='options') title: str votes_count: Optional[int] = None class Poll(SQLModel, table=True): id: int = Field(primary_key=True) - emojis: List['CustomEmoji'] = Field(default_factory=list) + #emojis: List["CustomEmoji"] = Field(default_factory=list) expires_at: Optional[datetime] = None expired: bool multiple: bool - options: List[PollOption] = Field(default_factory=list) + options: List[PollOption] = Relationship(back_populates='poll') own_votes: List[int] = Field(default_factory=list) voted: Optional[bool] = None votes_count: int voters_count: Optional[int] = None + + #status_id: Optional[int] = Field(default=None, foreign_key='status.id') + #status: 'Status' = Relationship(back_populates='poll') diff --git a/diyalgo/models/status.py b/diyalgo/models/status.py index ac6aca5..91f315f 100644 --- a/diyalgo/models/status.py +++ b/diyalgo/models/status.py @@ -1,17 +1,22 @@ from typing import Optional, Literal, List, TYPE_CHECKING from datetime import datetime -from sqlmodel import Field, SQLModel +from sqlmodel import Field, SQLModel, Relationship from bs4 import BeautifulSoup +from diyalgo.models.links import TagStatusLink, EmojiStatusLink + if TYPE_CHECKING: from diyalgo.models import Account, MediaAttachment, Tag, CustomEmoji, Poll class Mention(SQLModel, table=True): + mention_id: Optional[int] = Field(primary_key=True, default=None) acct: str id: int url: str username: str + status_id: Optional[int] = Field(default=None, foreign_key='status.id') + status: 'Status' = Relationship(back_populates='mentions') class Status(SQLModel, table=True): """ @@ -20,38 +25,36 @@ class Status(SQLModel, table=True): See: https://mastodonpy.readthedocs.io/en/stable/#toot-dicts """ id: int = Field(primary_key=True) - application: Optional[dict] = None - - account: 'Account' + # application: Optional[dict] = None + account_id: Optional[int] = Field(default=None, foreign_key='account.id') + account: 'Account' = Relationship(back_populates='statuses') bookmarked: Optional[bool] = None content: str created_at: datetime edited_at: Optional[datetime] = None - emojis: List['CustomEmoji'] = Field(default_factory=list) + emojis: List['CustomEmoji'] = Relationship(back_populates='statuses', link_model=EmojiStatusLink) favourited: Optional[bool] = None favourites_count: int - filtered: Optional[List[str]] = Field(default_factory=list) - in_reply_to_id: int - in_reply_to_account_id: int + filtered: List[str] = Field(default_factory=list) + in_reply_to_id: Optional[int] = None + in_reply_to_account_id: Optional[int] = None language: Optional[str] = None - media_attachments: List['MediaAttachment'] = Field(default_factory=list) - mentions: List[Mention] = Field(default_factory=list) + media_attachments: List['MediaAttachment'] = Relationship(back_populates='status') + mentions: List[Mention] = Relationship(back_populates='status') muted: Optional[bool] = None pinned: Optional[bool] = None - poll: Optional['Poll'] = None - reblog: bool + # poll: Optional['Poll'] = Relationship(back_populates='status') + reblog: Optional[bool] = None reblogged: Optional[bool] = None reblogs_count: int replies_count: int sensitive: bool spoiler_text: str - tags: List['Tag'] = Field(default_factory=list) + tags: List['Tag'] = Relationship(back_populates='statuses', link_model=TagStatusLink) text: Optional[str] = None uri: str url: str - visibility: Literal['public', 'unlisted', 'private', 'direct'] - in_reply_to_id: Optional[int] = None - in_reply_to_account_id: Optional[int] = None + visibility: str #Literal['public', 'unlisted', 'private', 'direct'] @property def soup(self) -> BeautifulSoup: diff --git a/diyalgo/models/tag.py b/diyalgo/models/tag.py index f4b6b22..71d0bdc 100644 --- a/diyalgo/models/tag.py +++ b/diyalgo/models/tag.py @@ -1,5 +1,14 @@ -from sqlmodel import Field, SQLModel +from typing import Optional, List, TYPE_CHECKING +from sqlmodel import Field, SQLModel, Relationship + +from diyalgo.models.links import TagStatusLink + +if TYPE_CHECKING: + from diyalgo.models import Status class Tag(SQLModel, table=True): + id: Optional[int] = Field(primary_key=True, default=None) name: str - url: str \ No newline at end of file + url: str + + statuses: List['Status'] = Relationship(back_populates='tags', link_model=TagStatusLink) \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 8be4837..8b00aa3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -345,6 +345,21 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-dotenv" +version = "0.21.0" +description = "Read key-value pairs from a .env file and set them as environment variables" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "python-dotenv-0.21.0.tar.gz", hash = "sha256:b77d08274639e3d34145dfa6c7008e66df0f04b7be7a75fd0d5292c191d79045"}, + {file = "python_dotenv-0.21.0-py3-none-any.whl", hash = "sha256:1684eb44636dd462b66c3ee016599815514527ad99965de77f43e0944634a7e5"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + [[package]] name = "python-magic" version = "0.4.27" @@ -542,4 +557,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "87440f24f73a01d5da686727e61561317d4c02b7e030fa15ce23769d4f9db5ee" +content-hash = "bcb1da145e3fb10a5e98a37ae6f4e9933e32b3152fe17799e964187294f7a890" diff --git a/pyproject.toml b/pyproject.toml index 05c3ad4..418e449 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,9 @@ description = "DIY Algorithms for Mastodon" authors = ["sneakers-the-rat "] license = "AGPL-3.0" readme = "README.md" +repository = "https://git.jon-e.net/jonny/diyalgo" +keywords = ["mastodon", "fediverse", "algorithm", "algorithms", "social media"] + [tool.poetry.dependencies] python = "^3.9" @@ -13,6 +16,7 @@ pydantic = "^1.10.4" sqlmodel = "^0.0.8" beautifulsoup4 = "^4.11.1" lxml = "^4.9.2" +python-dotenv = "^0.21.0" [build-system]