diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..313c984 --- /dev/null +++ b/.env.sample @@ -0,0 +1,3 @@ +MASTO_URL= +MASTO_TOKEN= +LOGDIR= \ No newline at end of file diff --git a/README.md b/README.md index 8f201d3..9c67b42 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,8 @@ # diyalgo -DIY Algoritms for mastodon \ No newline at end of file +![PyPI](https://img.shields.io/pypi/v/diyalgo) + +DIY Algoritms for mastodon + +Just uploading to PyPI for now to squat on the package name. Will release +ah um a version of this soon :) \ No newline at end of file diff --git a/diyalgo/__init__.py b/diyalgo/__init__.py index 8d1c8b6..3f7333e 100644 --- a/diyalgo/__init__.py +++ b/diyalgo/__init__.py @@ -1 +1 @@ - +from diyalgo.config import Config diff --git a/diyalgo/client/init.py b/diyalgo/client/init.py new file mode 100644 index 0000000..4b70750 --- /dev/null +++ b/diyalgo/client/init.py @@ -0,0 +1,15 @@ +from typing import Optional +from diyalgo import Config + +from mastodon import Mastodon + +def log_in(config:Optional[Config]=None) -> Mastodon: + if config is None: + config = Config() + + client = Mastodon( + access_token=config.MASTO_TOKEN, + api_base_url=config.MASTO_URL + ) + + return client \ No newline at end of file diff --git a/diyalgo/config.py b/diyalgo/config.py new file mode 100644 index 0000000..985bad2 --- /dev/null +++ b/diyalgo/config.py @@ -0,0 +1,12 @@ +from pathlib import Path +from typing import Optional +from pydantic import BaseSettings, AnyHttpUrl, EmailStr + +class Config(BaseSettings): + MASTO_URL:AnyHttpUrl + MASTO_TOKEN: Optional[str] = None + LOGDIR:Path = Path().home() / '.mastotools' + + class Config: + env_file = '.env' + env_file_encoding = 'utf-8' \ No newline at end of file diff --git a/diyalgo/expansions/timeline.py b/diyalgo/expansions/timeline.py new file mode 100644 index 0000000..8e4242e --- /dev/null +++ b/diyalgo/expansions/timeline.py @@ -0,0 +1,19 @@ +from typing import List, Literal +import pdb + +from mastodon import Mastodon + +from diyalgo.models import Status + +TIMELINES = Literal['home', 'local', 'public', 'tag', 'hashtag', 'list', 'id'] + +def fetch_timeline( + client:Mastodon, + timeline:TIMELINES="public", + **kwargs + ) -> List[Status]: + tl = client.timeline(timeline=timeline, **kwargs) + tl = client.fetch_remaining(tl) + pdb.set_trace() + tl = [Status(**status) for status in tl] + return tl diff --git a/diyalgo/models/account.py b/diyalgo/models/account.py index 0e0fa25..f7b9958 100644 --- a/diyalgo/models/account.py +++ b/diyalgo/models/account.py @@ -1,28 +1,14 @@ from pydantic import BaseModel, AnyHttpUrl -from sqlmodel import Field, SQLModel +from sqlmodel import Field, SQLModel, Relationship from datetime import datetime -from typing import Optional, List +from typing import Optional, List, TYPE_CHECKING from bs4 import BeautifulSoup -from diyalgo.models import CustomEmoji +from diyalgo.models.links import EmojiAccountLink, EmojiStatusLink -class AccountField(BaseModel): - name: str - value: str - verified_at: Optional[datetime] = None - url: Optional[AnyHttpUrl] = None - - def __init__(self, name:str, value:str, verified_at:Optional[datetime] = None): - soup = BeautifulSoup(value, 'lxml') - a = soup.find('a') - if a is not None: - url = a.get('href') - else: - url = None - super().__init__(name=name, value=value, url=url, verified_at=verified_at) - - class Config: - extra = "ignore" +if TYPE_CHECKING: + from diyalgo.models import Status + from diyalgo.models import CustomEmoji class Account(SQLModel, table=True): @@ -32,26 +18,50 @@ class Account(SQLModel, table=True): avatar: str avatar_static: str bot: bool - created_at:datetime + # created_at:datetime discoverable:bool display_name:str - emojis: List[CustomEmoji] = Field(default_factory=list) - fields: List[AccountField] = Field(default_factory=list) + emojis: List['CustomEmoji'] = Relationship(back_populates='accounts', link_model=EmojiAccountLink) + # fields: List["AccountField"] = Relationship(back_populates='account') followers_count:int following_count:int group: bool header: str - last_status_at: Optional[datetime] = None + # last_status_at: Optional[datetime] = None limited: Optional[bool] = None locked: bool - moved: Optional['Account'] = None + # moved: Optional['Account'] = Relationship() noindex: Optional[bool] = None header_static: str note: str + statuses: List['Status'] = Relationship(back_populates='account') statuses_count: int suspended: Optional[bool] = None url: AnyHttpUrl username: str - class Config: - extra = 'ignore' \ No newline at end of file + # class Config: + # extra = 'ignore' + +class AccountField(SQLModel): + id: Optional[int] = Field(primary_key=True, default=None) + name: str + value: str + # verified_at: Optional[datetime] = None + # url: Optional[AnyHttpUrl] = None + + account_id: Optional[int] = Field(default=None, foreign_key='account.id') + account: Account = Relationship(back_populates='fields') + + # def __init__(self, value:str, **kwargs): + # soup = BeautifulSoup(value, 'lxml') + # a = soup.find('a') + # if a is not None: + # url = a.get('href') + # else: + # url = None + # super().__init__(value=value, url=url, **kwargs) + # + # class Config: + # extra = "ignore" + diff --git a/diyalgo/models/attachment.py b/diyalgo/models/attachment.py index 5074ff2..84ef492 100644 --- a/diyalgo/models/attachment.py +++ b/diyalgo/models/attachment.py @@ -1,12 +1,20 @@ -from typing import Literal, Optional -from sqlmodel import Field, SQLModel +from typing import Literal, Optional, TYPE_CHECKING +from sqlmodel import Field, SQLModel, Relationship + +if TYPE_CHECKING: + from diyalgo.models import Status class MediaAttachment(SQLModel, table=True): id: int = Field(primary_key=True) blurhash: str description: str - meta: dict + # meta: dict preview_url: str remote_url: str - type: Literal['unknown', 'image', 'gifv', 'video', 'audio'] + type: str #Literal['unknown', 'image', 'gifv', 'video', 'audio'] url: str + status_id: Optional[int] = Field(default=None, foreign_key='status.id') + status: 'Status' = Relationship(back_populates='media_attachments') + + # class Config: + # extra = 'ignore' diff --git a/diyalgo/models/emojis.py b/diyalgo/models/emojis.py index 14384a3..24745b0 100644 --- a/diyalgo/models/emojis.py +++ b/diyalgo/models/emojis.py @@ -1,8 +1,19 @@ -from sqlmodel import Field, SQLModel +from typing import Optional, List, TYPE_CHECKING +from sqlmodel import Field, SQLModel, Relationship + +from diyalgo.models.links import EmojiAccountLink, EmojiStatusLink + +if TYPE_CHECKING: + from diyalgo.models import Account, Status + class CustomEmoji(SQLModel, table=True): - shortcode: str = Field(primary_key=True) + id: Optional[int] = Field(primary_key=True, default=None) + shortcode: str url: str static_url: str visible_in_picker: bool - category: str \ No newline at end of file + category: str + + accounts: List['Account'] = Relationship(back_populates='emojis', link_model=EmojiAccountLink) + statuses: List['Status'] = Relationship(back_populates='emojis', link_model=EmojiStatusLink) \ No newline at end of file diff --git a/diyalgo/models/links.py b/diyalgo/models/links.py new file mode 100644 index 0000000..c815d38 --- /dev/null +++ b/diyalgo/models/links.py @@ -0,0 +1,28 @@ +from typing import Optional + +from sqlmodel import SQLModel, Field + + +class EmojiAccountLink(SQLModel, table=True): + emoji_id: Optional[int] = Field( + default=None, foreign_key='customemoji.id', primary_key=True + ) + account_id: Optional[int] = Field( + default=None, foreign_key='account.id', primary_key=True + ) + +class EmojiStatusLink(SQLModel, table=True): + emoji_id: Optional[int] = Field( + default=None, foreign_key='customemoji.id', primary_key=True + ) + status_id: Optional[int] = Field( + default=None, foreign_key='status.id', primary_key=True + ) + +class TagStatusLink(SQLModel, table=True): + tag_id: Optional[int] = Field( + default=None, foreign_key='tag.id', primary_key=True + ) + status_id: Optional[int] = Field( + default=None, foreign_key='status.id', primary_key=True + ) \ No newline at end of file diff --git a/diyalgo/models/poll.py b/diyalgo/models/poll.py index 0d89296..ffb3af7 100644 --- a/diyalgo/models/poll.py +++ b/diyalgo/models/poll.py @@ -1,22 +1,28 @@ from datetime import datetime from typing import Optional, List, TYPE_CHECKING -from sqlmodel import Field, SQLModel +from sqlmodel import Field, SQLModel, Relationship if TYPE_CHECKING: - from diyalgo.models import CustomEmoji + from diyalgo.models import CustomEmoji, Status -class PollOption(SQLModel): +class PollOption(SQLModel, table=True): + id: Optional[int] = Field(primary_key=True, default=None) + poll_id: Optional[int] = Field(default=None, foreign_key='poll.id') + poll: 'Poll' = Relationship(back_populates='options') title: str votes_count: Optional[int] = None class Poll(SQLModel, table=True): id: int = Field(primary_key=True) - emojis: List['CustomEmoji'] = Field(default_factory=list) + #emojis: List["CustomEmoji"] = Field(default_factory=list) expires_at: Optional[datetime] = None expired: bool multiple: bool - options: List[PollOption] = Field(default_factory=list) + options: List[PollOption] = Relationship(back_populates='poll') own_votes: List[int] = Field(default_factory=list) voted: Optional[bool] = None votes_count: int voters_count: Optional[int] = None + + #status_id: Optional[int] = Field(default=None, foreign_key='status.id') + #status: 'Status' = Relationship(back_populates='poll') diff --git a/diyalgo/models/status.py b/diyalgo/models/status.py index ac6aca5..91f315f 100644 --- a/diyalgo/models/status.py +++ b/diyalgo/models/status.py @@ -1,17 +1,22 @@ from typing import Optional, Literal, List, TYPE_CHECKING from datetime import datetime -from sqlmodel import Field, SQLModel +from sqlmodel import Field, SQLModel, Relationship from bs4 import BeautifulSoup +from diyalgo.models.links import TagStatusLink, EmojiStatusLink + if TYPE_CHECKING: from diyalgo.models import Account, MediaAttachment, Tag, CustomEmoji, Poll class Mention(SQLModel, table=True): + mention_id: Optional[int] = Field(primary_key=True, default=None) acct: str id: int url: str username: str + status_id: Optional[int] = Field(default=None, foreign_key='status.id') + status: 'Status' = Relationship(back_populates='mentions') class Status(SQLModel, table=True): """ @@ -20,38 +25,36 @@ class Status(SQLModel, table=True): See: https://mastodonpy.readthedocs.io/en/stable/#toot-dicts """ id: int = Field(primary_key=True) - application: Optional[dict] = None - - account: 'Account' + # application: Optional[dict] = None + account_id: Optional[int] = Field(default=None, foreign_key='account.id') + account: 'Account' = Relationship(back_populates='statuses') bookmarked: Optional[bool] = None content: str created_at: datetime edited_at: Optional[datetime] = None - emojis: List['CustomEmoji'] = Field(default_factory=list) + emojis: List['CustomEmoji'] = Relationship(back_populates='statuses', link_model=EmojiStatusLink) favourited: Optional[bool] = None favourites_count: int - filtered: Optional[List[str]] = Field(default_factory=list) - in_reply_to_id: int - in_reply_to_account_id: int + filtered: List[str] = Field(default_factory=list) + in_reply_to_id: Optional[int] = None + in_reply_to_account_id: Optional[int] = None language: Optional[str] = None - media_attachments: List['MediaAttachment'] = Field(default_factory=list) - mentions: List[Mention] = Field(default_factory=list) + media_attachments: List['MediaAttachment'] = Relationship(back_populates='status') + mentions: List[Mention] = Relationship(back_populates='status') muted: Optional[bool] = None pinned: Optional[bool] = None - poll: Optional['Poll'] = None - reblog: bool + # poll: Optional['Poll'] = Relationship(back_populates='status') + reblog: Optional[bool] = None reblogged: Optional[bool] = None reblogs_count: int replies_count: int sensitive: bool spoiler_text: str - tags: List['Tag'] = Field(default_factory=list) + tags: List['Tag'] = Relationship(back_populates='statuses', link_model=TagStatusLink) text: Optional[str] = None uri: str url: str - visibility: Literal['public', 'unlisted', 'private', 'direct'] - in_reply_to_id: Optional[int] = None - in_reply_to_account_id: Optional[int] = None + visibility: str #Literal['public', 'unlisted', 'private', 'direct'] @property def soup(self) -> BeautifulSoup: diff --git a/diyalgo/models/tag.py b/diyalgo/models/tag.py index f4b6b22..71d0bdc 100644 --- a/diyalgo/models/tag.py +++ b/diyalgo/models/tag.py @@ -1,5 +1,14 @@ -from sqlmodel import Field, SQLModel +from typing import Optional, List, TYPE_CHECKING +from sqlmodel import Field, SQLModel, Relationship + +from diyalgo.models.links import TagStatusLink + +if TYPE_CHECKING: + from diyalgo.models import Status class Tag(SQLModel, table=True): + id: Optional[int] = Field(primary_key=True, default=None) name: str - url: str \ No newline at end of file + url: str + + statuses: List['Status'] = Relationship(back_populates='tags', link_model=TagStatusLink) \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 8be4837..8b00aa3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -345,6 +345,21 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-dotenv" +version = "0.21.0" +description = "Read key-value pairs from a .env file and set them as environment variables" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "python-dotenv-0.21.0.tar.gz", hash = "sha256:b77d08274639e3d34145dfa6c7008e66df0f04b7be7a75fd0d5292c191d79045"}, + {file = "python_dotenv-0.21.0-py3-none-any.whl", hash = "sha256:1684eb44636dd462b66c3ee016599815514527ad99965de77f43e0944634a7e5"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + [[package]] name = "python-magic" version = "0.4.27" @@ -542,4 +557,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "87440f24f73a01d5da686727e61561317d4c02b7e030fa15ce23769d4f9db5ee" +content-hash = "bcb1da145e3fb10a5e98a37ae6f4e9933e32b3152fe17799e964187294f7a890" diff --git a/pyproject.toml b/pyproject.toml index 05c3ad4..418e449 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,9 @@ description = "DIY Algorithms for Mastodon" authors = ["sneakers-the-rat "] license = "AGPL-3.0" readme = "README.md" +repository = "https://git.jon-e.net/jonny/diyalgo" +keywords = ["mastodon", "fediverse", "algorithm", "algorithms", "social media"] + [tool.poetry.dependencies] python = "^3.9" @@ -13,6 +16,7 @@ pydantic = "^1.10.4" sqlmodel = "^0.0.8" beautifulsoup4 = "^4.11.1" lxml = "^4.9.2" +python-dotenv = "^0.21.0" [build-system]