ok that's enough debugging for tonight.

getting the SQL models to work by using some simple local data and it's choking on the relationships.

by jove we've done it again and made it too complicated

.env.sample

@@ -0,0 +1,3 @@
+MASTO_URL=
+MASTO_TOKEN=
+LOGDIR=

README.md

@@ -1,3 +1,8 @@
 # diyalgo
 
-DIY Algoritms for mastodon
+![PyPI](https://img.shields.io/pypi/v/diyalgo)
+
+DIY Algoritms for mastodon
+
+Just uploading to PyPI for now to squat on the package name. Will release
+ah um a version of this soon :)

diyalgo/__init__.py

@@ -1 +1 @@
- 
+from diyalgo.config import Config

diyalgo/client/init.py

@@ -0,0 +1,15 @@
+from typing import Optional
+from diyalgo import Config
+
+from mastodon import Mastodon
+
+def log_in(config:Optional[Config]=None) -> Mastodon:
+    if config is None:
+        config = Config()
+
+    client = Mastodon(
+        access_token=config.MASTO_TOKEN,
+        api_base_url=config.MASTO_URL
+    )
+
+    return client

diyalgo/config.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+from typing import Optional
+from pydantic import BaseSettings, AnyHttpUrl, EmailStr
+
+class Config(BaseSettings):
+    MASTO_URL:AnyHttpUrl
+    MASTO_TOKEN: Optional[str] = None
+    LOGDIR:Path = Path().home() / '.mastotools'
+
+    class Config:
+        env_file = '.env'
+        env_file_encoding = 'utf-8'

diyalgo/expansions/timeline.py

@@ -0,0 +1,19 @@
+from typing import List, Literal
+import pdb
+
+from mastodon import Mastodon
+
+from diyalgo.models import Status
+
+TIMELINES = Literal['home', 'local', 'public', 'tag', 'hashtag', 'list', 'id']
+
+def fetch_timeline(
+        client:Mastodon,
+        timeline:TIMELINES="public",
+        **kwargs
+    ) -> List[Status]:
+    tl = client.timeline(timeline=timeline, **kwargs)
+    tl = client.fetch_remaining(tl)
+    pdb.set_trace()
+    tl = [Status(**status) for status in tl]
+    return tl

diyalgo/models/account.py

@@ -1,28 +1,14 @@
 from pydantic import BaseModel, AnyHttpUrl
-from sqlmodel import Field, SQLModel
+from sqlmodel import Field, SQLModel, Relationship
 from datetime import datetime
-from typing import Optional, List
+from typing import Optional, List, TYPE_CHECKING
 from bs4 import BeautifulSoup
 
-from diyalgo.models import CustomEmoji
+from diyalgo.models.links import EmojiAccountLink, EmojiStatusLink
 
-class AccountField(BaseModel):
+if TYPE_CHECKING:
-    name: str
+    from diyalgo.models import Status
-    value: str
+    from diyalgo.models import CustomEmoji
-    verified_at: Optional[datetime] = None
-    url: Optional[AnyHttpUrl] = None
-
-    def __init__(self, name:str, value:str, verified_at:Optional[datetime] = None):
-        soup = BeautifulSoup(value, 'lxml')
-        a = soup.find('a')
-        if a is not None:
-            url = a.get('href')
-        else:
-            url = None
-        super().__init__(name=name, value=value, url=url, verified_at=verified_at)
-
-    class Config:
-        extra = "ignore"
 
 
 class Account(SQLModel, table=True):
@@ -32,26 +18,50 @@ class Account(SQLModel, table=True):
     avatar: str
     avatar_static: str
     bot: bool
-    created_at:datetime
+    # created_at:datetime
     discoverable:bool
     display_name:str
-    emojis: List[CustomEmoji] = Field(default_factory=list)
+    emojis: List['CustomEmoji'] = Relationship(back_populates='accounts', link_model=EmojiAccountLink)
-    fields: List[AccountField] = Field(default_factory=list)
+    # fields: List["AccountField"] = Relationship(back_populates='account')
     followers_count:int
     following_count:int
     group: bool
     header: str
-    last_status_at: Optional[datetime] = None
+    # last_status_at: Optional[datetime] = None
     limited: Optional[bool] = None
     locked: bool
-    moved: Optional['Account'] = None
+    # moved: Optional['Account'] = Relationship()
     noindex: Optional[bool] = None
     header_static: str
     note: str
+    statuses: List['Status'] = Relationship(back_populates='account')
     statuses_count: int
     suspended: Optional[bool] = None
     url: AnyHttpUrl
     username: str
 
-    class Config:
+    # class Config:
-        extra = 'ignore'
+    #     extra = 'ignore'
+
+class AccountField(SQLModel):
+    id: Optional[int] = Field(primary_key=True, default=None)
+    name: str
+    value: str
+    # verified_at: Optional[datetime] = None
+    # url: Optional[AnyHttpUrl] = None
+
+    account_id: Optional[int] = Field(default=None, foreign_key='account.id')
+    account: Account = Relationship(back_populates='fields')
+
+    # def __init__(self, value:str, **kwargs):
+    #     soup = BeautifulSoup(value, 'lxml')
+    #     a = soup.find('a')
+    #     if a is not None:
+    #         url = a.get('href')
+    #     else:
+    #         url = None
+    #     super().__init__(value=value, url=url, **kwargs)
+    #
+    # class Config:
+    #     extra = "ignore"
+

diyalgo/models/attachment.py

@@ -1,12 +1,20 @@
-from typing import Literal, Optional
+from typing import Literal, Optional, TYPE_CHECKING
-from sqlmodel import Field, SQLModel
+from sqlmodel import Field, SQLModel, Relationship
+
+if TYPE_CHECKING:
+    from diyalgo.models import Status
 
 class MediaAttachment(SQLModel, table=True):
     id: int = Field(primary_key=True)
     blurhash: str
     description: str
-    meta: dict
+    # meta: dict
     preview_url: str
     remote_url: str
-    type: Literal['unknown', 'image', 'gifv', 'video', 'audio']
+    type: str #Literal['unknown', 'image', 'gifv', 'video', 'audio']
     url: str
+    status_id: Optional[int] = Field(default=None, foreign_key='status.id')
+    status: 'Status' = Relationship(back_populates='media_attachments')
+
+    # class Config:
+    #     extra = 'ignore'

diyalgo/models/emojis.py

@@ -1,8 +1,19 @@
-from sqlmodel import Field, SQLModel
+from typing import Optional, List, TYPE_CHECKING
+from sqlmodel import Field, SQLModel, Relationship
+
+from diyalgo.models.links import EmojiAccountLink, EmojiStatusLink
+
+if TYPE_CHECKING:
+    from diyalgo.models import Account, Status
+
 
 class CustomEmoji(SQLModel, table=True):
-    shortcode: str = Field(primary_key=True)
+    id: Optional[int] = Field(primary_key=True, default=None)
+    shortcode: str
     url: str
     static_url: str
     visible_in_picker: bool
-    category: str
+    category: str
+
+    accounts: List['Account'] = Relationship(back_populates='emojis', link_model=EmojiAccountLink)
+    statuses: List['Status'] = Relationship(back_populates='emojis', link_model=EmojiStatusLink)

diyalgo/models/links.py

@@ -0,0 +1,28 @@
+from typing import Optional
+
+from sqlmodel import SQLModel, Field
+
+
+class EmojiAccountLink(SQLModel, table=True):
+    emoji_id: Optional[int] = Field(
+        default=None, foreign_key='customemoji.id', primary_key=True
+    )
+    account_id: Optional[int] = Field(
+        default=None, foreign_key='account.id', primary_key=True
+    )
+
+class EmojiStatusLink(SQLModel, table=True):
+    emoji_id: Optional[int] = Field(
+        default=None, foreign_key='customemoji.id', primary_key=True
+    )
+    status_id: Optional[int] = Field(
+        default=None, foreign_key='status.id', primary_key=True
+    )
+
+class TagStatusLink(SQLModel, table=True):
+    tag_id: Optional[int] = Field(
+        default=None, foreign_key='tag.id', primary_key=True
+    )
+    status_id: Optional[int] = Field(
+        default=None, foreign_key='status.id', primary_key=True
+    )

diyalgo/models/poll.py

@@ -1,22 +1,28 @@
 from datetime import datetime
 from typing import Optional, List, TYPE_CHECKING
-from sqlmodel import Field, SQLModel
+from sqlmodel import Field, SQLModel, Relationship
 
 if TYPE_CHECKING:
-    from diyalgo.models import CustomEmoji
+    from diyalgo.models import CustomEmoji, Status
 
-class PollOption(SQLModel):
+class PollOption(SQLModel, table=True):
+    id: Optional[int] = Field(primary_key=True, default=None)
+    poll_id: Optional[int] = Field(default=None, foreign_key='poll.id')
+    poll: 'Poll' = Relationship(back_populates='options')
     title: str
     votes_count: Optional[int] = None
 
 class Poll(SQLModel, table=True):
     id: int = Field(primary_key=True)
-    emojis: List['CustomEmoji'] = Field(default_factory=list)
+    #emojis: List["CustomEmoji"] = Field(default_factory=list)
     expires_at: Optional[datetime] = None
     expired: bool
     multiple: bool
-    options: List[PollOption] = Field(default_factory=list)
+    options: List[PollOption] = Relationship(back_populates='poll')
     own_votes: List[int] = Field(default_factory=list)
     voted: Optional[bool] = None
     votes_count: int
     voters_count: Optional[int] = None
+
+    #status_id: Optional[int] = Field(default=None, foreign_key='status.id')
+    #status: 'Status' = Relationship(back_populates='poll')

diyalgo/models/status.py

@@ -1,17 +1,22 @@
 from typing import Optional, Literal, List, TYPE_CHECKING
 from datetime import datetime
-from sqlmodel import Field, SQLModel
+from sqlmodel import Field, SQLModel, Relationship
 from bs4 import BeautifulSoup
 
+from diyalgo.models.links import TagStatusLink, EmojiStatusLink
+
 if TYPE_CHECKING:
     from diyalgo.models import Account, MediaAttachment, Tag, CustomEmoji, Poll
 
 
 class Mention(SQLModel, table=True):
+    mention_id: Optional[int] = Field(primary_key=True, default=None)
     acct: str
     id: int
     url: str
     username: str
+    status_id: Optional[int] = Field(default=None, foreign_key='status.id')
+    status: 'Status' = Relationship(back_populates='mentions')
 
 class Status(SQLModel, table=True):
     """
@@ -20,38 +25,36 @@ class Status(SQLModel, table=True):
     See: https://mastodonpy.readthedocs.io/en/stable/#toot-dicts
     """
     id: int = Field(primary_key=True)
-    application: Optional[dict] = None
+    # application: Optional[dict] = None
-
+    account_id: Optional[int] = Field(default=None, foreign_key='account.id')
-    account: 'Account'
+    account: 'Account' = Relationship(back_populates='statuses')
     bookmarked: Optional[bool] = None
     content: str
     created_at: datetime
     edited_at: Optional[datetime] = None
-    emojis: List['CustomEmoji'] = Field(default_factory=list)
+    emojis: List['CustomEmoji'] = Relationship(back_populates='statuses', link_model=EmojiStatusLink)
     favourited: Optional[bool] = None
     favourites_count: int
-    filtered: Optional[List[str]] = Field(default_factory=list)
+    filtered: List[str] = Field(default_factory=list)
-    in_reply_to_id: int
+    in_reply_to_id: Optional[int] = None
-    in_reply_to_account_id: int
+    in_reply_to_account_id: Optional[int] = None
     language: Optional[str] = None
-    media_attachments: List['MediaAttachment'] = Field(default_factory=list)
+    media_attachments: List['MediaAttachment'] = Relationship(back_populates='status')
-    mentions: List[Mention] = Field(default_factory=list)
+    mentions: List[Mention] = Relationship(back_populates='status')
     muted: Optional[bool] = None
     pinned: Optional[bool] = None
-    poll: Optional['Poll'] = None
+    # poll: Optional['Poll'] = Relationship(back_populates='status')
-    reblog: bool
+    reblog: Optional[bool] = None
     reblogged: Optional[bool] = None
     reblogs_count: int
     replies_count: int
     sensitive: bool
     spoiler_text: str
-    tags: List['Tag'] = Field(default_factory=list)
+    tags: List['Tag'] = Relationship(back_populates='statuses', link_model=TagStatusLink)
     text: Optional[str] = None
     uri: str
     url: str
-    visibility: Literal['public', 'unlisted', 'private', 'direct']
+    visibility: str #Literal['public', 'unlisted', 'private', 'direct']
-    in_reply_to_id: Optional[int] = None
-    in_reply_to_account_id: Optional[int] = None
 
     @property
     def soup(self) -> BeautifulSoup:

diyalgo/models/tag.py

@@ -1,5 +1,14 @@
-from sqlmodel import Field, SQLModel
+from typing import Optional, List, TYPE_CHECKING
+from sqlmodel import Field, SQLModel, Relationship
+
+from diyalgo.models.links import TagStatusLink
+
+if TYPE_CHECKING:
+    from diyalgo.models import Status
 
 class Tag(SQLModel, table=True):
+    id: Optional[int] = Field(primary_key=True, default=None)
     name: str
-    url: str
+    url: str
+
+    statuses: List['Status'] = Relationship(back_populates='tags', link_model=TagStatusLink)

poetry.lock

@@ -345,6 +345,21 @@ files = [
 [package.dependencies]
 six = ">=1.5"
 
+[[package]]
+name = "python-dotenv"
+version = "0.21.0"
+description = "Read key-value pairs from a .env file and set them as environment variables"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "python-dotenv-0.21.0.tar.gz", hash = "sha256:b77d08274639e3d34145dfa6c7008e66df0f04b7be7a75fd0d5292c191d79045"},
+    {file = "python_dotenv-0.21.0-py3-none-any.whl", hash = "sha256:1684eb44636dd462b66c3ee016599815514527ad99965de77f43e0944634a7e5"},
+]
+
+[package.extras]
+cli = ["click (>=5.0)"]
+
 [[package]]
 name = "python-magic"
 version = "0.4.27"
@@ -542,4 +557,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "87440f24f73a01d5da686727e61561317d4c02b7e030fa15ce23769d4f9db5ee"
+content-hash = "bcb1da145e3fb10a5e98a37ae6f4e9933e32b3152fe17799e964187294f7a890"

pyproject.toml

@@ -5,6 +5,9 @@ description = "DIY Algorithms for Mastodon"
 authors = ["sneakers-the-rat <JLSaunders987@gmail.com>"]
 license = "AGPL-3.0"
 readme = "README.md"
+repository = "https://git.jon-e.net/jonny/diyalgo"
+keywords = ["mastodon", "fediverse", "algorithm", "algorithms", "social media"]
+
 
 [tool.poetry.dependencies]
 python = "^3.9"
@@ -13,6 +16,7 @@ pydantic = "^1.10.4"
 sqlmodel = "^0.0.8"
 beautifulsoup4 = "^4.11.1"
 lxml = "^4.9.2"
+python-dotenv = "^0.21.0"
 
 
 [build-system]