jonny/diyalgo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

ok that's enough debugging for tonight.

Browse source 
getting the SQL models to work by using some simple local data and it's choking on the relationships.

by jove we've done it again and made it too complicated
This commit is contained in:
sneakers-the-rat 2023-01-02 23:54:57 -08:00
parent d30275460d
commit b156c57ff1
15 changed files with 208 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.env.sample Normal file
View file

@ -0,0 +1,3 @@
MASTO_URL=
MASTO_TOKEN=
LOGDIR=

7
README.md
View file

@ -1,3 +1,8 @@
# diyalgo
DIY Algoritms for mastodon
![PyPI](https://img.shields.io/pypi/v/diyalgo)
DIY Algoritms for mastodon
Just uploading to PyPI for now to squat on the package name. Will release
ah um a version of this soon :)

2
diyalgo/__init__.py
View file

@ -1 +1 @@
from diyalgo.config import Config

15
diyalgo/client/init.py Normal file
View file

@ -0,0 +1,15 @@
from typing import Optional
from diyalgo import Config
from mastodon import Mastodon
def log_in(config:Optional[Config]=None) -> Mastodon:
if config is None:
config = Config()
client = Mastodon(
access_token=config.MASTO_TOKEN,
api_base_url=config.MASTO_URL
)
return client

12
diyalgo/config.py Normal file
View file

@ -0,0 +1,12 @@
from pathlib import Path
from typing import Optional
from pydantic import BaseSettings, AnyHttpUrl, EmailStr
class Config(BaseSettings):
MASTO_URL:AnyHttpUrl
MASTO_TOKEN: Optional[str] = None
LOGDIR:Path = Path().home() / '.mastotools'
class Config:
env_file = '.env'
env_file_encoding = 'utf-8'

19
diyalgo/expansions/timeline.py Normal file
View file

@ -0,0 +1,19 @@
from typing import List, Literal
import pdb
from mastodon import Mastodon
from diyalgo.models import Status
TIMELINES = Literal['home', 'local', 'public', 'tag', 'hashtag', 'list', 'id']
def fetch_timeline(
client:Mastodon,
timeline:TIMELINES="public",
**kwargs
) -> List[Status]:
tl = client.timeline(timeline=timeline, **kwargs)
tl = client.fetch_remaining(tl)
pdb.set_trace()
tl = [Status(**status) for status in tl]
return tl

64
diyalgo/models/account.py
View file

@ -1,28 +1,14 @@
from pydantic import BaseModel, AnyHttpUrl
from sqlmodel import Field, SQLModel
from sqlmodel import Field, SQLModel, Relationship
from datetime import datetime
from typing import Optional, List
from typing import Optional, List, TYPE_CHECKING
from bs4 import BeautifulSoup
from diyalgo.models import CustomEmoji
from diyalgo.models.links import EmojiAccountLink, EmojiStatusLink
class AccountField(BaseModel):
name: str
value: str
verified_at: Optional[datetime] = None
url: Optional[AnyHttpUrl] = None
def __init__(self, name:str, value:str, verified_at:Optional[datetime] = None):
soup = BeautifulSoup(value, 'lxml')
a = soup.find('a')
if a is not None:
url = a.get('href')
else:
url = None
super().__init__(name=name, value=value, url=url, verified_at=verified_at)
class Config:
extra = "ignore"
if TYPE_CHECKING:
from diyalgo.models import Status
from diyalgo.models import CustomEmoji
class Account(SQLModel, table=True):
@ -32,26 +18,50 @@ class Account(SQLModel, table=True):
avatar: str
avatar_static: str
bot: bool
created_at:datetime
# created_at:datetime
discoverable:bool
display_name:str
emojis: List[CustomEmoji] = Field(default_factory=list)
fields: List[AccountField] = Field(default_factory=list)
emojis: List['CustomEmoji'] = Relationship(back_populates='accounts', link_model=EmojiAccountLink)
# fields: List["AccountField"] = Relationship(back_populates='account')
followers_count:int
following_count:int
group: bool
header: str
last_status_at: Optional[datetime] = None
# last_status_at: Optional[datetime] = None
limited: Optional[bool] = None
locked: bool
moved: Optional['Account'] = None
# moved: Optional['Account'] = Relationship()
noindex: Optional[bool] = None
header_static: str
note: str
statuses: List['Status'] = Relationship(back_populates='account')
statuses_count: int
suspended: Optional[bool] = None
url: AnyHttpUrl
username: str
class Config:
extra = 'ignore'
# class Config:
# extra = 'ignore'
class AccountField(SQLModel):
id: Optional[int] = Field(primary_key=True, default=None)
name: str
value: str
# verified_at: Optional[datetime] = None
# url: Optional[AnyHttpUrl] = None
account_id: Optional[int] = Field(default=None, foreign_key='account.id')
account: Account = Relationship(back_populates='fields')
# def __init__(self, value:str, **kwargs):
# soup = BeautifulSoup(value, 'lxml')
# a = soup.find('a')
# if a is not None:
# url = a.get('href')
# else:
# url = None
# super().__init__(value=value, url=url, **kwargs)
#
# class Config:
# extra = "ignore"

16
diyalgo/models/attachment.py
View file

@ -1,12 +1,20 @@
from typing import Literal, Optional
from sqlmodel import Field, SQLModel
from typing import Literal, Optional, TYPE_CHECKING
from sqlmodel import Field, SQLModel, Relationship
if TYPE_CHECKING:
from diyalgo.models import Status
class MediaAttachment(SQLModel, table=True):
id: int = Field(primary_key=True)
blurhash: str
description: str
meta: dict
# meta: dict
preview_url: str
remote_url: str
type: Literal['unknown', 'image', 'gifv', 'video', 'audio']
type: str #Literal['unknown', 'image', 'gifv', 'video', 'audio']
url: str
status_id: Optional[int] = Field(default=None, foreign_key='status.id')
status: 'Status' = Relationship(back_populates='media_attachments')
# class Config:
# extra = 'ignore'

17
diyalgo/models/emojis.py
View file

@ -1,8 +1,19 @@
from sqlmodel import Field, SQLModel
from typing import Optional, List, TYPE_CHECKING
from sqlmodel import Field, SQLModel, Relationship
from diyalgo.models.links import EmojiAccountLink, EmojiStatusLink
if TYPE_CHECKING:
from diyalgo.models import Account, Status
class CustomEmoji(SQLModel, table=True):
shortcode: str = Field(primary_key=True)
id: Optional[int] = Field(primary_key=True, default=None)
shortcode: str
url: str
static_url: str
visible_in_picker: bool
category: str
category: str
accounts: List['Account'] = Relationship(back_populates='emojis', link_model=EmojiAccountLink)
statuses: List['Status'] = Relationship(back_populates='emojis', link_model=EmojiStatusLink)

28
diyalgo/models/links.py Normal file
View file

@ -0,0 +1,28 @@
from typing import Optional
from sqlmodel import SQLModel, Field
class EmojiAccountLink(SQLModel, table=True):
emoji_id: Optional[int] = Field(
default=None, foreign_key='customemoji.id', primary_key=True
)
account_id: Optional[int] = Field(
default=None, foreign_key='account.id', primary_key=True
)
class EmojiStatusLink(SQLModel, table=True):
emoji_id: Optional[int] = Field(
default=None, foreign_key='customemoji.id', primary_key=True
)
status_id: Optional[int] = Field(
default=None, foreign_key='status.id', primary_key=True
)
class TagStatusLink(SQLModel, table=True):
tag_id: Optional[int] = Field(
default=None, foreign_key='tag.id', primary_key=True
)
status_id: Optional[int] = Field(
default=None, foreign_key='status.id', primary_key=True
)

16
diyalgo/models/poll.py
View file

@ -1,22 +1,28 @@
from datetime import datetime
from typing import Optional, List, TYPE_CHECKING
from sqlmodel import Field, SQLModel
from sqlmodel import Field, SQLModel, Relationship
if TYPE_CHECKING:
from diyalgo.models import CustomEmoji
from diyalgo.models import CustomEmoji, Status
class PollOption(SQLModel):
class PollOption(SQLModel, table=True):
id: Optional[int] = Field(primary_key=True, default=None)
poll_id: Optional[int] = Field(default=None, foreign_key='poll.id')
poll: 'Poll' = Relationship(back_populates='options')
title: str
votes_count: Optional[int] = None
class Poll(SQLModel, table=True):
id: int = Field(primary_key=True)
emojis: List['CustomEmoji'] = Field(default_factory=list)
#emojis: List["CustomEmoji"] = Field(default_factory=list)
expires_at: Optional[datetime] = None
expired: bool
multiple: bool
options: List[PollOption] = Field(default_factory=list)
options: List[PollOption] = Relationship(back_populates='poll')
own_votes: List[int] = Field(default_factory=list)
voted: Optional[bool] = None
votes_count: int
voters_count: Optional[int] = None
#status_id: Optional[int] = Field(default=None, foreign_key='status.id')
#status: 'Status' = Relationship(back_populates='poll')

35
diyalgo/models/status.py
View file

@ -1,17 +1,22 @@
from typing import Optional, Literal, List, TYPE_CHECKING
from datetime import datetime
from sqlmodel import Field, SQLModel
from sqlmodel import Field, SQLModel, Relationship
from bs4 import BeautifulSoup
from diyalgo.models.links import TagStatusLink, EmojiStatusLink
if TYPE_CHECKING:
from diyalgo.models import Account, MediaAttachment, Tag, CustomEmoji, Poll
class Mention(SQLModel, table=True):
mention_id: Optional[int] = Field(primary_key=True, default=None)
acct: str
id: int
url: str
username: str
status_id: Optional[int] = Field(default=None, foreign_key='status.id')
status: 'Status' = Relationship(back_populates='mentions')
class Status(SQLModel, table=True):
"""
@ -20,38 +25,36 @@ class Status(SQLModel, table=True):
See: https://mastodonpy.readthedocs.io/en/stable/#toot-dicts
"""
id: int = Field(primary_key=True)
application: Optional[dict] = None
account: 'Account'
# application: Optional[dict] = None
account_id: Optional[int] = Field(default=None, foreign_key='account.id')
account: 'Account' = Relationship(back_populates='statuses')
bookmarked: Optional[bool] = None
content: str
created_at: datetime
edited_at: Optional[datetime] = None
emojis: List['CustomEmoji'] = Field(default_factory=list)
emojis: List['CustomEmoji'] = Relationship(back_populates='statuses', link_model=EmojiStatusLink)
favourited: Optional[bool] = None
favourites_count: int
filtered: Optional[List[str]] = Field(default_factory=list)
in_reply_to_id: int
in_reply_to_account_id: int
filtered: List[str] = Field(default_factory=list)
in_reply_to_id: Optional[int] = None
in_reply_to_account_id: Optional[int] = None
language: Optional[str] = None
media_attachments: List['MediaAttachment'] = Field(default_factory=list)
mentions: List[Mention] = Field(default_factory=list)
media_attachments: List['MediaAttachment'] = Relationship(back_populates='status')
mentions: List[Mention] = Relationship(back_populates='status')
muted: Optional[bool] = None
pinned: Optional[bool] = None
poll: Optional['Poll'] = None
reblog: bool
# poll: Optional['Poll'] = Relationship(back_populates='status')
reblog: Optional[bool] = None
reblogged: Optional[bool] = None
reblogs_count: int
replies_count: int
sensitive: bool
spoiler_text: str
tags: List['Tag'] = Field(default_factory=list)
tags: List['Tag'] = Relationship(back_populates='statuses', link_model=TagStatusLink)
text: Optional[str] = None
uri: str
url: str
visibility: Literal['public', 'unlisted', 'private', 'direct']
in_reply_to_id: Optional[int] = None
in_reply_to_account_id: Optional[int] = None
visibility: str #Literal['public', 'unlisted', 'private', 'direct']
@property
def soup(self) -> BeautifulSoup:

13
diyalgo/models/tag.py
View file

@ -1,5 +1,14 @@
from sqlmodel import Field, SQLModel
from typing import Optional, List, TYPE_CHECKING
from sqlmodel import Field, SQLModel, Relationship
from diyalgo.models.links import TagStatusLink
if TYPE_CHECKING:
from diyalgo.models import Status
class Tag(SQLModel, table=True):
id: Optional[int] = Field(primary_key=True, default=None)
name: str
url: str
url: str
statuses: List['Status'] = Relationship(back_populates='tags', link_model=TagStatusLink)

17
poetry.lock generated
View file

@ -345,6 +345,21 @@ files = [
[package.dependencies]
six = ">=1.5"
[[package]]
name = "python-dotenv"
version = "0.21.0"
description = "Read key-value pairs from a .env file and set them as environment variables"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
{file = "python-dotenv-0.21.0.tar.gz", hash = "sha256:b77d08274639e3d34145dfa6c7008e66df0f04b7be7a75fd0d5292c191d79045"},
{file = "python_dotenv-0.21.0-py3-none-any.whl", hash = "sha256:1684eb44636dd462b66c3ee016599815514527ad99965de77f43e0944634a7e5"},
]
[package.extras]
cli = ["click (>=5.0)"]
[[package]]
name = "python-magic"
version = "0.4.27"
@ -542,4 +557,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[metadata]
lock-version = "2.0"
python-versions = "^3.9"
content-hash = "87440f24f73a01d5da686727e61561317d4c02b7e030fa15ce23769d4f9db5ee"
content-hash = "bcb1da145e3fb10a5e98a37ae6f4e9933e32b3152fe17799e964187294f7a890"

4
pyproject.toml
View file

@ -5,6 +5,9 @@ description = "DIY Algorithms for Mastodon"
authors = ["sneakers-the-rat <JLSaunders987@gmail.com>"]
license = "AGPL-3.0"
readme = "README.md"
repository = "https://git.jon-e.net/jonny/diyalgo"
keywords = ["mastodon", "fediverse", "algorithm", "algorithms", "social media"]
[tool.poetry.dependencies]
python = "^3.9"
@ -13,6 +16,7 @@ pydantic = "^1.10.4"
sqlmodel = "^0.0.8"
beautifulsoup4 = "^4.11.1"
lxml = "^4.9.2"
python-dotenv = "^0.21.0"
[build-system]