From 6f25d06bbb8980b24fe363b2b39165c459298510 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 17 Jul 2022 18:43:08 +0200 Subject: [PATCH] Bootstrap Micropub support, and start support for Update activities --- .../versions/e58c1ffadf2e_update_support.py | 28 +++++ app/activitypub.py | 34 ++++-- app/ap_object.py | 4 + app/boxes.py | 71 ++++++++++++ app/database.py | 2 + app/incoming_activities.py | 12 +- app/indieauth.py | 7 ++ app/main.py | 2 + app/micropub.py | 109 ++++++++++++++++++ app/models.py | 25 ++++ app/templates/index.html | 1 + 11 files changed, 279 insertions(+), 16 deletions(-) create mode 100644 alembic/versions/e58c1ffadf2e_update_support.py create mode 100644 app/micropub.py diff --git a/alembic/versions/e58c1ffadf2e_update_support.py b/alembic/versions/e58c1ffadf2e_update_support.py new file mode 100644 index 0000000..4a3fdb2 --- /dev/null +++ b/alembic/versions/e58c1ffadf2e_update_support.py @@ -0,0 +1,28 @@ +"""Update support + +Revision ID: e58c1ffadf2e +Revises: fd23d95e5c16 +Create Date: 2022-07-17 18:19:42.362542 + +""" +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'e58c1ffadf2e' +down_revision = 'fd23d95e5c16' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('outbox', sa.Column('revisions', sa.JSON(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('outbox', 'revisions') + # ### end Alembic commands ### diff --git a/app/activitypub.py b/app/activitypub.py index 603e28a..c484c48 100644 --- a/app/activitypub.py +++ b/app/activitypub.py @@ -251,16 +251,30 @@ async def get_object(activity: RawObject) -> RawObject: def wrap_object(activity: RawObject) -> RawObject: - return { - "@context": AS_EXTENDED_CTX, - "actor": config.ID, - "to": activity.get("to", []), - "cc": activity.get("cc", []), - "id": activity["id"] + "/activity", - "object": remove_context(activity), - "published": activity["published"], - "type": "Create", - } + # TODO(ts): improve Create VS Update + if "updated" in activity: + return { + "@context": AS_EXTENDED_CTX, + "actor": config.ID, + "to": activity.get("to", []), + "cc": activity.get("cc", []), + "id": activity["id"] + "/update_activity/" + activity["updated"], + "object": remove_context(activity), + "published": activity["published"], + "updated": activity["updated"], + "type": "Update", + } + else: + return { + "@context": AS_EXTENDED_CTX, + "actor": config.ID, + "to": activity.get("to", []), + "cc": activity.get("cc", []), + "id": activity["id"] + "/activity", + "object": remove_context(activity), + "published": activity["published"], + "type": "Create", + } def wrap_object_if_needed(raw_object: RawObject) -> RawObject: diff --git a/app/ap_object.py b/app/ap_object.py index 5e226e5..c998de2 100644 --- a/app/ap_object.py +++ b/app/ap_object.py @@ -147,6 +147,10 @@ class Object: def summary(self) -> str | None: return self.ap_object.get("summary") + @property + def name(self) -> str | None: + return self.ap_object.get("name") + @cached_property def permalink_id(self) -> str: return ( diff --git a/app/boxes.py b/app/boxes.py index 83911cb..dbbe9af 100644 --- a/app/boxes.py +++ b/app/boxes.py @@ -392,6 +392,77 @@ async def send_create( return note_id +async def send_update( + db_session: AsyncSession, + ap_id: str, + source: str, +) -> str: + outbox_object = await get_outbox_object_by_ap_id(db_session, ap_id) + if not outbox_object: + raise ValueError(f"{ap_id} not found") + + revisions = outbox_object.revisions or [] + revisions.append( + { + "ap_object": outbox_object.ap_object, + "source": outbox_object.source, + "updated": ( + outbox_object.ap_object.get("updated") + or outbox_object.ap_object.get("published") + ), + } + ) + + updated = now().replace(microsecond=0).isoformat().replace("+00:00", "Z") + content, tags, mentioned_actors = await markdownify(db_session, source) + + note = { + "@context": ap.AS_EXTENDED_CTX, + "type": outbox_object.ap_type, + "id": outbox_object.ap_id, + "attributedTo": ID, + "content": content, + "to": outbox_object.ap_object["to"], + "cc": outbox_object.ap_object["cc"], + "published": outbox_object.ap_object["published"], + "context": outbox_object.ap_context, + "conversation": outbox_object.ap_context, + "url": outbox_object.url, + "tag": tags, + "summary": outbox_object.summary, + "inReplyTo": outbox_object.in_reply_to, + "sensitive": outbox_object.sensitive, + "attachment": outbox_object.ap_object["attachment"], + "updated": updated, + } + + outbox_object.ap_object = note + outbox_object.source = source + outbox_object.revisions = revisions + await db_session.commit() + + recipients = await _compute_recipients(db_session, note) + for rcp in recipients: + await new_outgoing_activity(db_session, rcp, outbox_object.id) + + # If the note is public, check if we need to send any webmentions + if outbox_object.visibility == ap.VisibilityEnum.PUBLIC: + possible_targets = opengraph._urls_from_note(note) + logger.info(f"webmentions possible targert {possible_targets}") + for target in possible_targets: + webmention_endpoint = await webmentions.discover_webmention_endpoint(target) + logger.info(f"{target=} {webmention_endpoint=}") + if webmention_endpoint: + await new_outgoing_activity( + db_session, + webmention_endpoint, + outbox_object_id=outbox_object.id, + webmention_target=target, + ) + + return outbox_object.public_id # type: ignore + + async def _compute_recipients( db_session: AsyncSession, ap_object: ap.RawObject ) -> set[str]: diff --git a/app/database.py b/app/database.py index ebfdd6f..661b275 100644 --- a/app/database.py +++ b/app/database.py @@ -1,6 +1,7 @@ from typing import Any from typing import AsyncGenerator +from sqlalchemy import MetaData from sqlalchemy import create_engine from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import create_async_engine @@ -20,6 +21,7 @@ async_engine = create_async_engine(DATABASE_URL, future=True, echo=False) async_session = sessionmaker(async_engine, class_=AsyncSession, expire_on_commit=False) Base: Any = declarative_base() +metadata_obj = MetaData() async def get_db_session() -> AsyncGenerator[AsyncSession, None]: diff --git a/app/incoming_activities.py b/app/incoming_activities.py index 6a365d3..21dbabd 100644 --- a/app/incoming_activities.py +++ b/app/incoming_activities.py @@ -100,12 +100,12 @@ async def process_next_incoming_activity(db_session: AsyncSession) -> bool: next_activity.last_try = now() try: - async with db_session.begin_nested(): - await save_to_inbox( - db_session, - next_activity.ap_object, - next_activity.sent_by_ap_actor_id, - ) + # async with db_session.begin_nested(): + await save_to_inbox( + db_session, + next_activity.ap_object, + next_activity.sent_by_ap_actor_id, + ) except Exception: logger.exception("Failed") next_activity.error = traceback.format_exc() diff --git a/app/indieauth.py b/app/indieauth.py index a0cf6d3..1b92a0d 100644 --- a/app/indieauth.py +++ b/app/indieauth.py @@ -292,6 +292,13 @@ async def verify_access_token( db_session: AsyncSession = Depends(get_db_session), ) -> AccessTokenInfo: token = request.headers.get("Authorization", "").removeprefix("Bearer ") + + # Check if the token is within the form data + if not token: + form_data = await request.form() + if "access_token" in form_data: + token = form_data.get("access_token") + is_token_valid, access_token = await _check_access_token(db_session, token) if not is_token_valid: raise HTTPException( diff --git a/app/main.py b/app/main.py index 6362f70..e07d6f2 100644 --- a/app/main.py +++ b/app/main.py @@ -44,6 +44,7 @@ from app import boxes from app import config from app import httpsig from app import indieauth +from app import micropub from app import models from app import templates from app import webmentions @@ -177,6 +178,7 @@ app.mount("/static", StaticFiles(directory="app/static"), name="static") app.include_router(admin.router, prefix="/admin") app.include_router(admin.unauthenticated_router, prefix="/admin") app.include_router(indieauth.router) +app.include_router(micropub.router) app.include_router(webmentions.router) app.add_middleware(ProxyHeadersMiddleware) app.add_middleware(CustomMiddleware) diff --git a/app/micropub.py b/app/micropub.py new file mode 100644 index 0000000..6d96b0a --- /dev/null +++ b/app/micropub.py @@ -0,0 +1,109 @@ +from typing import Any + +from fastapi import APIRouter +from fastapi import Depends +from fastapi import Request +from fastapi.responses import JSONResponse +from fastapi.responses import RedirectResponse + +from app import activitypub as ap +from app.boxes import get_outbox_object_by_ap_id +from app.boxes import send_create +from app.boxes import send_delete +from app.database import AsyncSession +from app.database import get_db_session +from app.indieauth import AccessTokenInfo +from app.indieauth import verify_access_token + +router = APIRouter() + + +@router.get("/micropub") +async def micropub_endpoint( + request: Request, + access_token_info: AccessTokenInfo = Depends(verify_access_token), + db_session: AsyncSession = Depends(get_db_session), +) -> dict[str, Any] | JSONResponse: + if request.query_params.get("q") == "config": + return {} + + elif request.query_params.get("q") == "source": + url = request.query_params.get("url") + outbox_object = await get_outbox_object_by_ap_id(db_session, url) + if not outbox_object: + return JSONResponse( + content={ + "error": "invalid_request", + "error_description": "No post with this URL", + }, + status_code=400, + ) + + extra_props: dict[str, list[str]] = {} + + return { + "type": ["h-entry"], + "properties": { + "published": [ + outbox_object.ap_published_at.isoformat() # type: ignore + ], + "content": [outbox_object.source], + **extra_props, + }, + } + + return {} + + +@router.post("/micropub") +async def post_micropub_endpoint( + request: Request, + access_token_info: AccessTokenInfo = Depends(verify_access_token), + db_session: AsyncSession = Depends(get_db_session), +) -> RedirectResponse | JSONResponse: + form_data = await request.form() + if "action" in form_data: + if form_data["action"] == "delete": + outbox_object = await get_outbox_object_by_ap_id( + db_session, form_data["url"] + ) + if not outbox_object: + return JSONResponse( + content={ + "error": "invalid_request", + "error_description": "No post with this URL", + }, + status_code=400, + ) + await send_delete(db_session, outbox_object.ap_id) # type: ignore + return JSONResponse(content={}, status_code=200) + + h = "entry" + if "h" in form_data: + h = form_data["h"] + + if h != "entry": + return JSONResponse( + content={ + "error": "invalid_request", + "error_description": "Only h-entry are supported", + }, + status_code=400, + ) + + content = form_data["content"] + public_id = await send_create( + db_session, + content, + uploads=[], + in_reply_to=None, + visibility=ap.VisibilityEnum.PUBLIC, + ) + + return JSONResponse( + content={}, + status_code=201, + headers={ + "Location": request.url_for("outbox_by_public_id", public_id=public_id) + }, + ) diff --git a/app/models.py b/app/models.py index 2ff1e14..1958561 100644 --- a/app/models.py +++ b/app/models.py @@ -3,6 +3,7 @@ from typing import Any from typing import Optional from typing import Union +import pydantic from loguru import logger from sqlalchemy import JSON from sqlalchemy import Boolean @@ -12,6 +13,7 @@ from sqlalchemy import Enum from sqlalchemy import ForeignKey from sqlalchemy import Integer from sqlalchemy import String +from sqlalchemy import Table from sqlalchemy import UniqueConstraint from sqlalchemy.orm import Mapped from sqlalchemy.orm import relationship @@ -23,10 +25,17 @@ from app.ap_object import Attachment from app.ap_object import Object as BaseObject from app.config import BASE_URL from app.database import Base +from app.database import metadata_obj from app.utils import webmentions from app.utils.datetime import now +class ObjectRevision(pydantic.BaseModel): + ap_object: ap.RawObject + source: str + updated_at: str + + class Actor(Base, BaseActor): __tablename__ = "actor" @@ -147,6 +156,7 @@ class OutboxObject(Base, BaseObject): # Source content for activities (like Notes) source = Column(String, nullable=True) + revisions: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True) ap_published_at = Column(DateTime(timezone=True), nullable=False, default=now) visibility = Column(Enum(ap.VisibilityEnum), nullable=False) @@ -491,3 +501,18 @@ class Webmention(Base): f"Failed to generate facefile item for Webmention id={self.id}" ) return None + + +outbox_fts = Table( + "outbox_fts", + metadata_obj, + Column("rowid", Integer), + Column("outbox_fts", String), + Column("summary", String, nullable=True), + Column("name", String, nullable=True), + Column("source", String), +) + +# db.execute(select(outbox_fts.c.rowid).where(outbox_fts.c.outbox_fts.op("MATCH")("toto AND omg"))).all() # noqa +# db.execute(select(models.OutboxObject).join(outbox_fts, outbox_fts.c.rowid == models.OutboxObject.id).where(outbox_fts.c.outbox_fts.op("MATCH")("toto2"))).scalars() # noqa +# db.execute(insert(outbox_fts).values({"outbox_fts": "delete", "rowid": 1, "source": dat[0].source})) # noqa diff --git a/app/templates/index.html b/app/templates/index.html index 23e2bc0..35b8648 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -6,6 +6,7 @@ +