v0.1.3 - add overwrite option

README.md

@@ -4,3 +4,45 @@ Bot to add tweets using an extended wikilink syntax to the wiki
 
 Starting with twitter, but then will add masto 
 
+# Mediawiki
+
+https://www.mediawiki.org/wiki/Manual:Creating_a_bot
+
+- Go to `Special:BotPasswords`
+- Permissions
+  - Edit existing pages
+  - Create, edit, and move pages
+  - 
+- save to `mediawiki_creds.json` like
+
+```json
+{
+	"user": "Jonny@wikibot",
+	"password": "<THE BOT PASSWORD>"
+}
+```
+
+# Slack
+
+Using the `python-slack-sdk` https://slack.dev/python-slack-sdk/
+
+## Make App
+
+https://slack.dev/python-slack-sdk/socket-mode/index.html
+
+- Make slack app - https://api.slack.com/apis/connections/socket#setup
+- Assign to workplace
+- Permissions
+  - `channels:history`
+  - `channels:read`
+  - `app_mentions:read`
+  - `chat:write`
+  - `links:read`
+  - `reactions:read`
+  - `reactions:write`
+  - `users:read`
+- Create App-level token with `connections:write`
+- Configure app
+  - Enable socket mode - https://api.slack.com/apis/connections/socket#toggling
+  - Enable Events - `message.channels`
+  - 

docs/changelog.md

@@ -0,0 +1,3 @@
+# 0.1.3
+
+- Add overwrite option to append_text in wiki interface

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wiki-postbot"
-version = "0.1.2"
+version = "0.1.3"
 description = "Add posts to the wiki!"
 authors = ["sneakers-the-rat <JLSaunders987@gmail.com>"]
 license = "GPL-3.0"
@@ -10,8 +10,10 @@ packages = [
 
 [tool.poetry.scripts]
 wikipostbot = "wiki_postbot.main:main"
-discord_bot = "wiki_postbot.clients.discord_client:main"
+discord_bot = "wiki_postbot.clients.discord:main"
 install_discord_bot = "wiki_postbot.service:main"
+slack_bot = "wiki_postbot.clients.slack:main"
+install_slack_bot = "wiki_postbot.service:main_slack"
 
 [tool.poetry.dependencies]
 python = "^3.9"
@@ -20,10 +22,21 @@ rich = "^12.4.4"
 parse = "^1.19.0"
 pywikibot = "^7.7.0"
 pyparsing = "^3.0.9"
-"discord.py" = "^2.0.1"
 wikitextparser = "^0.51.1"
 certifi = "^2022.9.24"
 
+[tool.poetry.group.discord]
+optional = true
+
+[tool.poetry.group.discord.dependencies]
+"discord.py" = "^2.0.1"
+
+[tool.poetry.group.slack]
+optional = true
+
+[tool.poetry.group.slack.dependencies]
+slack-sdk = "^3.21.3"
+
 [tool.poetry.dev-dependencies]
 pytest = "^7.1.2"
 Faker = "^15.1.0"

wiki_postbot/clients/client.py

@@ -0,0 +1,91 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from wiki_postbot.interfaces.mediawiki import Wiki
+from wiki_postbot.logger import init_logger
+from wiki_postbot.patterns.wikilink import Wikilink
+
+from typing import TYPE_CHECKING, List
+
+if TYPE_CHECKING:
+    from logging import Logger
+
+class Client(ABC):
+    """
+    Metaclass for different clients (things that receive inputs like discord/slack messages)
+    pending some actual formalization of the concepts in this library (like idk what do we
+    call the wiki, a sink? idk)
+
+    Subclasses must implement all the abstract methods (that is, i guess, the definition
+    of abstract methods) but may do whatever other logic they need to do to handle messages,
+    eg. discord bot subclasses
+    """
+
+    def __init__(
+            self,
+            wiki:Wiki,
+            name: str = "wikibot_client",
+            log_dir: Path = Path('/var/log/wikibot'),
+    ):
+        super(Client, self).__init__()
+
+        self._wiki = None
+        self.wiki = wiki
+        self.name = name
+        self.log_dir = Path(log_dir)
+
+        self.logger = self._init_logger(name=self.name, log_dir=self.log_dir)
+
+    # --------------------------------------------------
+    # abstract methods
+    # --------------------------------------------------
+
+    # @abstractmethod
+    # def _react_progress(self, message):
+    #     """React to a message indicating that we are processing it"""
+    #     pass
+
+    # @abstractmethod
+    # def _react_complete(self, message):
+    #     """React to a message indicating that we have completed processing it"""
+    #     pass
+    # @abstractmethod
+    # def _react_error(self, message):
+    #    """React to a message indicating there was some error"""
+
+    @abstractmethod
+    def parse_wikilinks(self, message) -> List[Wikilink]:
+        """Parse a given message, returning the wikilinks it contains"""
+
+    @abstractmethod
+    def handle_wikilinks(self, message, wl: List[Wikilink]):
+        """Handle the message with wikilinks, most likely by posting it to the wiki!"""
+
+
+
+    # --------------------------------------------------
+    # Properties
+    # --------------------------------------------------
+    @property
+    def wiki(self) -> Wiki:
+        return self._wiki
+
+    @wiki.setter
+    def wiki(self, wiki: Wiki):
+        if wiki.sess is None:
+            raise RuntimeError("Wiki client is not logged in! Login before passing to client")
+        self._wiki = wiki
+
+    # --------------------------------------------------
+    # Private methods
+    # --------------------------------------------------
+    def _init_logger(self, name:str, log_dir:Path) -> 'Logger':
+
+        # Try and make log directory, if we cant, it should fail.
+        log_dir.mkdir(exist_ok=True)
+
+        logger = init_logger(name=name, basedir=log_dir)
+        return logger
+
+
+

wiki_postbot/clients/discord.py

@@ -8,42 +8,40 @@ from wiki_postbot.creds import Discord_Creds, Mediawiki_Creds
 from wiki_postbot.patterns.wikilink import Wikilink
 from wiki_postbot.interfaces.mediawiki import Wiki
 from wiki_postbot.logger import init_logger
+from wiki_postbot.clients.client import Client as Wikibot_Client
 
 from discord.ext import commands
 from discord import Emoji
 import pdb
 
-class DiscordClient(Client):
+class DiscordClient(Client, Wikibot_Client):
 
     def __init__(
         self,
         wiki:Wiki,
+        name:str='discord_bot',
         intents=None,
         debug:bool=False,
         reply_channel:str="wikibot",
         log_dir:Path=Path('/var/log/wikibot'),
         **kwargs
     ):
+        Wikibot_Client.__init__(
+            self,
+            wiki=wiki,
+            name=name,
+            log_dir=log_dir)
+
         if intents is None:
             intents = Intents.default()
             intents.message_content = True
 
-        self.wiki = wiki
+        Client.__init__(self, intents=intents, **kwargs)
-        if self.wiki.sess is None:
-            raise RuntimeError("Wiki client is not logged in! Login before passing to discord client")
 
         self.debug = debug
         self.reply_channel_name = reply_channel
         self.reply_channel = None # type: Optional[discord.TextChannel]
 
-        self.log_dir = Path(log_dir)
-        # Try and make log directory, if we cant, it should fail.
-        self.log_dir.mkdir(exist_ok=True)
-
-        self.logger = init_logger(name="discord_bot", basedir=self.log_dir)
-
-        super(DiscordClient, self).__init__(intents=intents, **kwargs)
-
     async def get_channel(self, channel_name:str) -> discord.TextChannel:
         channel = discord.utils.get(self.get_all_channels(), name=channel_name)
         self.logger.debug(f"Got channel {channel}")
@@ -69,15 +67,15 @@ class DiscordClient(Client):
             await message.add_reaction("❤️‍🔥")
 
         try:
-            wl = Wikilink.parse(message.content)
+            wl = self.parse_wikilinks(message)
             self.logger.debug(f"Parsed wikilinks: {wl}")
             if len(wl)>0:
-                await self.handle_wikilink(message, wl)
+                await self.handle_wikilinks(message, wl)
         except Exception as e:
             self.logger.exception(f"Error parsing wikilink! got exception: ")
 
 
-    async def handle_wikilink(self, message:discord.message.Message, wl:List[Wikilink]):
+    async def handle_wikilinks(self, message:discord.message.Message, wl:List[Wikilink]):
         log_msg = f"Wikilinks detected: \n" + '\n'.join([str(l) for l in wl])
         self.logger.info(log_msg)
         if self.debug:
@@ -109,6 +107,10 @@ class DiscordClient(Client):
 
             # TODO: Logging!
 
+    def parse_wikilinks(self, message) -> List[Wikilink]:
+        wikilinks = Wikilink.parse(message.content)
+        return wikilinks
+
     # def add_links(self, links:Wikilink, msg:discord.message.Message):
     #     if 'testing links' in message.content:
     #         await message.channel.send(embed=Embed().add_field(name="Links", value="there are [links](https://example.com)"))

wiki_postbot/clients/slack.py

@@ -0,0 +1,264 @@
+import pdb
+from pprint import pformat
+
+import argparse
+from pathlib import Path
+from typing import List, Optional, Tuple, Union, Dict
+from threading import Event
+from dataclasses import dataclass
+from datetime import datetime
+
+from wiki_postbot.clients.client import Client
+from wiki_postbot.creds import Mediawiki_Creds, Slack_Creds
+from wiki_postbot.interfaces.mediawiki import Wiki
+from wiki_postbot.patterns.wikilink import Wikilink
+
+from slack_sdk.web import WebClient
+from slack_sdk.socket_mode import SocketModeClient
+from slack_sdk.socket_mode.response import SocketModeResponse
+from slack_sdk.socket_mode.request import SocketModeRequest
+
+
+class SlackClient(Client):
+
+    def __init__(
+            self,
+            creds: Slack_Creds,
+            wiki:Wiki,
+            name:str="wikibot_slack",
+            reply_channel="wikibot",
+            log_dir:Path = Path('/var/log/wikibot'),
+    ):
+        """Wikibot but for slack!"""
+        super(SlackClient, self).__init__(wiki=wiki, name=name, log_dir=log_dir)
+
+        self.creds = creds
+        self._initialized = False
+        self.web_client = None # type: Optional[WebClient]
+        self.socket_client = None # type: Optional[SocketModeClient]
+        self.reply_channel_name = reply_channel
+        self._reply_channel = None
+        self._channel_inverse = None
+        # self.web_client, self.socket_client = self._init_client(self.creds)
+
+    @property
+    def reply_channel(self) -> Union[str, None]:
+        if self._reply_channel is None:
+            self.logger.debug(f"Getting channel named {self.reply_channel_name}")
+            channels = self.web_client.conversations_list()
+            channel = [c for c in channels['channels'] if c['name'] == self.reply_channel_name]
+            # self.logger.debug(channel)
+            if len(channel) == 1:
+                self._reply_channel = channel[0]['id']
+            elif len(channel) > 1:
+                self.logger.exception(f"Got too many channels to reply to!")
+                self._reply_channel = None
+            else:
+                self.logger("Reply channel not found!")
+                self._reply_channel = None
+        return self._reply_channel
+
+    @property
+    def channel_inverse(self) -> Dict[str,str]:
+        """Maps channel IDs to channel names"""
+        if self._channel_inverse is None:
+            self.logger.debug("Getting inverse channel map")
+            channels = self.web_client.conversations_list()
+            self._channel_inverse = {c['id']: c['name'] for c in channels['channels']}
+        return self._channel_inverse
+
+
+    def _init_client(self, creds: Slack_Creds) -> Tuple[WebClient, SocketModeClient]:
+        web_client = WebClient(creds.bot_token)
+        socket_client = SocketModeClient(app_token=creds.app_token, web_client=web_client)
+        return web_client, socket_client
+
+    def handle_event(self, client:SocketModeClient, req: SocketModeRequest):
+        self.logger.debug(f"type: {req.type}, payload_type: {req.payload['type']}\n{pformat(req.payload)}")
+
+        if req.type == "events_api":
+            # acknowledge we got it so it don't get resent
+            response = SocketModeResponse(envelope_id=req.envelope_id)
+            client.send_socket_mode_response(response)
+        else:
+            self.logger.debug(f'Unhandled event type: {req.type}')
+            return
+
+
+        if req.type == "events_api" and \
+                req.payload['type'] == 'event_callback' and \
+                req.payload['event'].get('type', False) == 'message':
+            # Handle messages
+            self.logger.debug(f"Handling message")
+
+            message_text = req.payload['event']['text']
+
+            if 'good bot' in message_text.lower():
+                self.good_bot(client, req)
+
+            try:
+                wl = self.parse_wikilinks(message_text)
+                if len(wl) > 0:
+                    self.logger.debug(f"Parsed wikilinks: {wl}")
+                    self.handle_wikilinks(req, wl, client)
+                else:
+                    self.logger.debug("No wikilinks found")
+
+            except Exception:
+                self.logger.exception("Error parsing wikilinks! got exception...")
+
+        else:
+            self.logger.debug(f"Was event, but not a message. Payload type: {req.payload['type']}, Event type: {req.payload.get('event', {}).get('type', 'unknown')}")
+            return
+
+        # pdb.set_trace()
+        # if
+
+    def parse_wikilinks(self, message) -> List[Wikilink]:
+        wikilinks = Wikilink.parse(message)
+        return wikilinks
+
+    def handle_wikilinks(self, message, wl: List[Wikilink], client:SocketModeClient):
+        self.react('hourglass', client, message)
+        try:
+            # expand fields in message
+            channel_id = message.payload['event']['channel']
+            channel_name = self.channel_inverse[channel_id]
+            msg = SlackMessage(
+                content = message.payload['event']['text'],
+                user_id = message.payload['event']['user'],
+                channel_id=channel_id,
+                channel = channel_name,
+                timestamp = message.payload['event']['ts']
+            )
+            msg.complete(client.web_client)
+            self.logger.debug(f"Posting message:\n{msg}")
+            result = self.wiki.handle_slack(msg)
+            ok = result.ok
+
+        except Exception:
+            self.logger.exception("Error handling slack message")
+            result = None
+            ok = False
+
+        self.react('hourglass', client, message, remove=True)
+        if ok:
+            self.react('white_check_mark', client, message)
+        else:
+            self.react('x', client, message)
+
+        if result and result.reply:
+            if self.reply_channel is None:
+                self.logger.exception(f"Do not have channel to reply to!")
+            else:
+                # await self.reply_channel.send(embed=Embed().add_field(name="WikiLinks", value=result.reply))
+                self.logger.debug('TODO: should reply here!')
+
+
+    def good_bot(self, client:SocketModeClient, req: SocketModeRequest):
+        self.logger.info('Got told we are a good bot ^_^')
+        self.react('heart', client, req)
+        self.react('heavy_plus_sign', client, req)
+        self.react('fire', client, req)
+        self.react('heavy_equals_sign', client, req)
+        self.react('heart_on_fire', client, req)
+
+    def react(self, emoji:str, client:SocketModeClient, req: SocketModeRequest, remove:bool=False):
+        if remove:
+            client.web_client.reactions_remove(
+                name=emoji,
+                channel=req.payload["event"]["channel"],
+                timestamp=req.payload["event"]["ts"],
+            )
+        else:
+            client.web_client.reactions_add(
+                name=emoji,
+                channel=req.payload["event"]["channel"],
+                timestamp=req.payload["event"]["ts"],
+            )
+
+    def run(self, creds: Optional[Slack_Creds] = None):
+        if creds is None:
+            creds = self.creds
+        else:
+            self.creds = creds
+
+        self.web_client, self.socket_client = self._init_client(creds)
+
+        self.socket_client.socket_mode_request_listeners.append(self.handle_event)
+
+        self.logger.debug("Connecting...")
+
+        self.socket_client.connect()
+
+        self.logger.debug(f"Got Reply Channel ID {self.reply_channel} for {self.reply_channel_name}")
+
+        try:
+            self.logger.info("Slack Client Listening")
+            Event().wait()
+        except KeyboardInterrupt:
+            self.logger.info("Quitting Slack client!")
+
+@dataclass
+class SlackMessage:
+    content: str # the actual content of the message
+    user_id: str
+    channel_id: str
+    channel: str # currently expected to be passed at instantiation because the client keeps a reverse index. bad information hiding i know.
+    timestamp: str
+    """The unix epoch string timestamp stored as a slack event's ts attribute"""
+
+    # these need to be filled in after instantiation by passing a webclient to the completion methods
+    avatar: str = '' # URL of image
+    permalink: str = ''
+    author:str = '' # display name
+
+    _complete:bool=False
+
+    def get_permalink(self, client: WebClient):
+        permalink = client.chat_getPermalink(channel=self.channel_id, message_ts=self.timestamp)
+        self.permalink = permalink['permalink']
+
+    def get_user(self, client:WebClient):
+        user_info = client.users_info(user=self.user_id)
+        self.avatar = user_info['user']['profile']['image_192']
+        self.author = user_info['user']['profile']['display_name']
+
+    def complete(self, client:WebClient):
+        self.get_permalink(client)
+        self.get_user(client)
+        self._complete = True
+
+    @property
+    def date_sent(self) -> datetime:
+        return datetime.fromtimestamp(float(self.timestamp))
+
+
+
+def argparser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="slack_bot",
+        description="A slack bot for posting messages with wikilinks to an associated mediawiki wiki"
+    )
+    parser.add_argument('-d', '--directory', default='/etc/wikibot/', type=Path,
+                        help="Directory that stores credential files and logs")
+    parser.add_argument('-w', '--wiki', help="URL of wiki", type=str)
+    return parser
+
+def main():
+    parser = argparser()
+    args = parser.parse_args()
+    directory = Path(args.directory)
+    log_dir = directory / "logs"
+
+    slack_creds = Slack_Creds.from_json(directory / 'slack_creds.json')
+    wiki_creds = Mediawiki_Creds.from_json(directory / 'mediawiki_creds.json')
+
+    wiki = Wiki(url=args.wiki, log_dir=log_dir, creds=wiki_creds)
+    wiki.login(wiki_creds)
+
+    client = SlackClient(creds=slack_creds, wiki=wiki, log_dir=log_dir)
+    client.run()
+
+if __name__ == "__main__":
+    main()

wiki_postbot/creds.py

@@ -53,3 +53,16 @@ class Mediawiki_Creds:
         with open(path, 'r') as jfile:
             creds = json.load(jfile)
         return Mediawiki_Creds(**creds)
+
+@dataclass
+class Slack_Creds:
+    app_token:str
+    bot_token:str
+
+    @classmethod
+    @classmethod
+    def from_json(cls, path:Path) -> 'Slack_Creds':
+        """jesus christ this package is so sloppy"""
+        with open(path, 'r') as jfile:
+            creds = json.load(jfile)
+        return Slack_Creds(**creds)

wiki_postbot/interfaces/mediawiki.py

@@ -1,7 +1,7 @@
 """
 Class for interfacing with mediawiki
 """
-from typing import List, Optional
+from typing import List, Optional, TYPE_CHECKING
 from pathlib import Path
 from urllib.parse import urljoin
 from dataclasses import dataclass
@@ -16,6 +16,9 @@ import requests
 from discord.message import Message, Embed
 import pdb
 
+if TYPE_CHECKING:
+    from wiki_postbot.clients.slack import SlackMessage
+
 # creds = Mediawiki_Creds.from_json('mediawiki_creds.json')
 
 
@@ -83,7 +86,11 @@ class Wiki:
         self.logger.debug(content)
         return WikiPage.from_source(title=content['parse']['title'], source=content['parse']['wikitext'])
 
-    def insert_text(self, page, section, text):
+    def insert_text(self, page, section, text, overwrite:bool=False):
+        if overwrite:
+            operation_key = 'text'
+        else:
+            operation_key = 'appendtext'
 
         # TODO: Move finding section IDs into the page class!
         page_text = self.get_page(page)
@@ -116,7 +123,7 @@ class Wiki:
                     "action":"edit",
                     "title":page,
                     "section":str(matching_section),
-                    "appendtext":text,
+                    operation_key:text,
                     "format":"json",
                     "token":token
                 }
@@ -130,7 +137,7 @@ class Wiki:
                     "title":page,
                     "section":"new",
                     "sectiontitle":section,
-                    "appendtext":text,
+                    operation_key:text,
                     "format":"json",
                     "token":token
                 }
@@ -170,6 +177,39 @@ class Wiki:
         else:
             return Result(ok=False, log=f"Got exceptions: {errored_pages}")
 
+    def handle_slack(self, msg:'SlackMessage') -> Result:
+        """
+        Brooooo it's getting laaaaaaaaaate and I think it's obvious how this should be refactored
+        but I want to play some zeldaaaaaaaaaaaa so I am copy and pasting for now
+        """
+        self.login(self.creds)
+        # Get message in mediawiki template formatting
+        template_str = TemplateMessage.format_slack(msg)
+
+        # parse wikilinks, add to each page
+        wikilinks = Wikilink.parse(msg.content)
+        errored_pages = []
+        for link in wikilinks:
+            if link.section is None:
+                section = "Slack"
+            else:
+                section = link.section
+
+            res = self.insert_text(link.link, section, template_str)
+            if res.json()['edit']['result'] != 'Success':
+                errored_pages.append(res.json())
+
+        # Add to index page (only once)
+        self.add_to_index(template_str)
+
+        if len(errored_pages) == 0:
+            # gather links for a reply
+            reply = '\n'.join([f"[{l.link}]({urljoin(self.url, l.link.replace(' ', '_'))})" for l in wikilinks])
+
+            return Result(ok=True, log=f"Successfully posted message to {[l.link for l in wikilinks]}", reply=reply)
+        else:
+            return Result(ok=False, log=f"Got exceptions: {errored_pages}")
+
     def add_to_index(self, message):
         section = datetime.today().strftime("%y-%m-%d")
         self.insert_text(page=self.index_page, section=section, text=message)

wiki_postbot/service.py

@@ -64,3 +64,11 @@ def main():
 
     service = Service(wiki_url=args.wiki, directory=args.directory)
     service.install_service()
+
+def main_slack():
+    parser = argparser()
+    args = parser.parse_args()
+
+    service = Service(wiki_url=args.wiki, directory=args.directory)
+    service.bot_script = shutil.which('slack_bot')
+    service.install_service()

wiki_postbot/templates/wiki.py

@@ -1,12 +1,15 @@
 """
 Templates for representing different kinds of messages on mediawiki
 """
-
+from typing import TYPE_CHECKING
 from wiki_postbot.formats.wiki import WikiPage
 from abc import abstractmethod
 from discord.message import Message
 import warnings
 
+if TYPE_CHECKING:
+    from wiki_postbot.clients.slack import SlackMessage
+
 class WikiTemplate(WikiPage):
 
     @abstractmethod
@@ -37,6 +40,18 @@ class TemplateMessage(WikiTemplate):
             "}}"
             )
 
+    @classmethod
+    def format_slack(cls, msg:'SlackMessage') -> str:
+        return (
+            "{{Message\n"
+            f"|Author={msg.author}\n"
+            f"|Avatar={msg.avatar}\n"
+            f"|Date Sent={msg.date_sent.strftime('%y-%m-%d %H:%M:%S')}\n"
+            f"|Channel={msg.channel}\n"
+            f"|Text={msg.content}\n"
+            f"|Link={msg.permalink}\n"
+            "}}"
+            )
 
 #
 # template_message = TemplateMessage.from_source(