process manager draft

.gitignore

@@ -147,3 +147,6 @@ dist
 .DS_Store
 
 .idea
+
+matterbridge*64bit
+example.matterbridge.toml

README.md

@@ -8,6 +8,13 @@ Using matterbridge - https://github.com/42wim/matterbridge
 
 Very unfinished!! Mostly a programming exercise for me for now to practice fullstack. Theoretically it should be possible to reuse this but i make no guarantees especially while it is yno unfinished and potentially insecure. run at your own risk!
 
+## TODO
+
+- [ ] Manage matterbridge processes
+- [ ] Sanitize user inputs
+- [ ] Check status of matterbridge processes
+- [ ] Complete slack login workflow
+
 ## Supported Clients 
 
 (the ones that can use the website to join)

example.env

@@ -13,3 +13,7 @@ SLACK_CLIENT_SECRET=
 SLACK_SIGNING_SECRET=
 
 ADMIN_TOKEN=
+
+# Location of the matterbridge binary!
+MATTERBRIDGE_BINARY=
+MATTERBRIDGE_CONFIG_DIR=

server/config/custom-environment-variables.ts

@@ -16,5 +16,9 @@ export default {
     cookies:{
         key1: 'COOKIE_KEY_1',
         key2: 'COOKIE_KEY_2'
+    },
+    matterbridge: {
+        bin: 'MATTERBRIDGE_BINARY',
+        config: 'MATTERBRIDGE_CONFIG_DIR'
     }
 }

server/package.json

@@ -13,9 +13,11 @@
     "typeorm": "../node_modules/.bin/typeorm-ts-node-commonjs",
     "migrate": "rm -rf build && yarn build && yarn ../node_modules/.bin/typeorm migration:generate ./src/migrations/added-user-entity -d ./src/utils/data-source.ts",
     "db:push": "rm -rf build && yarn build && yarn ../node_modules/.bin/typeorm migration:run -d src/utils/data-source.ts",
-    "importData": "npx ts-node-dev --transpile-only --exit-child  src/data/seeder.ts"
+    "importData": "npx ts-node-dev --transpile-only --exit-child  src/data/seeder.ts",
+    "test": "jest"
   },
   "dependencies": {
+    "@ltd/j-toml": "^1.38.0",
     "@slack/oauth": "^2.6.1",
     "config": "^3.3.9",
     "cookie-session": "^2.0.0",
@@ -26,21 +28,27 @@
     "helmet": "^7.0.0",
     "jsonwebtoken": "^9.0.1",
     "pg": "^8.11.1",
+    "pm2": "^5.3.0",
     "pug": "^3.0.2",
     "redis": "^4.6.7",
     "reflect-metadata": "^0.1.13",
+    "slugify": "^1.6.6",
     "typeorm": "^0.3.17",
     "typescript": "^5.1.6",
     "zod": "^3.21.4"
   },
   "devDependencies": {
+    "@jest/globals": "^29.6.1",
     "@types/config": "^3.3.0",
     "@types/cookie-session": "^2.0.44",
     "@types/cors": "^2.8.13",
     "@types/express": "^4.17.17",
+    "@types/jest": "^29.5.3",
     "@types/jsonwebtoken": "^9.0.2",
     "@types/node": "^20.4.2",
     "@types/pug": "^2.0.6",
+    "jest": "^29.6.1",
+    "ts-jest": "^29.1.1",
     "ts-node": "^10.9.1",
     "ts-node-dev": "^2.0.0"
   }

server/src/app.ts

@@ -7,6 +7,8 @@ import cors from 'cors';
 import { AppDataSource } from './db/data-source';
 import AppError from './errors/appError';
 
+import MatterbridgeManager from "./matterbridge/process";
+
 import {cookieMiddleware} from "./middleware/cookies";
 import groupRoutes from "./routes/group.routes";
 import slackRoutes from "./routes/slack.routes";
@@ -17,6 +19,8 @@ import authRoutes from "./routes/auth.routes";
 AppDataSource.initialize()
   .then(async () => {
 
+
+
     const app = express();
     app.use(express.json({limit: "10kb"}));
     app.use(cookieMiddleware);
@@ -46,4 +50,8 @@ AppDataSource.initialize()
 
     console.log(`Server started on port: ${port}`)
 
+    await MatterbridgeManager.spawnAll();
+    console.log('Spawned group processes:');
+    console.log(MatterbridgeManager.processes);
+
   })

server/src/entities/bridge.entity.ts

@@ -45,7 +45,10 @@ export class Bridge extends Model {
 
 
 
-    @OneToMany(() => Channel, (channel) => channel.bridge)
+    @OneToMany(() => Channel, (channel) => channel.bridge,
+      {
+          cascade: ["remove"]
+      })
     channels: Channel[]
 
 }

server/src/entities/channel.entity.ts

@@ -8,10 +8,18 @@ export class Channel extends Model {
     @Column()
     name: string;
 
-    @ManyToOne(() => Bridge, (bridge) => bridge.channels)
+    @ManyToOne(() => Bridge, (bridge) => bridge.channels,
+{
+          eager: true
+        }
+    )
     bridge: Bridge
 
-    @ManyToOne(() => Group, (group) => group.channels)
+    @ManyToOne(() => Group, (group) => group.channels,
+{
+          eager: true
+        }
+    )
     group: Group
 
 }

server/src/matterbridge/config.ts

@@ -1,4 +1,223 @@
+// @ts-nocheck
 /*
 Creating and synchronizing the .toml configuration files for matterbridge
 
  */
+
+import {AppDataSource} from "../db/data-source";
+import {Group} from "../entities/group.entity";
+const slugify = require('slugify');
+import * as TOML from '@ltd/j-toml';
+const fs = require('fs');
+
+const groupRepository = AppDataSource.getRepository(Group)
+
+
+
+/*
+[{protocol}.{name}]
+  Token = {token}
+  PrefixMessagesWithNick = true
+  RemoteNickFormat = {RemoteNickFormat}
+ */
+type BridgeEntry = {
+  protocol: string;
+  name: string;
+  Label: string;
+  Token: string;
+  PrefixMessagesWithNick: boolean;
+  RemoteNickFormat: string
+}
+
+/*
+[[gateway.inout]]
+  account = {BridgeEntry.protocol}.{BridgeEntry.name}
+  channel = {channel}
+ */
+type GatewayInOut = {
+  account: string;
+  channel: string;
+}
+
+/*
+[[gateway]]
+  name = {name}
+  enable = {enable}
+ */
+type Gateway = {
+  name: string;
+  enable: boolean;
+  bridges: BridgeEntry[];
+  inOuts: GatewayInOut[];
+}
+
+
+export const getGroupConfig = async (group_name: string, group:object): Promise<Gateway> => {
+  let group = await groupRepository.findOne({
+    where: {name: group_name},
+    relations: {channels: true}
+  })
+
+  // Construct config in the gateway style for programmatic use
+  let gateway = <Gateway>{
+    name: group.name,
+    enable: group.enable,
+    bridges: group.channels.map((channel) => {
+      return {
+        protocol: channel.bridge.Protocol,
+        name: slugify(channel.bridge.Label),
+        Label: channel.bridge.Label,
+        Token: channel.bridge.Token,
+        PrefixMessagesWithNick: channel.bridge.PrefixMessagesWithNick,
+        RemoteNickFormat: channel.bridge.RemoteNickFormat
+      }
+    }),
+    inOuts: group.channels.map((channel) => {
+      return {
+        account: `${channel.bridge.Protocol}.${slugify(channel.bridge.Label)}`,
+        channel: channel.name
+      }
+    })
+  }
+  return gateway
+}
+
+/*
+[slack.0]
+        Token = ""
+        PrefixMessagesWithNick = true
+        RemoteNickFormat = "[{PROTOCOL}] <{NICK}> "
+
+[slack.1]
+        Token = ""
+        PrefixMessagesWithNick = true
+        RemoteNickFormat = "[{PROTOCOL}] <{NICK}> "
+
+[[gateway]]
+        name = "myGateway"
+        enable = true
+
+[[gateway.inout]]
+        account = "slack.0"
+        channel = "test"
+
+[[gateway.inout]]
+        account = "slack.1"
+        channel = "test"
+
+
+Becomes:
+
+Object <[Object: null prototype] {}> {
+  slack: Object <[Object: null prototype] {}> {
+    '0': Object <[Object: null prototype] {}> {
+      Token: '',
+      PrefixMessagesWithNick: true,
+      RemoteNickFormat: '[{PROTOCOL}] <{NICK}> '
+    },
+    '1': Object <[Object: null prototype] {}> {
+      Token: '',
+      PrefixMessagesWithNick: true,
+      RemoteNickFormat: '[{PROTOCOL}] <{NICK}> '
+    }
+  },
+  gateway: [
+    Object <[Object: null prototype] {}> {
+      name: 'myGateway',
+      enable: true,
+      inout: [Array]
+    }
+  ]
+}
+ */
+export const GatewayToTOML = (gateway: Gateway) => {
+  let protocols = {};
+  // Each bridge is prefixed by the protocol as a TOML object, so
+  // we have to do a sort of "groupBy" operation here
+  // The TOML.Section calls are just for formatting when we write the file out
+  // (ie. separate the different TOML table entries rather than representing them
+  // inline. See https://www.npmjs.com/package/@ltd/j-toml
+  gateway.bridges.forEach((bridge) => {
+
+    let bridgeEntry = {
+      Token: bridge.Token,
+      PrefixMessagesWithNick: bridge.PrefixMessagesWithNick,
+      RemoteNickFormat: bridge.RemoteNickFormat,
+      Label: bridge.Label
+    }
+
+    if (!protocols.hasOwnProperty(bridge.protocol)){
+      protocols[bridge.protocol] = {};
+    }
+
+    protocols[bridge.protocol][bridge.name] = TOML.Section(bridgeEntry)
+
+  })
+  console.log('protocols', protocols)
+
+  return {
+    ...protocols,
+    'gateway': [TOML.Section({
+      name: gateway.name,
+      enable: gateway.enable,
+      inout: gateway.inOuts.map((inout) => TOML.Section(inout))
+    })]
+  }
+}
+
+export const writeTOML = (gateway_toml: object, out_file: string) => {
+  let toml_string = TOML.stringify(
+    gateway_toml,
+    {
+      newline: '\n'
+    }
+  )
+
+  fs.writeFileSync(out_file, toml_string)
+
+}
+
+export const writeGroupConfig = async (group_name: string, out_file: string) => {
+  getGroupConfig(group_name)
+    .then((group_config) => {
+      writeTOML(GatewayToTOML(group_config), out_file)
+    })
+}
+
+
+
+
+// const testGroup = {
+//   name: 'testGroup',
+//   enable: true,
+//   channels: [
+//     {
+//       name: 'a-channel',
+//       bridge: {
+//         Protocol: 'slack',
+//         Label:" My Lab",
+//         Token: 'token',
+//         PrefixMessagesWithNick: true,
+//         RemoteNickFormat: 'hey'
+//       }
+//     },
+//     {
+//       name: 'b-channel',
+//       bridge: {
+//         Protocol: 'slack',
+//         Label:" My Lab2",
+//         Token: 'token',
+//         PrefixMessagesWithNick: true,
+//         RemoteNickFormat: 'hey'
+//       }
+//     }
+//   ]
+// }
+//
+// getGroupConfig('a', testGroup)
+//   .then((res) => {
+//     console.log('1', res);
+//     console.log('toml', GatewayToTOML(res));
+//     let toml_format = GatewayToTOML(res);
+//     writeTOML(toml_format, 'test.toml')
+//   })

server/src/matterbridge/process.ts

@@ -3,3 +3,114 @@
 Managing the matterbridge processes
 
 */
+
+import {AppDataSource} from "../db/data-source";
+
+const pm2 = require('pm2');
+import config from "config";
+
+import { writeGroupConfig } from "./config";
+import {Group} from "../entities/group.entity";
+const groupRepository = AppDataSource.getRepository(Group)
+
+
+enum ProcessStatus {
+  online = "online",
+  stopping = "stopping",
+  stopped = "stopped",
+  launching = "launching",
+  errored = "errored",
+  one_launch_status = "one-launch-status"
+}
+
+type Process = {
+  name: string;
+  pid: number;
+  pm_id: number;
+  status: ProcessStatus;
+  monit: {
+    memory: number;
+    cpu: number;
+  }
+  pm2_env: {
+    created_at: number;
+    exec_interpreter: string;
+    exec_mode: string;
+    instances: number;
+    pm_out_log_path: string;
+    pm_err_log_path: string;
+    pm_pid_path: string;
+  };
+
+}
+
+class MatterbridgeManager {
+
+  private process_list: string[] = []
+  constructor(
+    private matterbridge_bin: string,
+    private matterbridge_config_dir: string,
+  ){}
+
+  async spawnProcess(group_name: string) {
+
+    let group_filename = `${this.matterbridge_config_dir}/matterbridge-${group_name}.toml`
+    await writeGroupConfig(group_name, group_filename);
+    await pm2.start(
+      {
+        name: group_name,
+        script: this.matterbridge_bin,
+        args: `-conf ${group_filename}`,
+        interpreter: 'none'
+      },
+      (err:any, apps:object) => {
+        // TODO: Handle errors!
+      }
+    )
+    this.process_list.push(group_name)
+  }
+
+  async spawnAll(){
+    let groups = await groupRepository.find({
+      select: {name: true}
+    })
+    groups.map(
+      (group) => this.spawnProcess(group.name)
+    )
+  }
+
+  get processes(): Process[] {
+    let processes: Process[] = []
+    pm2.list()
+      .then((err:any, list:[]) => {
+        processes = list.map((proc:any) => {
+          return <Process>{
+            name: proc.name,
+            pid: proc.pid,
+            pm_id: proc.pm2_env.pm_id,
+            status: proc.pm2_env.status,
+            monit: proc.monit,
+            pm2_env: {
+              created_at: proc.pm2_env.proc.pm2_env.created_at,
+              exec_interpreter: proc.pm2_env.exec_interpreter,
+              exec_mode: proc.pm2_env.exec_mode,
+              instances: proc.pm2_env.instances,
+              pm_out_log_path: proc.pm2_env.pm_out_log_path,
+              pm_err_log_path: proc.pm2_env.pm_err_log_path,
+              pm_pid_path: proc.pm2_env.pm_pid_path,
+            }
+          }
+        })
+      })
+    return processes
+  }
+
+}
+
+const matterbridge_config = config.get<{bin:string, config:string}>('matterbridge')
+
+const manager = new MatterbridgeManager(
+  matterbridge_config.bin,
+  matterbridge_config.config
+)
+export default manager

server/tsconfig.json

@@ -14,7 +14,7 @@
     "rootDir": ".",
 
   },
-  "include": ["src/**/*"],
+  "include": ["src/**/*", "tests/**/*"],
   "exclude": [
     "../node_modules", "roles"]
 }