commit 43e113e420a19becdc8a0cb9a62e98ae85d05f07 Author: Thomas Sileo Date: Fri May 18 20:41:41 2018 +0200 Initial import diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8fce603 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +data/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e43f899 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*.sw[op] +key_*.pem +data/* +config/* +static/media/* + +.mypy_cache/ +__pycache__/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e3b0785 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +language: python +python: + - "3.6" +install: + - pip install pytest mypy flake8 +script: + - flake8 + - mypy --ignore-missing-imports + - pytest -v diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3fed815 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.6 +ADD . /app +WORKDIR /app +RUN pip install -r requirements.txt +ENV FLASK_APP=app.py +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5005", "app:app"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..be3f7b2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bc473fc --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +css: + python -c "import sass; sass.compile(dirname=('sass', 'static/css'), output_style='compressed')" + +password: + python -c "import bcrypt; from getpass import getpass; print(bcrypt.hashpw(getpass().encode('utf-8'), bcrypt.gensalt()).decode('utf-8'))" diff --git a/README.md b/README.md new file mode 100644 index 0000000..071be44 --- /dev/null +++ b/README.md @@ -0,0 +1,74 @@ +# microblog.pub + +

+ microblog.pub +

+

+Version +Build Status +License +

+ +

A self-hosted, single-user, ActivityPub powered microblog.

+ +## Features + + - Implements a basic [ActivityPub](https://activitypub.rocks/) server (with federation) + - Compatible with [Mastodon](https://github.com/tootsuite/mastodon) and others (Pleroma, Hubzilla...) + - Also implements a remote follow compatible with Mastodon instances + - Expose your outbox as a basic microblog + - [IndieAuth](https://indieauth.spec.indieweb.org/) endpoints (authorization and token endpoint) + - U2F support + - You can use your ActivityPub identity to login to other websites/app + - Admin UI with notifications and the stream of people you follow + - Attach files to your notes + - Privacy-aware upload that strip EXIF meta data before storing the file + - No JavaScript, that's it, even the admin UI is pure HTML/CSS + - Easy to customize (the theme is written Sass) + - Microformats aware (exports `h-feed`, `h-entry`, `h-cards`, ...) + - Exports RSS/Atom feeds + - Comes with a tiny HTTP API to help posting new content and performing basic actions + - Deployable with Docker + +## Running your instance + +### Installation + +```shell +$ git clone +$ make css +``` + +### Configuration + +```shell +$ make password +``` + +### Deployment + +```shell +$ docker-compose up -d +``` + +You should use a reverse proxy... + +## Development + +The most convenient way to hack on microblog.pub is to run the server locally, and run + + +```shell +# One-time setup +$ pip install -r requirements.txt +# Start the Celery worker, RabbitMQ and MongoDB +$ docker-compose -f docker-compose-dev.yml up -d +# Run the server locally +$ FLASK_APP=app.py flask run -p 5005 --with-threads +``` + +## Contributions + +PRs are welcome, please open an issue to start a discussion before your start any work. diff --git a/activitypub.py b/activitypub.py new file mode 100644 index 0000000..2a4a937 --- /dev/null +++ b/activitypub.py @@ -0,0 +1,978 @@ +import typing +import re +import json +import binascii +import os +from datetime import datetime +from enum import Enum + +import requests +from bleach.linkifier import Linker +from bson.objectid import ObjectId +from html2text import html2text +from feedgen.feed import FeedGenerator +from markdown import markdown + +from utils.linked_data_sig import generate_signature +from utils.actor_service import NotAnActorError +from utils.webfinger import get_actor_url +from utils.content_helper import parse_markdown +from config import USERNAME, BASE_URL, ID +from config import CTX_AS, CTX_SECURITY, AS_PUBLIC +from config import KEY, DB, ME, ACTOR_SERVICE +from config import OBJECT_SERVICE +from config import PUBLIC_INSTANCES +import tasks + +from typing import List, Optional, Tuple, Dict, Any, Union, Type +from typing import TypeVar + +A = TypeVar('A', bound='BaseActivity') +ObjectType = Dict[str, Any] +ObjectOrIDType = Union[str, ObjectType] + + +# Pleroma sample +# {'@context': ['https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1', {'Emoji': 'toot:Emoji', 'Hashtag': 'as:Hashtag', 'atomUri': 'ostatus:atomUri', 'conversation': 'ostatus:conversation', 'inReplyToAtomUri': 'ostatus:inReplyToAtomUri', 'manuallyApprovesFollowers': 'as:manuallyApprovesFollowers', 'ostatus': 'http://ostatus.org#', 'sensitive': 'as:sensitive', 'toot': 'http://joinmastodon.org/ns#'}], 'actor': 'https://soc.freedombone.net/users/bob', 'attachment': [{'mediaType': 'image/jpeg', 'name': 'stallmanlemote.jpg', 'type': 'Document', 'url': 'https://soc.freedombone.net/media/e1a3ca6f-df73-4f2d-a931-c389a221b008/stallmanlemote.jpg'}], 'attributedTo': 'https://soc.freedombone.net/users/bob', 'cc': ['https://cybre.space/users/vantablack', 'https://soc.freedombone.net/users/bob/followers'], 'content': '@vantablack
stallmanlemote.jpg', 'context': 'tag:cybre.space,2018-04-05:objectId=5756519:objectType=Conversation', 'conversation': 'tag:cybre.space,2018-04-05:objectId=5756519:objectType=Conversation', 'emoji': {}, 'id': 'https://soc.freedombone.net/objects/3f0faeca-4d37-4acf-b990-6a50146d23cc', 'inReplyTo': 'https://cybre.space/users/vantablack/statuses/99808953472969467', 'inReplyToStatusId': 300713, 'like_count': 1, 'likes': ['https://cybre.space/users/vantablack'], 'published': '2018-04-05T21:30:52.658817Z', 'sensitive': False, 'summary': None, 'tag': [{'href': 'https://cybre.space/users/vantablack', 'name': '@vantablack@cybre.space', 'type': 'Mention'}], 'to': ['https://www.w3.org/ns/activitystreams#Public'], 'type': 'Note'} + + +class ActivityTypes(Enum): + ANNOUNCE = 'Announce' + BLOCK = 'Block' + LIKE = 'Like' + CREATE = 'Create' + UPDATE = 'Update' + PERSON = 'Person' + ORDERED_COLLECTION = 'OrderedCollection' + ORDERED_COLLECTION_PAGE = 'OrderedCollectionPage' + COLLECTION_PAGE = 'CollectionPage' + COLLECTION = 'Collection' + NOTE = 'Note' + ACCEPT = 'Accept' + REJECT = 'Reject' + FOLLOW = 'Follow' + DELETE = 'Delete' + UNDO = 'Undo' + IMAGE = 'Image' + TOMBSTONE = 'Tombstone' + + +def random_object_id() -> str: + return binascii.hexlify(os.urandom(8)).decode('utf-8') + + +def _remove_id(doc: ObjectType) -> ObjectType: + doc = doc.copy() + if '_id' in doc: + del(doc['_id']) + return doc + + +def _to_list(data: Union[List[Any], Any]) -> List[Any]: + if isinstance(data, list): + return data + return [data] + + +def clean_activity(activity: ObjectType) -> Dict[str, Any]: + # Remove the hidden bco and bcc field + for field in ['bto', 'bcc']: + if field in activity: + del(activity[field]) + if activity['type'] == 'Create' and field in activity['object']: + del(activity['object'][field]) + return activity + + +def _get_actor_id(actor: ObjectOrIDType) -> str: + if isinstance(actor, dict): + return actor['id'] + return actor + + +class BaseActivity(object): + ACTIVITY_TYPE = None # type: Optional[ActivityTypes] + NO_CONTEXT = False + ALLOWED_OBJECT_TYPES = None # type: List[ActivityTypes] + + def __init__(self, **kwargs) -> None: + if not self.ACTIVITY_TYPE: + raise ValueError('Missing ACTIVITY_TYPE') + + if kwargs.get('type') is not None and kwargs.pop('type') != self.ACTIVITY_TYPE.value: + raise ValueError('Expect the type to be {}'.format(self.ACTIVITY_TYPE)) + + self._data = {'type': self.ACTIVITY_TYPE.value} # type: Dict[str, Any] + + if 'id' in kwargs: + self._data['id'] = kwargs.pop('id') + + if self.ACTIVITY_TYPE != ActivityTypes.PERSON: + actor = kwargs.get('actor') + if actor: + kwargs.pop('actor') + actor = self._validate_person(actor) + self._data['actor'] = actor + else: + if not self.NO_CONTEXT: + actor = ID + self._data['actor'] = actor + + if 'object' in kwargs: + obj = kwargs.pop('object') + if isinstance(obj, str): + self._data['object'] = obj + else: + if not self.ALLOWED_OBJECT_TYPES: + raise ValueError('unexpected object') + if 'type' not in obj or (self.ACTIVITY_TYPE != ActivityTypes.CREATE and 'id' not in obj): + raise ValueError('invalid object') + if ActivityTypes(obj['type']) not in self.ALLOWED_OBJECT_TYPES: + print(self, kwargs) + raise ValueError(f'unexpected object type {obj["type"]} (allowed={self.ALLOWED_OBJECT_TYPES})') + self._data['object'] = obj + + if '@context' not in kwargs: + if not self.NO_CONTEXT: + self._data['@context'] = CTX_AS + else: + self._data['@context'] = kwargs.pop('@context') + + # @context check + if not self.NO_CONTEXT: + if not isinstance(self._data['@context'], list): + self._data['@context'] = [self._data['@context']] + if not CTX_SECURITY in self._data['@context']: + self._data['@context'].append(CTX_SECURITY) + if isinstance(self._data['@context'][-1], dict): + self._data['@context'][-1]['Hashtag'] = 'as:Hashtag' + self._data['@context'][-1]['sensitive'] = 'as:sensitive' + else: + self._data['@context'].append({'Hashtag': 'as:Hashtag', 'sensitive': 'as:sensitive'}) + + allowed_keys = None + try: + allowed_keys = self._init(**kwargs) + except NotImplementedError: + pass + + if allowed_keys: + # Allows an extra to (like for Accept and Follow) + kwargs.pop('to', None) + if len(set(kwargs.keys()) - set(allowed_keys)) > 0: + raise ValueError('extra data left: {}'.format(kwargs)) + else: + self._data.update(**kwargs) + + def _init(self, **kwargs) -> Optional[List[str]]: + raise NotImplementedError + + def _verify(self) -> None: + raise NotImplementedError + + def verify(self) -> None: + try: + self._verify() + except NotImplementedError: + pass + + def __repr__(self) -> str: + return '{}({!r})'.format(self.__class__.__qualname__, self._data.get('id')) + + def __str__(self) -> str: + return str(self._data['id']) + + def __getattr__(self, name: str) -> Any: + if self._data.get(name): + return self._data.get(name) + + @property + def type_enum(self) -> ActivityTypes: + return ActivityTypes(self.type) + + def set_id(self, uri: str, obj_id: str) -> None: + self._data['id'] = uri + + def _actor_id(self, obj: ObjectOrIDType) -> str: + if isinstance(obj, dict) and obj['type'] == ActivityTypes.PERSON.value: + obj_id = obj.get('id') + if not obj_id: + raise ValueError('missing object id') + return obj_id + else: + return str(obj) + + def _validate_person(self, obj: ObjectOrIDType) -> str: + obj_id = self._actor_id(obj) + try: + actor = ACTOR_SERVICE.get(obj_id) + except Exception: + return obj_id # FIXME(tsileo): handle this + if not actor: + raise ValueError('Invalid actor') + return actor['id'] + + def get_object(self) -> 'BaseActivity': + if self.__obj: + return self.__obj + if isinstance(self._data['object'], dict): + p = parse_activity(self._data['object']) + else: + if self.ACTIVITY_TYPE == ActivityTypes.FOLLOW: + p = Person(**ACTOR_SERVICE.get(self._data['object'])) + else: + obj = OBJECT_SERVICE.get(self._data['object']) + if ActivityTypes(obj.get('type')) not in self.ALLOWED_OBJECT_TYPES: + raise ValueError('invalid object type') + + p = parse_activity(obj) + + self.__obj = p # type: BaseActivity + return p + + def _to_dict(self, data: ObjectType) -> ObjectType: + return data + + def to_dict(self, embed: bool = False) -> ObjectType: + data = dict(self._data) + if embed: + for k in ['@context', 'signature']: + if k in data: + del(data[k]) + return self._to_dict(data) + + def get_actor(self) -> 'BaseActivity': + actor = self._data.get('actor') + if not actor: + if self.type_enum == ActivityTypes.NOTE: + actor = str(self._data.get('attributedTo')) + else: + raise ValueError('failed to fetch actor') + + actor_id = self._actor_id(actor) + return Person(**ACTOR_SERVICE.get(actor_id)) + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + raise NotImplementedError + + def _undo_outbox(self) -> None: + raise NotImplementedError + + def _process_from_inbox(self) -> None: + raise NotImplementedError + + def _undo_inbox(self) -> None: + raise NotImplementedError + + def process_from_inbox(self) -> None: + self.verify() + actor = self.get_actor() + + if DB.outbox.find_one({'type': ActivityTypes.BLOCK.value, + 'activity.object': actor.id, + 'meta.undo': False}): + print('actor is blocked, drop activity') + return + + if DB.inbox.find_one({'remote_id': self.id}): + # The activity is already in the inbox + print('received duplicate activity') + return + + activity = self.to_dict() + DB.inbox.insert_one({ + 'activity': activity, + 'type': self.type, + 'remote_id': self.id, + 'meta': {'undo': False, 'deleted': False}, + }) + + try: + self._process_from_inbox() + except NotImplementedError: + pass + + def post_to_outbox(self) -> None: + obj_id = random_object_id() + self.set_id(f'{ID}/outbox/{obj_id}', obj_id) + self.verify() + activity = self.to_dict() + DB.outbox.insert_one({ + 'id': obj_id, + 'activity': activity, + 'type': self.type, + 'remote_id': self.id, + 'meta': {'undo': False, 'deleted': False}, + }) + + recipients = self.recipients() + activity = clean_activity(activity) + + try: + self._post_to_outbox(obj_id, activity, recipients) + except NotImplementedError: + pass + + #return + generate_signature(activity, KEY.privkey) + payload = json.dumps(activity) + print('will post') + for recp in recipients: + self._post_to_inbox(payload, recp) + print('done') + + def _post_to_inbox(self, payload: str, to: str): + tasks.post_to_inbox.delay(payload, to) + + def _recipients(self) -> List[str]: + return [] + + def recipients(self) -> List[str]: + recipients = self._recipients() + + out = [] # type: List[str] + for recipient in recipients: + if recipient in PUBLIC_INSTANCES: + if recipient not in out: + out.append(str(recipient)) + continue + if recipient in [ME, AS_PUBLIC, None]: + continue + if isinstance(recipient, Person): + if recipient.id == ME: + continue + actor = recipient + else: + try: + actor = Person(**ACTOR_SERVICE.get(recipient)) + except NotAnActorError as error: + # Is the activity a `Collection`/`OrderedCollection`? + if error.activity and error.activity['type'] in [ActivityTypes.COLLECTION.value, + ActivityTypes.ORDERED_COLLECTION.value]: + for item in parse_collection(error.activity): + if item in [ME, AS_PUBLIC]: + continue + try: + col_actor = Person(**ACTOR_SERVICE.get(item)) + except NotAnActorError: + pass + + if col_actor.endpoints: + shared_inbox = col_actor.endpoints.get('sharedInbox') + if shared_inbox not in out: + out.append(shared_inbox) + continue + if col_actor.inbox and col_actor.inbox not in out: + out.append(col_actor.inbox) + + continue + + if actor.endpoints: + shared_inbox = actor.endpoints.get('sharedInbox') + if shared_inbox not in out: + out.append(shared_inbox) + continue + + if actor.inbox and actor.inbox not in out: + out.append(actor.inbox) + + return out + + def build_undo(self) -> 'BaseActivity': + raise NotImplementedError + + +class Person(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.PERSON + + def _init(self, **kwargs): + #if 'icon' in kwargs: + # self._data['icon'] = Image(**kwargs.pop('icon')) + pass + + def _verify(self) -> None: + ACTOR_SERVICE.get(self._data['id']) + + def _to_dict(self, data): + #if 'icon' in data: + # data['icon'] = data['icon'].to_dict() + # + return data + + +class Block(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.BLOCK + + +class Collection(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.COLLECTION + + +class Image(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.IMAGE + NO_CONTEXT = True + + def _init(self, **kwargs): + self._data.update( + url=kwargs.pop('url'), + ) + + def __repr__(self): + return 'Image({!r})'.format(self._data.get('url')) + + +class Follow(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.FOLLOW + ALLOWED_OBJECT_TYPES = [ActivityTypes.PERSON] + + def _build_reply(self, reply_type: ActivityTypes) -> BaseActivity: + if reply_type == ActivityTypes.ACCEPT: + return Accept( + object=self.to_dict(embed=True), + ) + + raise ValueError(f'type {reply_type} is invalid for building a reply') + + def _recipients(self) -> List[str]: + return [self.get_object().id] + + def _process_from_inbox(self) -> None: + accept = self.build_accept() + accept.post_to_outbox() + + remote_actor = self.get_actor().id + + if DB.followers.find({'remote_actor': remote_actor}).count() == 0: + DB.followers.insert_one({'remote_actor': remote_actor}) + + def _undo_inbox(self) -> None: + DB.followers.delete_one({'remote_actor': self.get_actor().id}) + + def build_accept(self) -> BaseActivity: + return self._build_reply(ActivityTypes.ACCEPT) + + def build_undo(self) -> BaseActivity: + return Undo(object=self.to_dict(embed=True)) + + +class Accept(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.ACCEPT + ALLOWED_OBJECT_TYPES = [ActivityTypes.FOLLOW] + + def _recipients(self) -> List[str]: + return [self.get_object().get_actor().id] + + def _process_from_inbox(self) -> None: + remote_actor = self.get_actor().id + if DB.following.find({'remote_actor': remote_actor}).count() == 0: + DB.following.insert_one({'remote_actor': remote_actor}) + + +class Undo(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.UNDO + ALLOWED_OBJECT_TYPES = [ActivityTypes.FOLLOW, ActivityTypes.LIKE, ActivityTypes.ANNOUNCE] + + def _recipients(self) -> List[str]: + obj = self.get_object() + if obj.type_enum == ActivityTypes.FOLLOW: + return [obj.get_object().id] + else: + return [obj.get_object().get_actor().id] + # TODO(tsileo): handle like and announce + raise Exception('TODO') + + def _process_from_inbox(self) -> None: + obj = self.get_object() + DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) + + try: + obj._undo_inbox() + except NotImplementedError: + pass + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + obj = self.get_object() + DB.outbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) + + try: + obj._undo_outbox() + except NotImplementedError: + pass + +class Like(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.LIKE + ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE] + + def _recipients(self) -> List[str]: + return [self.get_object().get_actor().id] + + def _process_from_inbox(self): + obj = self.get_object() + # Update the meta counter if the object is published by the server + DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': 1}}) + + def _undo_inbox(self) -> None: + obj = self.get_object() + # Update the meta counter if the object is published by the server + DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': -1}}) + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]): + obj = self.get_object() + # Unlikely, but an actor can like it's own post + DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': 1}}) + + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': obj_id}}) + + def _undo_outbox(self) -> None: + obj = self.get_object() + # Unlikely, but an actor can like it's own post + DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': -1}}) + + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': False}}) + + def build_undo(self) -> BaseActivity: + return Undo(object=self.to_dict(embed=True)) + + +class Announce(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.ANNOUNCE + ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE] + + def _recipients(self) -> List[str]: + recipients = [] + + for field in ['to', 'cc']: + if field in self._data: + recipients.extend(_to_list(self._data[field])) + + return recipients + + def _process_from_inbox(self) -> None: + if isinstance(self._data['object'], str) and not self._data['object'].startswith('http'): + # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else + print(f'received an Annouce referencing an OStatus notice ({self._data["object"]}), dropping the message') + return + # Save/cache the object, and make it part of the stream so we can fetch it + if isinstance(self._data['object'], str): + raw_obj = OBJECT_SERVICE.get(self._data['object'], reload_cache=True, part_of_stream=True, announce_published=self._data['published']) + obj = parse_activity(raw_obj) + else: + obj = self.get_object() + DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_boost': 1}}) + + def _undo_inbox(self) -> None: + obj = self.get_object() + DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) + DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_boost': -1}}) + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + if isinstance(self._data['object'], str): + # Put the object in the cache + OBJECT_SERVICE.get(self._data['object'], reload_cache=True, part_of_stream=True, announce_published=self._data['published']) + + obj = self.get_object() + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': obj_id}}) + + def _undo_outbox(self) -> None: + obj = self.get_object() + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': False}}) + + def build_undo(self) -> BaseActivity: + return Undo(object=self.to_dict(embed=True)) + + +class Delete(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.DELETE + ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE, ActivityTypes.TOMBSTONE] + + def _recipients(self) -> List[str]: + return self.get_object().recipients() + + def _process_from_inbox(self): + DB.inbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) + # TODO(tsileo): also delete copies stored in parents' `meta.replies` + + def _post_to_outbox(self, obj_id, activity, recipients): + DB.outbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) + + +class Update(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.UPDATE + ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE, ActivityTypes.PERSON] + + # TODO(tsileo): ensure the actor updating is the same as the orinial activity + # (ensuring that the Update and its object are of same origin) + + def _process_from_inbox(self): + obj = self.get_object() + if obj.type_enum == ActivityTypes.NOTE: + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'activity.object': obj.to_dict()}}) + return + + # If the object is a Person, it means the profile was updated, we just refresh our local cache + ACTOR_SERVICE.get(obj.id, reload_cache=True) + + def _post_to_outbox(self, obj_id, activity, recipients): + obj = self.get_object() + + update_prefix = 'activity.object.' + update = {'$set': dict(), '$unset': dict()} + update['$set'][f'{update_prefix}updated'] = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' + for k, v in obj._data.items(): + if k in ['id', 'type']: + continue + if v is None: + update['$unset'][f'{update_prefix}{k}'] = '' + else: + update['$set'][f'{update_prefix}{k}'] = v + + if len(update['$unset']) == 0: + del(update['$unset']) + + DB.outbox.update_one({'remote_id': obj.id.replace('/activity', '')}, update) + # FIXME(tsileo): should send an Update (but not a partial one, to all the note's recipients + # (create a new Update with the result of the update, and send it without saving it?) + + +class Create(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.CREATE + ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE] + + def _set_id(self, uri, obj_id): + self._data['object']['id'] = uri + '/activity' + self._data['object']['url'] = ID + '/' + self.get_object().type.lower() + '/' + obj_id + + def _init(self, **kwargs): + obj = self.get_object() + if not obj.attributedTo: + self._data['object']['attributedTo'] = self.get_actor().id + if not obj.published: + if self.published: + self._data['object']['published'] = self.published + else: + now = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' + self._data['published'] = now + self._data['object']['published'] = now + + def _recipients(self) -> List[str]: + # TODO(tsileo): audience support? + recipients = [] + for field in ['to', 'cc', 'bto', 'bcc']: + if field in self._data: + recipients.extend(_to_list(self._data[field])) + + recipients.extend(self.get_object()._recipients()) + + return recipients + + def _process_from_inbox(self): + obj = self.get_object() + + tasks.fetch_og.delay('INBOX', self.id) + + in_reply_to = obj.inReplyTo + if in_reply_to: + parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) + if not parent: + DB.outbox.update_one( + {'activity.object.id': in_reply_to}, + {'$inc': {'meta.count_reply': 1}}, + ) + return + + # If the note is a "reply of a reply" update the parent message + # TODO(tsileo): review this code + while parent: + DB.inbox.update_one({'_id': parent['_id']}, {'$push': {'meta.replies': self.to_dict()}}) + in_reply_to = parent.get('activity', {}).get('object', {}).get('inReplyTo') + if in_reply_to: + parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) + if parent is None: + # The reply is a note from the outbox + data = DB.outbox.update_one( + {'activity.object.id': in_reply_to}, + {'$inc': {'meta.count_reply': 1}}, + ) + else: + parent = None + + +class Tombstone(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.TOMBSTONE + + +class Note(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.NOTE + + def _init(self, **kwargs): + print(self._data) + # Remove the `actor` field as `attributedTo` is used for `Note` instead + if 'actor' in self._data: + del(self._data['actor']) + # FIXME(tsileo): use kwarg + # TODO(tsileo): support mention tag + # TODO(tisleo): implement the tag endpoint + if 'sensitive' not in kwargs: + self._data['sensitive'] = False + + # FIXME(tsileo): add the tag in CC + # for t in kwargs.get('tag', []): + # if t['type'] == 'Mention': + # cc -> c['href'] + + def _recipients(self) -> List[str]: + # TODO(tsileo): audience support? + recipients = [] # type: List[str] + + # If the note is public, we publish it to the defined "public instances" + if AS_PUBLIC in self._data.get('to', []): + recipients.extend(PUBLIC_INSTANCES) + print('publishing to public instances') + print(recipients) + + for field in ['to', 'cc', 'bto', 'bcc']: + if field in self._data: + recipients.extend(_to_list(self._data[field])) + + return recipients + + def build_create(self) -> BaseActivity: + """Wraps an activity in a Create activity.""" + create_payload = { + 'object': self.to_dict(embed=True), + 'actor': self.attributedTo or ME, + } + for field in ['published', 'to', 'bto', 'cc', 'bcc', 'audience']: + if field in self._data: + create_payload[field] = self._data[field] + + return Create(**create_payload) + + def build_like(self) -> BaseActivity: + return Like(object=self.id) + + def build_announce(self) -> BaseActivity: + return Announce( + object=self.id, + to=[AS_PUBLIC], + cc=[ID+'/followers', self.attributedTo], + published=datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', + ) + +_ACTIVITY_TYPE_TO_CLS = { + ActivityTypes.IMAGE: Image, + ActivityTypes.PERSON: Person, + ActivityTypes.FOLLOW: Follow, + ActivityTypes.ACCEPT: Accept, + ActivityTypes.UNDO: Undo, + ActivityTypes.LIKE: Like, + ActivityTypes.ANNOUNCE: Announce, + ActivityTypes.UPDATE: Update, + ActivityTypes.DELETE: Delete, + ActivityTypes.CREATE: Create, + ActivityTypes.NOTE: Note, + ActivityTypes.BLOCK: Block, + ActivityTypes.COLLECTION: Collection, + ActivityTypes.TOMBSTONE: Tombstone, +} + +def parse_activity(payload: ObjectType) -> BaseActivity: + t = ActivityTypes(payload['type']) + if t not in _ACTIVITY_TYPE_TO_CLS: + raise ValueError('unsupported activity type') + + return _ACTIVITY_TYPE_TO_CLS[t](**payload) + + +def gen_feed(): + fg = FeedGenerator() + fg.id(f'{ID}') + fg.title(f'{USERNAME} notes') + fg.author( {'name': USERNAME,'email':'t@a4.io'} ) + fg.link(href=ID, rel='alternate') + fg.description(f'{USERNAME} notes') + fg.logo(ME.get('icon', {}).get('url')) + #fg.link( href='http://larskiesow.de/test.atom', rel='self' ) + fg.language('en') + for item in DB.outbox.find({'type': 'Create'}, limit=50): + fe = fg.add_entry() + fe.id(item['activity']['object'].get('url')) + fe.link(href=item['activity']['object'].get('url')) + fe.title(item['activity']['object']['content']) + fe.description(item['activity']['object']['content']) + return fg + + +def json_feed(path: str) -> Dict[str, Any]: + """JSON Feed (https://jsonfeed.org/) document.""" + data = [] + for item in DB.outbox.find({'type': 'Create'}, limit=50): + data.append({ + "id": item["id"], + "url": item['activity']['object'].get('url'), + "content_html": item['activity']['object']['content'], + "content_text": html2text(item['activity']['object']['content']), + "date_published": item['activity']['object'].get('published'), + }) + return { + "version": "https://jsonfeed.org/version/1", + "user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: " + ID + path, + "title": USERNAME, + "home_page_url": ID, + "feed_url": ID + path, + "author": { + "name": USERNAME, + "url": ID, + "avatar": ME.get('icon', {}).get('url'), + }, + "items": data, + } + + +def build_inbox_json_feed(path: str, request_cursor: Optional[str] = None) -> Dict[str, Any]: + data = [] + cursor = None + + q = {'type': 'Create'} # type: Dict[str, Any] + if request_cursor: + q['_id'] = {'$lt': request_cursor} + + for item in DB.inbox.find(q, limit=50).sort('_id', -1): + actor = ACTOR_SERVICE.get(item['activity']['actor']) + data.append({ + "id": item["activity"]["id"], + "url": item['activity']['object'].get('url'), + "content_html": item['activity']['object']['content'], + "content_text": html2text(item['activity']['object']['content']), + "date_published": item['activity']['object'].get('published'), + "author": { + "name": actor.get('name', actor.get('preferredUsername')), + "url": actor.get('url'), + 'avatar': actor.get('icon', {}).get('url'), + }, + }) + cursor = str(item['_id']) + + resp = { + "version": "https://jsonfeed.org/version/1", + "title": f'{USERNAME}\'s stream', + "home_page_url": ID, + "feed_url": ID + path, + "items": data, + } + if cursor and len(data) == 50: + resp['next_url'] = ID + path + '?cursor=' + cursor + + return resp + + +def parse_collection(payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None) -> List[str]: + """Resolve/fetch a `Collection`/`OrderedCollection`.""" + # Resolve internal collections via MongoDB directly + if url == ID + '/followers': + return [doc['remote_actor'] for doc in DB.followers.find()] + elif url == ID + '/following': + return [doc['remote_actor'] for doc in DB.following.find()] + + # Go through all the pages + out = [] # type: List[str] + if url: + resp = requests.get(url, headers={'Accept': 'application/activity+json'}) + resp.raise_for_status() + payload = resp.json() + + if not payload: + raise ValueError('must at least prove a payload or an URL') + + if payload['type'] in ['Collection', 'OrderedCollection']: + if 'orderedItems' in payload: + return payload['orderedItems'] + if 'items' in payload: + return payload['items'] + if 'first' in payload: + if 'orderedItems' in payload['first']: + out.extend(payload['first']['orderedItems']) + if 'items' in payload['first']: + out.extend(payload['first']['items']) + n = payload['first'].get('next') + if n: + out.extend(parse_collection(url=n)) + return out + + while payload: + if payload['type'] in ['CollectionPage', 'OrderedCollectionPage']: + if 'orderedItems' in payload: + out.extend(payload['orderedItems']) + if 'items' in payload: + out.extend(payload['items']) + n = payload.get('next') + if n is None: + break + resp = requests.get(n, headers={'Accept': 'application/activity+json'}) + resp.raise_for_status() + payload = resp.json() + else: + raise Exception('unexpected activity type {}'.format(payload['type'])) + + return out + + +def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, col_name=None): + col_name = col_name or col.name + if q is None: + q = {} + + if cursor: + q['_id'] = {'$lt': ObjectId(cursor)} + data = list(col.find(q, limit=limit).sort('_id', -1)) + + if not data: + return { + 'id': BASE_URL + '/' + col_name, + 'totalItems': 0, + 'type': 'OrderedCollection', + 'orederedItems': [], + } + + start_cursor = str(data[0]['_id']) + next_page_cursor = str(data[-1]['_id']) + total_items = col.find(q).count() + + data = [_remove_id(doc) for doc in data] + if map_func: + data = [map_func(doc) for doc in data] + + # No cursor, this is the first page and we return an OrderedCollection + if not cursor: + resp = { + '@context': CTX_AS, + 'first': { + 'id': BASE_URL + '/' + col_name + '?cursor=' + start_cursor, + 'orderedItems': data, + 'partOf': BASE_URL + '/' + col_name, + 'totalItems': total_items, + 'type': 'OrderedCollectionPage' + }, + 'id': BASE_URL + '/' + col_name, + 'totalItems': total_items, + 'type': 'OrderedCollection' + } + + if len(data) == limit: + resp['first']['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor + + return resp + + # If there's a cursor, then we return an OrderedCollectionPage + resp = { + '@context': CTX_AS, + 'type': 'OrderedCollectionPage', + 'id': BASE_URL + '/' + col_name + '?cursor=' + start_cursor, + 'totalItems': total_items, + 'partOf': BASE_URL + '/' + col_name, + 'orderedItems': data, + } + if len(data) == limit: + resp['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor + + return resp diff --git a/app.py b/app.py new file mode 100644 index 0000000..034f1ca --- /dev/null +++ b/app.py @@ -0,0 +1,1014 @@ +import binascii +import hashlib +import json +import urllib +import os +import mimetypes +import logging +from functools import wraps +from datetime import datetime + +import timeago +import bleach +import mf2py +import pymongo +import piexif +from bson.objectid import ObjectId +from flask import Flask +from flask import abort +from flask import request +from flask import redirect +from flask import Response +from flask import render_template +from flask import session +from flask import url_for +from html2text import html2text +from itsdangerous import JSONWebSignatureSerializer +from itsdangerous import BadSignature +from passlib.hash import bcrypt +from u2flib_server import u2f +from urllib.parse import urlparse, urlencode +from werkzeug.utils import secure_filename + +import activitypub +import config +from activitypub import ActivityTypes +from activitypub import clean_activity +from activitypub import parse_markdown +from config import KEY +from config import DB +from config import ME +from config import ID +from config import DOMAIN +from config import USERNAME +from config import BASE_URL +from config import ACTOR_SERVICE +from config import OBJECT_SERVICE +from config import PASS +from config import HEADERS +from utils.httpsig import HTTPSigAuth, verify_request +from utils.key import get_secret_key +from utils.webfinger import get_remote_follow_template +from utils.webfinger import get_actor_url + + +app = Flask(__name__) +app.secret_key = get_secret_key('flask') + +JWT_SECRET = get_secret_key('jwt') +JWT = JSONWebSignatureSerializer(JWT_SECRET) + +with open('config/jwt_token', 'wb+') as f: + f.write(JWT.dumps({'type': 'admin_token'})) + +SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) + + +def verify_pass(pwd): + return bcrypt.verify(pwd, PASS) + +@app.context_processor +def inject_config(): + return dict(config=config, logged_in=session.get('logged_in', False)) + +@app.after_request +def set_x_powered_by(response): + response.headers['X-Powered-By'] = 'microblog.pub' + return response + +# HTML/templates helper +ALLOWED_TAGS = [ + 'a', + 'abbr', + 'acronym', + 'b', + 'blockquote', + 'code', + 'pre', + 'em', + 'i', + 'li', + 'ol', + 'strong', + 'ul', + 'span', + 'div', + 'p', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', +] + + +def clean_html(html): + return bleach.clean(html, tags=ALLOWED_TAGS) + + +@app.template_filter() +def quote_plus(t): + return urllib.parse.quote_plus(t) + + +@app.template_filter() +def clean(html): + return clean_html(html) + + +@app.template_filter() +def html2plaintext(body): + return html2text(body) + + +@app.template_filter() +def domain(url): + return urlparse(url).netloc + + +@app.template_filter() +def get_actor(url): + if not url: + return None + print(f'GET_ACTOR {url}') + return ACTOR_SERVICE.get(url) + +@app.template_filter() +def format_time(val): + if val: + return datetime.strftime(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), '%B %d, %Y, %H:%M %p') + return val + + +@app.template_filter() +def format_timeago(val): + if val: + try: + return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), datetime.utcnow()) + except: + return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%S.%fZ'), datetime.utcnow()) + + return val + +def _is_img(filename): + filename = filename.lower() + if (filename.endswith('.png') or filename.endswith('.jpg') or filename.endswith('.jpeg') or + filename.endswith('.gif') or filename.endswith('.svg')): + return True + return False + +@app.template_filter() +def not_only_imgs(attachment): + for a in attachment: + if not _is_img(a['url']): + return True + return False + +@app.template_filter() +def is_img(filename): + return _is_img(filename) + + +def login_required(f): + @wraps(f) + def decorated_function(*args, **kwargs): + if not session.get('logged_in'): + return redirect(url_for('login', next=request.url)) + return f(*args, **kwargs) + return decorated_function + + +def api_required(f): + @wraps(f) + def decorated_function(*args, **kwargs): + if session.get('logged_in'): + return f(*args, **kwargs) + + # Token verification + token = request.headers.get('Authorization', '').replace('Bearer ', '') + if not token: + token = request.form.get('access_token', '') + + try: + payload = JWT.loads(token) + # TODO(tsileo): log payload + except BadSignature: + abort(401) + return f(*args, **kwargs) + return decorated_function + + +def jsonify(**data): + if '@context' not in data: + data['@context'] = config.CTX_AS + return Response( + response=json.dumps(data), + headers={'Content-Type': 'application/json' if app.debug else 'application/activity+json'}, + ) + + +def is_api_request(): + h = request.headers.get('Accept') + if h is None: + return False + h = h.split(',')[0] + if h in HEADERS or h == 'application/json': + return True + return False + +# App routes + +####### +# Login + +@app.route('/logout') +@login_required +def logout(): + session['logged_in'] = False + return redirect('/') + + +@app.route('/login', methods=['POST', 'GET']) +def login(): + devices = [doc['device'] for doc in DB.u2f.find()] + u2f_enabled = True if devices else False + if request.method == 'POST': + pwd = request.form.get('pass') + if pwd and verify_pass(pwd): + if devices: + resp = json.loads(request.form.get('resp')) + print(resp) + try: + u2f.complete_authentication(session['challenge'], resp) + except ValueError as exc: + print('failed', exc) + abort(401) + return + finally: + session['challenge'] = None + + session['logged_in'] = True + return redirect(request.args.get('redirect') or '/admin') + else: + abort(401) + + payload = None + if devices: + payload = u2f.begin_authentication(ID, devices) + session['challenge'] = payload + + return render_template( + 'login.html', + u2f_enabled=u2f_enabled, + me=ME, + payload=payload, + ) + + +@app.route('/remote_follow', methods=['GET', 'POST']) +@login_required +def remote_follow(): + if request.method == 'GET': + return render_template('remote_follow.html') + + return redirect(get_remote_follow_template('@'+request.form.get('profile')).format(uri=ID)) + + +@app.route('/authorize_follow', methods=['GET', 'POST']) +@login_required +def authorize_follow(): + if request.method == 'GET': + return render_template('authorize_remote_follow.html', profile=request.args.get('profile')) + + actor = get_actor_url(request.form.get('profile')) + if not actor: + abort(500) + if DB.following.find({'remote_actor': actor}).count() > 0: + return redirect('/following') + + follow = activitypub.Follow(object=actor) + follow.post_to_outbox() + return redirect('/following') + + +@app.route('/u2f/register', methods=['GET', 'POST']) +@login_required +def u2f_register(): + # TODO(tsileo): ensure no duplicates + if request.method == 'GET': + payload = u2f.begin_registration(ID) + session['challenge'] = payload + return render_template( + 'u2f.html', + payload=payload, + ) + else: + resp = json.loads(request.form.get('resp')) + device, device_cert = u2f.complete_registration(session['challenge'], resp) + session['challenge'] = None + DB.u2f.insert_one({'device': device, 'cert': device_cert}) + return '' + +####### +# Activity pub routes + +@app.route('/') +def index(): + print(request.headers.get('Accept')) + if is_api_request(): + return jsonify(**ME) + + # FIXME(tsileo): implements pagination, also for the followers/following page + limit = 50 + q = { + 'type': 'Create', + 'activity.object.type': 'Note', + 'meta.deleted': False, + } + c = request.args.get('cursor') + if c: + q['_id'] = {'$lt': ObjectId(c)} + + outbox_data = list(DB.outbox.find({'$or': [q, {'type': 'Announce', 'meta.undo': False}]}, limit=limit).sort('_id', -1)) + cursor = None + if outbox_data and len(outbox_data) == limit: + cursor = str(outbox_data[-1]['_id']) + + for data in outbox_data: + if data['type'] == 'Announce': + print(data) + if data['activity']['object'].startswith('http'): + data['ref'] = {'activity': {'object': OBJECT_SERVICE.get(data['activity']['object'])}, 'meta': {}} + + + return render_template( + 'index.html', + me=ME, + notes=DB.inbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False}).count(), + followers=DB.followers.count(), + following=DB.following.count(), + outbox_data=outbox_data, + cursor=cursor, + ) + + +@app.route('/note/') +def note_by_id(note_id): + data = DB.outbox.find_one({'id': note_id, 'meta.deleted': False}) + if not data: + return Response(status=404) + + replies = list(DB.inbox.find({ + 'type': 'Create', + 'activity.object.inReplyTo': data['activity']['object']['id'], + 'meta.deleted': False, + })) + + # Check for "replies of replies" + others = [] + for rep in replies: + for rep_reply in rep.get('meta', {}).get('replies', []): + others.append(rep_reply['id']) + + if others: + # Fetch the latest versions of the "replies of replies" + replies2 = list(DB.inbox.find({ + 'activity.id': {'$in': others}, + })) + + replies.extend(replies2) + + replies2 = list(DB.outbox.find({ + 'activity.id': {'$in': others}, + })) + + replies.extend(replies2) + + + # Re-sort everything + replies = sorted(replies, key=lambda o: o['activity']['object']['published']) + + + return render_template('note.html', me=ME, note=data, replies=replies) + + +@app.route('/.well-known/webfinger') +def webfinger(): + """Enable WebFinger support, required for Mastodon interopability.""" + resource = request.args.get('resource') + if resource not in ["acct:"+USERNAME+"@"+DOMAIN, ID]: + abort(404) + + out = { + "subject": "acct:"+USERNAME+"@"+DOMAIN, + "aliases": [ID], + "links": [ + {"rel": "http://webfinger.net/rel/profile-page", "type": "text/html", "href": BASE_URL}, + {"rel": "self", "type": "application/activity+json", "href": ID}, + {"rel":"http://ostatus.org/schema/1.0/subscribe","template": BASE_URL+"/authorize_follow?profile={uri}"}, + ], + } + + return Response( + response=json.dumps(out), + headers={'Content-Type': 'application/jrd+json; charset=utf-8' if not app.debug else 'application/json'}, + ) + +@app.route('/outbox', methods=['GET', 'POST']) +def outbox(): + if request.method == 'GET': + if not is_api_request(): + abort(404) + # TODO(tsileo): filter the outbox if not authenticated + # FIXME(tsileo): filter deleted, add query support for build_ordered_collection + q = { + 'meta.deleted': False, + 'type': {'$in': [ActivityTypes.CREATE.value, ActivityTypes.ANNOUNCE.value]}, + } + return jsonify(**activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get('cursor'), + map_func=lambda doc: clean_activity(doc['activity']), + )) + + # Handle POST request + # FIXME(tsileo): check auth + data = request.get_json(force=True) + print(data) + activity = activitypub.parse_activity(data) + + if activity.type_enum == ActivityTypes.NOTE: + activity = activity.build_create() + + activity.post_to_outbox() + + return Response(status=201, headers={'Location': activity.id}) + + +@app.route('/outbox/') +def outbox_detail(item_id): + doc = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + return jsonify(**clean_activity(doc['activity'])) + + +@app.route('/outbox//activity') +def outbox_activity(item_id): + data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + if not data: + abort(404) + obj = data['activity'] + if obj['type'] != ActivityTypes.CREATE.value: + abort(404) + return jsonify(**clean_activity(obj['object'])) + +@app.route('/admin', methods=['GET']) +@login_required +def admin(): + q = { + 'meta.deleted': False, + 'meta.undo': False, + 'type': ActivityTypes.LIKE.value, + } + col_liked = DB.outbox.count(q) + + return render_template( + 'admin.html', + instances=list(DB.instances.find()), + inbox_size=DB.inbox.count(), + outbox_size=DB.outbox.count(), + object_cache_size=DB.objects_cache.count(), + actor_cache_size=DB.actors_cache.count(), + col_liked=col_liked, + col_followers=DB.followers.count(), + col_following=DB.following.count(), + ) + + +@app.route('/new', methods=['GET', 'POST']) +@login_required +def new(): + if request.method == 'POST': + reply = None + if request.form.get('reply'): + reply = activitypub.parse_activity(OBJECT_SERVICE.get(request.form.get('reply'))) + source = request.form.get('content') + content, tags = parse_markdown(source) + to = request.form.get('to') + cc = [ID+'/followers'] + if reply: + cc.append(reply.attributedTo) + for tag in tags: + if tag['type'] == 'Mention': + cc.append(tag['href']) + + note = activitypub.Note( + cc=cc, + to=[to if to else config.AS_PUBLIC], + content=content, # TODO(tsileo): handle markdown + tag=tags, + source={'mediaType': 'text/markdown', 'content': source}, + inReplyTo=reply.id, + ) + create = note.build_create() + print(create.to_dict()) + create.post_to_outbox() + + reply_id = None + content = '' + if request.args.get('reply'): + reply = activitypub.parse_activity(OBJECT_SERVICE.get(request.args.get('reply'))) + reply_id = reply.id + actor = reply.get_actor() + domain = urlparse(actor.id).netloc + content = f'@{actor.preferredUsername}@{domain} ' + + return render_template('new.html', reply=reply_id, content=content) + + +@app.route('/notifications') +@login_required +def notifications(): + # FIXME(tsileo): implements pagination, also for the followers/following page + limit = 50 + q = { + 'type': 'Create', + 'activity.object.tag.type': 'Mention', + 'activity.object.tag.name': f'@{USERNAME}@{DOMAIN}', + 'meta.deleted': False, + } + # TODO(tsileo): also include replies via regex on Create replyTo + q = {'$or': [q, {'type': 'Follow'}, {'type': 'Accept'}, {'type': 'Undo', 'activity.object.type': 'Follow'}, + {'type': 'Announce', 'activity.object': {'$regex': f'^{BASE_URL}'}}, + {'type': 'Create', 'activity.object.inReplyTo': {'$regex': f'^{BASE_URL}'}}, + ]} + print(q) + c = request.args.get('cursor') + if c: + q['_id'] = {'$lt': ObjectId(c)} + + outbox_data = list(DB.inbox.find(q, limit=limit).sort('_id', -1)) + cursor = None + if outbox_data and len(outbox_data) == limit: + cursor = str(outbox_data[-1]['_id']) + + # TODO(tsileo): fix the annonce handling, copy it from /stream + #for data in outbox_data: + # if data['type'] == 'Announce': + # print(data) + # if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: + # data['ref'] = {'activity': {'object': objcache[data['activity']['object']]}, 'meta': {}} + # out.append(data) + # else: + # out.append(data) + + return render_template( + 'stream.html', + inbox_data=outbox_data, + cursor=cursor, + ) + +@app.route('/ui/boost') +@login_required +def ui_boost(): + oid = request.args.get('id') + obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid)) + announce = obj.build_announce() + announce.post_to_outbox() + return redirect(request.args.get('redirect')) + +@app.route('/ui/like') +@login_required +def ui_like(): + oid = request.args.get('id') + obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid)) + like = obj.build_like() + like.post_to_outbox() + return redirect(request.args.get('redirect')) + +@app.route('/ui/undo') +@login_required +def ui_undo(): + oid = request.args.get('id') + doc =DB.outbox.find_one({'id': oid}) + if doc: + obj = activitypub.parse_activity(doc.get('activity')) + undo = obj.build_undo() + undo.post_to_outbox() + return redirect(request.args.get('redirect')) + +@app.route('/stream') +@login_required +def stream(): + # FIXME(tsileo): implements pagination, also for the followers/following page + limit = 100 + q = { + 'type': 'Create', + 'activity.object.type': 'Note', + 'activity.object.inReplyTo': None, + 'meta.deleted': False, + } + c = request.args.get('cursor') + if c: + q['_id'] = {'$lt': ObjectId(c)} + + outbox_data = list(DB.inbox.find( + { + '$or': [ + q, + { + 'type': 'Announce', + }, + ] + }, limit=limit).sort('activity.published', -1)) + cursor = None + if outbox_data and len(outbox_data) == limit: + cursor = str(outbox_data[-1]['_id']) + + out = [] + objcache = {} + cached = list(DB.objects_cache.find({'meta.part_of_stream': True}, limit=limit*3).sort('meta.announce_published', -1)) + for c in cached: + objcache[c['object_id']] = c['cached_object'] + for data in outbox_data: + if data['type'] == 'Announce': + if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: + data['ref'] = {'activity': {'object': objcache[data['activity']['object']]}, 'meta': {}} + out.append(data) + else: + print('OMG', data) + else: + out.append(data) + return render_template( + 'stream.html', + inbox_data=out, + cursor=cursor, + ) + + +@app.route('/inbox', methods=['GET', 'POST']) +def inbox(): + if request.method == 'GET': + if not is_api_request(): + abort(404) + # TODO(tsileo): handle auth and only return 404 if unauthenticated + # abort(404) + return jsonify(**activitypub.build_ordered_collection( + DB.inbox, + q={'meta.deleted': False}, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['activity'], + )) + data = request.get_json(force=True) + # FIXME(tsileo): ensure verify_request() == True + print(data) + try: + print(verify_request(ACTOR_SERVICE)) + except Exception: + print('failed to verify request') + + activity = activitypub.parse_activity(data) + print(activity) + activity.process_from_inbox() + + return Response( + status=201, + ) + + +@app.route('/api/upload', methods=['POST']) +@api_required +def api_upload(): + file = request.files['file'] + rfilename = secure_filename(file.filename) + prefix = hashlib.sha256(os.urandom(32)).hexdigest()[:6] + mtype = mimetypes.guess_type(rfilename)[0] + filename = f'{prefix}_{rfilename}' + file.save(os.path.join('static', 'media', filename)) + + # Remove EXIF metadata + if filename.lower().endswith('.jpg') or filename.lower().endswith('.jpeg'): + piexif.remove(os.path.join('static', 'media', filename)) + + print('upload OK') + print(filename) + attachment = [ + {'mediaType': mtype, + 'name': rfilename, + 'type': 'Document', + 'url': BASE_URL + f'/static/media/{filename}' + }, + ] + print(attachment) + content = request.args.get('content') + to = request.args.get('to') + note = activitypub.Note( + cc=[ID+'/followers'], + to=[to if to else config.AS_PUBLIC], + content=content, # TODO(tsileo): handle markdown + attachment=attachment, + ) + print('post_note_init') + print(note) + create = note.build_create() + print(create) + print(create.to_dict()) + create.post_to_outbox() + print('posted') + + return Response( + status=201, + response='OK', + ) + + +@app.route('/api/new_note') +@api_required +def api_new_note(): + source = request.args.get('content') + content, tags = parse_markdown(source) + to = request.args.get('to') + cc = [ID+'/followers'] + for tag in tags: + if tag['type'] == 'Mention': + cc.append(tag['href']) + + note = activitypub.Note( + cc=cc, + to=[to if to else config.AS_PUBLIC], + content=content, # TODO(tsileo): handle markdown + tag=tags, + source={'mediaType': 'text/markdown', 'content': source}, + ) + create = note.build_create() + create.post_to_outbox() + return Response( + status=201, + response='OK', + ) + +@app.route('/api/stream') +def api_stream(): + return Response( + response=json.dumps(activitypub.build_inbox_json_feed('/api/stream', request.args.get('cursor'))), + headers={'Content-Type': 'application/json'}, + ) + +@app.route('/api/follow') +@api_required +def api_follow(): + actor = request.args.get('actor') + if DB.following.find({'remote_actor': actor}).count() > 0: + return Response(status=201) + + follow = activitypub.Follow(object=actor) + follow.post_to_outbox() + return Response( + status=201, + ) + + +@app.route('/followers') +def followers(): + if is_api_request(): + return jsonify( + **activitypub.build_ordered_collection( + DB.followers, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['remote_actor'], + ) + ) + + followers = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.followers.find(limit=50)] + return render_template( + 'followers.html', + me=ME, + notes=DB.inbox.find({'object.object.type': 'Note'}).count(), + followers=DB.followers.count(), + following=DB.following.count(), + followers_data=followers, + ) + + +@app.route('/following') +def following(): + if is_api_request(): + return jsonify( + **activitypub.build_ordered_collection( + DB.following, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['remote_actor'], + ), + ) + + following = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.following.find(limit=50)] + return render_template( + 'following.html', + me=ME, + notes=DB.inbox.find({'object.object.type': 'Note'}).count(), + followers=DB.followers.count(), + following=DB.following.count(), + following_data=following, + ) + + +@app.route('/tags/') +def tags(tag): + if not DB.outbox.count({'activity.object.tag.type': 'Hashtag', 'activity.object.tag.name': '#'+tag}): + abort(404) + if not is_api_request(): + return render_template( + 'tags.html', + tag=tag, + outbox_data=DB.outbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False, + 'activity.object.tag.type': 'Hashtag', + 'activity.object.tag.name': '#'+tag}), + ) + q = { + 'meta.deleted': False, + 'meta.undo': False, + 'type': ActivityTypes.CREATE.value, + 'activity.object.tag.type': 'Hashtag', + 'activity.object.tag.name': '#'+tag, + } + return jsonify(**activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['activity']['object']['id'], + col_name=f'tags/{tag}', + )) + + +@app.route('/liked') +def liked(): + if not is_api_request(): + abort(404) + q = { + 'meta.deleted': False, + 'meta.undo': False, + 'type': ActivityTypes.LIKE.value, + } + return jsonify(**activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['activity']['object'], + col_name='liked', + )) + +####### +# IndieAuth + + +def build_auth_resp(payload): + if request.headers.get('Accept') == 'application/json': + return Response( + status=200, + headers={'Content-Type': 'application/json'}, + response=json.dumps(payload), + ) + return Response( + status=200, + headers={'Content-Type': 'application/x-www-form-urlencoded'}, + response=urlencode(payload), + ) + + +def _get_prop(props, name, default=None): + if name in props: + items = props.get(name) + if isinstance(items, list): + return items[0] + return items + return default + +def get_client_id_data(url): + data = mf2py.parse(url=url) + for item in data['items']: + if 'h-x-app' in item['type'] or 'h-app' in item['type']: + props = item.get('properties', {}) + print(props) + return dict( + logo=_get_prop(props, 'logo'), + name=_get_prop(props, 'name'), + url=_get_prop(props, 'url'), + ) + return dict( + logo=None, + name=url, + url=url, + ) + + +@app.route('/indieauth/flow', methods=['POST']) +@login_required +def indieauth_flow(): + auth = dict( + scope=' '.join(request.form.getlist('scopes')), + me=request.form.get('me'), + client_id=request.form.get('client_id'), + state=request.form.get('state'), + redirect_uri=request.form.get('redirect_uri'), + response_type=request.form.get('response_type'), + ) + + code = binascii.hexlify(os.urandom(8)).decode('utf-8') + auth.update( + code=code, + verified=False, + ) + print(auth) + if not auth['redirect_uri']: + abort(500) + + DB.indieauth.insert_one(auth) + + # FIXME(tsileo): fetch client ID and validate redirect_uri + red = f'{auth["redirect_uri"]}?code={code}&state={auth["state"]}&me={auth["me"]}' + return redirect(red) + + +@app.route('/indieauth', methods=['GET', 'POST']) +def indieauth_endpoint(): + session['logged_in'] = True + if request.method == 'GET': + if not session.get('logged_in'): + return redirect(url_for('login', next=request.url)) + + me = request.args.get('me') + # FIXME(tsileo): ensure me == ID + client_id = request.args.get('client_id') + redirect_uri = request.args.get('redirect_uri') + state = request.args.get('state', '') + response_type = request.args.get('response_type', 'id') + scope = request.args.get('scope', '').split() + + print('STATE', state) + return render_template( + 'indieauth_flow.html', + client=get_client_id_data(client_id), + scopes=scope, + redirect_uri=redirect_uri, + state=state, + response_type=response_type, + client_id=client_id, + me=me, + ) + + # Auth verification via POST + code = request.form.get('code') + redirect_uri = request.form.get('redirect_uri') + client_id = request.form.get('client_id') + + auth = DB.indieauth.find_one_and_update( + {'code': code, 'redirect_uri': redirect_uri, 'client_id': client_id}, #}, # , 'verified': False}, + {'$set': {'verified': True}}, + sort=[('_id', pymongo.DESCENDING)], + ) + print(auth) + print(code, redirect_uri, client_id) + + if not auth: + abort(403) + return + + me = auth['me'] + state = auth['state'] + scope = ' '.join(auth['scope']) + print('STATE', state) + return build_auth_resp({'me': me, 'state': state, 'scope': scope}) + + +@app.route('/token', methods=['GET', 'POST']) +def token_endpoint(): + if request.method == 'POST': + code = request.form.get('code') + me = request.form.get('me') + redirect_uri = request.form.get('redirect_uri') + client_id = request.form.get('client_id') + + auth = DB.indieauth.find_one({'code': code, 'me': me, 'redirect_uri': redirect_uri, 'client_id': client_id}) + if not auth: + abort(403) + scope = ' '.join(auth['scope']) + payload = dict(me=me, client_id=client_id, scope=scope, ts=datetime.now().timestamp()) + token = JWT.dumps(payload).decode('utf-8') + + return build_auth_resp({'me': me, 'scope': scope, 'access_token': token}) + + # Token verification + token = request.headers.get('Authorization').replace('Bearer ', '') + try: + payload = JWT.loads(token) + except BadSignature: + abort(403) + + # TODO(tsileo): handle expiration + + return build_auth_resp({ + 'me': payload['me'], + 'scope': payload['scope'], + 'client_id': payload['client_id'], + }) diff --git a/config.py b/config.py new file mode 100644 index 0000000..b8ff84f --- /dev/null +++ b/config.py @@ -0,0 +1,83 @@ +import os +import yaml +from pymongo import MongoClient +import requests + +from utils.key import Key +from utils.actor_service import ActorService +from utils.object_service import ObjectService + + +VERSION = '1.0.0' + +CTX_AS = 'https://www.w3.org/ns/activitystreams' +CTX_SECURITY = 'https://w3id.org/security/v1' +AS_PUBLIC = 'https://www.w3.org/ns/activitystreams#Public' +HEADERS = [ + 'application/activity+json', + 'application/ld+json;profile=https://www.w3.org/ns/activitystreams', + 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + 'application/ld+json', +] + + +with open('config/me.yml') as f: + conf = yaml.load(f) + + USERNAME = conf['username'] + NAME = conf['name'] + DOMAIN = conf['domain'] + SCHEME = 'https' if conf.get('https', True) else 'http' + BASE_URL = SCHEME + '://' + DOMAIN + ID = BASE_URL + SUMMARY = conf['summary'] + ICON_URL = conf['icon_url'] + PASS = conf['pass'] + PUBLIC_INSTANCES = conf.get('public_instances') + +USER_AGENT = ( + f'{requests.utils.default_user_agent()} ' + f'(microblog.pub/{VERSION}; +{BASE_URL})' +) + +# TODO(tsileo): use 'mongo:27017; +# mongo_client = MongoClient(host=['mongo:27017']) +mongo_client = MongoClient( + host=[os.getenv('MICROBLOGPUB_MONGODB_HOST', 'localhost:27017')], +) + +DB = mongo_client['{}_{}'.format(USERNAME, DOMAIN.replace('.', '_'))] +KEY = Key(USERNAME, DOMAIN, create=True) + +ME = { + "@context": [ + CTX_AS, + CTX_SECURITY, + ], + "type": "Person", + "id": ID, + "following": ID+"/following", + "followers": ID+"/followers", + "liked": ID+"/liked", + "inbox": ID+"/inbox", + "outbox": ID+"/outbox", + "preferredUsername": USERNAME, + "name": NAME, + "summary": SUMMARY, + "endpoints": {}, + "url": ID, + "icon": { + "mediaType": "image/png", + "type": "Image", + "url": ICON_URL, + }, + "publicKey": { + "id": ID+"#main-key", + "owner": ID, + "publicKeyPem": KEY.pubkey_pem, + }, +} +print(ME) + +ACTOR_SERVICE = ActorService(USER_AGENT, DB.actors_cache, ID, ME, DB.instances) +OBJECT_SERVICE = ObjectService(USER_AGENT, DB.objects_cache, DB.inbox, DB.outbox, DB.instances) diff --git a/config/me.sample.yml b/config/me.sample.yml new file mode 100644 index 0000000..56e37b0 --- /dev/null +++ b/config/me.sample.yml @@ -0,0 +1,6 @@ +username: 'username' +name: 'You Name' +icon_url: 'https://you-avatar-url' +domain: 'your-domain.tld' +summary: 'your summary' +https: true diff --git a/data/.gitignore b/data/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 0000000..0967722 --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,21 @@ +version: '3' +services: + celery: + build: . + links: + - mongo + - rabbitmq + command: 'celery worker -l info -A tasks' + environment: + - MICROBLOGPUB_AMQP_BORKER=pyamqp://guest@rabbitmq// + - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + mongo: + image: "mongo:latest" + volumes: + - "./data:/data/db" + ports: + - "27017:27017" + rabbitmq: + image: "rabbitmq:latest" + ports: + - "5672:5672" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..50516aa --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +version: '3' +services: + web: + build: . + ports: + - "5005:5005" + links: + - mongo + - rabbitmq + volumes: + - "./config:/app/config" + - "./static:/app/static" + environment: + - MICROBLOGPUB_AMQP_BORKER=pyamqp://guest@rabbitmq// + - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + celery: + build: . + links: + - mongo + - rabbitmq + command: 'celery worker -l info -A tasks' + environment: + - MICROBLOGPUB_AMQP_BORKER=pyamqp://guest@rabbitmq// + - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + mongo: + image: "mongo:latest" + volumes: + - "./data:/data/db" + rabbitmq: + image: "rabbitmq:latest" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3e770ad --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +libsass +gunicorn +piexif +requests +markdown +python-u2flib-server +Flask +Celery +pymongo +pyld +timeago +bleach +pycryptodome +html2text +feedgen +itsdangerous +bcrypt +mf2py +passlib +pyyaml +git+https://github.com/erikriver/opengraph.git diff --git a/sass/base_theme.scss b/sass/base_theme.scss new file mode 100644 index 0000000..8a6fba0 --- /dev/null +++ b/sass/base_theme.scss @@ -0,0 +1,207 @@ +// a4.io dark theme +$background-color: #060606; +$background-light: #222; +$color: #808080; +$color-title-link: #fefefe; +$color-summary: #ddd; +$color-light: #bbb; +$color-menu-background: #222; +$color-note-link: #666; +$primary-color: #f7ca18; + +$background-color: #eee; +$background-light: #ccc; +$color: #111; +$color-title-link: #333; +$color-light: #555; +$color-summary: #111; +$color-note-link: #333; +$color-menu-background: #ddd; +$primary-color: #1d781d; + +.note-container p:first-child { + margin-top: 0; +} +html, body { + height: 100%; +} +body { + background-color: $background-color; + color: $color; + display: flex; + flex-direction: column; +} +.base-container { + flex: 1 0 auto; +} +.footer { + flex-shrink: 0; +} +a, h1, h2, h3, h4, h5, h6 { + color: $color-title-link; +} +a { + text-decoration: none; +} +a:hover { + text-decoration: underline; +} +.gold { + color: $primary-color; +} + +#header { + margin-bottom: 40px; + + .title { + font-size: 1.2em; + padding-right: 15px; + color: $color-title-link; + } + .title:hover { + text-decoration: none; + } + .subtitle-username { + color: $color; + } + .menu { + padding: 20px 0 10px 0; + ul { + display: inline; + list-style-type: none; + padding: 0; + li { + float:left; + padding-right:10px; + margin-bottom:10px; + } + } + a { + padding: 2px 7px; + } + a.selected { + background: $primary-color; + color: $background-color; + border-radius:2px; + } + a:hover { + background: $primary-color; + color: $background-color; + text-decoration: none; + } + } +} +#container { + width: 90%; + max-width: 720px; + margin: 40px auto; +} +#container #notes { + margin-top: 20px; +} +.actor-box { + display: block; + text-decoration: none; + margin-bottom: 40px; + + .actor-icon { + width: 100%; + max-width:120px; + border-radius:2px; + } + h3 { margin: 0; } +} +.note { + display: flex; + margin-bottom: 70px; + .l { + color: $color-note-link; + } + + .h-card { + flex: initial; + width: 50px; + } + + .u-photo { + width: 50px; + border-radius: 2px; + } + .note-wrapper { + flex: 1; + padding-left: 15px; + } + + .bottom-bar { margin-top:10px; } + + .img-attachment { + max-width:100%; + border-radius:2px; + } + + h3 { + font-size: 1.1em; + color: $color-light; + } + + strong { font-weight:600; } + + .note-container { + clear: right; + padding:10px 0; + } +} + +.bar-item { + background: $color-menu-background; + padding: 5px; + color: $color-light; + margin-right:5px; + border-radius:2px; +} + +.bottom-bar .perma-item { + margin-right:5px; +} +.bottom-bar a.bar-item:hover { + text-decoration: none; +} +.footer > div { + width: 90%; + max-width: 720px; + margin: 40px auto; +} +.footer a, .footer a:hover, .footer a:visited { + text-decoration: underline; + color: $color; +} +.summary { + color: $color-summary; + font-size:1.3em; + margin-top:50px; + margin-bottom:70px; +} +.summary a, .summay a:hover { + color: $color-summary; + text-decoration:underline; +} +#followers, #following, #new { + margin-top:50px; +} +#admin { + margin-top:50px; +} +textarea, input { + background: $color-menu-background; + padding: 10px; + color: $color-light; + border: 0px; + border-radius: 2px; +} +input { + padding: 10px; +} +input[type=submit] { + color: $primary-color; + text-transform: uppercase; +} diff --git a/sass/theme.scss b/sass/theme.scss new file mode 100644 index 0000000..6277839 --- /dev/null +++ b/sass/theme.scss @@ -0,0 +1 @@ +@import 'base_theme.scss' diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..6deafc2 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 120 diff --git a/static/base.css b/static/base.css new file mode 100644 index 0000000..aeab1f3 --- /dev/null +++ b/static/base.css @@ -0,0 +1,160 @@ +.note-container p:first-child { + margin-top: 0; +} +html, body { + height: 100%; +} +body { + background-color: #060606; + color: #808080; +display: flex; +flex-direction: column; +} +.base-container { + flex: 1 0 auto; +} +.footer { +flex-shrink: 0; +} +a, h1, h2, h3, h4, h5, h6 { + color: #fefefe; +} +a { + text-decoration: none; +} +a:hover { + text-decoration: underline; +} +.gold { + color: #f7ca18; +} + +#header { + margin-bottom: 40px; +} +#header .h-card { +} +#header .title { +font-size:1.2em;padding-right:15px;color:#fefefe; +} +#header .title:hover { +text-decoration: none; +} +#header .menu { +padding: 20px 0 10px 0; +} +#header .menu ul { +display:inline;list-style-type:none;padding:0; +} +#header .menu li { +float:left; +padding-right:10px; +margin-bottom:10px; +} +#header .menu a { + padding: 2px 7px; +} +#header .menu a.selected { +background:#f7ca18;color:#634806; +border-radius:2px; +} +#header .menu a:hover { +background:#f7ca18;color:#060606; +text-decoration: none; +} +#container { +width:90%;max-width: 720px;margin:40px auto; +} +#container #notes { +margin-top:20px; +} +.actor-box { +display:block;text-decoration:none;margin-bottom:40px; +} +.actor-box .actor-icon { +width: 100%; +max-width:120px; +border-radius:2px; +} +.actor-box h3 { margin:0; } +.note .l { +color:#666; +} +.note { +display:flex;margin-bottom:70px; +} +.note .h-card { +flex:initial;width:50px; +} +.note .u-photo { +width:50px;border-radius:2px; +} +.note .note-wrapper { +flex:1;padding-left:15px; +} +.note .bottom-bar { +margin-top:10px; +} +.bar-item { +background: #222; +padding: 5px; +color:#bbb; +margin-right:5px; +border-radius:2px; +} +.bottom-bar .perma-item { +margin-right:5px; +} +.bottom-bar a.bar-item:hover { + text-decoration: none; +} +.note .img-attachment { +max-width:100%; +border-radius:2px; +} +.note h3 { +font-size:1.1em;color:#ccc; +} +.note .note-container { +clear:right;padding:10px 0; +} +.note strong { +font-weight:600; +} +.footer > div { +width:90%;max-width: 720px;margin:40px auto; +} +.footer a, .footer a:hover, .footer a:visited { + text-decoration:underline; + color:#808080; +} +.summary { +color: #ddd; +font-size:1.3em; +margin-top:50px; +margin-bottom:70px; +} +.summary a, .summay a:hover { +color:#ddd; +text-decoration:underline; +} +#followers, #following, #new { + margin-top:50px; +} +#admin { + margin-top:50px; +} +textarea, input { + background: #222; + padding: 10px; + color: #bbb; + border: 0px; + border-radius: 2px; +} +input { + padding: 10px; +} +input[type=submit] { + color: #f7ca18; + text-transform: uppercase; +} diff --git a/static/css/.gitignore b/static/css/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/static/css/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/static/media/.gitignore b/static/media/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/static/media/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/static/nopic.png b/static/nopic.png new file mode 100644 index 0000000..988d806 Binary files /dev/null and b/static/nopic.png differ diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..7079491 --- /dev/null +++ b/tasks.py @@ -0,0 +1,56 @@ +import os +import logging +import random + +import requests +from celery import Celery +from requests.exceptions import HTTPError + +from config import HEADERS +from config import ID +from config import DB +from config import KEY +from config import USER_AGENT +from utils.httpsig import HTTPSigAuth +from utils.opengraph import fetch_og_metadata + + +log = logging.getLogger() +app = Celery('tasks', broker=os.getenv('MICROBLOGPUB_AMQP_BROKER', 'pyamqp://guest@localhost//')) +# app = Celery('tasks', broker='pyamqp://guest@rabbitmq//') +SigAuth = HTTPSigAuth(ID+'#main-key', KEY.privkey) + + +@app.task(bind=True, max_retries=12) +def post_to_inbox(self, payload, to): + try: + log.info('payload=%s', payload) + log.info('to=%s', to) + resp = requests.post(to, data=payload, auth=SigAuth, headers={ + 'Content-Type': HEADERS[1], + 'Accept': HEADERS[1], + 'User-Agent': USER_AGENT, + }) + print(resp) + log.info('resp=%s', resp) + log.info('resp_body=%s', resp.text) + resp.raise_for_status() + except HTTPError as err: + log.exception('request failed') + if 400 >= err.response.status_code >= 499: + log.info('client error, no retry') + return + self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) + + +@app.task(bind=True, max_retries=12) +def fetch_og(self, col, remote_id): + try: + log.info('fetch_og_meta remote_id=%s col=%s', remote_id, col) + if col == 'INBOX': + log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.inbox, remote_id)) + elif col == 'OUTBOX': + log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.outbox, remote_id)) + except Exception as err: + self.log.exception('failed') + self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) diff --git a/templates/about.html b/templates/about.html new file mode 100644 index 0000000..7625840 --- /dev/null +++ b/templates/about.html @@ -0,0 +1,22 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+ +{% include "header.html" %} + +
+
+ +
+ +
+{{ text | safe }} +
{{ me.summary }}
+
+
+ +
+{% endblock %} diff --git a/templates/admin.html b/templates/admin.html new file mode 100644 index 0000000..90e11c7 --- /dev/null +++ b/templates/admin.html @@ -0,0 +1,30 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block content %} +
+{% include "header.html" %} +
+

Stats

+

DB

+
    +
  • Inbox size: {{ inbox_size }}
    • +
  • Outbox size: {{ outbox_size }}
    • +
  • Object cache size: {{ object_cache_size }}
    • +
  • Actor cache size: {{ actor_cache_size }}
    • +
+

Collections

+
    +
  • followers: {{ col_followers }}
    • +
  • following: {{ col_following }}
    • +
  • liked: {{col_liked }}
    • +
+

Known Instances

+ +
+ +
+{% endblock %} diff --git a/templates/authorize_remote_follow.html b/templates/authorize_remote_follow.html new file mode 100644 index 0000000..78b7ef0 --- /dev/null +++ b/templates/authorize_remote_follow.html @@ -0,0 +1,15 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+

You're about to follow {{ profile}}

+ +
+ + +
+ +
+{% endblock %} diff --git a/templates/followers.html b/templates/followers.html new file mode 100644 index 0000000..c4d03e2 --- /dev/null +++ b/templates/followers.html @@ -0,0 +1,16 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+{% include "header.html" %} + +
+ {% for follower in followers_data %} + {{ utils.display_actor(follower) }} + {% endfor %} +
+ +
+{% endblock %} diff --git a/templates/following.html b/templates/following.html new file mode 100644 index 0000000..c783133 --- /dev/null +++ b/templates/following.html @@ -0,0 +1,16 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+{% include "header.html" %} + +
+ {% for followed in following_data %} + {{ utils.display_actor(followed) }} + {% endfor %} +
+ +
+{% endblock %} diff --git a/templates/header.html b/templates/header.html new file mode 100644 index 0000000..e3ca067 --- /dev/null +++ b/templates/header.html @@ -0,0 +1,29 @@ + diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..acf6b16 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,45 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} + + + + + + + + + + + + +{% endblock %} +{% block content %} +
+ + +{% include "header.html" %} + +
+{{ config.SUMMARY | safe }} +
+ +
+ {% for item in outbox_data %} + + {% if item.type == 'Announce' %} + {% set boost_actor = item.activity.actor | get_actor %} +

{{ boost_actor.name }} boosted

+ {% if item.ref %} + {{ utils.display_note(item.ref, ui=False) }} + {% endif %} + + {% elif item.type == 'Create' %} + {{ utils.display_note(item) }} + {% endif %} + + {% endfor %} +
+ +
+{% endblock %} diff --git a/templates/indieauth_flow.html b/templates/indieauth_flow.html new file mode 100644 index 0000000..c0ce2c9 --- /dev/null +++ b/templates/indieauth_flow.html @@ -0,0 +1,42 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+ {% if session.logged_in %}logged{% else%}not logged{%endif%} + +
+{% if client.logo %} +
+ +
+{% endif %} +
+
+{{ client.name }} +

wants you to login

+
+
+
+ +
+ {% if scopes %} +

Scopes

+
    + {% for scope in scopes %} +
  • +
    • + {% endfor %} +
+ {% endif %} + + + + + + +
+ +
+{% endblock %} diff --git a/templates/layout.html b/templates/layout.html new file mode 100644 index 0000000..a50d6fb --- /dev/null +++ b/templates/layout.html @@ -0,0 +1,29 @@ + + + + + + +{{ config.USERNAME }} + + + + + + + + + + + + +
+{% block content %}{% endblock %} +
+
+
+ Powered by microblog.pub (source code) and the ActivityPub protocol +
+
+ + diff --git a/templates/login.html b/templates/login.html new file mode 100644 index 0000000..2c16998 --- /dev/null +++ b/templates/login.html @@ -0,0 +1,36 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+ {% if session.logged_in %}logged{% else%}not logged{%endif%} + +
+ + {% if u2f_enabled %} + + + {% else %} + + {% endif %} +
+ +
+{% if u2f_enabled %} + +{% endif %} +{% endblock %} diff --git a/templates/new.html b/templates/new.html new file mode 100644 index 0000000..2e73f38 --- /dev/null +++ b/templates/new.html @@ -0,0 +1,17 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block content %} +
+{% include "header.html" %} +
+
+{% if reply %}{% endif %} + +
+ +
+
+ +
+
+{% endblock %} diff --git a/templates/note.html b/templates/note.html new file mode 100644 index 0000000..55bdfa2 --- /dev/null +++ b/templates/note.html @@ -0,0 +1,23 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} + + + + + + + + + + +{% endblock %} +{% block content %} +
+{% include "header.html" %} +{{ utils.display_note(note, perma=True) }} +{% for reply in replies %} +{{ utils.display_note(reply, perma=False) }} +{% endfor %} +
+{% endblock %} diff --git a/templates/remote_follow.html b/templates/remote_follow.html new file mode 100644 index 0000000..8b54d49 --- /dev/null +++ b/templates/remote_follow.html @@ -0,0 +1,16 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+{% include "header.html" %} +

You're about to follow me

+ +
+ + +
+ +
+{% endblock %} diff --git a/templates/stream.html b/templates/stream.html new file mode 100644 index 0000000..4759ae0 --- /dev/null +++ b/templates/stream.html @@ -0,0 +1,39 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block content %} +
+{% include "header.html" %} +
+ +
+ {% for item in inbox_data %} + {% if item.type == 'Create' %} + {{ utils.display_note(item, ui=True) }} + {% else %} + + {% if item.type == 'Announce' %} + + {% set boost_actor = item.activity.actor | get_actor %} +

{{ boost_actor.name or boost_actor.preferredUsername }} boosted

+ {% if item.ref %} + {{ utils.display_note(item.ref, ui=True) }} + {% endif %} + {% endif %} + + {% if item.type == 'Follow' %} +

{{ item.activity.actor }} followed you

+ {% elif item.type == 'Accept' %} +

you followed {{ item.activity.actor }}

+ {% elif item.type == 'Undo' %} +

{{ item.activity.actor }} unfollowed you

+ {% else %} + {% endif %} + + + {% endif %} + {% endfor %} +
+
+ +
+{% endblock %} diff --git a/templates/tags.html b/templates/tags.html new file mode 100644 index 0000000..fc02452 --- /dev/null +++ b/templates/tags.html @@ -0,0 +1,30 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} + + + + + + + + + + + + +{% endblock %} +{% block content %} +
+ + +{% include "header.html" %} +

#{{ tag }}

+
+ {% for item in outbox_data %} + {{ utils.display_note(item) }} + {% endfor %} +
+ +
+{% endblock %} diff --git a/templates/u2f.html b/templates/u2f.html new file mode 100644 index 0000000..0d94691 --- /dev/null +++ b/templates/u2f.html @@ -0,0 +1,29 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+ {% if session.logged_in %}logged{% else%}not logged{%endif%} + +
+ + +
+ +
+ +{% endblock %} diff --git a/templates/utils.html b/templates/utils.html new file mode 100644 index 0000000..5808374 --- /dev/null +++ b/templates/utils.html @@ -0,0 +1,91 @@ +{% macro display_actor(follower) -%} + +
+
+{% if not follower.icon %} + +{% else %} +{% endif %} +
+
+

{{ follower.name or follower.preferredUsername }}

+@{{ follower.preferredUsername }}@{{ follower.url | domain }} +
{{ follower.summary | safe }}
+
+
+ +{%- endmacro %} + +{% macro display_note(item, perma=False, ui=False) -%} +{% set actor = item.activity.object.attributedTo | get_actor %} +
+ +
+ + + +
+ +
+ {{ actor.name or actor.preferredUsername }} @{{ actor.preferredUsername }}@{{ actor.url | domain }} + + {% if not perma %} + + + + + {% endif %} + {% if item.activity.object.summary %}

{{ item.activity.object.summary }}

{% endif %} +
+ {{ item.activity.object.content | safe }} +
+ + {% if item.activity.object.attachment %} +
+ {% if item.activity.object.attachment | not_only_imgs %} +

Attachment

+ + {% endif %} +
+ {% endif %} + +
+{% if perma %}{{ item.activity.object.published | format_time }} {% endif %} +permalink +{% if item.meta.count_reply %}{{ item.meta.count_reply }} replies{% endif %} +{% if item.meta.count_boost %}{{ item.meta.count_boost }} boosts{% endif %} +{% if item.meta.count_like %}{{ item.meta.count_like }} likes{% endif %} +{% if ui %} +{% set aid = item.activity.object.id | quote_plus %} +reply + +{% set redir = request.path + "#activity-" + item['_id'].__str__() %} + +{% if item.meta.boosted %} +unboost +{% else %} +boost +{% endif %} + +{% if item.meta.liked %} +unlike +{% else %} +like +{% endif %} + +{% endif %} +
+
+ +
+{%- endmacro %} diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/actor_service.py b/utils/actor_service.py new file mode 100644 index 0000000..af7acc8 --- /dev/null +++ b/utils/actor_service.py @@ -0,0 +1,72 @@ +import logging + +import requests +from urllib.parse import urlparse +from Crypto.PublicKey import RSA + +logger = logging.getLogger(__name__) + + +class NotAnActorError(Exception): + def __init__(self, activity): + self.activity = activity + + +class ActorService(object): + def __init__(self, user_agent, col, actor_id, actor_data, instances): + logger.debug(f'Initializing ActorService user_agent={user_agent}') + self._user_agent = user_agent + self._col = col + self._in_mem = {actor_id: actor_data} + self._instances = instances + self._known_instances = set() + + def _fetch(self, actor_url): + logger.debug(f'fetching remote object {actor_url}') + resp = requests.get(actor_url, headers={ + 'Accept': 'application/activity+json', + 'User-Agent': self._user_agent, + }) + resp.raise_for_status() + return resp.json() + + def get(self, actor_url, reload_cache=False): + logger.info(f'get actor {actor_url} (reload_cache={reload_cache})') + + if actor_url in self._in_mem: + return self._in_mem[actor_url] + + instance = urlparse(actor_url)._replace(path='', query='', fragment='').geturl() + if instance not in self._known_instances: + self._known_instances.add(instance) + if not self._instances.find_one({'instance': instance}): + self._instances.insert({'instance': instance, 'first_object': actor_url}) + + if reload_cache: + actor = self._fetch(actor_url) + self._in_mem[actor_url] = actor + self._col.update({'actor_id': actor_url}, {'$set': {'cached_response': actor}}, upsert=True) + return actor + + cached_actor = self._col.find_one({'actor_id': actor_url}) + if cached_actor: + return cached_actor['cached_response'] + + actor = self._fetch(actor_url) + if not 'type' in actor: + raise NotAnActorError(None) + if actor['type'] != 'Person': + raise NotAnActorError(actor) + + self._col.update({'actor_id': actor_url}, {'$set': {'cached_response': actor}}, upsert=True) + self._in_mem[actor_url] = actor + return actor + + def get_public_key(self, actor_url, reload_cache=False): + profile = self.get(actor_url, reload_cache=reload_cache) + pub = profile['publicKey'] + return pub['id'], RSA.importKey(pub['publicKeyPem']) + + def get_inbox_url(self, actor_url, reload_cache=False): + profile = self.get(actor_url, reload_cache=reload_cache) + return profile.get('inbox') diff --git a/utils/content_helper.py b/utils/content_helper.py new file mode 100644 index 0000000..18fabf4 --- /dev/null +++ b/utils/content_helper.py @@ -0,0 +1,57 @@ +import typing +import re + +from bleach.linkifier import Linker +from markdown import markdown + +from utils.webfinger import get_actor_url +from config import USERNAME, BASE_URL, ID +from config import ACTOR_SERVICE + +from typing import List, Optional, Tuple, Dict, Any, Union, Type + + +def set_attrs(attrs, new=False): + attrs[(None, u'target')] = u'_blank' + attrs[(None, u'class')] = u'external' + attrs[(None, u'rel')] = u'noopener' + attrs[(None, u'title')] = attrs[(None, u'href')] + return attrs + + +LINKER = Linker(callbacks=[set_attrs]) +HASHTAG_REGEX = re.compile(r"(#[\d\w\.]+)") +MENTION_REGEX = re.compile(r"@[\d\w_.+-]+@[\d\w-]+\.[\d\w\-.]+") + + +def hashtagify(content: str) -> Tuple[str, List[Dict[str, str]]]: + tags = [] + for hashtag in re.findall(HASHTAG_REGEX, content): + tag = hashtag[1:] + link = f'' + tags.append(dict(href=f'{BASE_URL}/tags/{tag}', name=hashtag, type='Hashtag')) + content = content.replace(hashtag, link) + return content, tags + + +def mentionify(content: str) -> Tuple[str, List[Dict[str, str]]]: + tags = [] + for mention in re.findall(MENTION_REGEX, content): + _, username, domain = mention.split('@') + actor_url = get_actor_url(mention) + p = ACTOR_SERVICE.get(actor_url) + tags.append(dict(type='Mention', href=p['id'], name=mention)) + link = f'@{username}' + content = content.replace(mention, link) + return content, tags + + +def parse_markdown(content: str) -> Tuple[str, List[Dict[str, str]]]: + tags = [] + content = LINKER.linkify(content) + content, hashtag_tags = hashtagify(content) + tags.extend(hashtag_tags) + content, mention_tags = mentionify(content) + tags.extend(mention_tags) + content = markdown(content) + return content, tags diff --git a/utils/httpsig.py b/utils/httpsig.py new file mode 100644 index 0000000..a2e77c5 --- /dev/null +++ b/utils/httpsig.py @@ -0,0 +1,87 @@ +"""Implements HTTP signature for Flask requests. + +Mastodon instances won't accept requests that are not signed using this scheme. + +""" +from datetime import datetime +from urllib.parse import urlparse +from typing import Any, Dict +import base64 +import hashlib + +from flask import request +from requests.auth import AuthBase + +from Crypto.Signature import PKCS1_v1_5 +from Crypto.Hash import SHA256 + + +def _build_signed_string(signed_headers: str, method: str, path: str, headers: Any, body_digest: str) -> str: + out = [] + for signed_header in signed_headers.split(' '): + if signed_header == '(request-target)': + out.append('(request-target): '+method.lower()+' '+path) + elif signed_header == 'digest': + out.append('digest: '+body_digest) + else: + out.append(signed_header+': '+headers[signed_header]) + return '\n'.join(out) + + +def _parse_sig_header(val: str) -> Dict[str, str]: + out = {} + for data in val.split(','): + k, v = data.split('=', 1) + out[k] = v[1:len(v)-1] + return out + + +def _verify_h(signed_string, signature, pubkey): + signer = PKCS1_v1_5.new(pubkey) + digest = SHA256.new() + digest.update(signed_string.encode('utf-8')) + return signer.verify(digest, signature) + + +def _body_digest() -> str: + h = hashlib.new('sha256') + h.update(request.data) + return 'SHA-256='+base64.b64encode(h.digest()).decode('utf-8') + + +def verify_request(actor_service) -> bool: + hsig = _parse_sig_header(request.headers.get('Signature')) + signed_string = _build_signed_string(hsig['headers'], request.method, request.path, request.headers, _body_digest()) + _, rk = actor_service.get_public_key(hsig['keyId']) + return _verify_h(signed_string, base64.b64decode(hsig['signature']), rk) + + +class HTTPSigAuth(AuthBase): + def __init__(self, keyid, privkey): + self.keyid = keyid + self.privkey = privkey + + def __call__(self, r): + host = urlparse(r.url).netloc + bh = hashlib.new('sha256') + bh.update(r.body.encode('utf-8')) + bodydigest = 'SHA-256='+base64.b64encode(bh.digest()).decode('utf-8') + date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') + r.headers.update({'Digest': bodydigest, 'Date': date}) + r.headers.update({'Host': host}) + sigheaders = '(request-target) user-agent host date digest content-type' + to_be_signed = _build_signed_string(sigheaders, r.method, r.path_url, r.headers, bodydigest) + signer = PKCS1_v1_5.new(self.privkey) + digest = SHA256.new() + digest.update(to_be_signed.encode('utf-8')) + sig = base64.b64encode(signer.sign(digest)) + sig = sig.decode('utf-8') + headers = { + 'Signature': 'keyId="{keyid}",algorithm="rsa-sha256",headers="{headers}",signature="{signature}"'.format( + keyid=self.keyid, + signature=sig, + headers=sigheaders, + ), + } + r.headers.update(headers) + return r diff --git a/utils/key.py b/utils/key.py new file mode 100644 index 0000000..526b3be --- /dev/null +++ b/utils/key.py @@ -0,0 +1,39 @@ +import os +import binascii + +from Crypto.PublicKey import RSA + +KEY_DIR = 'config/' + + +def get_secret_key(name:str) -> str: + key_path = f'{KEY_DIR}{name}.key' + if not os.path.exists(key_path): + k = binascii.hexlify(os.urandom(32)).decode('utf-8') + with open(key_path, 'w+') as f: + f.write(k) + return k + + with open(key_path) as f: + return f.read() + + +class Key(object): + def __init__(self, user: str, domain: str, create: bool = True) -> None: + user = user.replace('.', '_') + domain = domain.replace('.', '_') + key_path = f'{KEY_DIR}/key_{user}_{domain}.pem' + if os.path.isfile(key_path): + with open(key_path) as f: + self.privkey_pem = f.read() + self.privkey = RSA.importKey(self.privkey_pem) + self.pubkey_pem = self.privkey.publickey().exportKey('PEM').decode('utf-8') + else: + if not create: + raise Exception('must init private key first') + k = RSA.generate(4096) + self.privkey_pem = k.exportKey('PEM').decode('utf-8') + self.pubkey_pem = k.publickey().exportKey('PEM').decode('utf-8') + with open(key_path, 'w') as f: + f.write(self.privkey_pem) + self.privkey = k diff --git a/utils/linked_data_sig.py b/utils/linked_data_sig.py new file mode 100644 index 0000000..9523ed4 --- /dev/null +++ b/utils/linked_data_sig.py @@ -0,0 +1,53 @@ +from pyld import jsonld +import hashlib +from datetime import datetime + +from Crypto.Signature import PKCS1_v1_5 +from Crypto.Hash import SHA256 +import base64 + + +def options_hash(doc): + doc = dict(doc['signature']) + for k in ['type', 'id', 'signatureValue']: + if k in doc: + del doc[k] + doc['@context'] = 'https://w3id.org/identity/v1' + normalized = jsonld.normalize(doc, {'algorithm': 'URDNA2015', 'format': 'application/nquads'}) + h = hashlib.new('sha256') + h.update(normalized.encode('utf-8')) + return h.hexdigest() + + +def doc_hash(doc): + doc = dict(doc) + if 'signature' in doc: + del doc['signature'] + normalized = jsonld.normalize(doc, {'algorithm': 'URDNA2015', 'format': 'application/nquads'}) + h = hashlib.new('sha256') + h.update(normalized.encode('utf-8')) + return h.hexdigest() + + +def verify_signature(doc, pubkey): + to_be_signed = options_hash(doc) + doc_hash(doc) + signature = doc['signature']['signatureValue'] + signer = PKCS1_v1_5.new(pubkey) + digest = SHA256.new() + digest.update(to_be_signed.encode('utf-8')) + return signer.verify(digest, base64.b64decode(signature)) + + +def generate_signature(doc, privkey): + options = { + 'type': 'RsaSignature2017', + 'creator': doc['actor'] + '#main-key', + 'created': datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', + } + doc['signature'] = options + to_be_signed = options_hash(doc) + doc_hash(doc) + signer = PKCS1_v1_5.new(privkey) + digest = SHA256.new() + digest.update(to_be_signed.encode('utf-8')) + sig = base64.b64encode(signer.sign(digest)) + options['signatureValue'] = sig.decode('utf-8') diff --git a/utils/object_service.py b/utils/object_service.py new file mode 100644 index 0000000..185488f --- /dev/null +++ b/utils/object_service.py @@ -0,0 +1,60 @@ +import requests +from urllib.parse import urlparse + + +class ObjectService(object): + def __init__(self, user_agent, col, inbox, outbox, instances): + self._user_agent = user_agent + self._col = col + self._inbox = inbox + self._outbox = outbox + self._instances = instances + self._known_instances = set() + + def _fetch_remote(self, object_id): + print(f'fetch remote {object_id}') + resp = requests.get(object_id, headers={ + 'Accept': 'application/activity+json', + 'User-Agent': self._user_agent, + }) + resp.raise_for_status() + return resp.json() + + def _fetch(self, object_id): + instance = urlparse(object_id)._replace(path='', query='', fragment='').geturl() + if instance not in self._known_instances: + self._known_instances.add(instance) + if not self._instances.find_one({'instance': instance}): + self._instances.insert({'instance': instance, 'first_object': object_id}) + + obj = self._inbox.find_one({'$or': [{'remote_id': object_id}, {'type': 'Create', 'activity.object.id': object_id}]}) + if obj: + if obj['remote_id'] == object_id: + return obj['activity'] + return obj['activity']['object'] + + obj = self._outbox.find_one({'$or': [{'remote_id': object_id}, {'type': 'Create', 'activity.object.id': object_id}]}) + if obj: + if obj['remote_id'] == object_id: + return obj['activity'] + return obj['activity']['object'] + + return self._fetch_remote(object_id) + + def get(self, object_id, reload_cache=False, part_of_stream=False, announce_published=None): + if reload_cache: + obj = self._fetch(object_id) + self._col.update({'object_id': object_id}, {'$set': {'cached_object': obj, 'meta.part_of_stream': part_of_stream, 'meta.announce_published': announce_published}}, upsert=True) + return obj + + cached_object = self._col.find_one({'object_id': object_id}) + if cached_object: + print(f'ObjectService: {cached_object}') + return cached_object['cached_object'] + + obj = self._fetch(object_id) + + self._col.update({'object_id': object_id}, {'$set': {'cached_object': obj, 'meta.part_of_stream': part_of_stream, 'meta.announce_published': announce_published}}, upsert=True) + # print(f'ObjectService: {obj}') + + return obj diff --git a/utils/opengraph.py b/utils/opengraph.py new file mode 100644 index 0000000..e67a5a8 --- /dev/null +++ b/utils/opengraph.py @@ -0,0 +1,46 @@ +from urllib.parse import urlparse + +import ipaddress +import opengraph +import requests +from bs4 import BeautifulSoup + +from .urlutils import is_url_valid + + +def links_from_note(note): + tags_href= set() + for t in note.get('tag', []): + h = t.get('href') + if h: + # TODO(tsileo): fetch the URL for Actor profile, type=mention + tags_href.add(h) + + links = set() + soup = BeautifulSoup(note['content']) + for link in soup.find_all('a'): + h = link.get('href') + if h.startswith('http') and h not in tags_href and is_url_valid(h): + links.add(h) + + return links + + +def fetch_og_metadata(user_agent, col, remote_id): + doc = col.find_one({'remote_id': remote_id}) + if not doc: + raise ValueError + note = doc['activity']['object'] + print(note) + links = links_from_note(note) + if not links: + return 0 + # FIXME(tsileo): set the user agent by giving HTML directly to OpenGraph + htmls = [] + for l in links: + r = requests.get(l, headers={'User-Agent': user_agent}) + r.raise_for_status() + htmls.append(r.text) + links_og_metadata = [dict(opengraph.OpenGraph(html=html)) for html in htmls] + col.update_one({'remote_id': remote_id}, {'$set': {'meta.og_metadata': links_og_metadata}}) + return len(links) diff --git a/utils/urlutils.py b/utils/urlutils.py new file mode 100644 index 0000000..b304f5e --- /dev/null +++ b/utils/urlutils.py @@ -0,0 +1,27 @@ +import logging +import socket +import ipaddress +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + + +def is_url_valid(url): + parsed = urlparse(url) + if parsed.scheme not in ['http', 'https']: + return False + + if parsed.hostname in ['localhost']: + return False + + try: + ip_address = socket.getaddrinfo(parsed.hostname, parsed.port or 80)[0][4][0] + except socket.gaierror: + logger.exception(f'failed to lookup url {url}') + return False + + if ipaddress.ip_address(ip_address).is_private: + logger.info(f'rejecting private URL {url}') + return False + + return True diff --git a/utils/webfinger.py b/utils/webfinger.py new file mode 100644 index 0000000..296ecae --- /dev/null +++ b/utils/webfinger.py @@ -0,0 +1,63 @@ +from typing import Optional +from urllib.parse import urlparse + +import requests + +def get_remote_follow_template(resource: str) -> Optional[str]: + """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL. + + Returns: + the Actor URL or None if the resolution failed. + """ + if resource.startswith('http'): + host = urlparse(resource).netloc + else: + if resource.startswith('acct:'): + resource = resource[5:] + if resource.startswith('@'): + resource = resource[1:] + _, host = resource.split('@', 1) + resource='acct:'+resource + resp = requests.get( + f'https://{host}/.well-known/webfinger', + {'resource': resource} + ) + print(resp, resp.request.url) + if resp.status_code == 404: + return None + resp.raise_for_status() + data = resp.json() + for link in data['links']: + if link.get('rel') == 'http://ostatus.org/schema/1.0/subscribe': + return link.get('template') + return None + + +def get_actor_url(resource: str) -> Optional[str]: + """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL. + + Returns: + the Actor URL or None if the resolution failed. + """ + if resource.startswith('http'): + host = urlparse(resource).netloc + else: + if resource.startswith('acct:'): + resource = resource[5:] + if resource.startswith('@'): + resource = resource[1:] + _, host = resource.split('@', 1) + resource='acct:'+resource + resp = requests.get( + f'https://{host}/.well-known/webfinger', + {'resource': resource} + ) + print(resp, resp.request.url) + if resp.status_code == 404: + return None + resp.raise_for_status() + data = resp.json() + for link in data['links']: + if link.get('rel') == 'self' and link.get('type') == 'application/activity+json': + return link.get('href') + return None