From 75f820900246b1a58d5b647c8434947d036f7a3d Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 4 Sep 2019 14:13:29 +0200 Subject: [PATCH 001/145] Initial commit --- .gitignore | 116 +++++ LICENSE | 674 +++++++++++++++++++++++++ MANIFEST.in | 3 + README.md | 51 ++ docs/.pyheader-milter.conf.example.swp | Bin 0 -> 12288 bytes docs/pyheader-milter.conf.example | 97 ++++ misc/openrc/pyheader-milter.confd | 17 + misc/openrc/pyheader-milter.initd | 50 ++ pyheadermilter/__init__.py | 336 ++++++++++++ setup.py | 35 ++ 10 files changed, 1379 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 README.md create mode 100644 docs/.pyheader-milter.conf.example.swp create mode 100644 docs/pyheader-milter.conf.example create mode 100644 misc/openrc/pyheader-milter.confd create mode 100755 misc/openrc/pyheader-milter.initd create mode 100644 pyheadermilter/__init__.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0447b8b --- /dev/null +++ b/.gitignore @@ -0,0 +1,116 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..ca4758c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE README.md +recursive-include docs * +recursive-include misc * diff --git a/README.md b/README.md new file mode 100644 index 0000000..bafe5cb --- /dev/null +++ b/README.md @@ -0,0 +1,51 @@ +# pyheader-milter +A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers. + +The project is currently in alpha status, but will soon be used in a productive enterprise environment and possibly existing bugs will be fixed. + +## Requirements +* pymilter +* netaddr + +## Configuration +The pyheader-milter uses an INI-style configuration file. The sections are described below. + +### Section "global" +Any available configuration option can be set in the global section as default instead of in a quarantine section. + +The following configuration options are mandatory in the global section: +* **rules** + Comma-separated, ordered list of active rules. For each, there must be a section of the same name in the configuration. + +### Rule sections +The following configuration options are mandatory for each rule: +* **action** + Set the action of this rule. Possible values are: + * **add** + * **del** + * **mod** +* **header** + Name of the header in case of adding a header, regular expression to match whole header lines in case of deleting or modifying a header. + +The following configuration options are mandatory for an add-rule: +* **value** + Value of the header. + +The following configuration options are mandatory for a mod-rule: +* **search** + Regular expression to match the value of header lines. You may use subgroups or named subgroups (python syntax) to include parts of the original value in the new value. +* **value** + New value of the header. + +The following configuration options are optional for each rule: +* **ignore_hosts** + Comma-separated list of host and network addresses. The rule will be skipped if the sending host is included here. +* **only_hosts** + Comma-separated list of host and network addresses. The rule will be skipped if the sending host is not included here. If a is included in both **ignore_hosts** and **only_hosts**, the rule will be skipped. +* **log** + Enable or disable logging of this rule. Possible values are: + * **true** + * **false** + +## Developer information +Everyone who wants to improve or extend this project is very welcome. diff --git a/docs/.pyheader-milter.conf.example.swp b/docs/.pyheader-milter.conf.example.swp new file mode 100644 index 0000000000000000000000000000000000000000..2f1e8655a8bba29316eb733de603f9a4aef740a3 GIT binary patch literal 12288 zcmeI2O>7%Q6vrosuhNFk6ClBZ9GsNcj*XNrD9EHCu|%4Xq@j?6YP>sZFJ14rJL5PO z6&z6oSHz`AzzKz8;lP0d0)Z$TkN|PwP!L?JoIYgxSywu0cVotpp_dWcyjlyy6(_e3tpZkoJro#ZkLPn^rqj@) zkI;Sh9^IqAeQXu53RnfK0#*U5fK|XMU=^?m+(-)O+5meU0o~LJinl%oc73*P?T1ys zDqt0`3RnfK0#*U5fK|XMU=^?mSOu&C*P#O3W$d$C7~8xJ&g1|8?Z5xOy^FCO@F92} zEP_ApWb9|~6ZjGQ0M3DLz^C9%Pz1Mv^Mj0i2hM@_!F%8|m;-+uVC;AB8+aF-0dIg6 za5K2LpRo(zBd`O`fRkVitb!t#2XkN=Oo3rA0506Y*stI`_#W(lcfi}=6nGV!0EfXC zcn~}Q?gt0Je(?7`#0EYGAA_^t1Mm`f790i{kOrgRVek+b1b^HPZNS&yEAR=Zf)~LH z;2BT=W8fk*{{?&rz5r*zTVNZQ_1FORvIrjC$9@GULqV<)f!ontPl%G`f z(ksHzPq86dGY{1?tusj$DNiPLU|r10Ipsu0=vA%u8BQm21psTu7{{#8$WQ#42M+2Uvf3 zB?)dibC3=7*j}EUn?2SS;MLluzz>Cry^^l)HM^ZijN{i9$0!0Ps?zN-jr%^sCM>l3 zp6~E@STeeM-lT=Gb@V%)7}-Puc4A!?j_|jH`~L;&ia==5?Nnc|=+YA`?yo6WjPDAz zoxRm2$wt^pw*NA3FR6Jfz0gcAHp29L{6gri>^V|cxO{blyejBli=$61?)Ihz?ypHL zjPFt_kY%Kwj_RW0J}P-VZlY7<@j!__d%o9`iHdk62ve1X+r9E=!RxqT&^%s|N-G%3 zE>M>aHJ_skpQ?ZrtwP#yHAXX|qgbHsYa6?HM7U z*>Z&H4z&{{V}3c5k+?BqHv3sgP!eL#Vs}T)SX2+N`5mH8wvBembhNWQb(S_hQwlSy z#kHht+Kao4naI?PqG}yW0qgb9KUt6Wn0BXC?2X1`+>uh{K$durB43dwqmu8sB8=Ct z;UiUg{cle(%W-tXbHrvUd`l1~g=5cb(G6cG2a7V*8<%iN1hUbWH|UOmN%+GiA$eMf z8jm=VZlpV7;x5>#6JO0XZBC!gWw!g|S=$LSx^ophL_{(^Q%#^qbTVh52sLsWuJ@c3 ztvHbO8nGlIMA){-i)1ynJOf`b7Ryja#ixc+MmjPPm>HA9%#=aui4+ZcGC=UAcpy=Y zJ7Og6?T4mcdypv*xI80kO=H#%kvp{jov{L#8|$dzMO>}GeS9OF)+_Ur723J%OZDXW H0S5g8ZdVkU literal 0 HcmV?d00001 diff --git a/docs/pyheader-milter.conf.example b/docs/pyheader-milter.conf.example new file mode 100644 index 0000000..f19dd53 --- /dev/null +++ b/docs/pyheader-milter.conf.example @@ -0,0 +1,97 @@ +# This is an example /etc/pyheader-milter.conf file. +# Copy it into place before use. +# +# Comments: use '#' for comment lines and ';' (following a space) for inline comments. +# +# If an option is not present in a modification section, it will be read from +# the global section. +# + + +[global] + +# Option: rules +# Notes: Set active rules (comma-separated). +# Each rule must have a section with the same name below. +# The rule name 'global' is forbidden and will be ignored. +# Rule names must be unique. +# Values: [ ACTIVE ] +# +rules = add_header,del_header,mod_header + +# Option: ignore_hosts +# Notes: Set a list of host and network addresses to be ignored. +# All the common host/network notations are supported, including IPv6. +# Value: [ HOST ] +# +ignore_hosts = 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 + +# Option: only_hosts +# Notes: Set a list of host and network addresses. +# All the common host/network notations are supported, including IPv6. +# Value: [ HOST ] +# +only_hosts = 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 + +# Option: log +# Notes: Set if modifications are logged. +# Value: [ true | false ] +# +log = true + + +[add_header] + +# Option: action +# Notes: Set the modification action. +# Values: [ add ] +action = add + +# Option: header +# Notes: Set the name of the header. +# Values: [ NAME ] +# +header = X-My-Own-Header + +# Option: value +# Notes: Set the value of the header. +# Values: [ VALUE ] +value = my own value + + +[del_header] + +# Option: action +# Notes: Set the modification action. +# Values: [ del ] +action = del + +# Option: header +# Notes: Set a regular expression to match the header lines to delete. +# Values: [ REGEX ] +# +header = ^Received:.* + + +[mod_header] + +# Option: action +# Notes: Set the modification action. +# Values: [ mod ] +action = mod + +# Option: header +# Notes: Set a regular expression to match the header lines to modify. +# Values: [ REGEX ] +# +header = ^Subject:.* + +# Option: search +# Notes: Set a regular expression to match the headers value. +# Values: [ VALUE ] +search = (?P.*) + +# Option: value +# Notes: Set the value of the header. +# Values: [ VALUE ] +value = [SPAM] \g diff --git a/misc/openrc/pyheader-milter.confd b/misc/openrc/pyheader-milter.confd new file mode 100644 index 0000000..b8a0c08 --- /dev/null +++ b/misc/openrc/pyheader-milter.confd @@ -0,0 +1,17 @@ +# /etc/conf.d/pyheader-milter: config file for /etc/init.d/pyheader-milter + +# Set the socket used to communicate with the MTA. +# Examples: +# unix:/path/to/socket a named pipe +# inet:8899 listen on ANY interface +# inet:8899@localhost listen on a specific interface +# inet6:8899 listen on ANY interface +# inet6:8899@[2001:db8:1234::1] listen on a specific interface +SOCKET="inet:8898@127.0.0.1" + +# Start the daemon as the user. You can optionally append a group name here also. +# USER="daemon" +# USER="daemon:nobody" + +# Optional parameters for pyheader-milter +# MILTER_OPTS="" diff --git a/misc/openrc/pyheader-milter.initd b/misc/openrc/pyheader-milter.initd new file mode 100755 index 0000000..8393089 --- /dev/null +++ b/misc/openrc/pyheader-milter.initd @@ -0,0 +1,50 @@ +#!/sbin/openrc-run + +user=${USER:-daemon} +socket="${SOCKET:-}" +milter_opts="${MILTER_OPTS:-}" + +pidfile="/run/${RC_SVCNAME}.pid" +command="/usr/bin/pyheader-milter" +command_args="-s ${socket} ${milter_opts}" +command_background=true +start_stop_daemon_args="--user ${user}" + +extra_commands="configtest" + +depend() { + need net + before mta +} + +checkconfig() { + if [ -z "${socket}" ]; then + eerror "No socket specified in config!" + fi + OUTPUT=$( ${command} ${command_args} -t 2>&1 ) + ret=$? + if [ $ret -ne 0 ]; then + eerror "${SVCNAME} has detected an error in your configuration:" + printf "%s\n" "${OUTPUT}" + fi + + return $ret +} + +configtest() { + ebegin "Checking ${SVCNAME} configuration" + checkconfig + eend $? +} + +start_pre() { + if [ "${RC_CMD}" != "restart" ]; then + checkconfig || return $? + fi +} + +stop_pre() { + if [ "${RC_CMD}" != "restart" ]; then + checkconfig || return $? + fi +} diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py new file mode 100644 index 0000000..9777061 --- /dev/null +++ b/pyheadermilter/__init__.py @@ -0,0 +1,336 @@ +# PyHeader-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyHeader-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyHeader-Milter. If not, see . +# + +__all__ = ["HeaderRule", "HeaderMilter"] + +import Milter +import argparse +import configparser +import logging +import logging.handlers +import os +import re +import sys + +from Milter.utils import parse_addr +from netaddr import IPAddress, IPNetwork, AddrFormatError + + +class HeaderRule: + def __init__(self, name, action, header, search=None, value=None, ignore_hosts=[], only_hosts=[], log=True): + self.logger = logging.getLogger(__name__) + self.name = name + self._action = action + if action == "add": + self.header = header + else: + try: + self.header = re.compile(header, re.MULTILINE + re.DOTALL + re.IGNORECASE) + except re.error as e: + raise RuntimeError("unable to parse option 'header' of rule '{}': {}".format(name, e)) + if action == "mod": + try: + self.search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE) + except re.error as e: + raise RuntimeError("unable to parse option 'search' of rule '{}': {}".format(name, e)) + else: + self.search = search + self.value = value + self.ignore_hosts = [] + try: + for ignore in ignore_hosts: + self.ignore_hosts.append(IPNetwork(ignore)) + except AddrFormatError as e: + raise RuntimeError("unable to parse option 'ignore_hosts' of rule '{}': {}".format(name, e)) + self.only_hosts = [] + try: + for only in only_hosts: + self.only_hosts.append(IPNetwork(only)) + except AddrFormatError as e: + raise RuntimeError("unable to parse option 'only_hosts' of rule '{}': {}".format(name, e)) + self.log = log + + def get_action(self): + return self._action + + def log_modification(self): + return self.log + + def ignore_host(self, host): + ip = IPAddress(host) + ignore = False + for ignored in self.ignore_hosts: + if ip in ignored: + ignore = True + break + if not ignore and self.only_hosts: + ignore = True + for only in self.only_hosts: + if ip in only: + ignore = False + break + if ignore: + self.logger.debug("host {} is ignored by rule {}".format(host, self.name)) + return ignore + + def execute(self, headers): + modified = [] + if self._action == "add": + modified.append((self.header, self.value, 0, 1)) + else: + index = 0 + occurrences = {} + for name, value in headers: + if name not in occurrences.keys(): + occurrences[name] = 1 + else: + occurrences[name] += 1 + if self.header.search("{}: {}".format(name, value)): + if self._action == "del": + value = "" + else: + value = self.search.sub(self.value, value) + modified.append((name, value, index, occurrences[name])) + index += 1 + return modified + + +class HeaderMilter(Milter.Base): + """HeaderMilter based on Milter.Base to implement milter communication""" + + _rules = [] + + @staticmethod + def set_rules(rules): + HeaderMilter._rules = rules + + def __init__(self): + self.logger = logging.getLogger(__name__) + # save rules, it must not change during runtime + self.rules = HeaderMilter._rules.copy() + + def connect(self, IPname, family, hostaddr): + self.logger.debug("accepted milter connection from {} port {}".format(*hostaddr)) + ip = IPAddress(hostaddr[0]) + for rule in self.rules.copy(): + if rule.ignore_host(ip): + self.rules.remove(rule) + if not self.rules: + self.logger.debug("host {} is ignored by all rules, skip further processing".format(hostaddr[0])) + return Milter.ACCEPT + return Milter.CONTINUE + + @Milter.noreply + def data(self): + self.queueid = self.getsymval('i') + self.logger.debug("{}: received queue-id from MTA".format(self.queueid)) + self.headers = [] + return Milter.CONTINUE + + @Milter.noreply + def header(self, name, value): + self.headers.append((name, value)) + return Milter.CONTINUE + + def eom(self): + try: + for rule in self.rules: + action = rule.get_action() + log = rule.log_modification() + self.logger.debug("{}: executing rule '{}'".format(self.queueid, rule.name)) + modified = rule.execute(self.headers) + for name, value, index, occurrence in modified: + if action == "add": + if log: + self.logger.info("{}: add: header: {}".format(self.queueid, "{}: {}".format(name, value)[0:50])) + else: + self.logger.debug("{}: add: header: {}".format(self.queueid, "{}: {}".format(name, value)[0:50])) + self.headers.insert(0, (name, value)) + self.addheader(name, value, 1) + else: + self.chgheader(name, occurrence, value) + if action == "mod": + if log: + self.logger.info("{}: modify: header: {}".format(self.queueid, "{}: {}".format(name, self.headers[index][1])[0:50])) + else: + self.logger.debug("{}: modify: header (occ. {}): {}".format(self.queueid, occurrence, "{}: {}".format(name, self.headers[index][1])[0:50])) + self.headers[index] = (name, value) + elif action == "del": + if log: + self.logger.info("{}: delete: header: {}".format(self.queueid, "{}: {}".format(name, self.headers[index][1])[0:50])) + else: + self.logger.debug("{}: delete: header (occ. {}): {}".format(self.queueid, occurrence, "{}: {}".format(name, self.headers[index][1])[0:50])) + del self.headers[index] + return Milter.ACCEPT + except Exception as e: + self.logger.exception("an exception occured in eom function: {}".format(e)) + return Milter.TEMPFAIL + + +def main(): + "Run PyHeader-Milter." + # parse command line + parser = argparse.ArgumentParser(description="PyHeader milter daemon", + formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=45, width=140)) + parser.add_argument("-c", "--config", help="Config file to read.", default="/etc/pyheader-milter.conf") + parser.add_argument("-s", "--socket", help="Socket used to communicate with the MTA.", required=True) + parser.add_argument("-d", "--debug", help="Log debugging messages.", action="store_true") + parser.add_argument("-t", "--test", help="Check configuration.", action="store_true") + args = parser.parse_args() + + # setup logging + loglevel = logging.INFO + logname = "pyheader-milter" + syslog_name = logname + if args.debug: + loglevel = logging.DEBUG + logname = "{}[%(name)s]".format(logname) + syslog_name = "{}: [%(name)s] %(levelname)s".format(syslog_name) + + # set config files for milter class + root_logger = logging.getLogger() + root_logger.setLevel(loglevel) + + # setup console log + stdouthandler = logging.StreamHandler(sys.stdout) + stdouthandler.setLevel(logging.DEBUG) + formatter = logging.Formatter("%(message)s".format(logname)) + stdouthandler.setFormatter(formatter) + root_logger.addHandler(stdouthandler) + logger = logging.getLogger(__name__) + + try: + # read config file + parser = configparser.ConfigParser() + if not parser.read(args.config): + raise RuntimeError("config file not found") + + # check if mandatory config options in global section are present + if "global" not in parser.sections(): + raise RuntimeError("mandatory section 'global' not present in config file") + for option in ["rules"]: + if not parser.has_option("global", option): + raise RuntimeError("mandatory option '{}' not present in config section 'global'".format(option)) + + # read global config section + global_config = dict(parser.items("global")) + + # read active rules + active_rules = [ r.strip() for r in global_config["rules"].split(",") ] + if len(active_rules) != len(set(active_rules)): + raise RuntimeError("at least one rule is specified multiple times in 'rules' option") + if "global" in active_rules: + active_rules.remove("global") + logger.warning("removed illegal rule name 'global' from list of active rules") + if not active_rules: + raise RuntimeError("no rules configured") + + logger.debug("preparing milter configuration ...") + rules = [] + # iterate active rules + for rule_name in active_rules: + # check if config section exists + if rule_name not in parser.sections(): + raise RuntimeError("config section '{}' does not exist".format(rule_name)) + config = dict(parser.items(rule_name)) + + # check if mandatory option action is present in config + option = "action" + if option not in config.keys() and \ + option in global_config.keys(): + config[option] = global_config[option] + if option not in config.keys(): + raise RuntimeError("mandatory option '{}' not specified for rule '{}'".format(option, rule_name)) + config["action"] = config["action"].lower() + if config["action"] not in ["add", "del", "mod"]: + raise RuntimeError("invalid action specified for rule '{}'".format(rule_name)) + + # check if mandatory options are present in config + mandatory = ["header"] + if config["action"] == "add": + mandatory += ["value"] + elif config["action"] == "mod": + mandatory += ["search", "value"] + for option in mandatory: + if option not in config.keys() and \ + option in global_config.keys(): + config[option] = global_config[option] + if option not in config.keys(): + raise RuntimeError("mandatory option '{}' not specified for rule '{}'".format(option, rule_name)) + + # check if optional config options are present in config + defaults = { + "ignore_hosts": [], + "only_hosts": [], + "log": "true" + } + for option in defaults.keys(): + if option not in config.keys() and \ + option in global_config.keys(): + config[option] = global_config[option] + if option not in config.keys(): + config[option] = defaults[option] + if config["ignore_hosts"]: + config["ignore_hosts"] = [ h.strip() for h in config["ignore_hosts"].split(",") ] + if config["only_hosts"]: + config["only_hosts"] = [ h.strip() for h in config["only_hosts"].split(",") ] + config["log"] = config["log"].lower() + if config["log"] == "true": + config["log"] = True + elif config["log"] == "false": + config["log"] = False + else: + raise RuntimeError("invalid value specified for option 'log' for rule '{}'".format(rule_name)) + # add rule + logging.debug("adding rule '{}'".format(rule_name)) + rules.append(HeaderRule(name=rule_name, **config)) + + except RuntimeError as e: + logger.error(e) + sys.exit(255) + + if args.test: + print("Configuration ok") + sys.exit(0) + + # change log format for runtime + formatter = logging.Formatter("%(asctime)s {}: [%(levelname)s] %(message)s".format(logname), datefmt="%Y-%m-%d %H:%M:%S") + stdouthandler.setFormatter(formatter) + + # setup syslog + sysloghandler = logging.handlers.SysLogHandler(address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) + sysloghandler.setLevel(loglevel) + formatter = logging.Formatter("{}: %(message)s".format(syslog_name)) + sysloghandler.setFormatter(formatter) + root_logger.addHandler(sysloghandler) + + logger.info("PyHeader-Milter starting") + HeaderMilter.set_rules(rules) + + # register milter factory class + Milter.factory = HeaderMilter + Milter.set_exception_policy(Milter.TEMPFAIL) + + rc = 0 + try: + Milter.runmilter("pyheader-milter", socketname=args.socket, timeout=30) + except Milter.milter.error as e: + logger.error(e) + rc = 255 + logger.info("PyHeader-Milter terminated") + sys.exit(rc) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c63f4ae --- /dev/null +++ b/setup.py @@ -0,0 +1,35 @@ +from setuptools import setup + +def read_file(fname): + with open(fname, 'r') as f: + return f.read() + +setup(name = "pyheadermilter", + version = "0.0.1", + author = "Thomas Oettli", + author_email = "spacefreak@noop.ch", + description = "A pymilter based sendmail/postfix pre-queue filter.", + license = "GPL 3", + keywords = "header milter", + url = "https://github.com/spacefreak86/pyheader-milter", + packages = ["pyheadermilter"], + long_description = read_file("README.md"), + classifiers = [ + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Topic :: Communications :: Email :: Header" + ], + entry_points = { + "console_scripts": [ + "pyheader-milter=pyheadermilter:main" + ] + }, + install_requires = ["pymilter"], + python_requires = ">=3" +) From d0ab24e7d689cb562da87bad6b9469ca00116f1f Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 4 Sep 2019 14:15:01 +0200 Subject: [PATCH 002/145] Fix typo in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bafe5cb..6eb275f 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ The project is currently in alpha status, but will soon be used in a productive The pyheader-milter uses an INI-style configuration file. The sections are described below. ### Section "global" -Any available configuration option can be set in the global section as default instead of in a quarantine section. +Any available configuration option can be set in the global section as default instead of in a rule section. The following configuration options are mandatory in the global section: * **rules** From 7fa6f05511189887017d81c54195d2f9cb6e4015 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 4 Sep 2019 14:54:48 +0200 Subject: [PATCH 003/145] Prepare for packaging --- docs/.pyheader-milter.conf.example.swp | Bin 12288 -> 0 bytes pyheadermilter/__init__.py | 1 + setup.py | 2 +- 3 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 docs/.pyheader-milter.conf.example.swp diff --git a/docs/.pyheader-milter.conf.example.swp b/docs/.pyheader-milter.conf.example.swp deleted file mode 100644 index 2f1e8655a8bba29316eb733de603f9a4aef740a3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2O>7%Q6vrosuhNFk6ClBZ9GsNcj*XNrD9EHCu|%4Xq@j?6YP>sZFJ14rJL5PO z6&z6oSHz`AzzKz8;lP0d0)Z$TkN|PwP!L?JoIYgxSywu0cVotpp_dWcyjlyy6(_e3tpZkoJro#ZkLPn^rqj@) zkI;Sh9^IqAeQXu53RnfK0#*U5fK|XMU=^?m+(-)O+5meU0o~LJinl%oc73*P?T1ys zDqt0`3RnfK0#*U5fK|XMU=^?mSOu&C*P#O3W$d$C7~8xJ&g1|8?Z5xOy^FCO@F92} zEP_ApWb9|~6ZjGQ0M3DLz^C9%Pz1Mv^Mj0i2hM@_!F%8|m;-+uVC;AB8+aF-0dIg6 za5K2LpRo(zBd`O`fRkVitb!t#2XkN=Oo3rA0506Y*stI`_#W(lcfi}=6nGV!0EfXC zcn~}Q?gt0Je(?7`#0EYGAA_^t1Mm`f790i{kOrgRVek+b1b^HPZNS&yEAR=Zf)~LH z;2BT=W8fk*{{?&rz5r*zTVNZQ_1FORvIrjC$9@GULqV<)f!ontPl%G`f z(ksHzPq86dGY{1?tusj$DNiPLU|r10Ipsu0=vA%u8BQm21psTu7{{#8$WQ#42M+2Uvf3 zB?)dibC3=7*j}EUn?2SS;MLluzz>Cry^^l)HM^ZijN{i9$0!0Ps?zN-jr%^sCM>l3 zp6~E@STeeM-lT=Gb@V%)7}-Puc4A!?j_|jH`~L;&ia==5?Nnc|=+YA`?yo6WjPDAz zoxRm2$wt^pw*NA3FR6Jfz0gcAHp29L{6gri>^V|cxO{blyejBli=$61?)Ihz?ypHL zjPFt_kY%Kwj_RW0J}P-VZlY7<@j!__d%o9`iHdk62ve1X+r9E=!RxqT&^%s|N-G%3 zE>M>aHJ_skpQ?ZrtwP#yHAXX|qgbHsYa6?HM7U z*>Z&H4z&{{V}3c5k+?BqHv3sgP!eL#Vs}T)SX2+N`5mH8wvBembhNWQb(S_hQwlSy z#kHht+Kao4naI?PqG}yW0qgb9KUt6Wn0BXC?2X1`+>uh{K$durB43dwqmu8sB8=Ct z;UiUg{cle(%W-tXbHrvUd`l1~g=5cb(G6cG2a7V*8<%iN1hUbWH|UOmN%+GiA$eMf z8jm=VZlpV7;x5>#6JO0XZBC!gWw!g|S=$LSx^ophL_{(^Q%#^qbTVh52sLsWuJ@c3 ztvHbO8nGlIMA){-i)1ynJOf`b7Ryja#ixc+MmjPPm>HA9%#=aui4+ZcGC=UAcpy=Y zJ7Og6?T4mcdypv*xI80kO=H#%kvp{jov{L#8|$dzMO>}GeS9OF)+_Ur723J%OZDXW H0S5g8ZdVkU diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 9777061..5dd6259 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -13,6 +13,7 @@ # __all__ = ["HeaderRule", "HeaderMilter"] +name = "pyheadermilter" import Milter import argparse diff --git a/setup.py b/setup.py index c63f4ae..4a11957 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,6 @@ setup(name = "pyheadermilter", "pyheader-milter=pyheadermilter:main" ] }, - install_requires = ["pymilter"], + install_requires = ["pymilter", "netaddr"], python_requires = ">=3" ) From cb83cb378f834122342d330996ca8ee4d99f411c Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 4 Sep 2019 15:22:26 +0200 Subject: [PATCH 004/145] Make setup.py compliance for pypi.org --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4a11957..b146542 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ setup(name = "pyheadermilter", url = "https://github.com/spacefreak86/pyheader-milter", packages = ["pyheadermilter"], long_description = read_file("README.md"), + long_description_content_type="text/markdown", classifiers = [ # 3 - Alpha # 4 - Beta @@ -23,7 +24,7 @@ setup(name = "pyheadermilter", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Topic :: Communications :: Email :: Header" + "Topic :: Communications :: Email :: Filters" ], entry_points = { "console_scripts": [ From 2ce6b2350e896711022c92d56ed8fbe14bbfff79 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 5 Sep 2019 13:57:13 +0200 Subject: [PATCH 005/145] Cleanup source code --- pyheadermilter/__init__.py | 104 +++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 38 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 5dd6259..e3cf8e7 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -29,38 +29,48 @@ from netaddr import IPAddress, IPNetwork, AddrFormatError class HeaderRule: - def __init__(self, name, action, header, search=None, value=None, ignore_hosts=[], only_hosts=[], log=True): + """HeaderRule to implement a rule to apply on e-mail headers.""" + + def __init__(self, name, action, header, search="", value="", ignore_hosts=[], only_hosts=[], log=True): self.logger = logging.getLogger(__name__) self.name = name self._action = action - if action == "add": - self.header = header - else: + self.header = header + self.search = search + self.value = value + self.ignore_hosts = ignore_hosts + self.only_hosts = only_hosts + self.log = log + + if action in ["del", "mod"]: + # compile header regex try: self.header = re.compile(header, re.MULTILINE + re.DOTALL + re.IGNORECASE) except re.error as e: raise RuntimeError("unable to parse option 'header' of rule '{}': {}".format(name, e)) - if action == "mod": - try: - self.search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE) - except re.error as e: - raise RuntimeError("unable to parse option 'search' of rule '{}': {}".format(name, e)) - else: - self.search = search - self.value = value - self.ignore_hosts = [] + + if action == "mod": + # compile search regex + try: + self.search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE) + except re.error as e: + raise RuntimeError("unable to parse option 'search' of rule '{}': {}".format(name, e)) + + if action in ["add", "mod"] and not value: + raise RuntimeError("value of option 'value' is empty") + + # replace strings in ignore_hosts and only_hosts with IPNetwork instances try: - for ignore in ignore_hosts: - self.ignore_hosts.append(IPNetwork(ignore)) + for index, ignore in enumerate(ignore_hosts): + self.ignore_hosts[index] = IPNetwork(ignore) except AddrFormatError as e: raise RuntimeError("unable to parse option 'ignore_hosts' of rule '{}': {}".format(name, e)) - self.only_hosts = [] + try: - for only in only_hosts: - self.only_hosts.append(IPNetwork(only)) + for index, only in enumerate(only_hosts): + self.only_hosts[index] = IPNetwork(only) except AddrFormatError as e: raise RuntimeError("unable to parse option 'only_hosts' of rule '{}': {}".format(name, e)) - self.log = log def get_action(self): return self._action @@ -71,39 +81,51 @@ class HeaderRule: def ignore_host(self, host): ip = IPAddress(host) ignore = False + + # check if host matches ignore_hosts for ignored in self.ignore_hosts: if ip in ignored: ignore = True break + if not ignore and self.only_hosts: + # host does not match ignore_hosts, check if it matches only_hosts ignore = True for only in self.only_hosts: if ip in only: ignore = False break + if ignore: self.logger.debug("host {} is ignored by rule {}".format(host, self.name)) return ignore def execute(self, headers): - modified = [] + """Execute rule on given headers and return list with modified headers.""" if self._action == "add": - modified.append((self.header, self.value, 0, 1)) - else: - index = 0 - occurrences = {} - for name, value in headers: - if name not in occurrences.keys(): - occurrences[name] = 1 + return [(self.header, self.value, 0, 1)] + + modified = [] + index = 0 + occurrences = {} + + # iterate headers + for name, value in headers: + # keep track of the occurrence of each header, needed by Milter.Base.chgheader + if name not in occurrences.keys(): + occurrences[name] = 1 + else: + occurrences[name] += 1 + + # check if header line matches regex + if self.header.search("{}: {}".format(name, value)): + if self._action == "del": + # set an empty value to delete the header + value = "" else: - occurrences[name] += 1 - if self.header.search("{}: {}".format(name, value)): - if self._action == "del": - value = "" - else: - value = self.search.sub(self.value, value) - modified.append((name, value, index, occurrences[name])) - index += 1 + value = self.search.sub(self.value, value) + modified.append((name, value, index, occurrences[name])) + index += 1 return modified @@ -124,9 +146,12 @@ class HeaderMilter(Milter.Base): def connect(self, IPname, family, hostaddr): self.logger.debug("accepted milter connection from {} port {}".format(*hostaddr)) ip = IPAddress(hostaddr[0]) + + # remove rules which ignore this host for rule in self.rules.copy(): if rule.ignore_host(ip): self.rules.remove(rule) + if not self.rules: self.logger.debug("host {} is ignored by all rules, skip further processing".format(hostaddr[0])) return Milter.ACCEPT @@ -147,20 +172,21 @@ class HeaderMilter(Milter.Base): def eom(self): try: for rule in self.rules: - action = rule.get_action() - log = rule.log_modification() self.logger.debug("{}: executing rule '{}'".format(self.queueid, rule.name)) modified = rule.execute(self.headers) + action = rule.get_action() + log = rule.log_modification() + for name, value, index, occurrence in modified: if action == "add": if log: self.logger.info("{}: add: header: {}".format(self.queueid, "{}: {}".format(name, value)[0:50])) else: self.logger.debug("{}: add: header: {}".format(self.queueid, "{}: {}".format(name, value)[0:50])) + self.headers.insert(0, (name, value)) self.addheader(name, value, 1) else: - self.chgheader(name, occurrence, value) if action == "mod": if log: self.logger.info("{}: modify: header: {}".format(self.queueid, "{}: {}".format(name, self.headers[index][1])[0:50])) @@ -173,6 +199,7 @@ class HeaderMilter(Milter.Base): else: self.logger.debug("{}: delete: header (occ. {}): {}".format(self.queueid, occurrence, "{}: {}".format(name, self.headers[index][1])[0:50])) del self.headers[index] + self.chgheader(name, occurrence, value) return Milter.ACCEPT except Exception as e: self.logger.exception("an exception occured in eom function: {}".format(e)) @@ -293,6 +320,7 @@ def main(): config["log"] = False else: raise RuntimeError("invalid value specified for option 'log' for rule '{}'".format(rule_name)) + # add rule logging.debug("adding rule '{}'".format(rule_name)) rules.append(HeaderRule(name=rule_name, **config)) From 9558c23922a491892c300988887a420d651efb2b Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 31 Jan 2020 23:02:34 +0100 Subject: [PATCH 006/145] Replace wrong encoded characters in header values --- pyheadermilter/__init__.py | 14 +++++++++----- setup.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index e3cf8e7..137f3f7 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -166,7 +166,7 @@ class HeaderMilter(Milter.Base): @Milter.noreply def header(self, name, value): - self.headers.append((name, value)) + self.headers.append((name, value.encode(errors="replace").decode())) return Milter.CONTINUE def eom(self): @@ -188,16 +188,20 @@ class HeaderMilter(Milter.Base): self.addheader(name, value, 1) else: if action == "mod": + old_line = "{}: {}".format(name, self.headers[index][1]) + new_line = "{}: {}".format(name, value) if log: - self.logger.info("{}: modify: header: {}".format(self.queueid, "{}: {}".format(name, self.headers[index][1])[0:50])) + self.logger.info("{}: modify: header: {}: {}".format( + self.queueid, old_line[0:40], new_line[0:40])) else: - self.logger.debug("{}: modify: header (occ. {}): {}".format(self.queueid, occurrence, "{}: {}".format(name, self.headers[index][1])[0:50])) + self.logger.debug("{}: modify: header (occ. {}): {}: {}: {}".format( + self.queueid, occurrence, name, old_line, new_line)) self.headers[index] = (name, value) elif action == "del": if log: - self.logger.info("{}: delete: header: {}".format(self.queueid, "{}: {}".format(name, self.headers[index][1])[0:50])) + self.logger.info("{}: delete: header: {}".format(self.queueid, "{}: {}".format(name, self.headers[index][1]))[0:50]) else: - self.logger.debug("{}: delete: header (occ. {}): {}".format(self.queueid, occurrence, "{}: {}".format(name, self.headers[index][1])[0:50])) + self.logger.debug("{}: delete: header (occ. {}): {}".format(self.queueid, occurrence, "{}: {}".format(name, self.headers[index][1]))[0:50]) del self.headers[index] self.chgheader(name, occurrence, value) return Milter.ACCEPT diff --git a/setup.py b/setup.py index b146542..49e4b87 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ def read_file(fname): return f.read() setup(name = "pyheadermilter", - version = "0.0.1", + version = "0.0.2", author = "Thomas Oettli", author_email = "spacefreak@noop.ch", description = "A pymilter based sendmail/postfix pre-queue filter.", From 9fb66e1a1d5ac19512f5925f7de5cfb9c5ee8ce1 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Sat, 1 Feb 2020 00:51:20 +0100 Subject: [PATCH 007/145] Some rework --- pyheadermilter/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 137f3f7..7cddb2e 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -166,7 +166,7 @@ class HeaderMilter(Milter.Base): @Milter.noreply def header(self, name, value): - self.headers.append((name, value.encode(errors="replace").decode())) + self.headers.append((name, value.encode(errors="surrogateescape").decode(errors="replace"))) return Milter.CONTINUE def eom(self): @@ -178,30 +178,30 @@ class HeaderMilter(Milter.Base): log = rule.log_modification() for name, value, index, occurrence in modified: + mod_header = "{}: {}".format(name, value) if action == "add": if log: - self.logger.info("{}: add: header: {}".format(self.queueid, "{}: {}".format(name, value)[0:50])) + self.logger.info("{}: add: header: {}".format(self.queueid, mod_header[0:70])) else: - self.logger.debug("{}: add: header: {}".format(self.queueid, "{}: {}".format(name, value)[0:50])) + self.logger.debug("{}: add: header: {}".format(self.queueid, mod_header)) self.headers.insert(0, (name, value)) self.addheader(name, value, 1) else: if action == "mod": - old_line = "{}: {}".format(name, self.headers[index][1]) - new_line = "{}: {}".format(name, value) + old_header = "{}: {}".format(name, self.headers[index][1]) if log: self.logger.info("{}: modify: header: {}: {}".format( - self.queueid, old_line[0:40], new_line[0:40])) + self.queueid, old_header[0:70], mod_header[0:70])) else: - self.logger.debug("{}: modify: header (occ. {}): {}: {}: {}".format( - self.queueid, occurrence, name, old_line, new_line)) + self.logger.debug("{}: modify: header (occ. {}): {}: {}".format( + self.queueid, occurrence, old_header, mod_header)) self.headers[index] = (name, value) elif action == "del": if log: - self.logger.info("{}: delete: header: {}".format(self.queueid, "{}: {}".format(name, self.headers[index][1]))[0:50]) + self.logger.info("{}: delete: header: {}".format(self.queueid, mod_header[0:70])) else: - self.logger.debug("{}: delete: header (occ. {}): {}".format(self.queueid, occurrence, "{}: {}".format(name, self.headers[index][1]))[0:50]) + self.logger.debug("{}: delete: header (occ. {}): {}".format(self.queueid, occurrence, mod_header)) del self.headers[index] self.chgheader(name, occurrence, value) return Milter.ACCEPT From b1fc52d878693b84fa1655ec63a55e90562812c0 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 19 Feb 2020 14:45:22 +0100 Subject: [PATCH 008/145] Add ability to whitelist envelope from addresses --- pyheadermilter/__init__.py | 29 ++++++++++++++++++++++++++++- setup.py | 2 +- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 7cddb2e..3e8d5f7 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -31,7 +31,7 @@ from netaddr import IPAddress, IPNetwork, AddrFormatError class HeaderRule: """HeaderRule to implement a rule to apply on e-mail headers.""" - def __init__(self, name, action, header, search="", value="", ignore_hosts=[], only_hosts=[], log=True): + def __init__(self, name, action, header, search="", value="", ignore_hosts=[], ignore_envfrom="", only_hosts=[], log=True): self.logger = logging.getLogger(__name__) self.name = name self._action = action @@ -39,6 +39,7 @@ class HeaderRule: self.search = search self.value = value self.ignore_hosts = ignore_hosts + self.ignore_envfrom = ignore_envfrom self.only_hosts = only_hosts self.log = log @@ -66,6 +67,11 @@ class HeaderRule: except AddrFormatError as e: raise RuntimeError("unable to parse option 'ignore_hosts' of rule '{}': {}".format(name, e)) + try: + self.ignore_envfrom = re.compile(ignore_envfrom, re.IGNORECASE) + except re.error as e: + raise RuntimeError("unable to parse option 'ignore_envfrom' of rule '{}': {}".format(name, e)) + try: for index, only in enumerate(only_hosts): self.only_hosts[index] = IPNetwork(only) @@ -100,6 +106,15 @@ class HeaderRule: self.logger.debug("host {} is ignored by rule {}".format(host, self.name)) return ignore + def ignore_from(self, envfrom): + ignore = False + + if self.ignore_envfrom: + if self.ignore_envfrom.search(envfrom): + ignore = True + self.logger.debug("envelope-from {} is ignored by rule {}".format(envfrom, self.name)) + return ignore + def execute(self, headers): """Execute rule on given headers and return list with modified headers.""" if self._action == "add": @@ -157,6 +172,17 @@ class HeaderMilter(Milter.Base): return Milter.ACCEPT return Milter.CONTINUE + def envfrom(self, mailfrom, *str): + mailfrom = "@".join(parse_addr(mailfrom)).lower() + for rule in self.rules.copy(): + if rule.ignore_from(mailfrom): + self.rules.remove(rule) + + if not self.rules: + self.logger.debug("mail from {} is ignored by all rules, skip further processing".format(mailfrom)) + return Milter.ACCEPT + return Milter.CONTINUE + @Milter.noreply def data(self): self.queueid = self.getsymval('i') @@ -304,6 +330,7 @@ def main(): # check if optional config options are present in config defaults = { "ignore_hosts": [], + "ignore_envfrom": "", "only_hosts": [], "log": "true" } diff --git a/setup.py b/setup.py index 49e4b87..c922066 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ def read_file(fname): return f.read() setup(name = "pyheadermilter", - version = "0.0.2", + version = "0.0.3", author = "Thomas Oettli", author_email = "spacefreak@noop.ch", description = "A pymilter based sendmail/postfix pre-queue filter.", From 92ff4ccf327c7ba8f004a621e9a8ebdcffc90bcb Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 19 Feb 2020 15:40:43 +0100 Subject: [PATCH 009/145] Fix ignore_from handling --- pyheadermilter/__init__.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 3e8d5f7..a1ab4b4 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -31,7 +31,7 @@ from netaddr import IPAddress, IPNetwork, AddrFormatError class HeaderRule: """HeaderRule to implement a rule to apply on e-mail headers.""" - def __init__(self, name, action, header, search="", value="", ignore_hosts=[], ignore_envfrom="", only_hosts=[], log=True): + def __init__(self, name, action, header, search="", value="", ignore_hosts=[], ignore_envfrom=None, only_hosts=[], log=True): self.logger = logging.getLogger(__name__) self.name = name self._action = action @@ -67,10 +67,11 @@ class HeaderRule: except AddrFormatError as e: raise RuntimeError("unable to parse option 'ignore_hosts' of rule '{}': {}".format(name, e)) - try: - self.ignore_envfrom = re.compile(ignore_envfrom, re.IGNORECASE) - except re.error as e: - raise RuntimeError("unable to parse option 'ignore_envfrom' of rule '{}': {}".format(name, e)) + if self.ignore_envfrom: + try: + self.ignore_envfrom = re.compile(ignore_envfrom, re.IGNORECASE) + except re.error as e: + raise RuntimeError("unable to parse option 'ignore_envfrom' of rule '{}': {}".format(name, e)) try: for index, only in enumerate(only_hosts): @@ -330,7 +331,7 @@ def main(): # check if optional config options are present in config defaults = { "ignore_hosts": [], - "ignore_envfrom": "", + "ignore_envfrom": None, "only_hosts": [], "log": "true" } From 13ca896a0f4dfa0c085ba3bf576acd84cfe91fae Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 20 Feb 2020 10:29:40 +0100 Subject: [PATCH 010/145] Skip log message when header value is unchanged --- pyheadermilter/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index a1ab4b4..bda16f6 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -137,10 +137,11 @@ class HeaderRule: if self.header.search("{}: {}".format(name, value)): if self._action == "del": # set an empty value to delete the header - value = "" + new_value = "" else: - value = self.search.sub(self.value, value) - modified.append((name, value, index, occurrences[name])) + new_value = self.search.sub(self.value, value) + if value != new_value: + modified.append((name, new_value, index, occurrences[name])) index += 1 return modified From c1ad06819759e7342b85ccbcc360b4e44ee98187 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 27 Feb 2020 16:37:50 +0100 Subject: [PATCH 011/145] Decode headers before processing --- pyheadermilter/__init__.py | 59 +++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index bda16f6..c35854d 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -25,6 +25,8 @@ import re import sys from Milter.utils import parse_addr +from email.message import EmailMessage +from email.policy import default as default_policy from netaddr import IPAddress, IPNetwork, AddrFormatError @@ -34,7 +36,7 @@ class HeaderRule: def __init__(self, name, action, header, search="", value="", ignore_hosts=[], ignore_envfrom=None, only_hosts=[], log=True): self.logger = logging.getLogger(__name__) self.name = name - self._action = action + self.action = action self.header = header self.search = search self.value = value @@ -79,12 +81,6 @@ class HeaderRule: except AddrFormatError as e: raise RuntimeError("unable to parse option 'only_hosts' of rule '{}': {}".format(name, e)) - def get_action(self): - return self._action - - def log_modification(self): - return self.log - def ignore_host(self, host): ip = IPAddress(host) ignore = False @@ -118,7 +114,7 @@ class HeaderRule: def execute(self, headers): """Execute rule on given headers and return list with modified headers.""" - if self._action == "add": + if self.action == "add": return [(self.header, self.value, 0, 1)] modified = [] @@ -126,7 +122,7 @@ class HeaderRule: occurrences = {} # iterate headers - for name, value in headers: + for name, hdr in headers: # keep track of the occurrence of each header, needed by Milter.Base.chgheader if name not in occurrences.keys(): occurrences[name] = 1 @@ -134,14 +130,18 @@ class HeaderRule: occurrences[name] += 1 # check if header line matches regex + value = hdr[name] if self.header.search("{}: {}".format(name, value)): - if self._action == "del": + if self.action == "del": # set an empty value to delete the header new_value = "" else: + str(hdr).split(": ", 1)[1].strip() new_value = self.search.sub(self.value, value) if value != new_value: - modified.append((name, new_value, index, occurrences[name])) + hdr = EmailMessage(policy=default_policy) + hdr[name] = new_value + modified.append((name, hdr, index, occurrences[name])) index += 1 return modified @@ -194,7 +194,11 @@ class HeaderMilter(Milter.Base): @Milter.noreply def header(self, name, value): - self.headers.append((name, value.encode(errors="surrogateescape").decode(errors="replace"))) + # remove surrogates from value + value = value.encode(errors="surrogateescape").decode(errors="replace") + hdr = EmailMessage(policy=default_policy) + hdr[name] = value + self.headers.append((name, hdr)) return Milter.CONTINUE def eom(self): @@ -202,36 +206,37 @@ class HeaderMilter(Milter.Base): for rule in self.rules: self.logger.debug("{}: executing rule '{}'".format(self.queueid, rule.name)) modified = rule.execute(self.headers) - action = rule.get_action() - log = rule.log_modification() - for name, value, index, occurrence in modified: + for name, hdr, index, occurrence in modified: + value = hdr[name] + encoded_value = bytes(header).decode().split(": ")[1].rstrip() mod_header = "{}: {}".format(name, value) - if action == "add": - if log: + if rule.action == "add": + if rule.log: self.logger.info("{}: add: header: {}".format(self.queueid, mod_header[0:70])) else: self.logger.debug("{}: add: header: {}".format(self.queueid, mod_header)) - - self.headers.insert(0, (name, value)) - self.addheader(name, value, 1) + self.headers.insert(0, (name, hdr)) + self.addheader(name, encoded_value, 1) else: - if action == "mod": - old_header = "{}: {}".format(name, self.headers[index][1]) - if log: + if rule.action == "mod": + old_value = self.headers[index][1][name] + old_header = "{}: {}".format(name, old_value) + if rule.log: self.logger.info("{}: modify: header: {}: {}".format( self.queueid, old_header[0:70], mod_header[0:70])) else: self.logger.debug("{}: modify: header (occ. {}): {}: {}".format( self.queueid, occurrence, old_header, mod_header)) - self.headers[index] = (name, value) - elif action == "del": - if log: + self.headers[index] = (name, hdr) + elif rule.action == "del": + if rule.log: self.logger.info("{}: delete: header: {}".format(self.queueid, mod_header[0:70])) else: self.logger.debug("{}: delete: header (occ. {}): {}".format(self.queueid, occurrence, mod_header)) del self.headers[index] - self.chgheader(name, occurrence, value) + + self.chgheader(name, occurrence, encoded_value) return Milter.ACCEPT except Exception as e: self.logger.exception("an exception occured in eom function: {}".format(e)) From cdc3b72220fa3981f7cf7b24c4f3e8c4eb53b920 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 27 Feb 2020 23:48:08 +0100 Subject: [PATCH 012/145] Properly encode header values after modify --- pyheadermilter/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index c35854d..f102e6a 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -26,6 +26,7 @@ import sys from Milter.utils import parse_addr from email.message import EmailMessage +from email.parser import HeaderParser from email.policy import default as default_policy from netaddr import IPAddress, IPNetwork, AddrFormatError @@ -136,11 +137,10 @@ class HeaderRule: # set an empty value to delete the header new_value = "" else: - str(hdr).split(": ", 1)[1].strip() new_value = self.search.sub(self.value, value) if value != new_value: hdr = EmailMessage(policy=default_policy) - hdr[name] = new_value + hdr.add_header(name, new_value) modified.append((name, hdr, index, occurrences[name])) index += 1 return modified @@ -196,8 +196,9 @@ class HeaderMilter(Milter.Base): def header(self, name, value): # remove surrogates from value value = value.encode(errors="surrogateescape").decode(errors="replace") - hdr = EmailMessage(policy=default_policy) - hdr[name] = value + self.logger.debug(f"{self.queueid}: received header: {name}: {value}") + hdr = HeaderParser(policy=default_policy).parsestr(f"{name}: {value}") + self.logger.debug(f"{self.queueid}: decoded header: {name}: {hdr[name]}") self.headers.append((name, hdr)) return Milter.CONTINUE @@ -209,7 +210,7 @@ class HeaderMilter(Milter.Base): for name, hdr, index, occurrence in modified: value = hdr[name] - encoded_value = bytes(header).decode().split(": ")[1].rstrip() + encoded_value = hdr.as_string().split(": ")[1].rstrip() mod_header = "{}: {}".format(name, value) if rule.action == "add": if rule.log: From 3b6cebb79cab195438e5c3620796aca547fca40f Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 28 Feb 2020 19:28:20 +0100 Subject: [PATCH 013/145] Replace 0-bytes and line breaks, pymilter does not like them --- pyheadermilter/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index f102e6a..56ac12d 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -210,7 +210,8 @@ class HeaderMilter(Milter.Base): for name, hdr, index, occurrence in modified: value = hdr[name] - encoded_value = hdr.as_string().split(": ")[1].rstrip() + # remove illegal characters, pymilter does not like them + encoded_value = hdr.as_string().replace("\r", "").replace("\n", "").replace("\x00", "").split(":", 1)[1].strip() mod_header = "{}: {}".format(name, value) if rule.action == "add": if rule.log: From f0110a5e18ad8332f3c0e091359442793794fb9a Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 28 Feb 2020 20:55:45 +0100 Subject: [PATCH 014/145] Change version to 0.0.4 and start beta --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c922066..8fe1df2 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ def read_file(fname): return f.read() setup(name = "pyheadermilter", - version = "0.0.3", + version = "0.0.4", author = "Thomas Oettli", author_email = "spacefreak@noop.ch", description = "A pymilter based sendmail/postfix pre-queue filter.", @@ -19,7 +19,7 @@ setup(name = "pyheadermilter", # 3 - Alpha # 4 - Beta # 5 - Production/Stable - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", "Operating System :: OS Independent", "Programming Language :: Python", From d6e2e4a5bbb5dff105716e91beafcc5d68906fef Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 28 Feb 2020 20:58:02 +0100 Subject: [PATCH 015/145] Change README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6eb275f..1cd2acf 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # pyheader-milter A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers. -The project is currently in alpha status, but will soon be used in a productive enterprise environment and possibly existing bugs will be fixed. +The project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month. ## Requirements * pymilter From d41978233e74c15dd252899598fa568d81ecdd9b Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 28 Feb 2020 22:02:23 +0100 Subject: [PATCH 016/145] Switch to python f-strings and autopep --- pyheadermilter/__init__.py | 210 +++++++++++++++++++++++-------------- 1 file changed, 131 insertions(+), 79 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 56ac12d..434eae3 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -2,18 +2,17 @@ # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. -# +# # PyHeader-Milter is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with PyHeader-Milter. If not, see . # __all__ = ["HeaderRule", "HeaderMilter"] -name = "pyheadermilter" import Milter import argparse @@ -34,7 +33,8 @@ from netaddr import IPAddress, IPNetwork, AddrFormatError class HeaderRule: """HeaderRule to implement a rule to apply on e-mail headers.""" - def __init__(self, name, action, header, search="", value="", ignore_hosts=[], ignore_envfrom=None, only_hosts=[], log=True): + def __init__(self, name, action, header, search="", value="", + ignore_hosts=[], ignore_envfrom=None, only_hosts=[], log=True): self.logger = logging.getLogger(__name__) self.name = name self.action = action @@ -49,38 +49,46 @@ class HeaderRule: if action in ["del", "mod"]: # compile header regex try: - self.header = re.compile(header, re.MULTILINE + re.DOTALL + re.IGNORECASE) + self.header = re.compile( + header, re.MULTILINE + re.DOTALL + re.IGNORECASE) except re.error as e: - raise RuntimeError("unable to parse option 'header' of rule '{}': {}".format(name, e)) + raise RuntimeError( + f"unable to parse option 'header' of rule '{name}': {e}") if action == "mod": # compile search regex try: - self.search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE) + self.search = re.compile( + search, re.MULTILINE + re.DOTALL + re.IGNORECASE) except re.error as e: - raise RuntimeError("unable to parse option 'search' of rule '{}': {}".format(name, e)) + raise RuntimeError( + f"unable to parse option 'search' of rule '{name}': {e}") if action in ["add", "mod"] and not value: raise RuntimeError("value of option 'value' is empty") - # replace strings in ignore_hosts and only_hosts with IPNetwork instances + # replace strings in ignore_hosts and only_hosts with IPNetwork + # instances try: for index, ignore in enumerate(ignore_hosts): self.ignore_hosts[index] = IPNetwork(ignore) except AddrFormatError as e: - raise RuntimeError("unable to parse option 'ignore_hosts' of rule '{}': {}".format(name, e)) + raise RuntimeError( + f"unable to parse option 'ignore_hosts' of rule '{name}': {e}") if self.ignore_envfrom: try: self.ignore_envfrom = re.compile(ignore_envfrom, re.IGNORECASE) except re.error as e: - raise RuntimeError("unable to parse option 'ignore_envfrom' of rule '{}': {}".format(name, e)) + raise RuntimeError( + f"unable to parse option 'ignore_envfrom' of rule '{name}': {e}") try: for index, only in enumerate(only_hosts): self.only_hosts[index] = IPNetwork(only) except AddrFormatError as e: - raise RuntimeError("unable to parse option 'only_hosts' of rule '{}': {}".format(name, e)) + raise RuntimeError( + f"unable to parse option 'only_hosts' of rule '{name}': {e}") def ignore_host(self, host): ip = IPAddress(host) @@ -101,7 +109,7 @@ class HeaderRule: break if ignore: - self.logger.debug("host {} is ignored by rule {}".format(host, self.name)) + self.logger.debug(f"host {host} is ignored by rule {self.name}") return ignore def ignore_from(self, envfrom): @@ -110,7 +118,8 @@ class HeaderRule: if self.ignore_envfrom: if self.ignore_envfrom.search(envfrom): ignore = True - self.logger.debug("envelope-from {} is ignored by rule {}".format(envfrom, self.name)) + self.logger.debug( + f"envelope-from {envfrom} is ignored by rule {self.name}") return ignore def execute(self, headers): @@ -123,25 +132,26 @@ class HeaderRule: occurrences = {} # iterate headers - for name, hdr in headers: - # keep track of the occurrence of each header, needed by Milter.Base.chgheader + for name, header in headers: + # keep track of the occurrence of each header, needed by + # Milter.Base.chgheader if name not in occurrences.keys(): occurrences[name] = 1 else: occurrences[name] += 1 # check if header line matches regex - value = hdr[name] - if self.header.search("{}: {}".format(name, value)): + value = header[name] + if self.header.search(f"{name}: {value}"): if self.action == "del": # set an empty value to delete the header new_value = "" else: new_value = self.search.sub(self.value, value) if value != new_value: - hdr = EmailMessage(policy=default_policy) - hdr.add_header(name, new_value) - modified.append((name, hdr, index, occurrences[name])) + header = EmailMessage(policy=default_policy) + header.add_header(name, new_value) + modified.append((name, header, index, occurrences[name])) index += 1 return modified @@ -161,7 +171,8 @@ class HeaderMilter(Milter.Base): self.rules = HeaderMilter._rules.copy() def connect(self, IPname, family, hostaddr): - self.logger.debug("accepted milter connection from {} port {}".format(*hostaddr)) + self.logger.debug( + f"accepted milter connection from {hostaddr[0]} port {hostaddr[1]}") ip = IPAddress(hostaddr[0]) # remove rules which ignore this host @@ -170,9 +181,10 @@ class HeaderMilter(Milter.Base): self.rules.remove(rule) if not self.rules: - self.logger.debug("host {} is ignored by all rules, skip further processing".format(hostaddr[0])) + self.logger.debug( + f"host {hostaddr[0]} is ignored by all rules, skip further processing") return Milter.ACCEPT - return Milter.CONTINUE + return Milter.CONTINUE def envfrom(self, mailfrom, *str): mailfrom = "@".join(parse_addr(mailfrom)).lower() @@ -181,14 +193,15 @@ class HeaderMilter(Milter.Base): self.rules.remove(rule) if not self.rules: - self.logger.debug("mail from {} is ignored by all rules, skip further processing".format(mailfrom)) + self.logger.debug( + f"mail from {mailfrom} is ignored by all rules, skip further processing") return Milter.ACCEPT - return Milter.CONTINUE + return Milter.CONTINUE @Milter.noreply def data(self): - self.queueid = self.getsymval('i') - self.logger.debug("{}: received queue-id from MTA".format(self.queueid)) + self.qid = self.getsymval('i') + self.logger.debug(f"{self.qid}: received queue-id from MTA") self.headers = [] return Milter.CONTINUE @@ -196,64 +209,89 @@ class HeaderMilter(Milter.Base): def header(self, name, value): # remove surrogates from value value = value.encode(errors="surrogateescape").decode(errors="replace") - self.logger.debug(f"{self.queueid}: received header: {name}: {value}") - hdr = HeaderParser(policy=default_policy).parsestr(f"{name}: {value}") - self.logger.debug(f"{self.queueid}: decoded header: {name}: {hdr[name]}") - self.headers.append((name, hdr)) + self.logger.debug(f"{self.qid}: received header: {name}: {value}") + header = HeaderParser( + policy=default_policy).parsestr(f"{name}: {value}") + self.logger.debug( + f"{self.qid}: decoded header: {name}: {header[name]}") + self.headers.append((name, header)) return Milter.CONTINUE def eom(self): try: for rule in self.rules: - self.logger.debug("{}: executing rule '{}'".format(self.queueid, rule.name)) + self.logger.debug(f"{self.qid}: executing rule '{rule.name}'") modified = rule.execute(self.headers) - for name, hdr, index, occurrence in modified: - value = hdr[name] + for name, header, index, occurrence in modified: + value = header[name] # remove illegal characters, pymilter does not like them - encoded_value = hdr.as_string().replace("\r", "").replace("\n", "").replace("\x00", "").split(":", 1)[1].strip() - mod_header = "{}: {}".format(name, value) + enc_value = header.as_string().replace( + "\r", "").replace( + "\n", "").replace( + "\x00", "").split( + ":", 1)[1].strip() + mod_header = f"{name}: {value}" if rule.action == "add": if rule.log: - self.logger.info("{}: add: header: {}".format(self.queueid, mod_header[0:70])) + self.logger.info( + f"{self.qid}: add: header: {mod_header[0:70]}") else: - self.logger.debug("{}: add: header: {}".format(self.queueid, mod_header)) - self.headers.insert(0, (name, hdr)) - self.addheader(name, encoded_value, 1) + self.logger.debug( + f"{self.qid}: add: header: {mod_header}") + self.headers.insert(0, (name, header)) + self.addheader(name, enc_value, 1) else: if rule.action == "mod": old_value = self.headers[index][1][name] - old_header = "{}: {}".format(name, old_value) + old_header = f"{name}: {old_value}" if rule.log: - self.logger.info("{}: modify: header: {}: {}".format( - self.queueid, old_header[0:70], mod_header[0:70])) + self.logger.info(f"{self.qid}: modify: header: {old_header[0:70]}: {mod_header[0:70]}") else: - self.logger.debug("{}: modify: header (occ. {}): {}: {}".format( - self.queueid, occurrence, old_header, mod_header)) - self.headers[index] = (name, hdr) + self.logger.debug( + f"{self.qid}: modify: header (occ. {occurrence}): {old_header}: {mod_header}") + self.headers[index] = (name, header) elif rule.action == "del": if rule.log: - self.logger.info("{}: delete: header: {}".format(self.queueid, mod_header[0:70])) + self.logger.info( + f"{self.qid}: delete: header: {mod_header[0:70]}") else: - self.logger.debug("{}: delete: header (occ. {}): {}".format(self.queueid, occurrence, mod_header)) + self.logger.debug( + f"{self.qid}: delete: header (occ. {occurrence}): {mod_header}") del self.headers[index] - self.chgheader(name, occurrence, encoded_value) + self.chgheader(name, occurrence, enc_value) return Milter.ACCEPT except Exception as e: - self.logger.exception("an exception occured in eom function: {}".format(e)) + self.logger.exception(f"an exception occured in eom function: {e}") return Milter.TEMPFAIL def main(): "Run PyHeader-Milter." # parse command line - parser = argparse.ArgumentParser(description="PyHeader milter daemon", - formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=45, width=140)) - parser.add_argument("-c", "--config", help="Config file to read.", default="/etc/pyheader-milter.conf") - parser.add_argument("-s", "--socket", help="Socket used to communicate with the MTA.", required=True) - parser.add_argument("-d", "--debug", help="Log debugging messages.", action="store_true") - parser.add_argument("-t", "--test", help="Check configuration.", action="store_true") + parser = argparse.ArgumentParser( + description="PyHeader milter daemon", + formatter_class=lambda prog: argparse.HelpFormatter( + prog, max_help_position=45, width=140)) + parser.add_argument( + "-c", "--config", help="Config file to read.", + default="/etc/pyheader-milter.conf") + parser.add_argument( + "-s", + "--socket", + help="Socket used to communicate with the MTA.", + required=True) + parser.add_argument( + "-d", + "--debug", + help="Log debugging messages.", + action="store_true") + parser.add_argument( + "-t", + "--test", + help="Check configuration.", + action="store_true") args = parser.parse_args() # setup logging @@ -262,8 +300,8 @@ def main(): syslog_name = logname if args.debug: loglevel = logging.DEBUG - logname = "{}[%(name)s]".format(logname) - syslog_name = "{}: [%(name)s] %(levelname)s".format(syslog_name) + logname = f"{logname}[%(name)s]" + syslog_name = f"{syslog_name}: [%(name)s] %(levelname)s" # set config files for milter class root_logger = logging.getLogger() @@ -272,11 +310,11 @@ def main(): # setup console log stdouthandler = logging.StreamHandler(sys.stdout) stdouthandler.setLevel(logging.DEBUG) - formatter = logging.Formatter("%(message)s".format(logname)) + formatter = logging.Formatter("%(message)s") stdouthandler.setFormatter(formatter) root_logger.addHandler(stdouthandler) logger = logging.getLogger(__name__) - + try: # read config file parser = configparser.ConfigParser() @@ -285,21 +323,25 @@ def main(): # check if mandatory config options in global section are present if "global" not in parser.sections(): - raise RuntimeError("mandatory section 'global' not present in config file") + raise RuntimeError( + "mandatory section 'global' not present in config file") for option in ["rules"]: if not parser.has_option("global", option): - raise RuntimeError("mandatory option '{}' not present in config section 'global'".format(option)) + raise RuntimeError( + f"mandatory option '{option}' not present in config section 'global'") # read global config section global_config = dict(parser.items("global")) # read active rules - active_rules = [ r.strip() for r in global_config["rules"].split(",") ] + active_rules = [r.strip() for r in global_config["rules"].split(",")] if len(active_rules) != len(set(active_rules)): - raise RuntimeError("at least one rule is specified multiple times in 'rules' option") + raise RuntimeError( + "at least one rule is specified multiple times in 'rules' option") if "global" in active_rules: active_rules.remove("global") - logger.warning("removed illegal rule name 'global' from list of active rules") + logger.warning( + "removed illegal rule name 'global' from list of active rules") if not active_rules: raise RuntimeError("no rules configured") @@ -307,9 +349,10 @@ def main(): rules = [] # iterate active rules for rule_name in active_rules: - # check if config section exists + # check if config section exists if rule_name not in parser.sections(): - raise RuntimeError("config section '{}' does not exist".format(rule_name)) + raise RuntimeError( + f"config section '{rule_name}' does not exist") config = dict(parser.items(rule_name)) # check if mandatory option action is present in config @@ -318,10 +361,12 @@ def main(): option in global_config.keys(): config[option] = global_config[option] if option not in config.keys(): - raise RuntimeError("mandatory option '{}' not specified for rule '{}'".format(option, rule_name)) + raise RuntimeError( + f"mandatory option '{option}' not specified for rule '{rule_name}'") config["action"] = config["action"].lower() if config["action"] not in ["add", "del", "mod"]: - raise RuntimeError("invalid action specified for rule '{}'".format(rule_name)) + raise RuntimeError( + f"invalid action specified for rule '{rule_name}'") # check if mandatory options are present in config mandatory = ["header"] @@ -334,7 +379,8 @@ def main(): option in global_config.keys(): config[option] = global_config[option] if option not in config.keys(): - raise RuntimeError("mandatory option '{}' not specified for rule '{}'".format(option, rule_name)) + raise RuntimeError( + f"mandatory option '{option}' not specified for rule '{rule_name}'") # check if optional config options are present in config defaults = { @@ -350,19 +396,22 @@ def main(): if option not in config.keys(): config[option] = defaults[option] if config["ignore_hosts"]: - config["ignore_hosts"] = [ h.strip() for h in config["ignore_hosts"].split(",") ] + config["ignore_hosts"] = [h.strip() + for h in config["ignore_hosts"].split(",")] if config["only_hosts"]: - config["only_hosts"] = [ h.strip() for h in config["only_hosts"].split(",") ] + config["only_hosts"] = [h.strip() + for h in config["only_hosts"].split(",")] config["log"] = config["log"].lower() if config["log"] == "true": config["log"] = True elif config["log"] == "false": config["log"] = False else: - raise RuntimeError("invalid value specified for option 'log' for rule '{}'".format(rule_name)) + raise RuntimeError( + f"invalid value specified for option 'log' for rule '{rule_name}'") # add rule - logging.debug("adding rule '{}'".format(rule_name)) + logging.debug(f"adding rule '{rule_name}'") rules.append(HeaderRule(name=rule_name, **config)) except RuntimeError as e: @@ -374,13 +423,16 @@ def main(): sys.exit(0) # change log format for runtime - formatter = logging.Formatter("%(asctime)s {}: [%(levelname)s] %(message)s".format(logname), datefmt="%Y-%m-%d %H:%M:%S") + formatter = logging.Formatter( + f"%(asctime)s {logname}: [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S") stdouthandler.setFormatter(formatter) # setup syslog - sysloghandler = logging.handlers.SysLogHandler(address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) + sysloghandler = logging.handlers.SysLogHandler( + address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) sysloghandler.setLevel(loglevel) - formatter = logging.Formatter("{}: %(message)s".format(syslog_name)) + formatter = logging.Formatter(f"{syslog_name}: %(message)s") sysloghandler.setFormatter(formatter) root_logger.addHandler(sysloghandler) From 05ec41158d5cb6014e6eddbd6931645ab186795b Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Sat, 29 Feb 2020 00:41:05 +0100 Subject: [PATCH 017/145] Fix python dependcy --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8fe1df2..6c318db 100644 --- a/setup.py +++ b/setup.py @@ -32,5 +32,5 @@ setup(name = "pyheadermilter", ] }, install_requires = ["pymilter", "netaddr"], - python_requires = ">=3" + python_requires = ">=3.6" ) From ad79efe1395bdb440c46994dbe86717fee869ac4 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Sat, 29 Feb 2020 17:46:16 +0100 Subject: [PATCH 018/145] Remove import of unused module os --- pyheadermilter/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 434eae3..95225bb 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -19,7 +19,6 @@ import argparse import configparser import logging import logging.handlers -import os import re import sys From 415c41ed7a4cf3b20ff4eb6a038e4654f8716b64 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Sun, 1 Mar 2020 22:25:13 +0100 Subject: [PATCH 019/145] Small bugfix and code cleanup --- pyheadermilter/__init__.py | 74 +++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 24 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 95225bb..278f2ad 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -33,7 +33,8 @@ class HeaderRule: """HeaderRule to implement a rule to apply on e-mail headers.""" def __init__(self, name, action, header, search="", value="", - ignore_hosts=[], ignore_envfrom=None, only_hosts=[], log=True): + ignore_hosts=[], ignore_envfrom=None, only_hosts=[], + log=True): self.logger = logging.getLogger(__name__) self.name = name self.action = action @@ -61,7 +62,8 @@ class HeaderRule: search, re.MULTILINE + re.DOTALL + re.IGNORECASE) except re.error as e: raise RuntimeError( - f"unable to parse option 'search' of rule '{name}': {e}") + f"unable to parse option 'search' of " + f"rule '{name}': {e}") if action in ["add", "mod"] and not value: raise RuntimeError("value of option 'value' is empty") @@ -80,7 +82,8 @@ class HeaderRule: self.ignore_envfrom = re.compile(ignore_envfrom, re.IGNORECASE) except re.error as e: raise RuntimeError( - f"unable to parse option 'ignore_envfrom' of rule '{name}': {e}") + f"unable to parse option 'ignore_envfrom' of " + f"rule '{name}': {e}") try: for index, only in enumerate(only_hosts): @@ -122,7 +125,10 @@ class HeaderRule: return ignore def execute(self, headers): - """Execute rule on given headers and return list with modified headers.""" + """ + Execute rule on given headers and return list + with modified headers. + """ if self.action == "add": return [(self.header, self.value, 0, 1)] @@ -139,14 +145,18 @@ class HeaderRule: else: occurrences[name] += 1 - # check if header line matches regex value = header[name] + # check if header line matches regex if self.header.search(f"{name}: {value}"): if self.action == "del": # set an empty value to delete the header new_value = "" else: - new_value = self.search.sub(self.value, value) + # Remove line breaks from new_value, EmailMessage object + # does not like them + new_value = self.search.sub(self.value, value).replace( + "\n", "").replace( + "\r", "") if value != new_value: header = EmailMessage(policy=default_policy) header.add_header(name, new_value) @@ -171,7 +181,8 @@ class HeaderMilter(Milter.Base): def connect(self, IPname, family, hostaddr): self.logger.debug( - f"accepted milter connection from {hostaddr[0]} port {hostaddr[1]}") + f"accepted milter connection from {hostaddr[0]} " + f"port {hostaddr[1]}") ip = IPAddress(hostaddr[0]) # remove rules which ignore this host @@ -181,7 +192,8 @@ class HeaderMilter(Milter.Base): if not self.rules: self.logger.debug( - f"host {hostaddr[0]} is ignored by all rules, skip further processing") + f"host {hostaddr[0]} is ignored by all rules, " + f"skip further processing") return Milter.ACCEPT return Milter.CONTINUE @@ -193,7 +205,8 @@ class HeaderMilter(Milter.Base): if not self.rules: self.logger.debug( - f"mail from {mailfrom} is ignored by all rules, skip further processing") + f"mail from {mailfrom} is ignored by all rules, " + f"skip further processing") return Milter.ACCEPT return Milter.CONTINUE @@ -245,24 +258,31 @@ class HeaderMilter(Milter.Base): old_value = self.headers[index][1][name] old_header = f"{name}: {old_value}" if rule.log: - self.logger.info(f"{self.qid}: modify: header: {old_header[0:70]}: {mod_header[0:70]}") + self.logger.info( + f"{self.qid}: modify: header: " + f"{old_header[0:70]}: {mod_header[0:70]}") else: self.logger.debug( - f"{self.qid}: modify: header (occ. {occurrence}): {old_header}: {mod_header}") + f"{self.qid}: modify: header " + f"(occ. {occurrence}): {old_header}: " + f"{mod_header}") self.headers[index] = (name, header) elif rule.action == "del": if rule.log: self.logger.info( - f"{self.qid}: delete: header: {mod_header[0:70]}") + f"{self.qid}: delete: header: " + f"{mod_header[0:70]}") else: self.logger.debug( - f"{self.qid}: delete: header (occ. {occurrence}): {mod_header}") + f"{self.qid}: delete: header " + f"(occ. {occurrence}): {mod_header}") del self.headers[index] self.chgheader(name, occurrence, enc_value) return Milter.ACCEPT except Exception as e: - self.logger.exception(f"an exception occured in eom function: {e}") + self.logger.exception( + f"an exception occured in eom function: {e}") return Milter.TEMPFAIL @@ -327,7 +347,8 @@ def main(): for option in ["rules"]: if not parser.has_option("global", option): raise RuntimeError( - f"mandatory option '{option}' not present in config section 'global'") + f"mandatory option '{option}' not present in config " + f"section 'global'") # read global config section global_config = dict(parser.items("global")) @@ -336,11 +357,13 @@ def main(): active_rules = [r.strip() for r in global_config["rules"].split(",")] if len(active_rules) != len(set(active_rules)): raise RuntimeError( - "at least one rule is specified multiple times in 'rules' option") + "at least one rule is specified multiple times " + "in 'rules' option") if "global" in active_rules: active_rules.remove("global") logger.warning( - "removed illegal rule name 'global' from list of active rules") + "removed illegal rule name 'global' from list of " + "active rules") if not active_rules: raise RuntimeError("no rules configured") @@ -361,7 +384,8 @@ def main(): config[option] = global_config[option] if option not in config.keys(): raise RuntimeError( - f"mandatory option '{option}' not specified for rule '{rule_name}'") + f"mandatory option '{option}' not specified for " + f"rule '{rule_name}'") config["action"] = config["action"].lower() if config["action"] not in ["add", "del", "mod"]: raise RuntimeError( @@ -379,7 +403,8 @@ def main(): config[option] = global_config[option] if option not in config.keys(): raise RuntimeError( - f"mandatory option '{option}' not specified for rule '{rule_name}'") + f"mandatory option '{option}' not specified for " + f"rule '{rule_name}'") # check if optional config options are present in config defaults = { @@ -395,11 +420,11 @@ def main(): if option not in config.keys(): config[option] = defaults[option] if config["ignore_hosts"]: - config["ignore_hosts"] = [h.strip() - for h in config["ignore_hosts"].split(",")] + config["ignore_hosts"] = [ + h.strip() for h in config["ignore_hosts"].split(",")] if config["only_hosts"]: - config["only_hosts"] = [h.strip() - for h in config["only_hosts"].split(",")] + config["only_hosts"] = [ + h.strip() for h in config["only_hosts"].split(",")] config["log"] = config["log"].lower() if config["log"] == "true": config["log"] = True @@ -407,7 +432,8 @@ def main(): config["log"] = False else: raise RuntimeError( - f"invalid value specified for option 'log' for rule '{rule_name}'") + f"invalid value specified for option 'log' for " + f"rule '{rule_name}'") # add rule logging.debug(f"adding rule '{rule_name}'") From 750c343b74fe39e768fa151465c3d83474ffb558 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 6 Mar 2020 15:07:10 +0100 Subject: [PATCH 020/145] Fix another problem with line breaks --- pyheadermilter/__init__.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 278f2ad..ca843c8 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -152,14 +152,12 @@ class HeaderRule: # set an empty value to delete the header new_value = "" else: - # Remove line breaks from new_value, EmailMessage object - # does not like them - new_value = self.search.sub(self.value, value).replace( - "\n", "").replace( - "\r", "") + new_value = self.search.sub(self.value, value) if value != new_value: header = EmailMessage(policy=default_policy) - header.add_header(name, new_value) + # Remove line breaks, EmailMessage object + #does not like them + header.add_header(name, " ".join(new_value.splitlines())) modified.append((name, header, index, occurrences[name])) index += 1 return modified From 62311612cd2daa3f64f87bfed0a02c68e60a0828 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 6 Mar 2020 15:07:28 +0100 Subject: [PATCH 021/145] Change version to 0.0.6 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6c318db..3785132 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ def read_file(fname): return f.read() setup(name = "pyheadermilter", - version = "0.0.4", + version = "0.0.6", author = "Thomas Oettli", author_email = "spacefreak@noop.ch", description = "A pymilter based sendmail/postfix pre-queue filter.", From 15ec705cb1915ec402f3ec1a96a50b21148b20d3 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 18 Mar 2020 16:12:43 +0100 Subject: [PATCH 022/145] Switch back to email.header lib because of error handling --- pyheadermilter/__init__.py | 90 +++++++++++++++++++++++--------------- setup.py | 2 +- 2 files changed, 55 insertions(+), 37 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index ca843c8..041f15b 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -23,12 +23,33 @@ import re import sys from Milter.utils import parse_addr -from email.message import EmailMessage -from email.parser import HeaderParser -from email.policy import default as default_policy +from email.charset import Charset +from email.header import Header, decode_header from netaddr import IPAddress, IPNetwork, AddrFormatError +def make_header(decoded_seq, maxlinelen=None, header_name=None, + continuation_ws=' ', errors='strict'): + """Create a Header from a sequence of pairs as returned by decode_header() + + decode_header() takes a header value string and returns a sequence of + pairs of the format (decoded_string, charset) where charset is the string + name of the character set. + + This function takes one of those sequence of pairs and returns a Header + instance. Optional maxlinelen, header_name, and continuation_ws are as in + the Header constructor. + """ + h = Header(maxlinelen=maxlinelen, header_name=header_name, + continuation_ws=continuation_ws) + for s, charset in decoded_seq: + # None means us-ascii but we can simply pass it on to h.append() + if charset is not None and not isinstance(charset, Charset): + charset = Charset(charset) + h.append(s, charset, errors=errors) + return h + + class HeaderRule: """HeaderRule to implement a rule to apply on e-mail headers.""" @@ -145,19 +166,19 @@ class HeaderRule: else: occurrences[name] += 1 - value = header[name] # check if header line matches regex - if self.header.search(f"{name}: {value}"): + header_line = str(header) + if self.header.search(header_line): + value = header_line.split(":", 1)[1].strip() if self.action == "del": # set an empty value to delete the header new_value = "" else: new_value = self.search.sub(self.value, value) if value != new_value: - header = EmailMessage(policy=default_policy) - # Remove line breaks, EmailMessage object - #does not like them - header.add_header(name, " ".join(new_value.splitlines())) + header = make_header( + decode_header( + f"{name}: {new_value}"), errors="replace") modified.append((name, header, index, occurrences[name])) index += 1 return modified @@ -215,15 +236,18 @@ class HeaderMilter(Milter.Base): self.headers = [] return Milter.CONTINUE - @Milter.noreply def header(self, name, value): - # remove surrogates from value - value = value.encode(errors="surrogateescape").decode(errors="replace") - self.logger.debug(f"{self.qid}: received header: {name}: {value}") - header = HeaderParser( - policy=default_policy).parsestr(f"{name}: {value}") - self.logger.debug( - f"{self.qid}: decoded header: {name}: {header[name]}") + try: + # remove surrogates from value + value = value.encode(errors="surrogateescape").decode(errors="replace") + self.logger.debug(f"{self.qid}: received header: {name}: {value}") + header = make_header(decode_header(f"{name}: {value}"), errors="replace") + self.logger.debug( + f"{self.qid}: decoded header: {header}") + except Exception as e: + self.logger.exception( + f"an exception occured in header function: {e}") + return Milter.TEMPFAIL self.headers.append((name, header)) return Milter.CONTINUE @@ -232,51 +256,45 @@ class HeaderMilter(Milter.Base): for rule in self.rules: self.logger.debug(f"{self.qid}: executing rule '{rule.name}'") modified = rule.execute(self.headers) - for name, header, index, occurrence in modified: - value = header[name] - # remove illegal characters, pymilter does not like them - enc_value = header.as_string().replace( - "\r", "").replace( - "\n", "").replace( - "\x00", "").split( - ":", 1)[1].strip() - mod_header = f"{name}: {value}" + header_line = str(header) + value = header.encode().split(":", 1)[1].strip() if rule.action == "add": if rule.log: self.logger.info( - f"{self.qid}: add: header: {mod_header[0:70]}") + f"{self.qid}: add: header: " + f"{header_line[0:70]}") else: self.logger.debug( - f"{self.qid}: add: header: {mod_header}") + f"{self.qid}: add: header: " + f"{header_line}") self.headers.insert(0, (name, header)) - self.addheader(name, enc_value, 1) + self.addheader(name, value, 1) else: if rule.action == "mod": - old_value = self.headers[index][1][name] - old_header = f"{name}: {old_value}" + old_header = str(self.headers[index][1]) if rule.log: self.logger.info( f"{self.qid}: modify: header: " - f"{old_header[0:70]}: {mod_header[0:70]}") + f"{old_header[0:70]}: {header_line[0:70]}") else: self.logger.debug( f"{self.qid}: modify: header " f"(occ. {occurrence}): {old_header}: " - f"{mod_header}") + f"{header_line}") self.headers[index] = (name, header) elif rule.action == "del": if rule.log: self.logger.info( f"{self.qid}: delete: header: " - f"{mod_header[0:70]}") + f"{header_line[0:70]}") else: self.logger.debug( f"{self.qid}: delete: header " - f"(occ. {occurrence}): {mod_header}") + f"(occ. {occurrence}): {header_line}") del self.headers[index] - self.chgheader(name, occurrence, enc_value) + self.chgheader(name, occurrence, value) return Milter.ACCEPT except Exception as e: self.logger.exception( diff --git a/setup.py b/setup.py index 3785132..765a949 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ def read_file(fname): return f.read() setup(name = "pyheadermilter", - version = "0.0.6", + version = "0.0.7", author = "Thomas Oettli", author_email = "spacefreak@noop.ch", description = "A pymilter based sendmail/postfix pre-queue filter.", From 45e39fcffc32cc902afefe9ae58c691c75c8c63c Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 18 Mar 2020 17:06:20 +0100 Subject: [PATCH 023/145] Note max line length --- pyheadermilter/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 041f15b..37d71c4 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -239,9 +239,12 @@ class HeaderMilter(Milter.Base): def header(self, name, value): try: # remove surrogates from value - value = value.encode(errors="surrogateescape").decode(errors="replace") + value = value.encode( + errors="surrogateescape").decode(errors="replace") self.logger.debug(f"{self.qid}: received header: {name}: {value}") - header = make_header(decode_header(f"{name}: {value}"), errors="replace") + header = make_header( + decode_header( + f"{name}: {value}"), errors="replace") self.logger.debug( f"{self.qid}: decoded header: {header}") except Exception as e: From 9741e93089c7df467ab2058b650f8751959fffbc Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 18 Mar 2020 18:26:07 +0100 Subject: [PATCH 024/145] Bugfix --- pyheadermilter/__init__.py | 51 ++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 37d71c4..54e6719 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -158,7 +158,7 @@ class HeaderRule: occurrences = {} # iterate headers - for name, header in headers: + for name, value in headers: # keep track of the occurrence of each header, needed by # Milter.Base.chgheader if name not in occurrences.keys(): @@ -167,19 +167,15 @@ class HeaderRule: occurrences[name] += 1 # check if header line matches regex - header_line = str(header) - if self.header.search(header_line): - value = header_line.split(":", 1)[1].strip() + if self.header.search(f"{name}: {value}"): if self.action == "del": # set an empty value to delete the header new_value = "" else: - new_value = self.search.sub(self.value, value) + new_value = self.search.sub(self.value, str(value)) if value != new_value: - header = make_header( - decode_header( - f"{name}: {new_value}"), errors="replace") - modified.append((name, header, index, occurrences[name])) + modified.append( + (name, Header(s=new_value), index, occurrences[name])) index += 1 return modified @@ -242,62 +238,59 @@ class HeaderMilter(Milter.Base): value = value.encode( errors="surrogateescape").decode(errors="replace") self.logger.debug(f"{self.qid}: received header: {name}: {value}") - header = make_header( - decode_header( - f"{name}: {value}"), errors="replace") + value = make_header(decode_header(value), errors="replace") self.logger.debug( - f"{self.qid}: decoded header: {header}") + f"{self.qid}: decoded header: {name}: {value}") + self.headers.append((name, value)) + return Milter.CONTINUE except Exception as e: self.logger.exception( f"an exception occured in header function: {e}") return Milter.TEMPFAIL - self.headers.append((name, header)) - return Milter.CONTINUE def eom(self): try: for rule in self.rules: self.logger.debug(f"{self.qid}: executing rule '{rule.name}'") modified = rule.execute(self.headers) - for name, header, index, occurrence in modified: - header_line = str(header) - value = header.encode().split(":", 1)[1].strip() + for name, value, index, occurrence in modified: + header = f"{name}: {value}" if rule.action == "add": if rule.log: self.logger.info( f"{self.qid}: add: header: " - f"{header_line[0:70]}") + f"{header[0:70]}") else: self.logger.debug( f"{self.qid}: add: header: " - f"{header_line}") - self.headers.insert(0, (name, header)) - self.addheader(name, value, 1) + f"{header}") + self.headers.insert(0, (name, value)) + self.addheader(name, value.encode(), 1) else: if rule.action == "mod": - old_header = str(self.headers[index][1]) + old_header = "{}: {}".format(*self.headers[index]) if rule.log: self.logger.info( f"{self.qid}: modify: header: " - f"{old_header[0:70]}: {header_line[0:70]}") + f"{old_header[0:70]}: {header[0:70]}") else: self.logger.debug( f"{self.qid}: modify: header " f"(occ. {occurrence}): {old_header}: " - f"{header_line}") - self.headers[index] = (name, header) + f"{header}") + self.headers[index] = (name, value) elif rule.action == "del": if rule.log: self.logger.info( f"{self.qid}: delete: header: " - f"{header_line[0:70]}") + f"{header[0:70]}") else: self.logger.debug( f"{self.qid}: delete: header " - f"(occ. {occurrence}): {header_line}") + f"(occ. {occurrence}): {header}") del self.headers[index] - self.chgheader(name, occurrence, value) + self.chgheader(name, occurrence, value.encode()) return Milter.ACCEPT except Exception as e: self.logger.exception( From 30d9d2b779861d920fa0dc82d0e9fd5f7a4d8f69 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 19 Mar 2020 10:33:00 +0100 Subject: [PATCH 025/145] Cleanup source --- pyheadermilter/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 54e6719..90c9e0b 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -172,10 +172,10 @@ class HeaderRule: # set an empty value to delete the header new_value = "" else: - new_value = self.search.sub(self.value, str(value)) + new_value = self.search.sub(self.value, value) if value != new_value: modified.append( - (name, Header(s=new_value), index, occurrences[name])) + (name, new_value, index, occurrences[name])) index += 1 return modified @@ -238,7 +238,7 @@ class HeaderMilter(Milter.Base): value = value.encode( errors="surrogateescape").decode(errors="replace") self.logger.debug(f"{self.qid}: received header: {name}: {value}") - value = make_header(decode_header(value), errors="replace") + value = str(make_header(decode_header(value), errors="replace")) self.logger.debug( f"{self.qid}: decoded header: {name}: {value}") self.headers.append((name, value)) @@ -255,6 +255,7 @@ class HeaderMilter(Milter.Base): modified = rule.execute(self.headers) for name, value, index, occurrence in modified: header = f"{name}: {value}" + enc_value = Header(s=value).encode() if rule.action == "add": if rule.log: self.logger.info( @@ -265,7 +266,7 @@ class HeaderMilter(Milter.Base): f"{self.qid}: add: header: " f"{header}") self.headers.insert(0, (name, value)) - self.addheader(name, value.encode(), 1) + self.addheader(name, enc_value, 1) else: if rule.action == "mod": old_header = "{}: {}".format(*self.headers[index]) @@ -290,7 +291,7 @@ class HeaderMilter(Milter.Base): f"(occ. {occurrence}): {header}") del self.headers[index] - self.chgheader(name, occurrence, value.encode()) + self.chgheader(name, occurrence, enc_value) return Milter.ACCEPT except Exception as e: self.logger.exception( From 0856c39442bf8b417eb71e522b8a6f41f30f69eb Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 20 Mar 2020 11:27:36 +0100 Subject: [PATCH 026/145] Remove 0-bytes from headers before processing --- pyheadermilter/__init__.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pyheadermilter/__init__.py b/pyheadermilter/__init__.py index 90c9e0b..dea4c65 100644 --- a/pyheadermilter/__init__.py +++ b/pyheadermilter/__init__.py @@ -12,7 +12,11 @@ # along with PyHeader-Milter. If not, see . # -__all__ = ["HeaderRule", "HeaderMilter"] +__all__ = [ + "make_header", + "replace_illegal_chars", + "HeaderRule", + "HeaderMilter"] import Milter import argparse @@ -50,6 +54,13 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None, return h +def replace_illegal_chars(string): + return string.replace( + "\x00", "").replace( + "\r", "").replace( + "\n", "") + + class HeaderRule: """HeaderRule to implement a rule to apply on e-mail headers.""" @@ -238,7 +249,8 @@ class HeaderMilter(Milter.Base): value = value.encode( errors="surrogateescape").decode(errors="replace") self.logger.debug(f"{self.qid}: received header: {name}: {value}") - value = str(make_header(decode_header(value), errors="replace")) + header = make_header(decode_header(value), errors="replace") + value = str(header).replace("\x00", "") self.logger.debug( f"{self.qid}: decoded header: {name}: {value}") self.headers.append((name, value)) @@ -255,7 +267,8 @@ class HeaderMilter(Milter.Base): modified = rule.execute(self.headers) for name, value, index, occurrence in modified: header = f"{name}: {value}" - enc_value = Header(s=value).encode() + enc_value = replace_illegal_chars( + Header(s=value).encode()) if rule.action == "add": if rule.log: self.logger.info( From c6e1408732bcfb4290fbd57a1c6c534bb5c07d69 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Mon, 20 Apr 2020 10:33:49 +0200 Subject: [PATCH 027/145] Add missing documentation for option ignore_envfrom --- README.md | 2 ++ docs/pyheader-milter.conf.example | 6 ++++++ setup.py | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1cd2acf..a8e01a7 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ The following configuration options are mandatory for a mod-rule: New value of the header. The following configuration options are optional for each rule: +* **ignore_envfrom** + Regular expression to match envelop-from addresses. The rule will be skipped if the expression matches. * **ignore_hosts** Comma-separated list of host and network addresses. The rule will be skipped if the sending host is included here. * **only_hosts** diff --git a/docs/pyheader-milter.conf.example b/docs/pyheader-milter.conf.example index f19dd53..faa4713 100644 --- a/docs/pyheader-milter.conf.example +++ b/docs/pyheader-milter.conf.example @@ -19,6 +19,12 @@ # rules = add_header,del_header,mod_header +# Option: ignore_envfrom +# Notes: Set a regular expression to match envelope-from addresses to be ignored. +# Value: [ REGEX ] +# +ignore_envfrom = ^.*@localhost$ + # Option: ignore_hosts # Notes: Set a list of host and network addresses to be ignored. # All the common host/network notations are supported, including IPv6. diff --git a/setup.py b/setup.py index 765a949..d006ca0 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ def read_file(fname): return f.read() setup(name = "pyheadermilter", - version = "0.0.7", + version = "0.0.8", author = "Thomas Oettli", author_email = "spacefreak@noop.ch", description = "A pymilter based sendmail/postfix pre-queue filter.", From 594d3a466bc7e2bbc508ab9fd5e52791a77b31d3 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Mon, 20 Apr 2020 11:50:01 +0200 Subject: [PATCH 028/145] Rename project to pymodmilter --- README.md | 4 ++-- ....conf.example => pymodmilter.conf.example} | 2 +- ...yheader-milter.confd => pymodmilter.confd} | 4 ++-- ...yheader-milter.initd => pymodmilter.initd} | 2 +- {pyheadermilter => pymodmilter}/__init__.py | 20 +++++++++---------- setup.py | 8 ++++---- 6 files changed, 20 insertions(+), 20 deletions(-) rename docs/{pyheader-milter.conf.example => pymodmilter.conf.example} (97%) rename misc/openrc/{pyheader-milter.confd => pymodmilter.confd} (82%) rename misc/openrc/{pyheader-milter.initd => pymodmilter.initd} (96%) rename {pyheadermilter => pymodmilter}/__init__.py (97%) diff --git a/README.md b/README.md index a8e01a7..acf0810 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# pyheader-milter +# pymodmilter A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers. The project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month. @@ -8,7 +8,7 @@ The project is currently in beta status, but it is already used in a productive * netaddr ## Configuration -The pyheader-milter uses an INI-style configuration file. The sections are described below. +The pymodmilter uses an INI-style configuration file. The sections are described below. ### Section "global" Any available configuration option can be set in the global section as default instead of in a rule section. diff --git a/docs/pyheader-milter.conf.example b/docs/pymodmilter.conf.example similarity index 97% rename from docs/pyheader-milter.conf.example rename to docs/pymodmilter.conf.example index faa4713..3f0a350 100644 --- a/docs/pyheader-milter.conf.example +++ b/docs/pymodmilter.conf.example @@ -1,4 +1,4 @@ -# This is an example /etc/pyheader-milter.conf file. +# This is an example /etc/pymodmilter.conf file. # Copy it into place before use. # # Comments: use '#' for comment lines and ';' (following a space) for inline comments. diff --git a/misc/openrc/pyheader-milter.confd b/misc/openrc/pymodmilter.confd similarity index 82% rename from misc/openrc/pyheader-milter.confd rename to misc/openrc/pymodmilter.confd index b8a0c08..b4e923c 100644 --- a/misc/openrc/pyheader-milter.confd +++ b/misc/openrc/pymodmilter.confd @@ -1,4 +1,4 @@ -# /etc/conf.d/pyheader-milter: config file for /etc/init.d/pyheader-milter +# /etc/conf.d/pymodmilter: config file for /etc/init.d/pymodmilter # Set the socket used to communicate with the MTA. # Examples: @@ -13,5 +13,5 @@ SOCKET="inet:8898@127.0.0.1" # USER="daemon" # USER="daemon:nobody" -# Optional parameters for pyheader-milter +# Optional parameters for pymodmilter # MILTER_OPTS="" diff --git a/misc/openrc/pyheader-milter.initd b/misc/openrc/pymodmilter.initd similarity index 96% rename from misc/openrc/pyheader-milter.initd rename to misc/openrc/pymodmilter.initd index 8393089..a487a22 100755 --- a/misc/openrc/pyheader-milter.initd +++ b/misc/openrc/pymodmilter.initd @@ -5,7 +5,7 @@ socket="${SOCKET:-}" milter_opts="${MILTER_OPTS:-}" pidfile="/run/${RC_SVCNAME}.pid" -command="/usr/bin/pyheader-milter" +command="/usr/bin/pymodmilter" command_args="-s ${socket} ${milter_opts}" command_background=true start_stop_daemon_args="--user ${user}" diff --git a/pyheadermilter/__init__.py b/pymodmilter/__init__.py similarity index 97% rename from pyheadermilter/__init__.py rename to pymodmilter/__init__.py index dea4c65..c46c955 100644 --- a/pyheadermilter/__init__.py +++ b/pymodmilter/__init__.py @@ -1,15 +1,15 @@ -# PyHeader-Milter is free software: you can redistribute it and/or modify +# PyMod-Milter is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyHeader-Milter is distributed in the hope that it will be useful, +# PyMod-Milter is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyHeader-Milter. If not, see . +# along with PyMod-Milter. If not, see . # __all__ = [ @@ -313,15 +313,15 @@ class HeaderMilter(Milter.Base): def main(): - "Run PyHeader-Milter." + "Run PyMod-Milter." # parse command line parser = argparse.ArgumentParser( - description="PyHeader milter daemon", + description="PyMod milter daemon", formatter_class=lambda prog: argparse.HelpFormatter( prog, max_help_position=45, width=140)) parser.add_argument( "-c", "--config", help="Config file to read.", - default="/etc/pyheader-milter.conf") + default="/etc/pymodmilter.conf") parser.add_argument( "-s", "--socket", @@ -341,7 +341,7 @@ def main(): # setup logging loglevel = logging.INFO - logname = "pyheader-milter" + logname = "pymodmilter" syslog_name = logname if args.debug: loglevel = logging.DEBUG @@ -487,7 +487,7 @@ def main(): sysloghandler.setFormatter(formatter) root_logger.addHandler(sysloghandler) - logger.info("PyHeader-Milter starting") + logger.info("pymodmilter starting") HeaderMilter.set_rules(rules) # register milter factory class @@ -496,11 +496,11 @@ def main(): rc = 0 try: - Milter.runmilter("pyheader-milter", socketname=args.socket, timeout=30) + Milter.runmilter("pymodmilter", socketname=args.socket, timeout=30) except Milter.milter.error as e: logger.error(e) rc = 255 - logger.info("PyHeader-Milter terminated") + logger.info("pymodmilter terminated") sys.exit(rc) diff --git a/setup.py b/setup.py index d006ca0..962c6b7 100644 --- a/setup.py +++ b/setup.py @@ -4,15 +4,15 @@ def read_file(fname): with open(fname, 'r') as f: return f.read() -setup(name = "pyheadermilter", +setup(name = "pymodmilter", version = "0.0.8", author = "Thomas Oettli", author_email = "spacefreak@noop.ch", description = "A pymilter based sendmail/postfix pre-queue filter.", license = "GPL 3", keywords = "header milter", - url = "https://github.com/spacefreak86/pyheader-milter", - packages = ["pyheadermilter"], + url = "https://github.com/spacefreak86/pymodmilter", + packages = ["pymodmilter"], long_description = read_file("README.md"), long_description_content_type="text/markdown", classifiers = [ @@ -28,7 +28,7 @@ setup(name = "pyheadermilter", ], entry_points = { "console_scripts": [ - "pyheader-milter=pyheadermilter:main" + "pymodmilter=pymodmilter:main" ] }, install_requires = ["pymilter", "netaddr"], From 1bcfbb2414db9b2a1388d7cd2b0adcddd6376f88 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 19:50:25 +0200 Subject: [PATCH 029/145] switch config to JSON format and new rules/modifications logic --- README.md | 99 +++--- docs/pymodmilter.conf.example | 206 ++++++++----- pymodmilter/__init__.py | 558 ++++++++++++++-------------------- pymodmilter/run.py | 189 ++++++++++++ pymodmilter/version.py | 1 + setup.py | 7 +- test-pymodmilter | 9 + 7 files changed, 619 insertions(+), 450 deletions(-) create mode 100644 pymodmilter/run.py create mode 100644 pymodmilter/version.py create mode 100755 test-pymodmilter diff --git a/README.md b/README.md index acf0810..6b1a9fa 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,82 @@ # pymodmilter A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers. -The project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month. +This project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month. +The basic idea is to define rules with conditions and do modifications make changes when all conditions are met. ## Requirements * pymilter * netaddr ## Configuration -The pymodmilter uses an INI-style configuration file. The sections are described below. +Pymodmilter uses a configuration file in JSON format. The options are described below. Make a copy of the example configuration file in the docs folder to start with. -### Section "global" -Any available configuration option can be set in the global section as default instead of in a rule section. +### global (Object) +The following optional global configuration options are available: +* **local_addrs (Array of Strings)** + A list of hosts and network addresses which are considered local. It is used to for the condition option 'local'. This option may be overriden within a rule object. +* **log (Bool)** + Enable or disable logging. This option may be overriden by a rule or modification object. -The following configuration options are mandatory in the global section: -* **rules** - Comma-separated, ordered list of active rules. For each, there must be a section of the same name in the configuration. +### rules (Array) +A mandatory list of rule objects which are processed in the given order. -### Rule sections +### rule (Object) The following configuration options are mandatory for each rule: -* **action** - Set the action of this rule. Possible values are: - * **add** - * **del** - * **mod** -* **header** - Name of the header in case of adding a header, regular expression to match whole header lines in case of deleting or modifying a header. - -The following configuration options are mandatory for an add-rule: -* **value** - Value of the header. - -The following configuration options are mandatory for a mod-rule: -* **search** - Regular expression to match the value of header lines. You may use subgroups or named subgroups (python syntax) to include parts of the original value in the new value. -* **value** - New value of the header. +* **modifications (Array of Objects)** + A list of modification objects which are processed in the given order. The following configuration options are optional for each rule: -* **ignore_envfrom** - Regular expression to match envelop-from addresses. The rule will be skipped if the expression matches. -* **ignore_hosts** - Comma-separated list of host and network addresses. The rule will be skipped if the sending host is included here. -* **only_hosts** - Comma-separated list of host and network addresses. The rule will be skipped if the sending host is not included here. If a is included in both **ignore_hosts** and **only_hosts**, the rule will be skipped. -* **log** - Enable or disable logging of this rule. Possible values are: - * **true** - * **false** +* **name (String)** + Name of the rule. +* **conditions (Object)** + A list of conditions which all have to be true to process the rule. +* **local_addrs (Array of Strings)** + As described above in the global object section. +* **log (Bool)** + As described above in the global object section. + +### modification (Object) +The following configuration options are mandatory for each modification: +* **type (String)** + Set the modification type. Possible values are: + * **add_header** + * **del_header** + * **mod_header** + +Additional parameters are mandatory based on the modification type. +* **add_header** + * **header (String)** + Name of the header. + * **value (String)** + Value of the header. + +* **del_header** + * **header (String)** + Regular expression to match against header lines. + +* **mod_header** + * **header (String)** + Regular expression to match against header lines. + * **search (String)** + Regular expression to match against the value of header lines. You may use subgroups or named subgroups (python syntax) to include parts of the original value in the new value. + * **value (String)** + New value of the header. + +The following configuration options are optional for each modification: +* **name (String)** + Name of the modification. +* **log (Bool)** + As described above in the global object section. + +### conditions (Object) +The following configuration options are optional: +* **local (Bool)** + If set to true, the rule is only executed for emails originating from addresses defined in local_addrs and vice versa. +* **hosts (Array of Strings)** + A list of hosts and network addresses for which the rule should be executed. +* **envfrom (String)** + A regular expression to match against the evenlope-from addresses for which the rule should be executed. ## Developer information Everyone who wants to improve or extend this project is very welcome. diff --git a/docs/pymodmilter.conf.example b/docs/pymodmilter.conf.example index 3f0a350..a5d3035 100644 --- a/docs/pymodmilter.conf.example +++ b/docs/pymodmilter.conf.example @@ -1,103 +1,143 @@ # This is an example /etc/pymodmilter.conf file. # Copy it into place before use. # -# Comments: use '#' for comment lines and ';' (following a space) for inline comments. +# The file is in JSON format. # -# If an option is not present in a modification section, it will be read from -# the global section. +# The global option 'log' can be overriden per rule or per modification. # +{ + # Section: global + # Notes: Set default options. + # + "global": { + # Option: local_addrs + # Type: List + # Notes: Set a list of local hosts and networks. + # Value: [ LIST ] + # + "local_addrs": ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], + # Option: log + # Type: Bool + # Notes: Set if processing of rules and modifications is logged. + # Value: [ true | false ] + # + "log": true + }, -[global] + # Section: rules + # Notes: Set rules and related modifications. + # + "rules": [ + { + # Option: name + # Type: String + # Notes: Set the name of the rule. + # Value: [ NAME ] + # + "name": "MyRule", -# Option: rules -# Notes: Set active rules (comma-separated). -# Each rule must have a section with the same name below. -# The rule name 'global' is forbidden and will be ignored. -# Rule names must be unique. -# Values: [ ACTIVE ] -# -rules = add_header,del_header,mod_header + # Section: conditions + # Notes: Optionally set conditions to run the rule. + # If multiple conditions are specified, they all + # have to be true to run the rule. + # + "conditions": { + # Option: local + # Type: Bool + # Notes: Set a condition on the senders host address. + # Set to true to execute the rule only for emails originating + # from addresses defined in local_addrs and vice versa. + # Value: [ true | false ] + # + "local": false, -# Option: ignore_envfrom -# Notes: Set a regular expression to match envelope-from addresses to be ignored. -# Value: [ REGEX ] -# -ignore_envfrom = ^.*@localhost$ + # Option: hosts + # Type: String + # Notes: Set a condition on the senders host address. + # The rule will only be executed if the list contains the + # senders host address. + # Value: [ LIST ] + # + "hosts": [ "127.0.0.1" ], -# Option: ignore_hosts -# Notes: Set a list of host and network addresses to be ignored. -# All the common host/network notations are supported, including IPv6. -# Value: [ HOST ] -# -ignore_hosts = 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 + # Option: envfrom + # Type: String + # Notes: Set a regular expression to match against the envelope-from address. + # Value: [ REGEX ] + # + "envfrom": "^(?!.+@mycompany\\.com).+$" + }, -# Option: only_hosts -# Notes: Set a list of host and network addresses. -# All the common host/network notations are supported, including IPv6. -# Value: [ HOST ] -# -only_hosts = 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 + # Section: modifications + # Notes: Set modifications for the rule. + # + "modifications": [ + { + # Option: name + # Type: String + # Notes: Set the name of the modification. + # Value: [ NAME ] + # + "name": "AddHeader", -# Option: log -# Notes: Set if modifications are logged. -# Value: [ true | false ] -# -log = true + # Option: type + # Type: String + # Notes: Set the modification type. + # Value: [ add_header | del_header | mod_header ] + # + "type": "add_header", + # Option: header + # Type: String + # Notes: Set the name of the new header. + # Value: [ NAME ] + # + "header": "X-Test-Header", -[add_header] + # Option: value + # Type: String + # Notes: Set the value of the new header. + # Value: [ VALUE ] + # + "value": "true" + }, { + "name": "ModifyHeader", -# Option: action -# Notes: Set the modification action. -# Values: [ add ] -action = add + "type": "mod_header", -# Option: header -# Notes: Set the name of the header. -# Values: [ NAME ] -# -header = X-My-Own-Header + # Option: header + # Type: String + # Notes: Set a regular expression to match against header lines (e.g. Subject: Test-Subject). + # Value: [ REGEX ] + # + "header": "^Subject:", -# Option: value -# Notes: Set the value of the header. -# Values: [ VALUE ] -value = my own value + # Option: search + # Type: String + # Notes: Set a regular expression to match against the headers value. + # Values: [ VALUE ] + # + "search": "(?P.*)", + # Option: value + # Type: String + # Notes: Set the value of the header. + # Values: [ VALUE ] + "value": "[EXTERNAL] \\g" + }, { + "name": "DeleteHeader", -[del_header] + "type": "del_header", -# Option: action -# Notes: Set the modification action. -# Values: [ del ] -action = del - -# Option: header -# Notes: Set a regular expression to match the header lines to delete. -# Values: [ REGEX ] -# -header = ^Received:.* - - -[mod_header] - -# Option: action -# Notes: Set the modification action. -# Values: [ mod ] -action = mod - -# Option: header -# Notes: Set a regular expression to match the header lines to modify. -# Values: [ REGEX ] -# -header = ^Subject:.* - -# Option: search -# Notes: Set a regular expression to match the headers value. -# Values: [ VALUE ] -search = (?P.*) - -# Option: value -# Notes: Set the value of the header. -# Values: [ VALUE ] -value = [SPAM] \g + # Option: header + # Type: String + # Notes: Set a regular expression to match against header lines (e.g. Subject: Test-Subject). + # Value: [ REGEX ] + # + "header": "^Received:" + } + ] + } + ] +} diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index c46c955..781f79c 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -15,16 +15,16 @@ __all__ = [ "make_header", "replace_illegal_chars", - "HeaderRule", - "HeaderMilter"] + "run", + "version", + "Modification", + "Rule", + "ModifyMilter"] import Milter -import argparse -import configparser import logging import logging.handlers import re -import sys from Milter.utils import parse_addr from email.charset import Charset @@ -61,108 +61,74 @@ def replace_illegal_chars(string): "\n", "") -class HeaderRule: - """HeaderRule to implement a rule to apply on e-mail headers.""" +class Modification: + """Modification to implement a modification to apply on e-mail headers.""" - def __init__(self, name, action, header, search="", value="", - ignore_hosts=[], ignore_envfrom=None, only_hosts=[], - log=True): + types = { + "add_header": ["header", "value"], + "del_header": ["header"], + "mod_header": ["header", "search", "value"] + } + + def __init__(self, name, mod_type, log, **params): self.logger = logging.getLogger(__name__) + self.logger.debug(f"initializing modification '{name}'") self.name = name - self.action = action - self.header = header - self.search = search - self.value = value - self.ignore_hosts = ignore_hosts - self.ignore_envfrom = ignore_envfrom - self.only_hosts = only_hosts self.log = log + # check mod_type + if mod_type not in Modification.types: + raise RuntimeError( + f"{self.name}: invalid modification type '{mod_type}'") + self.mod_type = mod_type + # check if mandatory modification options are present in config + for option in Modification.types[self.mod_type]: + if option not in params: + raise RuntimeError( + f"{self.name}: mandatory config " + f"option '{option}' not found") + if option == "value" and not params["value"]: + raise RuntimeError( + f"{self.name}: empty value specified") - if action in ["del", "mod"]: + if mod_type == "add_header": + self.header = params["header"] + self.value = params["value"] + elif mod_type in ["del_header", "mod_header"]: # compile header regex try: self.header = re.compile( - header, re.MULTILINE + re.DOTALL + re.IGNORECASE) + params["header"], re.MULTILINE + re.DOTALL + re.IGNORECASE) except re.error as e: raise RuntimeError( - f"unable to parse option 'header' of rule '{name}': {e}") + f"{self.name}: unable to parse regular expression of " + f"option 'header': {e}") - if action == "mod": + if mod_type == "mod_header": # compile search regex try: self.search = re.compile( - search, re.MULTILINE + re.DOTALL + re.IGNORECASE) + params["search"], + re.MULTILINE + re.DOTALL + re.IGNORECASE) except re.error as e: raise RuntimeError( - f"unable to parse option 'search' of " - f"rule '{name}': {e}") + f"{self.name}: unable to parse regular expression of " + f"option 'search': {e}") + self.value = params["value"] - if action in ["add", "mod"] and not value: - raise RuntimeError("value of option 'value' is empty") - - # replace strings in ignore_hosts and only_hosts with IPNetwork - # instances - try: - for index, ignore in enumerate(ignore_hosts): - self.ignore_hosts[index] = IPNetwork(ignore) - except AddrFormatError as e: - raise RuntimeError( - f"unable to parse option 'ignore_hosts' of rule '{name}': {e}") - - if self.ignore_envfrom: - try: - self.ignore_envfrom = re.compile(ignore_envfrom, re.IGNORECASE) - except re.error as e: - raise RuntimeError( - f"unable to parse option 'ignore_envfrom' of " - f"rule '{name}': {e}") - - try: - for index, only in enumerate(only_hosts): - self.only_hosts[index] = IPNetwork(only) - except AddrFormatError as e: - raise RuntimeError( - f"unable to parse option 'only_hosts' of rule '{name}': {e}") - - def ignore_host(self, host): - ip = IPAddress(host) - ignore = False - - # check if host matches ignore_hosts - for ignored in self.ignore_hosts: - if ip in ignored: - ignore = True - break - - if not ignore and self.only_hosts: - # host does not match ignore_hosts, check if it matches only_hosts - ignore = True - for only in self.only_hosts: - if ip in only: - ignore = False - break - - if ignore: - self.logger.debug(f"host {host} is ignored by rule {self.name}") - return ignore - - def ignore_from(self, envfrom): - ignore = False - - if self.ignore_envfrom: - if self.ignore_envfrom.search(envfrom): - ignore = True - self.logger.debug( - f"envelope-from {envfrom} is ignored by rule {self.name}") - return ignore - - def execute(self, headers): + def execute(self, qid, headers): """ Execute rule on given headers and return list with modified headers. """ - if self.action == "add": - return [(self.header, self.value, 0, 1)] + if self.mod_type == "add_header": + header = f"{self.header}: {self.value}" + if self.log: + self.logger.info( + f"{qid}: {self.name}: add_header: {header[0:70]}") + else: + self.logger.debug( + f"{qid}: {self.name}: add_header: {header}") + return [(self.mod_type, self.header, self.value, 0, 1)] modified = [] index = 0 @@ -178,32 +144,184 @@ class HeaderRule: occurrences[name] += 1 # check if header line matches regex - if self.header.search(f"{name}: {value}"): - if self.action == "del": + header = f"{name}: {value}" + if self.header.search(header): + if self.mod_type == "del_header": # set an empty value to delete the header new_value = "" + if self.log: + self.logger.info( + f"{qid}: {self.name}: del_header: " + f"{header[0:70]}") + else: + self.logger.debug( + f"{qid}: {self.name}: del_header: " + f"(occ. {occurrences[name]}): {header}") else: + old_header = header new_value = self.search.sub(self.value, value) - if value != new_value: - modified.append( - (name, new_value, index, occurrences[name])) + if value == new_value: + continue + header = f"{name}: {new_value}" + if self.log: + self.logger.info( + f"{qid}: {self.name}: mod_header: " + f"{old_header[0:70]}: {header[0:70]}") + else: + self.logger.debug( + f"{qid}: {self.name}: mod_header: " + f"(occ. {occurrences[name]}): {old_header}: " + f"{header}") + modified.append( + (self.mod_type, name, new_value, index, occurrences[name])) index += 1 return modified -class HeaderMilter(Milter.Base): - """HeaderMilter based on Milter.Base to implement milter communication""" +class Rule: + def __init__(self, name, modifications, local_addrs, log, conditions={}): + self.logger = logging.getLogger(__name__) + self.name = name + self.log = log + + self.logger.debug(f"initializing rule '{self.name}'") + + self._local_addrs = [] + # replace strings in local_addrs list with IPNetwork instances + try: + for addr in local_addrs: + self._local_addrs.append(IPNetwork(addr)) + except AddrFormatError as e: + raise RuntimeError( + f"{self.name}: unable to parse entry of " + f"option local_addrs: {e}") + + self.conditions = {} + for option, value in conditions.items(): + if option == "local": + self.conditions[option] = value + self.logger.debug( + f"{self.name}: added condition: {option} = {value}") + elif option == "hosts": + self.conditions[option] = [] + try: + for host in value: + self.conditions[option].append(IPNetwork(host)) + except AddrFormatError as e: + raise RuntimeError( + f"{self.name}: unable to parse entry of " + f"condition '{option}': {e}") + self.logger.debug( + f"{self.name}: added condition: {option} = {value}") + elif option == "envfrom": + try: + self.conditions[option] = re.compile(value, re.IGNORECASE) + except re.error as e: + raise RuntimeError( + f"{self.name}: unable to parse regular expression of " + f"condition '{option}': {e}") + self.logger.debug( + f"{self.name}: added condition: {option} = {value}") + + self.modifications = [] + for mod_idx, mod in enumerate(modifications): + params = {} + # set default values if not specified in config + if "name" not in mod: + mod["name"] = f"Modification #{mod_idx}" + + if self.name: + params["name"] = f"{self.name}/{mod['name']}" + else: + params["name"] = mod["name"] + + if "log" in mod: + params["log"] = mod["log"] + else: + params["log"] = self.log + + if "type" in mod: + params["mod_type"] = mod["type"] + else: + raise RuntimeError( + f"{params['name']}: mandatory config " + f"option 'type' not found") + + if "header" in mod: + params["header"] = mod["header"] + + if "search" in mod: + params["search"] = mod["search"] + + if "value" in mod: + params["value"] = mod["value"] + + self.modifications.append(Modification(**params)) + self.logger.debug( + f"{self.name}: added modification: {mod['name']}") + + def ignore_host(self, host): + ip = IPAddress(host) + + if "local" in self.conditions: + is_local = False + for addr in self._local_addrs: + if ip in addr: + is_local = True + break + if is_local != self.conditions["local"]: + return True + + if "hosts" in self.conditions: + # check if host is in list + for accepted in self.conditions["hosts"]: + if ip in accepted: + return False + return True + + return False + + def ignore_envfrom(self, envfrom): + if "envfrom" in self.conditions: + if not self.conditions["envfrom"].search(envfrom): + return True + return False + + def execute(self, qid, headers): + changes = [] + if self.log: + self.logger.info(f"{qid}: executing rule '{self.name}'") + else: + self.logger.debug(f"{qid}: executing rule '{self.name}'") + + for mod in self.modifications: + self.logger.debug(f"{qid}: executing modification '{mod.name}'") + result = mod.execute(qid, headers) + changes += result + for mod_type, name, value, index, occurrence in result: + if mod_type == "add_header": + headers.append((name, value)) + else: + if mod_type == "mod_header": + headers[index] = (name, value) + elif mod_type == "del_header": + del headers[index] + return changes + + +class ModifyMilter(Milter.Base): + """ModifyMilter based on Milter.Base to implement milter communication""" _rules = [] @staticmethod def set_rules(rules): - HeaderMilter._rules = rules + ModifyMilter._rules = rules def __init__(self): self.logger = logging.getLogger(__name__) # save rules, it must not change during runtime - self.rules = HeaderMilter._rules.copy() + self.rules = ModifyMilter._rules.copy() def connect(self, IPname, family, hostaddr): self.logger.debug( @@ -214,6 +332,8 @@ class HeaderMilter(Milter.Base): # remove rules which ignore this host for rule in self.rules.copy(): if rule.ignore_host(ip): + self.logger.debug( + f"host {hostaddr[0]} is ignored by rule '{rule.name}'") self.rules.remove(rule) if not self.rules: @@ -226,7 +346,10 @@ class HeaderMilter(Milter.Base): def envfrom(self, mailfrom, *str): mailfrom = "@".join(parse_addr(mailfrom)).lower() for rule in self.rules.copy(): - if rule.ignore_from(mailfrom): + if rule.ignore_envfrom(mailfrom): + self.logger.debug( + f"envelope-from {mailfrom} is ignored by " + f"rule '{rule.name}'") self.rules.remove(rule) if not self.rules: @@ -263,246 +386,21 @@ class HeaderMilter(Milter.Base): def eom(self): try: for rule in self.rules: - self.logger.debug(f"{self.qid}: executing rule '{rule.name}'") - modified = rule.execute(self.headers) - for name, value, index, occurrence in modified: - header = f"{name}: {value}" + changes = rule.execute(self.qid, self.headers) + for mod_type, name, value, index, occurrence in changes: enc_value = replace_illegal_chars( Header(s=value).encode()) - if rule.action == "add": - if rule.log: - self.logger.info( - f"{self.qid}: add: header: " - f"{header[0:70]}") - else: - self.logger.debug( - f"{self.qid}: add: header: " - f"{header}") - self.headers.insert(0, (name, value)) - self.addheader(name, enc_value, 1) + if mod_type == "add_header": + self.logger.debug(f"{self.qid}: milter: adding " + f"header: {name}: {enc_value}") + self.addheader(name, enc_value, -1) else: - if rule.action == "mod": - old_header = "{}: {}".format(*self.headers[index]) - if rule.log: - self.logger.info( - f"{self.qid}: modify: header: " - f"{old_header[0:70]}: {header[0:70]}") - else: - self.logger.debug( - f"{self.qid}: modify: header " - f"(occ. {occurrence}): {old_header}: " - f"{header}") - self.headers[index] = (name, value) - elif rule.action == "del": - if rule.log: - self.logger.info( - f"{self.qid}: delete: header: " - f"{header[0:70]}") - else: - self.logger.debug( - f"{self.qid}: delete: header " - f"(occ. {occurrence}): {header}") - del self.headers[index] - + self.logger.debug(f"{self.qid}: milter: modify " + f"header (occ. {occurrence}): " + f"{name}: {enc_value}") self.chgheader(name, occurrence, enc_value) return Milter.ACCEPT except Exception as e: self.logger.exception( f"an exception occured in eom function: {e}") return Milter.TEMPFAIL - - -def main(): - "Run PyMod-Milter." - # parse command line - parser = argparse.ArgumentParser( - description="PyMod milter daemon", - formatter_class=lambda prog: argparse.HelpFormatter( - prog, max_help_position=45, width=140)) - parser.add_argument( - "-c", "--config", help="Config file to read.", - default="/etc/pymodmilter.conf") - parser.add_argument( - "-s", - "--socket", - help="Socket used to communicate with the MTA.", - required=True) - parser.add_argument( - "-d", - "--debug", - help="Log debugging messages.", - action="store_true") - parser.add_argument( - "-t", - "--test", - help="Check configuration.", - action="store_true") - args = parser.parse_args() - - # setup logging - loglevel = logging.INFO - logname = "pymodmilter" - syslog_name = logname - if args.debug: - loglevel = logging.DEBUG - logname = f"{logname}[%(name)s]" - syslog_name = f"{syslog_name}: [%(name)s] %(levelname)s" - - # set config files for milter class - root_logger = logging.getLogger() - root_logger.setLevel(loglevel) - - # setup console log - stdouthandler = logging.StreamHandler(sys.stdout) - stdouthandler.setLevel(logging.DEBUG) - formatter = logging.Formatter("%(message)s") - stdouthandler.setFormatter(formatter) - root_logger.addHandler(stdouthandler) - logger = logging.getLogger(__name__) - - try: - # read config file - parser = configparser.ConfigParser() - if not parser.read(args.config): - raise RuntimeError("config file not found") - - # check if mandatory config options in global section are present - if "global" not in parser.sections(): - raise RuntimeError( - "mandatory section 'global' not present in config file") - for option in ["rules"]: - if not parser.has_option("global", option): - raise RuntimeError( - f"mandatory option '{option}' not present in config " - f"section 'global'") - - # read global config section - global_config = dict(parser.items("global")) - - # read active rules - active_rules = [r.strip() for r in global_config["rules"].split(",")] - if len(active_rules) != len(set(active_rules)): - raise RuntimeError( - "at least one rule is specified multiple times " - "in 'rules' option") - if "global" in active_rules: - active_rules.remove("global") - logger.warning( - "removed illegal rule name 'global' from list of " - "active rules") - if not active_rules: - raise RuntimeError("no rules configured") - - logger.debug("preparing milter configuration ...") - rules = [] - # iterate active rules - for rule_name in active_rules: - # check if config section exists - if rule_name not in parser.sections(): - raise RuntimeError( - f"config section '{rule_name}' does not exist") - config = dict(parser.items(rule_name)) - - # check if mandatory option action is present in config - option = "action" - if option not in config.keys() and \ - option in global_config.keys(): - config[option] = global_config[option] - if option not in config.keys(): - raise RuntimeError( - f"mandatory option '{option}' not specified for " - f"rule '{rule_name}'") - config["action"] = config["action"].lower() - if config["action"] not in ["add", "del", "mod"]: - raise RuntimeError( - f"invalid action specified for rule '{rule_name}'") - - # check if mandatory options are present in config - mandatory = ["header"] - if config["action"] == "add": - mandatory += ["value"] - elif config["action"] == "mod": - mandatory += ["search", "value"] - for option in mandatory: - if option not in config.keys() and \ - option in global_config.keys(): - config[option] = global_config[option] - if option not in config.keys(): - raise RuntimeError( - f"mandatory option '{option}' not specified for " - f"rule '{rule_name}'") - - # check if optional config options are present in config - defaults = { - "ignore_hosts": [], - "ignore_envfrom": None, - "only_hosts": [], - "log": "true" - } - for option in defaults.keys(): - if option not in config.keys() and \ - option in global_config.keys(): - config[option] = global_config[option] - if option not in config.keys(): - config[option] = defaults[option] - if config["ignore_hosts"]: - config["ignore_hosts"] = [ - h.strip() for h in config["ignore_hosts"].split(",")] - if config["only_hosts"]: - config["only_hosts"] = [ - h.strip() for h in config["only_hosts"].split(",")] - config["log"] = config["log"].lower() - if config["log"] == "true": - config["log"] = True - elif config["log"] == "false": - config["log"] = False - else: - raise RuntimeError( - f"invalid value specified for option 'log' for " - f"rule '{rule_name}'") - - # add rule - logging.debug(f"adding rule '{rule_name}'") - rules.append(HeaderRule(name=rule_name, **config)) - - except RuntimeError as e: - logger.error(e) - sys.exit(255) - - if args.test: - print("Configuration ok") - sys.exit(0) - - # change log format for runtime - formatter = logging.Formatter( - f"%(asctime)s {logname}: [%(levelname)s] %(message)s", - datefmt="%Y-%m-%d %H:%M:%S") - stdouthandler.setFormatter(formatter) - - # setup syslog - sysloghandler = logging.handlers.SysLogHandler( - address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) - sysloghandler.setLevel(loglevel) - formatter = logging.Formatter(f"{syslog_name}: %(message)s") - sysloghandler.setFormatter(formatter) - root_logger.addHandler(sysloghandler) - - logger.info("pymodmilter starting") - HeaderMilter.set_rules(rules) - - # register milter factory class - Milter.factory = HeaderMilter - Milter.set_exception_policy(Milter.TEMPFAIL) - - rc = 0 - try: - Milter.runmilter("pymodmilter", socketname=args.socket, timeout=30) - except Milter.milter.error as e: - logger.error(e) - rc = 255 - logger.info("pymodmilter terminated") - sys.exit(rc) - - -if __name__ == "__main__": - main() diff --git a/pymodmilter/run.py b/pymodmilter/run.py new file mode 100644 index 0000000..3afd970 --- /dev/null +++ b/pymodmilter/run.py @@ -0,0 +1,189 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + + +import Milter +import argparse +import logging +import logging.handlers +import sys + +from json import loads +from re import sub + +from pymodmilter import Rule, ModifyMilter +from pymodmilter.version import __version__ as version + + +def main(): + "Run PyMod-Milter." + # parse command line + parser = argparse.ArgumentParser( + description="PyMod milter daemon", + formatter_class=lambda prog: argparse.HelpFormatter( + prog, max_help_position=45, width=140)) + parser.add_argument( + "-c", "--config", help="Config file to read.", + default="/etc/pymodmilter.conf") + parser.add_argument( + "-s", + "--socket", + help="Socket used to communicate with the MTA.", + required=True) + parser.add_argument( + "-d", + "--debug", + help="Log debugging messages.", + action="store_true") + parser.add_argument( + "-t", + "--test", + help="Check configuration.", + action="store_true") + parser.add_argument( + "-v", "--version", + help="Print version.", + action="version", + version=f"%(prog)s ({version})") + args = parser.parse_args() + + # setup logging + loglevel = logging.INFO + logname = "pymodmilter" + syslog_name = logname + if args.debug: + loglevel = logging.DEBUG + logname = f"{logname}[%(name)s]" + syslog_name = f"{syslog_name}: [%(name)s] %(levelname)s" + + root_logger = logging.getLogger() + root_logger.setLevel(loglevel) + + # setup console log + stdouthandler = logging.StreamHandler(sys.stdout) + stdouthandler.setLevel(logging.DEBUG) + formatter = logging.Formatter("%(message)s") + stdouthandler.setFormatter(formatter) + root_logger.addHandler(stdouthandler) + logger = logging.getLogger(__name__) + + try: + # read config file + logger.debug("parsing config file") + try: + with open(args.config, "r") as fh: + config = loads( + sub(r"(?m)^\s*#.*\n?", "", fh.read())) + except Exception as e: + raise RuntimeError( + f"unable to parse config file: {e}") + + logger.debug("preparing milter configuration ...") + + # default values for global config if not set + if "global" not in config: + config["global"] = {} + + if "local_addrs" not in config["global"]: + config["global"]["local_addrs"] = [ + "127.0.0.0/8", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16"] + + if "log" not in config["global"]: + config["global"]["log"] = True + + # check if mandatory sections are present in config + for section in ["rules"]: + if section not in config: + raise RuntimeError( + f"mandatory config section '{section}' not found") + + if not config["rules"]: + raise RuntimeError("no rules configured") + + rules = [] + # iterate configured rules + for rule_idx, rule in enumerate(config["rules"]): + params = {} + # set default values if not specified in config + if "name" in rule: + params["name"] = rule["name"] + else: + params["name"] = f"Rule #{rule_idx}" + + if "log" in rule: + params["log"] = rule["log"] + else: + params["log"] = config["global"]["log"] + + if "local_addrs" in rule: + params["local_addrs"] = rule["local_addrs"] + else: + params["local_addrs"] = config["global"]["local_addrs"] + + if "conditions" in rule: + params["conditions"] = rule["conditions"] + + if "modifications" in rule: + params["modifications"] = rule["modifications"] + else: + raise RuntimeError( + f"{rule['name']}: mandatory config section " + f"'modifications' not found") + + rules.append(Rule(**params)) + + except RuntimeError as e: + logger.error(e) + sys.exit(255) + + if args.test: + print("Configuration ok") + sys.exit(0) + + # change log format for runtime + formatter = logging.Formatter( + f"%(asctime)s {logname}: [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S") + stdouthandler.setFormatter(formatter) + + # setup syslog + sysloghandler = logging.handlers.SysLogHandler( + address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) + sysloghandler.setLevel(loglevel) + formatter = logging.Formatter(f"{syslog_name}: %(message)s") + sysloghandler.setFormatter(formatter) + root_logger.addHandler(sysloghandler) + + logger.info("pymodmilter starting") + ModifyMilter.set_rules(rules) + + # register milter factory class + Milter.factory = ModifyMilter + Milter.set_exception_policy(Milter.TEMPFAIL) + + rc = 0 + try: + Milter.runmilter("pymodmilter", socketname=args.socket, timeout=30) + except Milter.milter.error as e: + logger.error(e) + rc = 255 + logger.info("pymodmilter terminated") + sys.exit(rc) + + +if __name__ == "__main__": + main() diff --git a/pymodmilter/version.py b/pymodmilter/version.py new file mode 100644 index 0000000..a73339b --- /dev/null +++ b/pymodmilter/version.py @@ -0,0 +1 @@ +__version__ = "0.0.8" diff --git a/setup.py b/setup.py index 962c6b7..ba29fe6 100644 --- a/setup.py +++ b/setup.py @@ -4,8 +4,11 @@ def read_file(fname): with open(fname, 'r') as f: return f.read() +version = {} +exec(read_file("pymodmilter/version.py"), version) + setup(name = "pymodmilter", - version = "0.0.8", + version = version["__version__"], author = "Thomas Oettli", author_email = "spacefreak@noop.ch", description = "A pymilter based sendmail/postfix pre-queue filter.", @@ -28,7 +31,7 @@ setup(name = "pymodmilter", ], entry_points = { "console_scripts": [ - "pymodmilter=pymodmilter:main" + "pymodmilter=pymodmilter.run:main" ] }, install_requires = ["pymilter", "netaddr"], diff --git a/test-pymodmilter b/test-pymodmilter new file mode 100755 index 0000000..fbe25a8 --- /dev/null +++ b/test-pymodmilter @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +import sys +import pymodmilter.run + +if __name__ == '__main__': + sys.exit( + pymodmilter.run.main() + ) From 2a3c557eeb636891454995b27f5b7ea5329d7c08 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 19:54:33 +0200 Subject: [PATCH 030/145] fix typo in README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6b1a9fa..cb443d0 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # pymodmilter -A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers. +A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers. +The project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month. -This project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month. -The basic idea is to define rules with conditions and do modifications make changes when all conditions are met. +The basic idea is to define rules with conditions and do modifications when all conditions are met. ## Requirements * pymilter From e31c87ba6b029e654907e3285f35e80c638c01cb Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 20:59:01 +0200 Subject: [PATCH 031/145] Change README.md --- README.md | 70 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index cb443d0..c3136b1 100644 --- a/README.md +++ b/README.md @@ -4,41 +4,51 @@ The project is currently in beta status, but it is already used in a productive The basic idea is to define rules with conditions and do modifications when all conditions are met. -## Requirements +## Dependencies +Pymodmilter is depending on these python packages, but they are installed automatically if you are working with pip. * pymilter * netaddr -## Configuration -Pymodmilter uses a configuration file in JSON format. The options are described below. Make a copy of the example configuration file in the docs folder to start with. +## Installation +* Install pymodmilter with pip. +```sh +pip install pymodmilter +``` +* Download the example config file. +```sh +curl https://raw.githubusercontent.com/spacefreak86/pymodmilter/master/docs/pymodmilter.conf.example --output /etc/pymodmilter.conf +``` +* Modify pymodmilter.conf according to your needs and you are ready to go. -### global (Object) +## Configuration options +Pymodmilter uses a configuration file in JSON format. The options are described below. Make a copy of the example configuration file in the [docs](https://github.com/spacefreak86/pymodmilter/tree/master/docs) folder to start with. +Rules and modifications are processed in the given order. + +### Global The following optional global configuration options are available: -* **local_addrs (Array of Strings)** +* **local_addrs** A list of hosts and network addresses which are considered local. It is used to for the condition option 'local'. This option may be overriden within a rule object. -* **log (Bool)** +* **log** Enable or disable logging. This option may be overriden by a rule or modification object. -### rules (Array) -A mandatory list of rule objects which are processed in the given order. - -### rule (Object) +### Rules The following configuration options are mandatory for each rule: -* **modifications (Array of Objects)** +* **modifications** A list of modification objects which are processed in the given order. The following configuration options are optional for each rule: -* **name (String)** +* **name** Name of the rule. -* **conditions (Object)** +* **conditions** A list of conditions which all have to be true to process the rule. -* **local_addrs (Array of Strings)** +* **local_addrs** As described above in the global object section. -* **log (Bool)** +* **log** As described above in the global object section. -### modification (Object) +### Modifications The following configuration options are mandatory for each modification: -* **type (String)** +* **type** Set the modification type. Possible values are: * **add_header** * **del_header** @@ -46,36 +56,36 @@ The following configuration options are mandatory for each modification: Additional parameters are mandatory based on the modification type. * **add_header** - * **header (String)** + * **header** Name of the header. - * **value (String)** + * **value** Value of the header. * **del_header** - * **header (String)** + * **header** Regular expression to match against header lines. * **mod_header** - * **header (String)** + * **header** Regular expression to match against header lines. - * **search (String)** + * **search** Regular expression to match against the value of header lines. You may use subgroups or named subgroups (python syntax) to include parts of the original value in the new value. - * **value (String)** + * **value** New value of the header. The following configuration options are optional for each modification: -* **name (String)** +* **name** Name of the modification. -* **log (Bool)** +* **log** As described above in the global object section. -### conditions (Object) -The following configuration options are optional: -* **local (Bool)** +### Conditions +The following condition options are optional: +* **local** If set to true, the rule is only executed for emails originating from addresses defined in local_addrs and vice versa. -* **hosts (Array of Strings)** +* **hosts** A list of hosts and network addresses for which the rule should be executed. -* **envfrom (String)** +* **envfrom** A regular expression to match against the evenlope-from addresses for which the rule should be executed. ## Developer information From f7db51b5b2f311b6036d0d8204d7a2898af5d57e Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 21:04:01 +0200 Subject: [PATCH 032/145] change README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c3136b1..d774110 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ curl https://raw.githubusercontent.com/spacefreak86/pymodmilter/master/docs/pymo * Modify pymodmilter.conf according to your needs and you are ready to go. ## Configuration options -Pymodmilter uses a configuration file in JSON format. The options are described below. Make a copy of the example configuration file in the [docs](https://github.com/spacefreak86/pymodmilter/tree/master/docs) folder to start with. +Pymodmilter uses a configuration file in JSON format. The options are described below. Make a copy of the [example configuration file](https://github.com/spacefreak86/pymodmilter/blob/master/docs/pymodmilter.conf.example) in the [docs](https://github.com/spacefreak86/pymodmilter/tree/master/docs) folder to start with. Rules and modifications are processed in the given order. ### Global @@ -42,9 +42,9 @@ The following configuration options are optional for each rule: * **conditions** A list of conditions which all have to be true to process the rule. * **local_addrs** - As described above in the global object section. + As described above in the [Global](#Global) section. * **log** - As described above in the global object section. + As described above in the [Global](#Global) section. ### Modifications The following configuration options are mandatory for each modification: @@ -54,7 +54,7 @@ The following configuration options are mandatory for each modification: * **del_header** * **mod_header** -Additional parameters are mandatory based on the modification type. +The following configuration options are mandatory based on the modification type in use. * **add_header** * **header** Name of the header. From b20a37cf7cc0d8874bbe99fdc19fa32c369ee3d2 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 21:08:20 +0200 Subject: [PATCH 033/145] change README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d774110..0a85ac0 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ The basic idea is to define rules with conditions and do modifications when all ## Dependencies Pymodmilter is depending on these python packages, but they are installed automatically if you are working with pip. -* pymilter -* netaddr +* [pymilter](https://pythonhosted.org/pymilter/) +* [netaddr](https://github.com/drkjam/netaddr/) ## Installation * Install pymodmilter with pip. @@ -27,7 +27,7 @@ Rules and modifications are processed in the given order. ### Global The following optional global configuration options are available: * **local_addrs** - A list of hosts and network addresses which are considered local. It is used to for the condition option 'local'. This option may be overriden within a rule object. + A list of hosts and network addresses which are considered local. It is used to for the condition option [local](#Conditions). This option may be overriden within a rule object. * **log** Enable or disable logging. This option may be overriden by a rule or modification object. From aa135190b997798bf9f01b264ef2fb59bbcb5dbb Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 21:40:28 +0200 Subject: [PATCH 034/145] add some comments and change setup.py --- pymodmilter/__init__.py | 11 +++++++++-- setup.py | 11 ++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 781f79c..f7e7d2a 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -62,8 +62,9 @@ def replace_illegal_chars(string): class Modification: - """Modification to implement a modification to apply on e-mail headers.""" + """Modification to implement a modification to apply on e-mails.""" + # mandatory parameters for each modification type types = { "add_header": ["header", "value"], "del_header": ["header"], @@ -117,7 +118,7 @@ class Modification: def execute(self, qid, headers): """ - Execute rule on given headers and return list + Execute modification and return list with modified headers. """ if self.mod_type == "add_header": @@ -179,6 +180,9 @@ class Modification: class Rule: + """ + Rule to implement multiple modifications on emails based on conditions. + """ def __init__(self, name, modifications, local_addrs, log, conditions={}): self.logger = logging.getLogger(__name__) self.name = name @@ -261,6 +265,7 @@ class Rule: f"{self.name}: added modification: {mod['name']}") def ignore_host(self, host): + """Check if host is ignored by this rule.""" ip = IPAddress(host) if "local" in self.conditions: @@ -282,12 +287,14 @@ class Rule: return False def ignore_envfrom(self, envfrom): + """Check if envelope-from address is ignored by this rule.""" if "envfrom" in self.conditions: if not self.conditions["envfrom"].search(envfrom): return True return False def execute(self, qid, headers): + """Execute all modifications of this rule.""" changes = [] if self.log: self.logger.info(f"{qid}: executing rule '{self.name}'") diff --git a/setup.py b/setup.py index ba29fe6..45fff62 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup(name = "pymodmilter", url = "https://github.com/spacefreak86/pymodmilter", packages = ["pymodmilter"], long_description = read_file("README.md"), - long_description_content_type="text/markdown", + long_description_content_type = "text/markdown", classifiers = [ # 3 - Alpha # 4 - Beta @@ -29,11 +29,20 @@ setup(name = "pymodmilter", "Programming Language :: Python :: 3", "Topic :: Communications :: Email :: Filters" ], + include_package_data = True, entry_points = { "console_scripts": [ "pymodmilter=pymodmilter.run:main" ] }, + data_files = [ + ( + 'usr/share/docs/', + [ + 'docs/pymodmilter.conf.example' + ] + ) + ], install_requires = ["pymilter", "netaddr"], python_requires = ">=3.6" ) From 2d73d525b8b63aec22e801dc2ef8d62ae21c93ee Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 21:51:02 +0200 Subject: [PATCH 035/145] fix setup.py and release 0.0.9 --- README.md | 6 +++--- pymodmilter/version.py | 2 +- setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0a85ac0..29dd483 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,11 @@ Pymodmilter is depending on these python packages, but they are installed automa ```sh pip install pymodmilter ``` -* Download the example config file. +* Copy the example configuration file. ```sh -curl https://raw.githubusercontent.com/spacefreak86/pymodmilter/master/docs/pymodmilter.conf.example --output /etc/pymodmilter.conf +cp /usr/share/doc/pymodmilter/pymodmilter.conf.example /etc/pymodmilter.conf ``` -* Modify pymodmilter.conf according to your needs and you are ready to go. +* Modify /etc/pymodmilter.conf according to your needs and you are ready to go. ## Configuration options Pymodmilter uses a configuration file in JSON format. The options are described below. Make a copy of the [example configuration file](https://github.com/spacefreak86/pymodmilter/blob/master/docs/pymodmilter.conf.example) in the [docs](https://github.com/spacefreak86/pymodmilter/tree/master/docs) folder to start with. diff --git a/pymodmilter/version.py b/pymodmilter/version.py index a73339b..00ec2dc 100644 --- a/pymodmilter/version.py +++ b/pymodmilter/version.py @@ -1 +1 @@ -__version__ = "0.0.8" +__version__ = "0.0.9" diff --git a/setup.py b/setup.py index 45fff62..1459663 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ setup(name = "pymodmilter", }, data_files = [ ( - 'usr/share/docs/', + 'usr/share/doc/pymodmilter', [ 'docs/pymodmilter.conf.example' ] From 01b516ce7ccabe0ff31333f1faa08c03db600cdb Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 23:02:05 +0200 Subject: [PATCH 036/145] add config option socket, rework example config and release 1.0.0 --- README.md | 11 +++---- docs/pymodmilter.conf.example | 57 ++++++++++++++++++++--------------- misc/openrc/pymodmilter.confd | 9 ------ misc/openrc/pymodmilter.initd | 3 +- pymodmilter/run.py | 13 ++++++-- pymodmilter/version.py | 2 +- 6 files changed, 51 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 29dd483..0059bfa 100644 --- a/README.md +++ b/README.md @@ -10,22 +10,21 @@ Pymodmilter is depending on these python packages, but they are installed automa * [netaddr](https://github.com/drkjam/netaddr/) ## Installation -* Install pymodmilter with pip. +* Install pymodmilter with pip and copy the example configuration file. ```sh pip install pymodmilter -``` -* Copy the example configuration file. -```sh cp /usr/share/doc/pymodmilter/pymodmilter.conf.example /etc/pymodmilter.conf ``` -* Modify /etc/pymodmilter.conf according to your needs and you are ready to go. +* Modify /etc/pymodmilter.conf according to your needs. ## Configuration options Pymodmilter uses a configuration file in JSON format. The options are described below. Make a copy of the [example configuration file](https://github.com/spacefreak86/pymodmilter/blob/master/docs/pymodmilter.conf.example) in the [docs](https://github.com/spacefreak86/pymodmilter/tree/master/docs) folder to start with. Rules and modifications are processed in the given order. ### Global -The following optional global configuration options are available: +The following global configuration options are optional: +* **socket** + The socket used to communicate with the MTA. * **local_addrs** A list of hosts and network addresses which are considered local. It is used to for the condition option [local](#Conditions). This option may be overriden within a rule object. * **log** diff --git a/docs/pymodmilter.conf.example b/docs/pymodmilter.conf.example index a5d3035..00cde86 100644 --- a/docs/pymodmilter.conf.example +++ b/docs/pymodmilter.conf.example @@ -7,97 +7,106 @@ # { # Section: global - # Notes: Set default options. + # Notes: Global options. # "global": { + # Option: socket + # Type: String + # Notes: The socket used to communicate with the MTA. + # + # Examples: + # unix:/path/to/socket a named pipe + # inet:8899 listen on ANY interface + # inet:8899@localhost listen on a specific interface + # inet6:8899 listen on ANY interface + # inet6:8899@[2001:db8:1234::1] listen on a specific interface + # Value: [ SOCKET ] + "socket": "inet:8898@127.0.0.1", + # Option: local_addrs # Type: List - # Notes: Set a list of local hosts and networks. + # Notes: A list of local hosts and networks. # Value: [ LIST ] # "local_addrs": ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], # Option: log # Type: Bool - # Notes: Set if processing of rules and modifications is logged. + # Notes: Enable or disable logging of rules and modifications. # Value: [ true | false ] # "log": true }, # Section: rules - # Notes: Set rules and related modifications. + # Notes: Rules and related modifications. # "rules": [ { # Option: name # Type: String - # Notes: Set the name of the rule. + # Notes: Name of the rule. # Value: [ NAME ] # "name": "MyRule", # Section: conditions - # Notes: Optionally set conditions to run the rule. - # If multiple conditions are specified, they all - # have to be true to run the rule. + # Notes: Optional conditions to process the rule. + # If multiple conditions are set, they all + # have to be true to process the rule. # "conditions": { # Option: local # Type: Bool - # Notes: Set a condition on the senders host address. - # Set to true to execute the rule only for emails originating - # from addresses defined in local_addrs and vice versa. + # Notes: Condition wheter the senders host address is listed in local_addrs. # Value: [ true | false ] # "local": false, # Option: hosts # Type: String - # Notes: Set a condition on the senders host address. - # The rule will only be executed if the list contains the - # senders host address. + # Notes: Condition wheter the senders host address is listed in this list. # Value: [ LIST ] # "hosts": [ "127.0.0.1" ], # Option: envfrom # Type: String - # Notes: Set a regular expression to match against the envelope-from address. + # Notes: Condition wheter the envelop-from address matches this regular expression. # Value: [ REGEX ] # "envfrom": "^(?!.+@mycompany\\.com).+$" }, # Section: modifications - # Notes: Set modifications for the rule. + # Notes: Modifications of the rule. # "modifications": [ { # Option: name # Type: String - # Notes: Set the name of the modification. + # Notes: Name of the modification. # Value: [ NAME ] # "name": "AddHeader", # Option: type # Type: String - # Notes: Set the modification type. + # Notes: Type of the modification. # Value: [ add_header | del_header | mod_header ] # "type": "add_header", # Option: header # Type: String - # Notes: Set the name of the new header. + # Notes: Name of the header. # Value: [ NAME ] # "header": "X-Test-Header", # Option: value # Type: String - # Notes: Set the value of the new header. + # Notes: Value of the header. # Value: [ VALUE ] # "value": "true" @@ -108,21 +117,21 @@ # Option: header # Type: String - # Notes: Set a regular expression to match against header lines (e.g. Subject: Test-Subject). + # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). # Value: [ REGEX ] # "header": "^Subject:", # Option: search # Type: String - # Notes: Set a regular expression to match against the headers value. + # Notes: Regular expression to match against the headers value. # Values: [ VALUE ] # "search": "(?P.*)", # Option: value # Type: String - # Notes: Set the value of the header. + # Notes: New value of the header. # Values: [ VALUE ] "value": "[EXTERNAL] \\g" }, { @@ -132,7 +141,7 @@ # Option: header # Type: String - # Notes: Set a regular expression to match against header lines (e.g. Subject: Test-Subject). + # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). # Value: [ REGEX ] # "header": "^Received:" diff --git a/misc/openrc/pymodmilter.confd b/misc/openrc/pymodmilter.confd index b4e923c..6c5e71d 100644 --- a/misc/openrc/pymodmilter.confd +++ b/misc/openrc/pymodmilter.confd @@ -1,14 +1,5 @@ # /etc/conf.d/pymodmilter: config file for /etc/init.d/pymodmilter -# Set the socket used to communicate with the MTA. -# Examples: -# unix:/path/to/socket a named pipe -# inet:8899 listen on ANY interface -# inet:8899@localhost listen on a specific interface -# inet6:8899 listen on ANY interface -# inet6:8899@[2001:db8:1234::1] listen on a specific interface -SOCKET="inet:8898@127.0.0.1" - # Start the daemon as the user. You can optionally append a group name here also. # USER="daemon" # USER="daemon:nobody" diff --git a/misc/openrc/pymodmilter.initd b/misc/openrc/pymodmilter.initd index a487a22..bb9a646 100755 --- a/misc/openrc/pymodmilter.initd +++ b/misc/openrc/pymodmilter.initd @@ -1,12 +1,11 @@ #!/sbin/openrc-run user=${USER:-daemon} -socket="${SOCKET:-}" milter_opts="${MILTER_OPTS:-}" pidfile="/run/${RC_SVCNAME}.pid" command="/usr/bin/pymodmilter" -command_args="-s ${socket} ${milter_opts}" +command_args="${milter_opts}" command_background=true start_stop_daemon_args="--user ${user}" diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 3afd970..6e3c8e2 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -40,7 +40,7 @@ def main(): "-s", "--socket", help="Socket used to communicate with the MTA.", - required=True) + default="") parser.add_argument( "-d", "--debug", @@ -95,6 +95,15 @@ def main(): if "global" not in config: config["global"] = {} + if args.socket: + socket = args.socket + elif "socket" in config["global"]: + socket = config["global"]["socket"] + else: + raise RuntimeError( + f"listening socket is neither specified on the command line " + f"nor in the configuration file") + if "local_addrs" not in config["global"]: config["global"]["local_addrs"] = [ "127.0.0.0/8", @@ -177,7 +186,7 @@ def main(): rc = 0 try: - Milter.runmilter("pymodmilter", socketname=args.socket, timeout=30) + Milter.runmilter("pymodmilter", socketname=socket, timeout=30) except Milter.milter.error as e: logger.error(e) rc = 255 diff --git a/pymodmilter/version.py b/pymodmilter/version.py index 00ec2dc..5becc17 100644 --- a/pymodmilter/version.py +++ b/pymodmilter/version.py @@ -1 +1 @@ -__version__ = "0.0.9" +__version__ = "1.0.0" From 295aeee83ef70e2304be3e67292a67f5051d6538 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 23:13:32 +0200 Subject: [PATCH 037/145] change default config location and release 1.0.1 --- README.md | 6 +++--- pymodmilter/run.py | 2 +- pymodmilter/version.py | 2 +- setup.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0059bfa..ebcc178 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers. The project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month. -The basic idea is to define rules with conditions and do modifications when all conditions are met. +The basic idea is to define rules with conditions and modifications which are processed when all conditions are true. ## Dependencies Pymodmilter is depending on these python packages, but they are installed automatically if you are working with pip. @@ -13,9 +13,9 @@ Pymodmilter is depending on these python packages, but they are installed automa * Install pymodmilter with pip and copy the example configuration file. ```sh pip install pymodmilter -cp /usr/share/doc/pymodmilter/pymodmilter.conf.example /etc/pymodmilter.conf +cp /etc/pymodmilter/pymodmilter.conf.example /etc/pymodmilter/pymodmilter.conf ``` -* Modify /etc/pymodmilter.conf according to your needs. +* Modify /etc/pymodmilter/pymodmilter.conf according to your needs. ## Configuration options Pymodmilter uses a configuration file in JSON format. The options are described below. Make a copy of the [example configuration file](https://github.com/spacefreak86/pymodmilter/blob/master/docs/pymodmilter.conf.example) in the [docs](https://github.com/spacefreak86/pymodmilter/tree/master/docs) folder to start with. diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 6e3c8e2..fe2faee 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -35,7 +35,7 @@ def main(): prog, max_help_position=45, width=140)) parser.add_argument( "-c", "--config", help="Config file to read.", - default="/etc/pymodmilter.conf") + default="/etc/pymodmilter/pymodmilter.conf") parser.add_argument( "-s", "--socket", diff --git a/pymodmilter/version.py b/pymodmilter/version.py index 5becc17..5c4105c 100644 --- a/pymodmilter/version.py +++ b/pymodmilter/version.py @@ -1 +1 @@ -__version__ = "1.0.0" +__version__ = "1.0.1" diff --git a/setup.py b/setup.py index 1459663..693433c 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ setup(name = "pymodmilter", }, data_files = [ ( - 'usr/share/doc/pymodmilter', + 'etc/pymodmilter', [ 'docs/pymodmilter.conf.example' ] From 7ff2965ce8b03bfe30d274f2e6eaa25503ebb82f Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 23:16:59 +0200 Subject: [PATCH 038/145] change rule/modification name separator --- docs/pymodmilter.conf.example | 8 ++++---- pymodmilter/__init__.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/pymodmilter.conf.example b/docs/pymodmilter.conf.example index 00cde86..8d329eb 100644 --- a/docs/pymodmilter.conf.example +++ b/docs/pymodmilter.conf.example @@ -48,7 +48,7 @@ # Notes: Name of the rule. # Value: [ NAME ] # - "name": "MyRule", + "name": "myrule", # Section: conditions # Notes: Optional conditions to process the rule. @@ -88,7 +88,7 @@ # Notes: Name of the modification. # Value: [ NAME ] # - "name": "AddHeader", + "name": "add_test_header", # Option: type # Type: String @@ -111,7 +111,7 @@ # "value": "true" }, { - "name": "ModifyHeader", + "name": "modify_subject", "type": "mod_header", @@ -135,7 +135,7 @@ # Values: [ VALUE ] "value": "[EXTERNAL] \\g" }, { - "name": "DeleteHeader", + "name": "delete_received_header", "type": "del_header", diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index f7e7d2a..e4d7774 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -235,7 +235,7 @@ class Rule: mod["name"] = f"Modification #{mod_idx}" if self.name: - params["name"] = f"{self.name}/{mod['name']}" + params["name"] = f"{self.name}: {mod['name']}" else: params["name"] = mod["name"] From a7917e201b378e055773aca59c5edc334d851265 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 23:32:56 +0200 Subject: [PATCH 039/145] fix setup.py and release 1.0.2 --- pymodmilter/version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymodmilter/version.py b/pymodmilter/version.py index 5c4105c..7863915 100644 --- a/pymodmilter/version.py +++ b/pymodmilter/version.py @@ -1 +1 @@ -__version__ = "1.0.1" +__version__ = "1.0.2" diff --git a/setup.py b/setup.py index 693433c..6e482f0 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ setup(name = "pymodmilter", }, data_files = [ ( - 'etc/pymodmilter', + '/etc/pymodmilter', [ 'docs/pymodmilter.conf.example' ] From 466cbd32e69e285bcc172558fcf9b25de5a6325e Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 23:45:05 +0200 Subject: [PATCH 040/145] fix gentoo init script --- misc/openrc/pymodmilter.initd | 3 --- 1 file changed, 3 deletions(-) diff --git a/misc/openrc/pymodmilter.initd b/misc/openrc/pymodmilter.initd index bb9a646..f8e8096 100755 --- a/misc/openrc/pymodmilter.initd +++ b/misc/openrc/pymodmilter.initd @@ -17,9 +17,6 @@ depend() { } checkconfig() { - if [ -z "${socket}" ]; then - eerror "No socket specified in config!" - fi OUTPUT=$( ${command} ${command_args} -t 2>&1 ) ret=$? if [ $ret -ne 0 ]; then From 390ef046a1bf1c890fea7d3d1cbf34882392d066 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 5 May 2020 10:56:59 +0200 Subject: [PATCH 041/145] change config file format/style and add disclaimer functionality --- README.md | 25 +- docs/pymodmilter.conf.example | 52 +- docs/templates/disclaimer_html.template | 9 + docs/templates/disclaimer_text.template | 4 + pymodmilter/__init__.py | 609 +++++++++++++++++++----- pymodmilter/run.py | 8 + pymodmilter/version.py | 2 +- setup.py | 10 +- 8 files changed, 580 insertions(+), 139 deletions(-) create mode 100644 docs/templates/disclaimer_html.template create mode 100644 docs/templates/disclaimer_text.template diff --git a/README.md b/README.md index ebcc178..d90e816 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,8 @@ The following configuration options are optional for each rule: As described above in the [Global](#Global) section. * **log** As described above in the [Global](#Global) section. +* **pretend** + Just pretend to make the modifications, for test purposes. ### Modifications The following configuration options are mandatory for each modification: @@ -52,6 +54,7 @@ The following configuration options are mandatory for each modification: * **add_header** * **del_header** * **mod_header** + * **add_disclaimer** The following configuration options are mandatory based on the modification type in use. * **add_header** @@ -72,6 +75,24 @@ The following configuration options are mandatory based on the modification type * **value** New value of the header. +* **add_disclaimer** + * **action** + Action to perform with the disclaimer. Possible values are: + * append + * prepend + * **html_template** + Path to a file that contains the html representation of the disclaimer. + * **text_template** + Path to a file that contains the text representation of the disclaimer. + * **error_policy** + Set what should be done if the disclaimer could not be added (e.g. no body text found). Possible values are: + * wrap + The original e-mail will be attached to a new one containing the disclaimer. + * ignore + Ignore the error and do nothing. + * reject + Reject the e-mail. + The following configuration options are optional for each modification: * **name** Name of the modification. @@ -81,11 +102,13 @@ The following configuration options are optional for each modification: ### Conditions The following condition options are optional: * **local** - If set to true, the rule is only executed for emails originating from addresses defined in local_addrs and vice versa. + If set to true, the rule is only executed for e-mails originating from addresses defined in local_addrs and vice versa. * **hosts** A list of hosts and network addresses for which the rule should be executed. * **envfrom** A regular expression to match against the evenlope-from addresses for which the rule should be executed. +* **envto** + A regular expression to match against all evenlope-to addresses. All addresses must match to fulfill the condition. ## Developer information Everyone who wants to improve or extend this project is very welcome. diff --git a/docs/pymodmilter.conf.example b/docs/pymodmilter.conf.example index 8d329eb..51874ee 100644 --- a/docs/pymodmilter.conf.example +++ b/docs/pymodmilter.conf.example @@ -75,7 +75,14 @@ # Notes: Condition wheter the envelop-from address matches this regular expression. # Value: [ REGEX ] # - "envfrom": "^(?!.+@mycompany\\.com).+$" + "envfrom": "^.+@mypartner\\.com$", + + # Option: envto + # Type: String + # Notes: Condition wheter the envelop-to address matches this regular expression. + # Value: [ REGEX ] + # + "envto": "^postmaster@.+$" }, # Section: modifications @@ -109,7 +116,7 @@ # Notes: Value of the header. # Value: [ VALUE ] # - "value": "true" + "value": "true", }, { "name": "modify_subject", @@ -145,8 +152,47 @@ # Value: [ REGEX ] # "header": "^Received:" + }, { + "name": "add_disclaimer", + + "type": "add_disclaimer", + + # Option: action + # Type: String + # Notes: Action to perform with the disclaimer. + # Value: [ append | prepend ] + # + "action": "prepend", + + # Option: html_template + # Type: String + # Notes: Path to a file that contains the html representation of the disclaimer. + # Value: [ FILE_PATH ] + # + "html_template": "/etc/pymodmilter/templates/disclaimer_html.template", + + # Option: text_template + # Type: String + # Notes: Path to a file that contains the text representation of the disclaimer. + # Value: [ FILE_PATH ] + # + "text_template": "/etc/pymodmilter/templates/disclaimer_text.template", + + # Option: error_policy + # Type: String + # Notes: Set what should be done if the modification fails (e.g. no message body present). + # Value: [ wrap | ignore | reject ] + # + "error_policy": "wrap" } - ] + ], + + # Option: pretend + # Type: Bool + # Notes: Just pretend to do the modifications, for test purposes. + # Value: [ true | false ] + # + "pretend": true } ] } diff --git a/docs/templates/disclaimer_html.template b/docs/templates/disclaimer_html.template new file mode 100644 index 0000000..eec02a0 --- /dev/null +++ b/docs/templates/disclaimer_html.template @@ -0,0 +1,9 @@ + + + + +
+ CAUTION: This email originated from outside the organization. + Do not follow guidance, click links or open attachments unless you recognize the sender and know the content is safe. +
+

diff --git a/docs/templates/disclaimer_text.template b/docs/templates/disclaimer_text.template new file mode 100644 index 0000000..6d9532d --- /dev/null +++ b/docs/templates/disclaimer_text.template @@ -0,0 +1,4 @@ + +CAUTION: This email originated from outside the organization. Do not follow guidance, click links or open attachments unless you recognize the sender and know the content is safe. + + diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index e4d7774..278f985 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -27,8 +27,13 @@ import logging.handlers import re from Milter.utils import parse_addr +from bs4 import BeautifulSoup from email.charset import Charset from email.header import Header, decode_header +from email import message_from_binary_file +from email.message import MIMEPart +from email.policy import default as default_policy, SMTP +from io import BytesIO from netaddr import IPAddress, IPNetwork, AddrFormatError @@ -55,6 +60,7 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None, def replace_illegal_chars(string): + """Replace illegal characters in header values.""" return string.replace( "\x00", "").replace( "\r", "").replace( @@ -62,136 +68,367 @@ def replace_illegal_chars(string): class Modification: - """Modification to implement a modification to apply on e-mails.""" + """ + Modification to implement certain modifications on e-mails. - # mandatory parameters for each modification type - types = { - "add_header": ["header", "value"], - "del_header": ["header"], - "mod_header": ["header", "search", "value"] - } + Each modification function returns the necessary changes for ModifyMilter + so they can be applied to the email passing the MTA. + """ def __init__(self, name, mod_type, log, **params): self.logger = logging.getLogger(__name__) self.logger.debug(f"initializing modification '{name}'") self.name = name self.log = log - # check mod_type - if mod_type not in Modification.types: + # needs for each modification type + self.types = { + "add_header": { + "needs": ["headers"]}, + "del_header": { + "needs": ["headers"]}, + "mod_header": { + "needs": ["headers"]}, + "add_disclaimer": { + "needs": ["headers", "data"]}} + + if mod_type not in self.types: raise RuntimeError( f"{self.name}: invalid modification type '{mod_type}'") + self.mod_type = mod_type - # check if mandatory modification options are present in config - for option in Modification.types[self.mod_type]: - if option not in params: - raise RuntimeError( - f"{self.name}: mandatory config " - f"option '{option}' not found") - if option == "value" and not params["value"]: - raise RuntimeError( - f"{self.name}: empty value specified") - if mod_type == "add_header": - self.header = params["header"] - self.value = params["value"] - elif mod_type in ["del_header", "mod_header"]: - # compile header regex - try: - self.header = re.compile( - params["header"], re.MULTILINE + re.DOTALL + re.IGNORECASE) - except re.error as e: - raise RuntimeError( - f"{self.name}: unable to parse regular expression of " - f"option 'header': {e}") - - if mod_type == "mod_header": - # compile search regex + try: + if mod_type == "add_header": + self.header = params["header"] + self.value = params["value"] + elif mod_type in ["del_header", "mod_header"]: try: - self.search = re.compile( - params["search"], + self.header = re.compile( + params["header"], re.MULTILINE + re.DOTALL + re.IGNORECASE) except re.error as e: raise RuntimeError( - f"{self.name}: unable to parse regular expression of " - f"option 'search': {e}") - self.value = params["value"] + f"{self.name}: unable to parse regex of " + f"option 'header': {e}") - def execute(self, qid, headers): - """ - Execute modification and return list - with modified headers. - """ - if self.mod_type == "add_header": - header = f"{self.header}: {self.value}" - if self.log: - self.logger.info( - f"{qid}: {self.name}: add_header: {header[0:70]}") - else: - self.logger.debug( - f"{qid}: {self.name}: add_header: {header}") - return [(self.mod_type, self.header, self.value, 0, 1)] + if mod_type == "mod_header": + try: + self.search = re.compile( + params["search"], + re.MULTILINE + re.DOTALL + re.IGNORECASE) + except re.error as e: + raise RuntimeError( + f"{self.name}: unable to parse regex of " + f"option 'search': {e}") - modified = [] + self.value = params["value"] + elif mod_type == "add_disclaimer": + if params["action"] not in ["append", "prepend"]: + raise RuntimeError( + f"{self.name}: unknown action specified") + + self.action = params["action"] + + if params["error_policy"] not in ["wrap", "ignore", "reject"]: + raise RuntimeError( + f"{self.name}: unknown error_policy specified") + + self.error_policy = params["error_policy"] + + try: + with open(params["html_template"], "r") as f: + self.html = BeautifulSoup(f.read(), "html.parser") + body = self.html.find('body') + if body: + # just use content within the body tag if present + self.html = body + with open(params["text_template"], "r") as f: + self.text = f.read() + except IOError as e: + raise RuntimeError(f"unable to read template: {e}") + except KeyError as e: + raise RuntimeError( + f"{self.name}: mandatory configuration option not found: {e}") + + def needs(self): + """Return the needs of this modification to work.""" + return self.types[self.mod_type]["needs"] + + def add_header(self, qid, headers, header, value, pos=-1): + """Add header to email.""" + hdr = f"{header}: {value}" + if self.log: + self.logger.info( + f"{qid}: {self.name}: add_header: {hdr[0:70]}") + else: + self.logger.debug( + f"{qid}: {self.name}: add_header: {hdr}") + + headers.append((header, value)) + params = [header, value, pos] + return [("add_header", *params)] + + def mod_header(self, qid, headers, header, search, replace): + """Modify an email header.""" + if isinstance(header, str): + header = re.compile( + header, re.MULTILINE + re.DOTALL + re.IGNORECASE) + + if isinstance(search, str): + search = re.compile( + search, re.MULTILINE + re.DOTALL + re.IGNORECASE) + + changes = [] index = 0 occurrences = {} - - # iterate headers - for name, value in headers: - # keep track of the occurrence of each header, needed by - # Milter.Base.chgheader + # iterate a copy of headers because headers may be modified + for name, value in headers.copy(): + # keep track of the occurrence of each header + # needed by Milter.Base.chgheader if name not in occurrences.keys(): occurrences[name] = 1 else: occurrences[name] += 1 - # check if header line matches regex - header = f"{name}: {value}" - if self.header.search(header): - if self.mod_type == "del_header": - # set an empty value to delete the header - new_value = "" - if self.log: - self.logger.info( - f"{qid}: {self.name}: del_header: " - f"{header[0:70]}") - else: - self.logger.debug( - f"{qid}: {self.name}: del_header: " - f"(occ. {occurrences[name]}): {header}") - else: - old_header = header - new_value = self.search.sub(self.value, value) - if value == new_value: - continue - header = f"{name}: {new_value}" + hdr = f"{name}: {value}" + if header.search(hdr): + new_value = search.sub(replace, value).strip() + if new_value == "": + self.logger.warning( + f"{qid}: {self.name}: mod_header: resulting value is " + f"empty: del_header: {hdr[0:70]}") + del headers[index] + params = [name, new_value, index, occurrences[name]] + changes.append(("mod_header", *params)) + index -= 1 + occurrences[name] -= 1 + elif value != new_value: + old_hdr = hdr + hdr = f"{name}: {new_value}" if self.log: self.logger.info( f"{qid}: {self.name}: mod_header: " - f"{old_header[0:70]}: {header[0:70]}") + f"{old_hdr[0:70]}: {hdr[0:70]}") else: self.logger.debug( f"{qid}: {self.name}: mod_header: " - f"(occ. {occurrences[name]}): {old_header}: " - f"{header}") - modified.append( - (self.mod_type, name, new_value, index, occurrences[name])) + f"(occ. {occurrences[name]}): {old_hdr}: " + f"{hdr}") + + headers[index] = (name, new_value) + params = [name, new_value, occurrences[name]] + changes.append(("mod_header", *params)) + index += 1 - return modified + + return changes + + def del_header(self, qid, headers, header): + """Delete an email header.""" + if isinstance(header, str): + header = re.compile( + header, re.MULTILINE + re.DOTALL + re.IGNORECASE) + + changes = [] + index = 0 + occurrences = {} + # iterate a copy of headers because headers may be modified + for name, value in headers.copy(): + # keep track of the occurrence of each header, + # needed by Milter.Base.chgheader + if name not in occurrences.keys(): + occurrences[name] = 1 + else: + occurrences[name] += 1 + + hdr = f"{name}: {value}" + if header.search(hdr): + if self.log: + self.logger.info( + f"{qid}: {self.name}: del_header: " + f"{hdr[0:70]}") + else: + self.logger.debug( + f"{qid}: {self.name}: del_header: " + f"(occ. {occurrences[name]}): {hdr}") + + del headers[index] + params = [name, "", occurrences[name]] + changes.append(("mod_header", *params)) + index -= 1 + occurrences[name] -= 1 + + index += 1 + + return changes + + def add_disclaimer(self, qid, headers, fp, text_template, html_template, + error_policy): + """Append or prepend a disclaimer to the email body.""" + changes = [] + + fp.seek(0) + msg = message_from_binary_file(fp, policy=default_policy) + + html_body = None + text_body = None + update_headers = False + + try: + html_body = msg.get_body(preferencelist=("html")) + text_body = msg.get_body(preferencelist=("plain")) + except Exception as e: + self.logger.error( + f"{qid}: {self.name}: an error occured in " + f"email.message.EmailMessage.get_body: {e}") + + if html_body is None and text_body is None: + if self.error_policy == "ignore": + self.logger.info( + f"{qid}: {self.name}: unable to find email body, " + f"ignore according to policy") + return changes + elif self.error_policy == "reject": + self.logger.info( + f"{qid}: {self.name}: unable to find email body, " + f"reject message according to policy") + return [ + ("reject", "Message rejected due to missing email body")] + + self.logger.info( + f"{qid}: {self.name}: unable to find email body, " + f"wrapping original email in a new message envelope") + msg = MIMEPart() + msg.add_header("MIME-Version", "1.0") + msg.set_content( + "Please see the original email attached to this email.") + msg.add_alternative( + "Please see the original email attached to this email.", + subtype="html") + fp.seek(0) + msg.add_attachment( + fp.read(), maintype="plain", subtype="text", + filename="original_email.eml") + html_body = msg.get_body(preferencelist=("html")) + text_body = msg.get_body(preferencelist=("plain")) + # content and mime headers may have to be updated because + # a new message has been created + update_headers = True + elif not msg.is_multipart(): + # content and mime headers may have to be updated because + # we operate on a non-multipart email + update_headers = True + + if text_body is not None: + if self.log: + self.logger.info( + f"{qid}: {self.name}: {self.action} text disclaimer") + else: + self.logger.debug( + f"{qid}: {self.name}: {self.action} text disclaimer") + + text = text_body.get_content() + if self.action == "prepend": + text = f"{text_template}{text}" + else: + text = f"{text}{text_template}" + + text_body.set_content( + text.encode(), maintype="text", subtype="plain") + text_body.set_param("charset", "UTF-8", header="Content-Type") + + if html_body is not None: + if self.log: + self.logger.info( + f"{qid}: {self.name}: {self.action} html disclaimer") + else: + self.logger.debug( + f"{qid}: {self.name}: {self.action} html disclaimer") + + soup = BeautifulSoup(html_body.get_content(), "html.parser") + body = soup.find('body') + if body: + # work within the body tag if it is present + soup = body + + if self.action == "prepend": + soup.insert(0, html_template) + else: + soup.append(html_template) + + html_body.set_content( + str(soup).encode(), maintype="text", subtype="html") + html_body.set_param("charset", "UTF-8", header="Content-Type") + + if update_headers: + for name, value in msg.items(): + name_lower = name.lower() + if not name_lower.startswith("content-") and \ + name_lower != "mime-version": + continue + + defined = False + for n, v in headers: + if n.lower() == name_lower: + changes += self.mod_header( + qid, headers, f"^{n}:", ".*", value) + defined = True + break + + if not defined: + changes += self.add_header( + qid, headers, name, value) + + msg = msg.as_string(policy=SMTP).encode("ascii", errors="replace") + fp.seek(0) + fp.write(msg) + fp.truncate() + body_pos = msg.find(b"\r\n\r\n") + 2 + changes.append(("mod_body", body_pos)) + return changes + + def execute(self, qid, headers, fp): + """ + Execute configured modification. + """ + changes = [] + + if self.mod_type == "add_header": + changes = self.add_header( + qid, headers, self.header, self.value) + elif self.mod_type == "mod_header": + changes = self.mod_header( + qid, headers, self.header, self.search, self.value) + elif self.mod_type == "del_header": + changes = self.del_header( + qid, headers, self.header) + elif self.mod_type == "add_disclaimer": + changes = self.add_disclaimer( + qid, headers, fp, self.text, + self.html, self.error_policy) + + return changes class Rule: """ Rule to implement multiple modifications on emails based on conditions. """ - def __init__(self, name, modifications, local_addrs, log, conditions={}): + + def __init__(self, name, modifications, local_addrs, log, conditions={}, + pretend=False): self.logger = logging.getLogger(__name__) - self.name = name - self.log = log + if pretend: + self.name = f"{name} (pretend)" + else: + self.name = name self.logger.debug(f"initializing rule '{self.name}'") - + self.log = log + self.pretend = pretend + self._needs = [] self._local_addrs = [] - # replace strings in local_addrs list with IPNetwork instances + try: for addr in local_addrs: self._local_addrs.append(IPNetwork(addr)) @@ -215,22 +452,23 @@ class Rule: raise RuntimeError( f"{self.name}: unable to parse entry of " f"condition '{option}': {e}") + self.logger.debug( f"{self.name}: added condition: {option} = {value}") - elif option == "envfrom": + elif option in ["envfrom", "envto"]: try: self.conditions[option] = re.compile(value, re.IGNORECASE) except re.error as e: raise RuntimeError( - f"{self.name}: unable to parse regular expression of " + f"{self.name}: unable to parse regex of " f"condition '{option}': {e}") + self.logger.debug( f"{self.name}: added condition: {option} = {value}") self.modifications = [] for mod_idx, mod in enumerate(modifications): params = {} - # set default values if not specified in config if "name" not in mod: mod["name"] = f"Modification #{mod_idx}" @@ -251,19 +489,28 @@ class Rule: f"{params['name']}: mandatory config " f"option 'type' not found") - if "header" in mod: - params["header"] = mod["header"] + for param in [ + "header", "search", "value", "action", "html_template", + "text_template", "error_policy"]: + if param in mod: + params[param] = mod[param] - if "search" in mod: - params["search"] = mod["search"] + modification = Modification(**params) + for need in modification.needs(): + if need not in self._needs: + self._needs.append(need) - if "value" in mod: - params["value"] = mod["value"] - - self.modifications.append(Modification(**params)) + self.modifications.append(modification) self.logger.debug( f"{self.name}: added modification: {mod['name']}") + self.logger.debug( + f"{self.name}: rule needs: {self._needs}") + + def needs(self): + """Return the needs of this rule.""" + return self._needs + def ignore_host(self, host): """Check if host is ignored by this rule.""" ip = IPAddress(host) @@ -274,6 +521,7 @@ class Rule: if ip in addr: is_local = True break + if is_local != self.conditions["local"]: return True @@ -282,6 +530,7 @@ class Rule: for accepted in self.conditions["hosts"]: if ip in accepted: return False + return True return False @@ -291,9 +540,21 @@ class Rule: if "envfrom" in self.conditions: if not self.conditions["envfrom"].search(envfrom): return True + return False - def execute(self, qid, headers): + def ignore_envto(self, envto): + """Check if envelope-to address is ignored by this rule.""" + if "envto" in self.conditions: + if not isinstance(envto, set): + envto = set(envto) + for to in envto: + if not self.conditions["envto"].search(to): + return True + + return False + + def execute(self, qid, headers, data): """Execute all modifications of this rule.""" changes = [] if self.log: @@ -303,16 +564,10 @@ class Rule: for mod in self.modifications: self.logger.debug(f"{qid}: executing modification '{mod.name}'") - result = mod.execute(qid, headers) - changes += result - for mod_type, name, value, index, occurrence in result: - if mod_type == "add_header": - headers.append((name, value)) - else: - if mod_type == "mod_header": - headers[index] = (name, value) - elif mod_type == "del_header": - del headers[index] + changes += mod.execute(qid, headers, data) + + if self.pretend: + changes = [] return changes @@ -348,6 +603,7 @@ class ModifyMilter(Milter.Base): f"host {hostaddr[0]} is ignored by all rules, " f"skip further processing") return Milter.ACCEPT + return Milter.CONTINUE def envfrom(self, mailfrom, *str): @@ -361,16 +617,52 @@ class ModifyMilter(Milter.Base): if not self.rules: self.logger.debug( - f"mail from {mailfrom} is ignored by all rules, " + f"envelope-from address {mailfrom} is ignored by all rules, " f"skip further processing") return Milter.ACCEPT + + self.recipients = set() return Milter.CONTINUE @Milter.noreply + def envrcpt(self, to, *str): + self.recipients.add("@".join(parse_addr(to)).lower()) + return Milter.CONTINUE + def data(self): - self.qid = self.getsymval('i') - self.logger.debug(f"{self.qid}: received queue-id from MTA") - self.headers = [] + try: + for rule in self.rules.copy(): + if rule.ignore_envto(self.recipients): + self.logger.debug( + f"envelope-to addresses are ignored by " + f"rule '{rule.name}'") + self.rules.remove(rule) + + if not self.rules: + self.logger.debug( + f"envelope-to addresses are ignored by all rules, " + f"skip further processing") + return Milter.ACCEPT + + self.qid = self.getsymval('i') + self.logger.debug(f"{self.qid}: received queue-id from MTA") + self.headers = None + self.fp = None + for rule in self.rules: + if "headers" in rule.needs() and self.headers is None: + self.headers = [] + + if "data" in rule.needs() and self.fp is None: + self.fp = BytesIO() + + if None not in [self.headers, self.fp]: + break + + except Exception as e: + self.logger.exception( + f"an exception occured in data function: {e}") + return Milter.TEMPFAIL + return Milter.CONTINUE def header(self, name, value): @@ -378,36 +670,89 @@ class ModifyMilter(Milter.Base): # remove surrogates from value value = value.encode( errors="surrogateescape").decode(errors="replace") - self.logger.debug(f"{self.qid}: received header: {name}: {value}") - header = make_header(decode_header(value), errors="replace") - value = str(header).replace("\x00", "") - self.logger.debug( - f"{self.qid}: decoded header: {name}: {value}") - self.headers.append((name, value)) - return Milter.CONTINUE + if self.fp is not None: + self.fp.write(f"{name}: {value}\r\n".encode( + encoding="ascii", errors="replace")) + + if self.headers is not None: + self.logger.debug(f"{self.qid}: received header: " + f"{name}: {value}") + header = make_header(decode_header(value), errors="replace") + value = str(header).replace("\x00", "") + self.logger.debug( + f"{self.qid}: decoded header: {name}: {value}") + self.headers.append((name, value)) except Exception as e: self.logger.exception( f"an exception occured in header function: {e}") return Milter.TEMPFAIL + return Milter.CONTINUE + + def eoh(self): + try: + if self.fp is not None: + self.fp.write(b"\r\n") + except Exception as e: + self.logger.exception( + f"an exception occured in eoh function: {e}") + return Milter.TEMPFAIL + + return Milter.CONTINUE + + def body(self, chunk): + try: + if self.fp is not None: + self.fp.write(chunk) + except Exception as e: + self.logger.exception( + f"an exception occured in body function: {e}") + return Milter.TEMPFAIL + + return Milter.CONTINUE + def eom(self): try: + changes = [] for rule in self.rules: - changes = rule.execute(self.qid, self.headers) - for mod_type, name, value, index, occurrence in changes: + changes += rule.execute(self.qid, self.headers, self.fp) + + mod_body_pos = None + for mod_type, *params in changes: + if mod_type in ["add_header", "mod_header", "del_header"]: + header, value, occurrence = params enc_value = replace_illegal_chars( Header(s=value).encode()) if mod_type == "add_header": - self.logger.debug(f"{self.qid}: milter: adding " - f"header: {name}: {enc_value}") - self.addheader(name, enc_value, -1) + self.logger.debug(f"{self.qid}: milter: add " + f"header: {header}: {enc_value}") + self.addheader(header, enc_value, occurrence) else: - self.logger.debug(f"{self.qid}: milter: modify " - f"header (occ. {occurrence}): " - f"{name}: {enc_value}") - self.chgheader(name, occurrence, enc_value) - return Milter.ACCEPT + if enc_value == "": + self.logger.debug( + f"{self.qid}: milter: delete " + f"header (occ. {occurrence}): " + f"{header}") + else: + self.logger.debug( + f"{self.qid}: milter: modify " + f"header (occ. {occurrence}): " + f"{header}: {enc_value}") + + self.chgheader(header, occurrence, enc_value) + elif mod_type == "mod_body": + mod_body_pos = params[0] + elif mod_type == "reject": + self.setreply("554", "5.7.0", params[0]) + return Milter.REJECT + + if mod_body_pos is not None: + self.fp.seek(mod_body_pos) + self.logger.debug(f"{self.qid}: milter: replace body") + self.replacebody(self.fp.read()) except Exception as e: self.logger.exception( f"an exception occured in eom function: {e}") return Milter.TEMPFAIL + + return Milter.ACCEPT diff --git a/pymodmilter/run.py b/pymodmilter/run.py index fe2faee..924ab2a 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -114,6 +114,9 @@ def main(): if "log" not in config["global"]: config["global"]["log"] = True + if "pretend" not in config["global"]: + config["global"]["pretend"] = False + # check if mandatory sections are present in config for section in ["rules"]: if section not in config: @@ -138,6 +141,11 @@ def main(): else: params["log"] = config["global"]["log"] + if "pretend" in rule: + params["pretend"] = rule["pretend"] + else: + params["pretend"] = config["global"]["pretend"] + if "local_addrs" in rule: params["local_addrs"] = rule["local_addrs"] else: diff --git a/pymodmilter/version.py b/pymodmilter/version.py index 7863915..976498a 100644 --- a/pymodmilter/version.py +++ b/pymodmilter/version.py @@ -1 +1 @@ -__version__ = "1.0.2" +__version__ = "1.0.3" diff --git a/setup.py b/setup.py index 6e482f0..3417ec4 100644 --- a/setup.py +++ b/setup.py @@ -37,9 +37,15 @@ setup(name = "pymodmilter", }, data_files = [ ( - '/etc/pymodmilter', + "/etc/pymodmilter", [ - 'docs/pymodmilter.conf.example' + "docs/pymodmilter.conf.example" + ] + ), ( + "/etc/pymodmilter/templates", + [ + "docs/templates/disclaimer_html.template", + "docs/templates/disclaimer_txt.template" ] ) ], From 708cef085ebd23e7247720bebf6ac4a215ea9db5 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 5 May 2020 12:05:17 +0200 Subject: [PATCH 042/145] fix setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3417ec4..c158fd6 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ setup(name = "pymodmilter", "/etc/pymodmilter/templates", [ "docs/templates/disclaimer_html.template", - "docs/templates/disclaimer_txt.template" + "docs/templates/disclaimer_text.template" ] ) ], From 4a8d416d1c87a9675945fa1063b36a404db09161 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 5 May 2020 13:29:18 +0200 Subject: [PATCH 043/145] fix mod_header function if resulting value is empty --- pymodmilter/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 278f985..1e3faf3 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -196,12 +196,7 @@ class Modification: if new_value == "": self.logger.warning( f"{qid}: {self.name}: mod_header: resulting value is " - f"empty: del_header: {hdr[0:70]}") - del headers[index] - params = [name, new_value, index, occurrences[name]] - changes.append(("mod_header", *params)) - index -= 1 - occurrences[name] -= 1 + f"empty, skip modification") elif value != new_value: old_hdr = hdr hdr = f"{name}: {new_value}" From 4af37798e36c07ace3822b16e4e0d4bb1dad26de Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 8 May 2020 11:45:57 +0200 Subject: [PATCH 044/145] fix internal template handling --- pymodmilter/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 1e3faf3..fb4fac1 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -28,6 +28,7 @@ import re from Milter.utils import parse_addr from bs4 import BeautifulSoup +from copy import copy from email.charset import Charset from email.header import Header, decode_header from email import message_from_binary_file @@ -347,7 +348,7 @@ class Modification: soup = body if self.action == "prepend": - soup.insert(0, html_template) + soup.insert(0, copy(html_template)) else: soup.append(html_template) From 9019ad37aba7b899fdc7d02084f8e898a94ec5b1 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 8 May 2020 13:51:52 +0200 Subject: [PATCH 045/145] rename original email attachment and change notification text --- pymodmilter/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index fb4fac1..3fe5b69 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -297,14 +297,14 @@ class Modification: msg = MIMEPart() msg.add_header("MIME-Version", "1.0") msg.set_content( - "Please see the original email attached to this email.") + "Please see the original email attached.") msg.add_alternative( - "Please see the original email attached to this email.", + "Please see the original email attached.", subtype="html") fp.seek(0) msg.add_attachment( fp.read(), maintype="plain", subtype="text", - filename="original_email.eml") + filename=f"{qid}.eml") html_body = msg.get_body(preferencelist=("html")) text_body = msg.get_body(preferencelist=("plain")) # content and mime headers may have to be updated because From d60ea5282cf80188f9d8947730899d50bf4bca86 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Mon, 11 May 2020 13:38:54 +0200 Subject: [PATCH 046/145] change version to 1.0.4 --- pymodmilter/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/version.py b/pymodmilter/version.py index 976498a..92192ee 100644 --- a/pymodmilter/version.py +++ b/pymodmilter/version.py @@ -1 +1 @@ -__version__ = "1.0.3" +__version__ = "1.0.4" From 0651ceba628ff9b5584fd80e7d1e20f4b640f192 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Jun 2020 01:18:00 +0200 Subject: [PATCH 047/145] massive refactoring of the source --- README.md | 117 +++-- docs/pymodmilter.conf.example | 48 +- docs/templates/disclaimer_html.template | 2 +- pymodmilter/__init__.py | 650 ++++-------------------- pymodmilter/actions.py | 456 +++++++++++++++++ pymodmilter/conditions.py | 134 +++++ pymodmilter/run.py | 212 +++++--- 7 files changed, 905 insertions(+), 714 deletions(-) create mode 100644 pymodmilter/actions.py create mode 100644 pymodmilter/conditions.py diff --git a/README.md b/README.md index d90e816..e103d79 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers. The project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month. -The basic idea is to define rules with conditions and modifications which are processed when all conditions are true. +The basic idea is to define rules with conditions and actions which are processed when all conditions are true. ## Dependencies Pymodmilter is depending on these python packages, but they are installed automatically if you are working with pip. @@ -10,7 +10,7 @@ Pymodmilter is depending on these python packages, but they are installed automa * [netaddr](https://github.com/drkjam/netaddr/) ## Installation -* Install pymodmilter with pip and copy the example configuration file. +* Install pymodmilter with pip and copy the example config file. ```sh pip install pymodmilter cp /etc/pymodmilter/pymodmilter.conf.example /etc/pymodmilter/pymodmilter.conf @@ -18,96 +18,107 @@ cp /etc/pymodmilter/pymodmilter.conf.example /etc/pymodmilter/pymodmilter.conf * Modify /etc/pymodmilter/pymodmilter.conf according to your needs. ## Configuration options -Pymodmilter uses a configuration file in JSON format. The options are described below. Make a copy of the [example configuration file](https://github.com/spacefreak86/pymodmilter/blob/master/docs/pymodmilter.conf.example) in the [docs](https://github.com/spacefreak86/pymodmilter/tree/master/docs) folder to start with. -Rules and modifications are processed in the given order. +Pymodmilter uses a config file in JSON format. The config file has to be JSON valid with the exception of allowed comment lines starting with **#**. The options are described below. +Rules and actions are processed in the given order. ### Global -The following global configuration options are optional: -* **socket** - The socket used to communicate with the MTA. -* **local_addrs** - A list of hosts and network addresses which are considered local. It is used to for the condition option [local](#Conditions). This option may be overriden within a rule object. -* **log** - Enable or disable logging. This option may be overriden by a rule or modification object. +Config options in **global** section: +* **socket** (optional) + The socket used to communicate with the MTA. If it is not specified in the config, it has to be set as command line option. +* **local_addrs** (optional) + A list of hosts and network addresses which are considered local. It is used to for the condition option [local](#Conditions). + Default: **::1/128, 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16** +* **loglevel** (optional) + Set the log level. This option may be overriden by any rule or action object. Possible values are: + * **error** + * **warning** + * **info** + * **debug** + Default: **info** +* **pretend** (optional) + Pretend actions, for test purposes. This option may be overriden by any rule or action object. ### Rules -The following configuration options are mandatory for each rule: -* **modifications** - A list of modification objects which are processed in the given order. - -The following configuration options are optional for each rule: -* **name** - Name of the rule. -* **conditions** - A list of conditions which all have to be true to process the rule. -* **local_addrs** +Config options for **rule** objects: +* **name** (optional) + Name of the rule. + Default: **Rule #n** +* **actions** + A list of action objects which are processed in the given order. +* **conditions** (optional) + A list of conditions which all have to be true to process the actions. +* **loglevel** (optional) As described above in the [Global](#Global) section. -* **log** +* **pretend** (optional) As described above in the [Global](#Global) section. -* **pretend** - Just pretend to make the modifications, for test purposes. -### Modifications -The following configuration options are mandatory for each modification: +### Actions +Config options for **action** objects: +* **name** (optional) + Name of the action. + Default: **Action #n** * **type** - Set the modification type. Possible values are: + Action type. Possible values are: * **add_header** * **del_header** * **mod_header** * **add_disclaimer** +* **conditions** (optional) + A list of conditions which all have to be true to process the action. +* **pretend** (optional) + Just pretend all actions of this rule, for test purposes. +* **loglevel** (optional) + As described above in the [Global](#Global) section. -The following configuration options are mandatory based on the modification type in use. -* **add_header** +Config options for **add_header** actions: * **header** Name of the header. * **value** Value of the header. -* **del_header** +Config options for **del_header** actions: * **header** - Regular expression to match against header lines. + Regular expression to match against header names. + * **value** (optional) + Regular expression to match against the headers value. -* **mod_header** +Config options for **mod_header** actions: * **header** - Regular expression to match against header lines. - * **search** - Regular expression to match against the value of header lines. You may use subgroups or named subgroups (python syntax) to include parts of the original value in the new value. + Regular expression to match against header names. + * **search** (optional) + Regular expression to match against header values. You may use subgroups or named subgroups (python syntax) to include parts of the original value in the new value. * **value** New value of the header. -* **add_disclaimer** +Config options for **add_disclaimer** actions: * **action** Action to perform with the disclaimer. Possible values are: * append * prepend - * **html_template** - Path to a file that contains the html representation of the disclaimer. - * **text_template** - Path to a file that contains the text representation of the disclaimer. - * **error_policy** - Set what should be done if the disclaimer could not be added (e.g. no body text found). Possible values are: + * **html_file** + Path to a file which contains the html representation of the disclaimer. + * **text_file** + Path to a file which contains the text representation of the disclaimer. + * **error_policy** (optional) + Set the error policy in case the disclaimer cannot be added (e.g. if no body part is present in the e-mail). Possible values are: * wrap - The original e-mail will be attached to a new one containing the disclaimer. + A new e-mail body is generated with the disclaimer as body and the original e-mail attached. * ignore Ignore the error and do nothing. * reject Reject the e-mail. + Default: **wrap** -The following configuration options are optional for each modification: -* **name** - Name of the modification. -* **log** - As described above in the global object section. ### Conditions -The following condition options are optional: -* **local** +Config options for **conditions** objects: +* **local** (optional) If set to true, the rule is only executed for e-mails originating from addresses defined in local_addrs and vice versa. -* **hosts** +* **hosts** (optional) A list of hosts and network addresses for which the rule should be executed. -* **envfrom** +* **envfrom** (optional) A regular expression to match against the evenlope-from addresses for which the rule should be executed. -* **envto** +* **envto** (optional) A regular expression to match against all evenlope-to addresses. All addresses must match to fulfill the condition. ## Developer information diff --git a/docs/pymodmilter.conf.example b/docs/pymodmilter.conf.example index 51874ee..1afaa43 100644 --- a/docs/pymodmilter.conf.example +++ b/docs/pymodmilter.conf.example @@ -30,16 +30,23 @@ # "local_addrs": ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], - # Option: log + # Option: loglevel + # Type: String + # Notes: Set loglevel for rules and actions. + # Value: [ error | warning | info | debug ] + # + "loglevel": "info", + + # Option: pretend # Type: Bool - # Notes: Enable or disable logging of rules and modifications. + # Notes: Just pretend to do the actions, for test purposes. # Value: [ true | false ] # - "log": true + "pretend": true }, # Section: rules - # Notes: Rules and related modifications. + # Notes: Rules and related actions. # "rules": [ { @@ -85,10 +92,10 @@ "envto": "^postmaster@.+$" }, - # Section: modifications - # Notes: Modifications of the rule. + # Section: actions + # Notes: Actions of the rule. # - "modifications": [ + "actions": [ { # Option: name # Type: String @@ -116,7 +123,7 @@ # Notes: Value of the header. # Value: [ VALUE ] # - "value": "true", + "value": "true" }, { "name": "modify_subject", @@ -127,7 +134,7 @@ # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). # Value: [ REGEX ] # - "header": "^Subject:", + "header": "^Subject$", # Option: search # Type: String @@ -151,7 +158,7 @@ # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). # Value: [ REGEX ] # - "header": "^Received:" + "header": "^Received$" }, { "name": "add_disclaimer", @@ -164,19 +171,19 @@ # "action": "prepend", - # Option: html_template + # Option: html_file # Type: String - # Notes: Path to a file that contains the html representation of the disclaimer. + # Notes: Path to a file which contains the html representation of the disclaimer. # Value: [ FILE_PATH ] # - "html_template": "/etc/pymodmilter/templates/disclaimer_html.template", + "html_file": "/etc/pymodmilter/templates/disclaimer_html.template", - # Option: text_template + # Option: text_file # Type: String - # Notes: Path to a file that contains the text representation of the disclaimer. + # Notes: Path to a file which contains the text representation of the disclaimer. # Value: [ FILE_PATH ] # - "text_template": "/etc/pymodmilter/templates/disclaimer_text.template", + "text_file": "/etc/pymodmilter/templates/disclaimer_text.template", # Option: error_policy # Type: String @@ -185,14 +192,7 @@ # "error_policy": "wrap" } - ], - - # Option: pretend - # Type: Bool - # Notes: Just pretend to do the modifications, for test purposes. - # Value: [ true | false ] - # - "pretend": true + ] } ] } diff --git a/docs/templates/disclaimer_html.template b/docs/templates/disclaimer_html.template index eec02a0..827f148 100644 --- a/docs/templates/disclaimer_html.template +++ b/docs/templates/disclaimer_html.template @@ -6,4 +6,4 @@ -

+

diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 3fe5b69..1b255e6 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -14,28 +14,23 @@ __all__ = [ "make_header", - "replace_illegal_chars", + "actions", + "conditions", "run", "version", - "Modification", + "CustomLogger", "Rule", "ModifyMilter"] import Milter import logging -import logging.handlers -import re from Milter.utils import parse_addr -from bs4 import BeautifulSoup -from copy import copy from email.charset import Charset from email.header import Header, decode_header -from email import message_from_binary_file -from email.message import MIMEPart -from email.policy import default as default_policy, SMTP from io import BytesIO -from netaddr import IPAddress, IPNetwork, AddrFormatError + +from pymodmilter.conditions import Conditions def make_header(decoded_seq, maxlinelen=None, header_name=None, @@ -60,524 +55,101 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None, return h -def replace_illegal_chars(string): - """Replace illegal characters in header values.""" - return string.replace( - "\x00", "").replace( - "\r", "").replace( - "\n", "") +class CustomLogger(logging.LoggerAdapter): + def process(self, msg, kwargs): + if "name" in self.extra: + msg = "{}: {}".format(self.extra["name"], msg) + if "qid" in self.extra: + msg = "{}: {}".format(self.extra["qid"], msg) -class Modification: - """ - Modification to implement certain modifications on e-mails. + if self.logger.getEffectiveLevel() != logging.DEBUG: + msg = msg.replace("\n", "").replace("\r", "") - Each modification function returns the necessary changes for ModifyMilter - so they can be applied to the email passing the MTA. - """ - - def __init__(self, name, mod_type, log, **params): - self.logger = logging.getLogger(__name__) - self.logger.debug(f"initializing modification '{name}'") - self.name = name - self.log = log - # needs for each modification type - self.types = { - "add_header": { - "needs": ["headers"]}, - "del_header": { - "needs": ["headers"]}, - "mod_header": { - "needs": ["headers"]}, - "add_disclaimer": { - "needs": ["headers", "data"]}} - - if mod_type not in self.types: - raise RuntimeError( - f"{self.name}: invalid modification type '{mod_type}'") - - self.mod_type = mod_type - - try: - if mod_type == "add_header": - self.header = params["header"] - self.value = params["value"] - elif mod_type in ["del_header", "mod_header"]: - try: - self.header = re.compile( - params["header"], - re.MULTILINE + re.DOTALL + re.IGNORECASE) - except re.error as e: - raise RuntimeError( - f"{self.name}: unable to parse regex of " - f"option 'header': {e}") - - if mod_type == "mod_header": - try: - self.search = re.compile( - params["search"], - re.MULTILINE + re.DOTALL + re.IGNORECASE) - except re.error as e: - raise RuntimeError( - f"{self.name}: unable to parse regex of " - f"option 'search': {e}") - - self.value = params["value"] - elif mod_type == "add_disclaimer": - if params["action"] not in ["append", "prepend"]: - raise RuntimeError( - f"{self.name}: unknown action specified") - - self.action = params["action"] - - if params["error_policy"] not in ["wrap", "ignore", "reject"]: - raise RuntimeError( - f"{self.name}: unknown error_policy specified") - - self.error_policy = params["error_policy"] - - try: - with open(params["html_template"], "r") as f: - self.html = BeautifulSoup(f.read(), "html.parser") - body = self.html.find('body') - if body: - # just use content within the body tag if present - self.html = body - with open(params["text_template"], "r") as f: - self.text = f.read() - except IOError as e: - raise RuntimeError(f"unable to read template: {e}") - except KeyError as e: - raise RuntimeError( - f"{self.name}: mandatory configuration option not found: {e}") - - def needs(self): - """Return the needs of this modification to work.""" - return self.types[self.mod_type]["needs"] - - def add_header(self, qid, headers, header, value, pos=-1): - """Add header to email.""" - hdr = f"{header}: {value}" - if self.log: - self.logger.info( - f"{qid}: {self.name}: add_header: {hdr[0:70]}") - else: - self.logger.debug( - f"{qid}: {self.name}: add_header: {hdr}") - - headers.append((header, value)) - params = [header, value, pos] - return [("add_header", *params)] - - def mod_header(self, qid, headers, header, search, replace): - """Modify an email header.""" - if isinstance(header, str): - header = re.compile( - header, re.MULTILINE + re.DOTALL + re.IGNORECASE) - - if isinstance(search, str): - search = re.compile( - search, re.MULTILINE + re.DOTALL + re.IGNORECASE) - - changes = [] - index = 0 - occurrences = {} - # iterate a copy of headers because headers may be modified - for name, value in headers.copy(): - # keep track of the occurrence of each header - # needed by Milter.Base.chgheader - if name not in occurrences.keys(): - occurrences[name] = 1 - else: - occurrences[name] += 1 - - hdr = f"{name}: {value}" - if header.search(hdr): - new_value = search.sub(replace, value).strip() - if new_value == "": - self.logger.warning( - f"{qid}: {self.name}: mod_header: resulting value is " - f"empty, skip modification") - elif value != new_value: - old_hdr = hdr - hdr = f"{name}: {new_value}" - if self.log: - self.logger.info( - f"{qid}: {self.name}: mod_header: " - f"{old_hdr[0:70]}: {hdr[0:70]}") - else: - self.logger.debug( - f"{qid}: {self.name}: mod_header: " - f"(occ. {occurrences[name]}): {old_hdr}: " - f"{hdr}") - - headers[index] = (name, new_value) - params = [name, new_value, occurrences[name]] - changes.append(("mod_header", *params)) - - index += 1 - - return changes - - def del_header(self, qid, headers, header): - """Delete an email header.""" - if isinstance(header, str): - header = re.compile( - header, re.MULTILINE + re.DOTALL + re.IGNORECASE) - - changes = [] - index = 0 - occurrences = {} - # iterate a copy of headers because headers may be modified - for name, value in headers.copy(): - # keep track of the occurrence of each header, - # needed by Milter.Base.chgheader - if name not in occurrences.keys(): - occurrences[name] = 1 - else: - occurrences[name] += 1 - - hdr = f"{name}: {value}" - if header.search(hdr): - if self.log: - self.logger.info( - f"{qid}: {self.name}: del_header: " - f"{hdr[0:70]}") - else: - self.logger.debug( - f"{qid}: {self.name}: del_header: " - f"(occ. {occurrences[name]}): {hdr}") - - del headers[index] - params = [name, "", occurrences[name]] - changes.append(("mod_header", *params)) - index -= 1 - occurrences[name] -= 1 - - index += 1 - - return changes - - def add_disclaimer(self, qid, headers, fp, text_template, html_template, - error_policy): - """Append or prepend a disclaimer to the email body.""" - changes = [] - - fp.seek(0) - msg = message_from_binary_file(fp, policy=default_policy) - - html_body = None - text_body = None - update_headers = False - - try: - html_body = msg.get_body(preferencelist=("html")) - text_body = msg.get_body(preferencelist=("plain")) - except Exception as e: - self.logger.error( - f"{qid}: {self.name}: an error occured in " - f"email.message.EmailMessage.get_body: {e}") - - if html_body is None and text_body is None: - if self.error_policy == "ignore": - self.logger.info( - f"{qid}: {self.name}: unable to find email body, " - f"ignore according to policy") - return changes - elif self.error_policy == "reject": - self.logger.info( - f"{qid}: {self.name}: unable to find email body, " - f"reject message according to policy") - return [ - ("reject", "Message rejected due to missing email body")] - - self.logger.info( - f"{qid}: {self.name}: unable to find email body, " - f"wrapping original email in a new message envelope") - msg = MIMEPart() - msg.add_header("MIME-Version", "1.0") - msg.set_content( - "Please see the original email attached.") - msg.add_alternative( - "Please see the original email attached.", - subtype="html") - fp.seek(0) - msg.add_attachment( - fp.read(), maintype="plain", subtype="text", - filename=f"{qid}.eml") - html_body = msg.get_body(preferencelist=("html")) - text_body = msg.get_body(preferencelist=("plain")) - # content and mime headers may have to be updated because - # a new message has been created - update_headers = True - elif not msg.is_multipart(): - # content and mime headers may have to be updated because - # we operate on a non-multipart email - update_headers = True - - if text_body is not None: - if self.log: - self.logger.info( - f"{qid}: {self.name}: {self.action} text disclaimer") - else: - self.logger.debug( - f"{qid}: {self.name}: {self.action} text disclaimer") - - text = text_body.get_content() - if self.action == "prepend": - text = f"{text_template}{text}" - else: - text = f"{text}{text_template}" - - text_body.set_content( - text.encode(), maintype="text", subtype="plain") - text_body.set_param("charset", "UTF-8", header="Content-Type") - - if html_body is not None: - if self.log: - self.logger.info( - f"{qid}: {self.name}: {self.action} html disclaimer") - else: - self.logger.debug( - f"{qid}: {self.name}: {self.action} html disclaimer") - - soup = BeautifulSoup(html_body.get_content(), "html.parser") - body = soup.find('body') - if body: - # work within the body tag if it is present - soup = body - - if self.action == "prepend": - soup.insert(0, copy(html_template)) - else: - soup.append(html_template) - - html_body.set_content( - str(soup).encode(), maintype="text", subtype="html") - html_body.set_param("charset", "UTF-8", header="Content-Type") - - if update_headers: - for name, value in msg.items(): - name_lower = name.lower() - if not name_lower.startswith("content-") and \ - name_lower != "mime-version": - continue - - defined = False - for n, v in headers: - if n.lower() == name_lower: - changes += self.mod_header( - qid, headers, f"^{n}:", ".*", value) - defined = True - break - - if not defined: - changes += self.add_header( - qid, headers, name, value) - - msg = msg.as_string(policy=SMTP).encode("ascii", errors="replace") - fp.seek(0) - fp.write(msg) - fp.truncate() - body_pos = msg.find(b"\r\n\r\n") + 2 - changes.append(("mod_body", body_pos)) - return changes - - def execute(self, qid, headers, fp): - """ - Execute configured modification. - """ - changes = [] - - if self.mod_type == "add_header": - changes = self.add_header( - qid, headers, self.header, self.value) - elif self.mod_type == "mod_header": - changes = self.mod_header( - qid, headers, self.header, self.search, self.value) - elif self.mod_type == "del_header": - changes = self.del_header( - qid, headers, self.header) - elif self.mod_type == "add_disclaimer": - changes = self.add_disclaimer( - qid, headers, fp, self.text, - self.html, self.error_policy) - - return changes + return msg, kwargs class Rule: """ - Rule to implement multiple modifications on emails based on conditions. + Rule to implement multiple actions on emails. """ - def __init__(self, name, modifications, local_addrs, log, conditions={}, - pretend=False): - self.logger = logging.getLogger(__name__) - if pretend: - self.name = f"{name} (pretend)" - else: - self.name = name + def __init__(self, name, local_addrs, conditions, actions, pretend=False, + loglevel=logging.INFO): + logger = logging.getLogger(name) + self.logger = CustomLogger(logger, {"name": name}) + self.logger.setLevel(loglevel) - self.logger.debug(f"initializing rule '{self.name}'") - self.log = log + if logger is None: + logger = logging.getLogger(__name__) + + self.logger = CustomLogger(logger, {"name": name}) + self.conditions = Conditions( + local_addrs=local_addrs, + args=conditions, + logger=self.logger) + self.actions = actions self.pretend = pretend + self._needs = [] - self._local_addrs = [] - - try: - for addr in local_addrs: - self._local_addrs.append(IPNetwork(addr)) - except AddrFormatError as e: - raise RuntimeError( - f"{self.name}: unable to parse entry of " - f"option local_addrs: {e}") - - self.conditions = {} - for option, value in conditions.items(): - if option == "local": - self.conditions[option] = value - self.logger.debug( - f"{self.name}: added condition: {option} = {value}") - elif option == "hosts": - self.conditions[option] = [] - try: - for host in value: - self.conditions[option].append(IPNetwork(host)) - except AddrFormatError as e: - raise RuntimeError( - f"{self.name}: unable to parse entry of " - f"condition '{option}': {e}") - - self.logger.debug( - f"{self.name}: added condition: {option} = {value}") - elif option in ["envfrom", "envto"]: - try: - self.conditions[option] = re.compile(value, re.IGNORECASE) - except re.error as e: - raise RuntimeError( - f"{self.name}: unable to parse regex of " - f"condition '{option}': {e}") - - self.logger.debug( - f"{self.name}: added condition: {option} = {value}") - - self.modifications = [] - for mod_idx, mod in enumerate(modifications): - params = {} - if "name" not in mod: - mod["name"] = f"Modification #{mod_idx}" - - if self.name: - params["name"] = f"{self.name}: {mod['name']}" - else: - params["name"] = mod["name"] - - if "log" in mod: - params["log"] = mod["log"] - else: - params["log"] = self.log - - if "type" in mod: - params["mod_type"] = mod["type"] - else: - raise RuntimeError( - f"{params['name']}: mandatory config " - f"option 'type' not found") - - for param in [ - "header", "search", "value", "action", "html_template", - "text_template", "error_policy"]: - if param in mod: - params[param] = mod[param] - - modification = Modification(**params) - for need in modification.needs(): + for action in actions: + for need in action.needs(): if need not in self._needs: self._needs.append(need) - self.modifications.append(modification) - self.logger.debug( - f"{self.name}: added modification: {mod['name']}") - - self.logger.debug( - f"{self.name}: rule needs: {self._needs}") + self.logger.debug("needs: {}".format(", ".join(self._needs))) def needs(self): """Return the needs of this rule.""" return self._needs - def ignore_host(self, host): - """Check if host is ignored by this rule.""" - ip = IPAddress(host) + def ignores(self, host=None, envfrom=None, envto=None): + args = {} - if "local" in self.conditions: - is_local = False - for addr in self._local_addrs: - if ip in addr: - is_local = True - break + if host is not None: + args["host"] = host - if is_local != self.conditions["local"]: - return True + if envfrom is not None: + args["envfrom"] = envfrom - if "hosts" in self.conditions: - # check if host is in list - for accepted in self.conditions["hosts"]: - if ip in accepted: + if envto is not None: + args["envto"] = envto + + if self.conditions.match(args): + for action in self.actions: + if action.conditions.match(args): return False - return True + return True - return False + def execute(self, milter, pretend=None): + """Execute all actions of this rule.""" + if pretend is None: + pretend = self.pretend - def ignore_envfrom(self, envfrom): - """Check if envelope-from address is ignored by this rule.""" - if "envfrom" in self.conditions: - if not self.conditions["envfrom"].search(envfrom): - return True - - return False - - def ignore_envto(self, envto): - """Check if envelope-to address is ignored by this rule.""" - if "envto" in self.conditions: - if not isinstance(envto, set): - envto = set(envto) - for to in envto: - if not self.conditions["envto"].search(to): - return True - - return False - - def execute(self, qid, headers, data): - """Execute all modifications of this rule.""" - changes = [] - if self.log: - self.logger.info(f"{qid}: executing rule '{self.name}'") - else: - self.logger.debug(f"{qid}: executing rule '{self.name}'") - - for mod in self.modifications: - self.logger.debug(f"{qid}: executing modification '{mod.name}'") - changes += mod.execute(qid, headers, data) - - if self.pretend: - changes = [] - return changes + for action in self.actions: + milter_action = action.execute(milter) + if milter_action is not None: + return milter_action class ModifyMilter(Milter.Base): """ModifyMilter based on Milter.Base to implement milter communication""" _rules = [] + _loglevel = logging.INFO @staticmethod def set_rules(rules): ModifyMilter._rules = rules + def set_loglevel(level): + ModifyMilter._loglevel = level + def __init__(self): self.logger = logging.getLogger(__name__) + self.logger.setLevel(ModifyMilter._loglevel) + # save rules, it must not change during runtime self.rules = ModifyMilter._rules.copy() @@ -585,13 +157,10 @@ class ModifyMilter(Milter.Base): self.logger.debug( f"accepted milter connection from {hostaddr[0]} " f"port {hostaddr[1]}") - ip = IPAddress(hostaddr[0]) # remove rules which ignore this host for rule in self.rules.copy(): - if rule.ignore_host(ip): - self.logger.debug( - f"host {hostaddr[0]} is ignored by rule '{rule.name}'") + if rule.ignores(host=hostaddr[0]): self.rules.remove(rule) if not self.rules: @@ -605,10 +174,7 @@ class ModifyMilter(Milter.Base): def envfrom(self, mailfrom, *str): mailfrom = "@".join(parse_addr(mailfrom)).lower() for rule in self.rules.copy(): - if rule.ignore_envfrom(mailfrom): - self.logger.debug( - f"envelope-from {mailfrom} is ignored by " - f"rule '{rule.name}'") + if rule.ignores(envfrom=mailfrom): self.rules.remove(rule) if not self.rules: @@ -628,10 +194,7 @@ class ModifyMilter(Milter.Base): def data(self): try: for rule in self.rules.copy(): - if rule.ignore_envto(self.recipients): - self.logger.debug( - f"envelope-to addresses are ignored by " - f"rule '{rule.name}'") + if rule.ignores(envto=[*self.recipients]): self.rules.remove(rule) if not self.rules: @@ -641,17 +204,18 @@ class ModifyMilter(Milter.Base): return Milter.ACCEPT self.qid = self.getsymval('i') - self.logger.debug(f"{self.qid}: received queue-id from MTA") - self.headers = None + self.logger = CustomLogger(self.logger, {"qid": self.qid}) + self.logger.debug("received queue-id from MTA") + self.fields = None self.fp = None for rule in self.rules: - if "headers" in rule.needs() and self.headers is None: - self.headers = [] + if "fields" in rule.needs() and self.fields is None: + self.fields = [] - if "data" in rule.needs() and self.fp is None: + if "body" in rule.needs() and self.fp is None: self.fp = BytesIO() - if None not in [self.headers, self.fp]: + if None not in [self.fields, self.fp]: break except Exception as e: @@ -663,21 +227,15 @@ class ModifyMilter(Milter.Base): def header(self, name, value): try: - # remove surrogates from value - value = value.encode( - errors="surrogateescape").decode(errors="replace") - if self.fp is not None: - self.fp.write(f"{name}: {value}\r\n".encode( - encoding="ascii", errors="replace")) - - if self.headers is not None: - self.logger.debug(f"{self.qid}: received header: " - f"{name}: {value}") + if self.fields is not None: + # remove surrogates from value + value = value.encode( + errors="surrogateescape").decode(errors="replace") + self.logger.debug(f"received header: {name}: {value}") header = make_header(decode_header(value), errors="replace") value = str(header).replace("\x00", "") - self.logger.debug( - f"{self.qid}: decoded header: {name}: {value}") - self.headers.append((name, value)) + self.logger.debug(f"decoded header: {name}: {value}") + self.fields.append((name, value)) except Exception as e: self.logger.exception( f"an exception occured in header function: {e}") @@ -685,17 +243,6 @@ class ModifyMilter(Milter.Base): return Milter.CONTINUE - def eoh(self): - try: - if self.fp is not None: - self.fp.write(b"\r\n") - except Exception as e: - self.logger.exception( - f"an exception occured in eoh function: {e}") - return Milter.TEMPFAIL - - return Milter.CONTINUE - def body(self, chunk): try: if self.fp is not None: @@ -709,43 +256,20 @@ class ModifyMilter(Milter.Base): def eom(self): try: - changes = [] for rule in self.rules: - changes += rule.execute(self.qid, self.headers, self.fp) + milter_action = rule.execute(self) - mod_body_pos = None - for mod_type, *params in changes: - if mod_type in ["add_header", "mod_header", "del_header"]: - header, value, occurrence = params - enc_value = replace_illegal_chars( - Header(s=value).encode()) - if mod_type == "add_header": - self.logger.debug(f"{self.qid}: milter: add " - f"header: {header}: {enc_value}") - self.addheader(header, enc_value, occurrence) - else: - if enc_value == "": - self.logger.debug( - f"{self.qid}: milter: delete " - f"header (occ. {occurrence}): " - f"{header}") - else: - self.logger.debug( - f"{self.qid}: milter: modify " - f"header (occ. {occurrence}): " - f"{header}: {enc_value}") + if milter_action is not None: + if milter_action["action"] == "reject": + self.setreply("554", "5.7.0", milter_action["reason"]) + return Milter.REJECT - self.chgheader(header, occurrence, enc_value) - elif mod_type == "mod_body": - mod_body_pos = params[0] - elif mod_type == "reject": - self.setreply("554", "5.7.0", params[0]) - return Milter.REJECT + if milter_action["action"] == "accept": + return Milter.ACCEPT + + if milter_action["action"] == "discard": + return Milter.DISCARD - if mod_body_pos is not None: - self.fp.seek(mod_body_pos) - self.logger.debug(f"{self.qid}: milter: replace body") - self.replacebody(self.fp.read()) except Exception as e: self.logger.exception( f"an exception occured in eom function: {e}") diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py new file mode 100644 index 0000000..c9a1119 --- /dev/null +++ b/pymodmilter/actions.py @@ -0,0 +1,456 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +import logging +import re + +from bs4 import BeautifulSoup +from collections import defaultdict +from copy import copy +from email.header import Header +from email.parser import BytesFeedParser +from email.message import MIMEPart +from email.policy import default as default_policy, SMTP + +from pymodmilter import CustomLogger, Conditions + + +def _replace_illegal_chars(string): + """Replace illegal characters in header values.""" + return string.replace( + "\x00", "").replace( + "\r", "").replace( + "\n", "") + + +def add_header(field, value, milter, idx=-1, pretend=False, + logger=logging.getLogger(__name__)): + """Add a mail header field.""" + header = f"{field}: {value}" + if logger.getEffectiveLevel() == logging.DEBUG: + logger.debug(f"add_header: {header}") + else: + logger.info(f"add_header: {header[0:70]}") + + if idx == -1: + milter.fields.append((field, value)) + else: + milter.fields.insert(idx, (field, value)) + + if pretend: + return + + encoded_value = _replace_illegal_chars( + Header(s=value).encode()) + milter.logger.debug(f"milter: addheader: {field}[{idx}]: {encoded_value}") + milter.addheader(field, encoded_value, idx) + + +def mod_header(field, value, milter, search=None, pretend=False, + logger=logging.getLogger(__name__)): + """Change the value of a mail header field.""" + if not isinstance(field, re.Pattern): + field = re.compile(field, re.IGNORECASE) + + if search is not None and not isinstance(search, re.Pattern): + search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE) + + occ = defaultdict(int) + + for idx, (f, v) in enumerate(milter.fields): + occ[f] += 1 + + if not field.match(f): + continue + + if search is not None: + new_v = search.sub(value, v).strip() + else: + new_v = value.strip() + + if new_v == v: + continue + + if not new_v: + logger.warning( + f"mod_header: resulting value is empty, " + f"skip modification") + continue + + header = f"{f}: {v}" + new_header = f"{f}: {new_v}" + if logger.getEffectiveLevel() == logging.DEBUG: + logger.debug(f"mod_header: {header}: {new_header}") + else: + logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}") + + milter.fields[idx] = (f, new_v) + + if pretend: + continue + + encoded_value = _replace_illegal_chars( + Header(s=new_v).encode()) + milter.logger.debug( + f"milter: chgheader: {f}[{occ[f]}]: {encoded_value}") + milter.chgheader(f, occ[f], encoded_value) + + +def del_header(field, milter, value=None, pretend=False, + logger=logging.getLogger(__name__)): + """Delete a mail header field.""" + if not isinstance(field, re.Pattern): + field = re.compile(field, re.IGNORECASE) + + if value is not None and not isinstance(value, re.Pattern): + value = re.compile(value, re.MULTILINE + re.DOTALL + re.IGNORECASE) + + idx = -1 + occ = defaultdict(int) + + # iterate a copy of milter.fields because elements may get removed + # during iteration + for f, v in milter.fields.copy(): + idx += 1 + occ[f] += 1 + + if not field.match(f): + continue + + if value is not None and not value.search(v): + continue + + header = f"{f}: {v}" + if logger.getEffectiveLevel() == logging.DEBUG: + logger.debug(f"del_header: {header}") + else: + logger.info(f"del_header: {header[0:70]}") + + del milter.fields[idx] + + if not pretend: + encoded_value = "" + milter.logger.debug( + f"milter: chgheader: {f}[{occ[f]}]: {encoded_value}") + milter.chgheader(f, occ[f], encoded_value) + + idx -= 1 + occ[f] -= 1 + + +def _get_body_content(msg, body_type): + content = None + body_part = msg.get_body(preferencelist=(body_type)) + if body_part is not None: + content = body_part.get_content() + + return (body_part, content) + + +def _wrap_message(milter): + msg = MIMEPart() + msg.add_header("MIME-Version", "1.0") + + msg.set_content( + "Please see the original email attached.") + msg.add_alternative( + "Please see the original email attached.", + subtype="html") + + data = b"" + for field, value in milter.fields: + encoded_value = _replace_illegal_chars( + Header(s=value).encode()) + data += field.encode("ascii", errors="replace") + data += b": " + data += encoded_value.encode("ascii", errors="replace") + data += b"\r\n" + + milter.fp.seek(0) + data += b"\r\n" + milter.fp.read() + + msg.add_attachment( + data, maintype="plain", subtype="text", + filename=f"{milter.qid}.eml") + + return msg + + +def _inject_body(milter, msg): + if not msg.is_multipart(): + msg.make_mixed() + + new_msg = MIMEPart() + new_msg.add_header("MIME-Version", "1.0") + new_msg.set_content("") + new_msg.add_alternative("", subtype="html") + new_msg.make_mixed() + for attachment in msg.iter_attachments(): + new_msg.attach(attachment) + + return new_msg + + +def add_disclaimer(text, html, action, policy, milter, pretend=False, + logger=logging.getLogger(__name__)): + """Append or prepend a disclaimer to the mail body.""" + milter.fp.seek(0) + fp = BytesFeedParser(policy=default_policy) + + for field, value in milter.fields: + field_lower = field.lower() + if not field_lower.startswith("content-") and \ + field_lower != "mime-version": + continue + logger.debug( + f"feed content header to message object: {field}: {value}") + encoded_value = _replace_illegal_chars( + Header(s=value).encode()) + fp.feed(field.encode("ascii", errors="replace")) + fp.feed(b": ") + fp.feed(encoded_value.encode("ascii", errors="replace")) + fp.feed(b"\r\n") + + fp.feed(b"\r\n") + logger.debug(f"feed body to message object: {field}: {value}") + fp.feed(milter.fp.read()) + + logger.debug("parse message") + msg = fp.close() + + text_content = None + html_content = None + + try: + try: + logger.debug("try to find a plain and/or html body part") + text_body, text_content = _get_body_content(msg, "plain") + html_body, html_content = _get_body_content(msg, "html") + if text_content is None and html_content is None: + raise RuntimeError() + except RuntimeError: + logger.info( + "message does not contain any body part, " + "inject empty plain and html body parts") + msg = _inject_body(milter, msg) + text_body, text_content = _get_body_content(msg, "plain") + html_body, html_content = _get_body_content(msg, "html") + if text_content is None and html_content is None: + raise RuntimeError("no message body present after injecting") + except Exception as e: + logger.warning(e) + if policy == "ignore": + logger.info( + f"unable to add disclaimer to message body, " + f"ignore error according to policy") + return + elif policy == "reject": + logger.info( + f"unable to add disclaimer to message body, " + f"reject message according to policy") + return [ + ("reject", "Message rejected due to error")] + + logger.info("wrap original message in a new message envelope") + msg = _wrap_message(milter) + text_body, text_content = _get_body_content(msg, "plain") + html_body, html_content = _get_body_content(msg, "html") + if text_content is None and html_content is None: + raise Exception("no message body present after wrapping, " + "give up ...") + + if text_content is not None: + logger.info(f"{action} text disclaimer") + + if action == "prepend": + content = f"{text}{text_content}" + else: + content = f"{text_content}{text}" + + text_body.set_content( + content.encode(), maintype="text", subtype="plain") + text_body.set_param("charset", "UTF-8", header="Content-Type") + + if html_content is not None: + logger.info(f"{action} html disclaimer") + + soup = BeautifulSoup(html_content, "html.parser") + + body = soup.find('body') + if body: + soup = body + + if action == "prepend": + soup.insert(0, copy(html)) + else: + soup.append(html) + + html_body.set_content( + str(soup).encode(), maintype="text", subtype="html") + html_body.set_param("charset", "UTF-8", header="Content-Type") + + try: + logger.debug("serialize message as bytes") + data = msg.as_bytes(policy=SMTP) + except Exception as e: + logger.waring( + f"unable to serialize message as bytes: {e}") + try: + logger.warning("try to serialize message as string") + data = msg.as_string(policy=SMTP) + data = data.encode("ascii", errors="replace") + except Exception as e: + raise e + + body_pos = data.find(b"\r\n\r\n") + 4 + milter.fp.seek(0) + milter.fp.write(data[body_pos:]) + milter.fp.truncate() + + if pretend: + return + + logger.debug("milter: replacebody") + milter.replacebody(data[body_pos:]) + del data + + fields = { + "mime-version": { + "field": "MIME-Version", + "value": msg.get("MIME-Version"), + "modified": False}, + "content-type": { + "field": "Content-Type", + "value": msg.get("Content-Type"), + "modified": False}, + "content-transfer-encoding": { + "field": "Content-Transfer-Encoding", + "value": msg.get("Content-Transfer-Encoding"), + "modified": False}} + + for field, value in milter.fields.copy(): + field_lower = field.lower() + if field_lower in fields and fields[field_lower]["value"] is not None: + mod_header(field=f"^{field}$", value=fields[field_lower]["value"], + milter=milter, pretend=pretend, logger=logger) + fields[field_lower]["modified"] = True + + elif field_lower.startswith("content-"): + del_header(field=f"^{field}$", milter=milter, + pretend=pretend, logger=logger) + + for field in fields.values(): + if not field["modified"] and field["value"] is not None: + add_header(field=field["field"], value=field["value"], + milter=milter, pretend=pretend, logger=logger) + + +class Action: + """Action to implement a pre-configured action to perform on e-mails.""" + _types = { + "add_header": ["fields"], + "del_header": ["fields"], + "mod_header": ["fields"], + "add_disclaimer": ["fields", "body"]} + + def __init__(self, name, local_addrs, conditions, action_type, args, + loglevel=logging.INFO, pretend=False): + logger = logging.getLogger(name) + self.logger = CustomLogger(logger, {"name": name}) + self.logger.setLevel(loglevel) + + self.conditions = Conditions( + local_addrs=local_addrs, + args=conditions, + logger=self.logger) + self.pretend = pretend + self._args = {} + + if action_type not in self._types: + raise RuntimeError(f"invalid action_type '{action_type}'") + self._needs = self._types[action_type] + + try: + if action_type == "add_header": + self._func = add_header + self._args["field"] = args["header"] + self._args["value"] = args["value"] + if "idx" in args: + self._args["idx"] = args["idx"] + + elif action_type in ["mod_header", "del_header"]: + args["field"] = args["header"] + del args["header"] + regex_args = ["field"] + + if action_type == "mod_header": + self._func = mod_header + self._args["value"] = args["value"] + regex_args.append("search") + elif action_type == "del_header" and "value" in args: + self._func = del_header + regex_args.append("value") + + for arg in regex_args: + try: + self._args[arg] = re.compile( + args[arg], + re.MULTILINE + re.DOTALL + re.IGNORECASE) + except re.error as e: + raise RuntimeError( + f"unable to parse {arg} regex: {e}") + + elif action_type == "add_disclaimer": + self._func = add_disclaimer + if args["action"] not in ["append", "prepend"]: + raise RuntimeError(f"invalid action '{args['action']}'") + + self._args["action"] = args["action"] + + if args["error_policy"] not in ["wrap", "ignore", "reject"]: + raise RuntimeError(f"invalid policy '{args['policy']}'") + + self._args["policy"] = args["error_policy"] + + try: + with open(args["html_file"], "r") as f: + html = BeautifulSoup( + f.read(), "html.parser") + body = html.find('body') + if body: + # just use content within the body tag if present + html = body + self._args["html"] = html + with open(args["text_file"], "r") as f: + self._args["text"] = f.read() + except IOError as e: + raise RuntimeError(f"unable to read template: {e}") + + except KeyError as e: + raise RuntimeError( + f"mandatory argument not found: {e}") + + def needs(self): + """Return the needs of this action.""" + return self._needs + + def execute(self, milter, pretend=None): + """Execute configured action.""" + if pretend is None: + pretend = self.pretend + + logger = CustomLogger(self.logger, {"qid": milter.qid}) + + return self._func( + milter=milter, pretend=pretend, logger=logger, **self._args) diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py new file mode 100644 index 0000000..c0fbf9a --- /dev/null +++ b/pymodmilter/conditions.py @@ -0,0 +1,134 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +import logging +import re + +from netaddr import IPAddress, IPNetwork, AddrFormatError + + +class Conditions: + """Conditions to implement conditions for rules and actions.""" + + def __init__(self, local_addrs, args, logger=None): + if logger is None: + logger = logging.getLogger(__name__) + + self._local_addrs = [] + self.logger = logger + self._args = {} + + try: + for addr in local_addrs: + self._local_addrs.append(IPNetwork(addr)) + except AddrFormatError as e: + raise RuntimeError(f"invalid address in local_addrs: {e}") + + try: + if "local" in args: + logger.debug(f"condition: local = {args['local']}") + self._args["local"] = args["local"] + + if "hosts" in args: + logger.debug(f"condition: hosts = {args['hosts']}") + self._args["hosts"] = [] + try: + for host in args["hosts"]: + self._args["hosts"].append(IPNetwork(host)) + except AddrFormatError as e: + raise RuntimeError(f"invalid address in hosts: {e}") + + if "envfrom" in args: + logger.debug(f"condition: envfrom = {args['envfrom']}") + try: + self._args["envfrom"] = re.compile( + args["envfrom"], re.IGNORECASE) + except re.error as e: + raise RuntimeError(f"unable to parse envfrom regex: {e}") + + if "envto" in args: + logger.debug(f"condition: envto = {args['envto']}") + try: + self._args["envto"] = re.compile( + args["envto"], re.IGNORECASE) + except re.error as e: + raise RuntimeError(f"unable to parse envto regex: {e}") + + except KeyError as e: + raise RuntimeError( + f"mandatory argument not found: {e}") + + def match(self, args): + if "host" in args: + ip = IPAddress(args["host"]) + + if "local" in self._args: + is_local = False + for addr in self._local_addrs: + if ip in addr: + is_local = True + break + + if is_local != self._args["local"]: + self.logger.debug( + f"ignore host {args['host']}, " + f"condition local does not match") + return False + + self.logger.debug( + f"condition local matches for host {args['host']}") + + if "hosts" in self._args: + found = False + for addr in self._args["hosts"]: + if ip in addr: + found = True + break + + if not found: + self.logger.debug( + f"ignore host {args['host']}, " + f"condition hosts does not match") + return False + + self.logger.debug( + f"condition hosts matches for host {args['host']}") + + if "envfrom" in args and "envfrom" in self._args: + if not self._args["envfrom"].match(args["envfrom"]): + self.logger.debug( + f"ignore envelope-from address {args['envfrom']}, " + f"condition envfrom does not match") + return False + + self.logger.debug( + f"condition envfrom matches for " + f"envelope-from address {args['envfrom']}") + + if "envto" in args and "envto" in self._args: + if not isinstance(args["envto"], list): + args["envto"] = [args["envto"]] + + for envto in args["envto"]: + if not self._args["envto"].match(envto): + self.logger.debug( + f"ignore envelope-to address {args['envto']}, " + f"condition envto does not match") + return False + + self.logger.debug( + f"condition envto matches for " + f"envelope-to address {args['envto']}") + + return True diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 924ab2a..d5a3666 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -12,7 +12,6 @@ # along with PyMod-Milter. If not, see . # - import Milter import argparse import logging @@ -24,77 +23,103 @@ from re import sub from pymodmilter import Rule, ModifyMilter from pymodmilter.version import __version__ as version +from pymodmilter.actions import Action def main(): "Run PyMod-Milter." - # parse command line parser = argparse.ArgumentParser( description="PyMod milter daemon", formatter_class=lambda prog: argparse.HelpFormatter( prog, max_help_position=45, width=140)) + parser.add_argument( "-c", "--config", help="Config file to read.", default="/etc/pymodmilter/pymodmilter.conf") + parser.add_argument( "-s", "--socket", help="Socket used to communicate with the MTA.", default="") + parser.add_argument( "-d", "--debug", help="Log debugging messages.", action="store_true") + parser.add_argument( "-t", "--test", help="Check configuration.", action="store_true") + parser.add_argument( "-v", "--version", help="Print version.", action="version", version=f"%(prog)s ({version})") + args = parser.parse_args() - # setup logging - loglevel = logging.INFO - logname = "pymodmilter" - syslog_name = logname - if args.debug: - loglevel = logging.DEBUG - logname = f"{logname}[%(name)s]" - syslog_name = f"{syslog_name}: [%(name)s] %(levelname)s" + loglevels = { + "error": logging.ERROR, + "warning": logging.WARNING, + "info": logging.INFO, + "debug": logging.DEBUG + } root_logger = logging.getLogger() - root_logger.setLevel(loglevel) + root_logger.setLevel(logging.DEBUG) # setup console log stdouthandler = logging.StreamHandler(sys.stdout) - stdouthandler.setLevel(logging.DEBUG) - formatter = logging.Formatter("%(message)s") - stdouthandler.setFormatter(formatter) + stdouthandler.setFormatter( + logging.Formatter("%(asctime)s - %(levelname)s: %(message)s")) root_logger.addHandler(stdouthandler) + + # setup syslog + sysloghandler = logging.handlers.SysLogHandler( + address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) + sysloghandler.setFormatter( + logging.Formatter("pymodmilter: %(message)s")) + root_logger.addHandler(sysloghandler) + logger = logging.getLogger(__name__) + if not args.debug: + logger.setLevel(logging.INFO) + try: - # read config file - logger.debug("parsing config file") try: with open(args.config, "r") as fh: - config = loads( - sub(r"(?m)^\s*#.*\n?", "", fh.read())) + config = sub(r"(?m)^\s*#.*\n?", "", fh.read()) + config = loads(config) except Exception as e: + for num, line in enumerate(config.splitlines()): + logger.error(f"{num+1}: {line}") raise RuntimeError( f"unable to parse config file: {e}") - logger.debug("preparing milter configuration ...") - - # default values for global config if not set if "global" not in config: config["global"] = {} + if "loglevel" not in config["global"]: + config["global"]["loglevel"] = "info" + + if args.debug: + loglevel = logging.DEBUG + else: + loglevel = loglevels[config["global"]["loglevel"]] + + logger.setLevel(loglevel) + + logger.debug("prepar milter configuration") + + if "pretend" not in config["global"]: + config["global"]["pretend"] = False + if args.socket: socket = args.socket elif "socket" in config["global"]: @@ -104,64 +129,110 @@ def main(): f"listening socket is neither specified on the command line " f"nor in the configuration file") - if "local_addrs" not in config["global"]: - config["global"]["local_addrs"] = [ + if "local_addrs" in config["global"]: + local_addrs = config["global"]["local_addrs"] + else: + local_addrs = [ + "::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] - if "log" not in config["global"]: - config["global"]["log"] = True - - if "pretend" not in config["global"]: - config["global"]["pretend"] = False - - # check if mandatory sections are present in config - for section in ["rules"]: - if section not in config: - raise RuntimeError( - f"mandatory config section '{section}' not found") + if "rules" not in config: + raise RuntimeError( + f"mandatory config section 'rules' not found") if not config["rules"]: raise RuntimeError("no rules configured") - rules = [] - # iterate configured rules - for rule_idx, rule in enumerate(config["rules"]): - params = {} - # set default values if not specified in config - if "name" in rule: - params["name"] = rule["name"] - else: - params["name"] = f"Rule #{rule_idx}" + logger.debug("initialize rules ...") - if "log" in rule: - params["log"] = rule["log"] + rules = [] + for rule_idx, rule in enumerate(config["rules"]): + if "name" in rule: + rule_name = rule["name"] else: - params["log"] = config["global"]["log"] + rule_name = f"Rule #{rule_idx}" + + logger.debug(f"prepare rule {rule_name} ...") + + if "actions" not in rule: + raise RuntimeError( + f"{rule_name}: mandatory config " + f"section 'actions' not found") + + if not rule["actions"]: + raise RuntimeError("{rule_name}: no actions configured") + + if args.debug: + rule_loglevel = logging.DEBUG + elif "loglevel" in rule: + rule_loglevel = loglevels[rule["loglevel"]] + else: + rule_loglevel = loglevels[config["global"]["loglevel"]] if "pretend" in rule: - params["pretend"] = rule["pretend"] + rule_pretend = rule["pretend"] else: - params["pretend"] = config["global"]["pretend"] + rule_pretend = config["global"]["pretend"] - if "local_addrs" in rule: - params["local_addrs"] = rule["local_addrs"] - else: - params["local_addrs"] = config["global"]["local_addrs"] + actions = [] + for action_idx, action in enumerate(rule["actions"]): + if "name" in action: + action_name = action["name"] + else: + action_name = f"Action #{action_idx}" - if "conditions" in rule: - params["conditions"] = rule["conditions"] + if args.debug: + action_loglevel = logging.DEBUG + elif "loglevel" in action: + action_loglevel = loglevels[action["loglevel"]] + else: + action_loglevel = rule_loglevel - if "modifications" in rule: - params["modifications"] = rule["modifications"] - else: - raise RuntimeError( - f"{rule['name']}: mandatory config section " - f"'modifications' not found") + if "pretend" in action: + action_pretend = action["pretend"] + else: + action_pretend = rule_pretend - rules.append(Rule(**params)) + if "type" not in action: + raise RuntimeError( + f"{rule_name}: {action_name}: mandatory config " + f"section 'actions' not found") + + if "conditions" not in action: + action["conditions"] = {} + + try: + actions.append( + Action( + name=action_name, + local_addrs=local_addrs, + conditions=action["conditions"], + action_type=action["type"], + args=action, + loglevel=action_loglevel, + pretend=action_pretend)) + except RuntimeError as e: + logger.error(f"{action_name}: {e}") + sys.exit(253) + + if "conditions" not in rule: + rule["conditions"] = {} + + try: + rules.append( + Rule( + name=rule_name, + local_addrs=local_addrs, + conditions=rule["conditions"], + actions=actions, + loglevel=rule_loglevel, + pretend=rule_pretend)) + except RuntimeError as e: + logger.error(f"{rule_name}: {e}") + sys.exit(254) except RuntimeError as e: logger.error(e) @@ -171,27 +242,22 @@ def main(): print("Configuration ok") sys.exit(0) - # change log format for runtime - formatter = logging.Formatter( - f"%(asctime)s {logname}: [%(levelname)s] %(message)s", - datefmt="%Y-%m-%d %H:%M:%S") + # setup console log for runtime + formatter = logging.Formatter("%(asctime)s - %(levelname)s: %(message)s") stdouthandler.setFormatter(formatter) - - # setup syslog - sysloghandler = logging.handlers.SysLogHandler( - address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) - sysloghandler.setLevel(loglevel) - formatter = logging.Formatter(f"{syslog_name}: %(message)s") - sysloghandler.setFormatter(formatter) - root_logger.addHandler(sysloghandler) + stdouthandler.setLevel(logging.DEBUG) logger.info("pymodmilter starting") ModifyMilter.set_rules(rules) + ModifyMilter.set_loglevel(loglevels[config["global"]["loglevel"]]) # register milter factory class Milter.factory = ModifyMilter Milter.set_exception_policy(Milter.TEMPFAIL) + if args.debug: + Milter.setdbg(1) + rc = 0 try: Milter.runmilter("pymodmilter", socketname=socket, timeout=30) From 699e76acba019fa38ac35327f7522e7c386c990c Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Jun 2020 01:29:23 +0200 Subject: [PATCH 048/145] extend local_addrs in example config with ::1/128 --- docs/pymodmilter.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pymodmilter.conf.example b/docs/pymodmilter.conf.example index 1afaa43..9760544 100644 --- a/docs/pymodmilter.conf.example +++ b/docs/pymodmilter.conf.example @@ -28,7 +28,7 @@ # Notes: A list of local hosts and networks. # Value: [ LIST ] # - "local_addrs": ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], + "local_addrs": ["::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], # Option: loglevel # Type: String From 2f8865cd66019c36ea9972c20009800524b63ebc Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Jun 2020 01:56:35 +0200 Subject: [PATCH 049/145] fix some stuff --- pymodmilter/actions.py | 8 ++++---- pymodmilter/run.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index c9a1119..0c06bf8 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -60,10 +60,10 @@ def add_header(field, value, milter, idx=-1, pretend=False, def mod_header(field, value, milter, search=None, pretend=False, logger=logging.getLogger(__name__)): """Change the value of a mail header field.""" - if not isinstance(field, re.Pattern): + if isinstance(field, str): field = re.compile(field, re.IGNORECASE) - if search is not None and not isinstance(search, re.Pattern): + if isinstance(search, str): search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE) occ = defaultdict(int) @@ -110,10 +110,10 @@ def mod_header(field, value, milter, search=None, pretend=False, def del_header(field, milter, value=None, pretend=False, logger=logging.getLogger(__name__)): """Delete a mail header field.""" - if not isinstance(field, re.Pattern): + if isinstance(field, str): field = re.compile(field, re.IGNORECASE) - if value is not None and not isinstance(value, re.Pattern): + if isinstance(value, str): value = re.compile(value, re.MULTILINE + re.DOTALL + re.IGNORECASE) idx = -1 diff --git a/pymodmilter/run.py b/pymodmilter/run.py index d5a3666..686fa69 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -105,12 +105,12 @@ def main(): if "global" not in config: config["global"] = {} - if "loglevel" not in config["global"]: - config["global"]["loglevel"] = "info" if args.debug: loglevel = logging.DEBUG else: + if "loglevel" not in config["global"]: + config["global"]["loglevel"] = "info" loglevel = loglevels[config["global"]["loglevel"]] logger.setLevel(loglevel) @@ -249,7 +249,7 @@ def main(): logger.info("pymodmilter starting") ModifyMilter.set_rules(rules) - ModifyMilter.set_loglevel(loglevels[config["global"]["loglevel"]]) + ModifyMilter.set_loglevel(loglevel) # register milter factory class Milter.factory = ModifyMilter From 0f1b48a5c4a0b096385b1efe9f02da619b85294f Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Jun 2020 10:04:50 +0200 Subject: [PATCH 050/145] update headers only if neccessary --- pymodmilter/actions.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 0c06bf8..6304d11 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -231,6 +231,7 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, text_content = None html_content = None + update_headers = False try: try: @@ -239,6 +240,9 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, html_body, html_content = _get_body_content(msg, "html") if text_content is None and html_content is None: raise RuntimeError() + + if not msg.is_multipart(): + update_headers = True except RuntimeError: logger.info( "message does not contain any body part, " @@ -248,6 +252,9 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, html_body, html_content = _get_body_content(msg, "html") if text_content is None and html_content is None: raise RuntimeError("no message body present after injecting") + + update_headers = True + except Exception as e: logger.warning(e) if policy == "ignore": @@ -270,6 +277,8 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, raise Exception("no message body present after wrapping, " "give up ...") + update_headers = True + if text_content is not None: logger.info(f"{action} text disclaimer") @@ -325,6 +334,9 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, milter.replacebody(data[body_pos:]) del data + if not update_headers: + return + fields = { "mime-version": { "field": "MIME-Version", From 7a7c55f14fb78b58a51be5c707e1f0c93ff018de Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Jun 2020 10:32:10 +0200 Subject: [PATCH 051/145] fix logging --- pymodmilter/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 686fa69..516db59 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -180,7 +180,7 @@ def main(): actions = [] for action_idx, action in enumerate(rule["actions"]): if "name" in action: - action_name = action["name"] + action_name = f"{rule_name}: {action['name']}" else: action_name = f"Action #{action_idx}" From 745027b2d5536d1553816b0952cb58d5b170e34d Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Jun 2020 14:55:15 +0200 Subject: [PATCH 052/145] enhance error handling when adding disclaimer --- pymodmilter/actions.py | 134 +++++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 65 deletions(-) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 6304d11..53c2996 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -158,6 +158,61 @@ def _get_body_content(msg, body_type): return (body_part, content) +def _patch_message_body(msg, action, text, html, logger): + text_body, text_content = _get_body_content(msg, "plain") + html_body, html_content = _get_body_content(msg, "html") + + if text_content is None and html_content is None: + raise RuntimeError("message does not contain any body part") + + if text_content is not None: + logger.info(f"{action} text disclaimer") + + if action == "prepend": + content = f"{text}{text_content}" + else: + content = f"{text_content}{text}" + + text_body.set_content( + content.encode(), maintype="text", subtype="plain") + text_body.set_param("charset", "UTF-8", header="Content-Type") + + if html_content is not None: + logger.info(f"{action} html disclaimer") + + soup = BeautifulSoup(html_content, "html.parser") + + body = soup.find('body') + if not body: + body = soup + + if action == "prepend": + body.insert(0, copy(html)) + else: + body.append(html) + + html_body.set_content( + str(body).encode(), maintype="text", subtype="html") + html_body.set_param("charset", "UTF-8", header="Content-Type") + + +def _serialize_msg(msg, logger): + try: + logger.debug("serialize message as bytes") + data = msg.as_bytes(policy=SMTP) + except Exception as e: + logger.waring( + f"unable to serialize message as bytes: {e}") + try: + logger.warning("try to serialize message as string") + data = msg.as_string(policy=SMTP) + data = data.encode("ascii", errors="replace") + except Exception as e: + raise e + + return data + + def _wrap_message(milter): msg = MIMEPart() msg.add_header("MIME-Version", "1.0") @@ -235,26 +290,16 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, try: try: - logger.debug("try to find a plain and/or html body part") - text_body, text_content = _get_body_content(msg, "plain") - html_body, html_content = _get_body_content(msg, "html") - if text_content is None and html_content is None: - raise RuntimeError() - + _patch_message_body(msg, action, text, html, logger) + data = _serialize_msg(msg, logger) if not msg.is_multipart(): update_headers = True - except RuntimeError: - logger.info( - "message does not contain any body part, " - "inject empty plain and html body parts") + except RuntimeError as e: + logger.info("inject empty plain and html body parts") msg = _inject_body(milter, msg) - text_body, text_content = _get_body_content(msg, "plain") - html_body, html_content = _get_body_content(msg, "html") - if text_content is None and html_content is None: - raise RuntimeError("no message body present after injecting") - + _patch_message_body(msg, action, text, html, logger) + data = _serialize_msg(msg, logger) update_headers = True - except Exception as e: logger.warning(e) if policy == "ignore": @@ -270,57 +315,14 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, ("reject", "Message rejected due to error")] logger.info("wrap original message in a new message envelope") - msg = _wrap_message(milter) - text_body, text_content = _get_body_content(msg, "plain") - html_body, html_content = _get_body_content(msg, "html") - if text_content is None and html_content is None: - raise Exception("no message body present after wrapping, " - "give up ...") - - update_headers = True - - if text_content is not None: - logger.info(f"{action} text disclaimer") - - if action == "prepend": - content = f"{text}{text_content}" - else: - content = f"{text_content}{text}" - - text_body.set_content( - content.encode(), maintype="text", subtype="plain") - text_body.set_param("charset", "UTF-8", header="Content-Type") - - if html_content is not None: - logger.info(f"{action} html disclaimer") - - soup = BeautifulSoup(html_content, "html.parser") - - body = soup.find('body') - if body: - soup = body - - if action == "prepend": - soup.insert(0, copy(html)) - else: - soup.append(html) - - html_body.set_content( - str(soup).encode(), maintype="text", subtype="html") - html_body.set_param("charset", "UTF-8", header="Content-Type") - - try: - logger.debug("serialize message as bytes") - data = msg.as_bytes(policy=SMTP) - except Exception as e: - logger.waring( - f"unable to serialize message as bytes: {e}") try: - logger.warning("try to serialize message as string") - data = msg.as_string(policy=SMTP) - data = data.encode("ascii", errors="replace") + msg = _wrap_message(milter) + _patch_message_body(msg, action, text, html, logger) + data = _serialize_msg(msg, logger) + update_headers = True except Exception as e: - raise e + raise Exception("unable to wrap message in a new message envelope, " + "give up ...") body_pos = data.find(b"\r\n\r\n") + 4 milter.fp.seek(0) @@ -448,6 +450,8 @@ class Action: self._args["text"] = f.read() except IOError as e: raise RuntimeError(f"unable to read template: {e}") + else: + raise RuntimeError(f"unknown action type: {action_type}") except KeyError as e: raise RuntimeError( From 5a59ab69b9b42fc4744c72f56c531776959ff78d Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Jun 2020 16:31:46 +0200 Subject: [PATCH 053/145] fix del_header function --- pymodmilter/actions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 53c2996..668dd7d 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -295,7 +295,7 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, if not msg.is_multipart(): update_headers = True except RuntimeError as e: - logger.info("inject empty plain and html body parts") + logger.info(f"{e}, inject empty plain and html body") msg = _inject_body(milter, msg) _patch_message_body(msg, action, text, html, logger) data = _serialize_msg(msg, logger) @@ -412,9 +412,10 @@ class Action: self._func = mod_header self._args["value"] = args["value"] regex_args.append("search") - elif action_type == "del_header" and "value" in args: + elif action_type == "del_header": self._func = del_header - regex_args.append("value") + if "value" in args: + regex_args.append("value") for arg in regex_args: try: From 2c1d5c17e0efe30d004f92ef9e70f319c9cb594a Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Jun 2020 19:47:47 +0200 Subject: [PATCH 054/145] cleanup source --- pymodmilter/actions.py | 8 ++++---- pymodmilter/run.py | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 668dd7d..ab87f96 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -284,8 +284,6 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, logger.debug("parse message") msg = fp.close() - text_content = None - html_content = None update_headers = False try: @@ -321,8 +319,10 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, data = _serialize_msg(msg, logger) update_headers = True except Exception as e: - raise Exception("unable to wrap message in a new message envelope, " - "give up ...") + logger.error(e) + raise Exception( + "unable to wrap message in a new message envelope, " + "give up ...") body_pos = data.find(b"\r\n\r\n") + 4 milter.fp.seek(0) diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 516db59..3bf4f28 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -105,7 +105,6 @@ def main(): if "global" not in config: config["global"] = {} - if args.debug: loglevel = logging.DEBUG else: From 7402ad9b1f5c5967b76380b4c65970935111f499 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Jun 2020 10:33:15 +0200 Subject: [PATCH 055/145] imporove disclaimer placement and change version to 1.0.5 --- pymodmilter/actions.py | 4 +++- pymodmilter/version.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index ab87f96..ea17304 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -185,6 +185,8 @@ def _patch_message_body(msg, action, text, html, logger): body = soup.find('body') if not body: body = soup + elif body.text != soup.text: + body = soup if action == "prepend": body.insert(0, copy(html)) @@ -192,7 +194,7 @@ def _patch_message_body(msg, action, text, html, logger): body.append(html) html_body.set_content( - str(body).encode(), maintype="text", subtype="html") + str(soup).encode(), maintype="text", subtype="html") html_body.set_param("charset", "UTF-8", header="Content-Type") diff --git a/pymodmilter/version.py b/pymodmilter/version.py index 92192ee..68cdeee 100644 --- a/pymodmilter/version.py +++ b/pymodmilter/version.py @@ -1 +1 @@ -__version__ = "1.0.4" +__version__ = "1.0.5" From d52e3965a5bae39df88931ce7acef065464eb7cf Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Jun 2020 17:35:50 +0200 Subject: [PATCH 056/145] fix handling of html bodies with content before body tag --- pymodmilter/actions.py | 14 +++++++++++++- pymodmilter/version.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index ea17304..8dcce14 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -22,6 +22,7 @@ from email.header import Header from email.parser import BytesFeedParser from email.message import MIMEPart from email.policy import default as default_policy, SMTP +from os import linesep from pymodmilter import CustomLogger, Conditions @@ -158,6 +159,17 @@ def _get_body_content(msg, body_type): return (body_part, content) +def _has_content_before_body_tag(soup): + s = copy(soup) + for element in s.find_all("head") + s.find_all("body"): + element.extract() + + if len(s.text.strip()) > 0: + return True + + return False + + def _patch_message_body(msg, action, text, html, logger): text_body, text_content = _get_body_content(msg, "plain") html_body, html_content = _get_body_content(msg, "html") @@ -185,7 +197,7 @@ def _patch_message_body(msg, action, text, html, logger): body = soup.find('body') if not body: body = soup - elif body.text != soup.text: + elif _has_content_before_body_tag(soup): body = soup if action == "prepend": diff --git a/pymodmilter/version.py b/pymodmilter/version.py index 68cdeee..382021f 100644 --- a/pymodmilter/version.py +++ b/pymodmilter/version.py @@ -1 +1 @@ -__version__ = "1.0.5" +__version__ = "1.0.6" From 533fef1d63c7e4847fae21655f7dd8ab6d125edb Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 11 Jun 2020 12:03:02 +0200 Subject: [PATCH 057/145] improve exception handling --- pymodmilter/__init__.py | 67 +++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 1b255e6..960ee81 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -154,41 +154,62 @@ class ModifyMilter(Milter.Base): self.rules = ModifyMilter._rules.copy() def connect(self, IPname, family, hostaddr): - self.logger.debug( - f"accepted milter connection from {hostaddr[0]} " - f"port {hostaddr[1]}") + try: + if hostaddr is None: + self.logger.error("unable to proceed, host address is None") + return Milter.TEMPFAIL - # remove rules which ignore this host - for rule in self.rules.copy(): - if rule.ignores(host=hostaddr[0]): - self.rules.remove(rule) - - if not self.rules: self.logger.debug( - f"host {hostaddr[0]} is ignored by all rules, " - f"skip further processing") - return Milter.ACCEPT + f"accepted milter connection from {hostaddr[0]} " + f"port {hostaddr[1]}") + + # remove rules which ignore this host + for rule in self.rules.copy(): + if rule.ignores(host=hostaddr[0]): + self.rules.remove(rule) + + if not self.rules: + self.logger.debug( + f"host {hostaddr[0]} is ignored by all rules, " + f"skip further processing") + return Milter.ACCEPT + except Exception as e: + self.logger.exception( + f"an exception occured in connect function: {e}") + return Milter.TEMPFAIL return Milter.CONTINUE def envfrom(self, mailfrom, *str): - mailfrom = "@".join(parse_addr(mailfrom)).lower() - for rule in self.rules.copy(): - if rule.ignores(envfrom=mailfrom): - self.rules.remove(rule) + try: + mailfrom = "@".join(parse_addr(mailfrom)).lower() + for rule in self.rules.copy(): + if rule.ignores(envfrom=mailfrom): + self.rules.remove(rule) - if not self.rules: - self.logger.debug( - f"envelope-from address {mailfrom} is ignored by all rules, " - f"skip further processing") - return Milter.ACCEPT + if not self.rules: + self.logger.debug( + f"envelope-from address {mailfrom} is ignored by " + f"all rules, skip further processing") + return Milter.ACCEPT + + self.recipients = set() + except Exception as e: + self.logger.exception( + f"an exception occured in envfrom function: {e}") + return Milter.TEMPFAIL - self.recipients = set() return Milter.CONTINUE @Milter.noreply def envrcpt(self, to, *str): - self.recipients.add("@".join(parse_addr(to)).lower()) + try: + self.recipients.add("@".join(parse_addr(to)).lower()) + except Exception as e: + self.logger.exception( + f"an exception occured in envrcpt function: {e}") + return Milter.TEMPFAIL + return Milter.CONTINUE def data(self): From 3d34dbbbf36a82e1bfb3f5eb10768e263d143c6b Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 11 Jun 2020 12:03:28 +0200 Subject: [PATCH 058/145] change version to 1.0.7 --- pymodmilter/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/version.py b/pymodmilter/version.py index 382021f..9e604c0 100644 --- a/pymodmilter/version.py +++ b/pymodmilter/version.py @@ -1 +1 @@ -__version__ = "1.0.6" +__version__ = "1.0.7" From 3c40776542670f42346db3d47ea9b23e89b135c8 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 11 Jun 2020 12:43:07 +0200 Subject: [PATCH 059/145] fix segfault during shutdown --- pymodmilter/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 3bf4f28..87d1ac8 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -263,7 +263,6 @@ def main(): except Milter.milter.error as e: logger.error(e) rc = 255 - logger.info("pymodmilter terminated") sys.exit(rc) From d592ca59cf580872f74d739c8ac69101c9bf0a1f Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 17 Jun 2020 15:04:25 +0200 Subject: [PATCH 060/145] add action to store message in file --- pymodmilter/actions.py | 40 ++++++++++++++++++++++++++++++++++++---- pymodmilter/version.py | 2 +- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 8dcce14..62600a7 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -13,16 +13,17 @@ # import logging +import os import re from bs4 import BeautifulSoup from collections import defaultdict from copy import copy +from datetime import datetime from email.header import Header from email.parser import BytesFeedParser from email.message import MIMEPart from email.policy import default as default_policy, SMTP -from os import linesep from pymodmilter import CustomLogger, Conditions @@ -384,13 +385,36 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, milter=milter, pretend=pretend, logger=logger) +def store(directory, milter, pretend=False, + logger=logging.getLogger(__name__)): + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + store_id = f"{timestamp}_{milter.qid}" + datafile = os.path.join(directory, store_id) + milter.fp.seek(0) + logger.info("store message in file {datafile}") + try: + with open(datafile, "wb") as fp: + for field, value in milter.fields: + encoded_value = _replace_illegal_chars( + Header(s=value).encode()) + fp.write(field.encode("ascii", errors="replace")) + fp.write(b": ") + fp.write(encoded_value.encode("ascii", errors="replace")) + fp.write(b"\r\n") + fp.write(b"\r\n") + fp.write(milter.fp.read()) + except IOError as e: + raise RuntimeError(f"unable to store message: {e}") + + class Action: """Action to implement a pre-configured action to perform on e-mails.""" _types = { "add_header": ["fields"], "del_header": ["fields"], "mod_header": ["fields"], - "add_disclaimer": ["fields", "body"]} + "add_disclaimer": ["fields", "body"], + "store": ["fields", "body"]} def __init__(self, name, local_addrs, conditions, action_type, args, loglevel=logging.INFO, pretend=False): @@ -406,7 +430,7 @@ class Action: self._args = {} if action_type not in self._types: - raise RuntimeError(f"invalid action_type '{action_type}'") + raise RuntimeError(f"invalid action type '{action_type}'") self._needs = self._types[action_type] try: @@ -465,8 +489,16 @@ class Action: self._args["text"] = f.read() except IOError as e: raise RuntimeError(f"unable to read template: {e}") + elif action_type == "store": + self._func = store + if args["storage_type"] not in ["file"]: + raise RuntimeError( + f"invalid storage_type 'args['storage_type']'") + + if args["storage_type"] == "file": + self._args["directory"] = args["directory"] else: - raise RuntimeError(f"unknown action type: {action_type}") + raise RuntimeError(f"invalid action type: {action_type}") except KeyError as e: raise RuntimeError( diff --git a/pymodmilter/version.py b/pymodmilter/version.py index 9e604c0..e13bd59 100644 --- a/pymodmilter/version.py +++ b/pymodmilter/version.py @@ -1 +1 @@ -__version__ = "1.0.7" +__version__ = "1.0.8" From 3a97b649e051f9d3aa7b4c4ffe2effb6f6b54578 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 17 Jun 2020 16:06:44 +0200 Subject: [PATCH 061/145] keep original headers if needed by any action --- pymodmilter/__init__.py | 31 ++++++++++++++++++++++--------- pymodmilter/actions.py | 32 ++++++++++++++------------------ 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 960ee81..7905d9b 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -227,17 +227,22 @@ class ModifyMilter(Milter.Base): self.qid = self.getsymval('i') self.logger = CustomLogger(self.logger, {"qid": self.qid}) self.logger.debug("received queue-id from MTA") + self.fields = None - self.fp = None + self.fields_data = None + self.body_data = None + needs = [] for rule in self.rules: - if "fields" in rule.needs() and self.fields is None: - self.fields = [] + needs += rule.needs() - if "body" in rule.needs() and self.fp is None: - self.fp = BytesIO() + if "fields" in needs: + self.fields = [] - if None not in [self.fields, self.fp]: - break + if "original_fields" in needs: + self.fields_data = BytesIO() + + if "body" in needs: + self.body_data = BytesIO() except Exception as e: self.logger.exception( @@ -248,6 +253,14 @@ class ModifyMilter(Milter.Base): def header(self, name, value): try: + if self.fields_data != None: + self.fields_data.write( + name.encode("ascii", errors="surrogateescape")) + self.fields_data.write(b": ") + self.fields_data.write( + value.encode("ascii", errors="surrogateescape")) + self.fields_data.write(b"\r\n") + if self.fields is not None: # remove surrogates from value value = value.encode( @@ -266,8 +279,8 @@ class ModifyMilter(Milter.Base): def body(self, chunk): try: - if self.fp is not None: - self.fp.write(chunk) + if self.body_data is not None: + self.body_data.write(chunk) except Exception as e: self.logger.exception( f"an exception occured in body function: {e}") diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 62600a7..a568bb6 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -24,6 +24,7 @@ from email.header import Header from email.parser import BytesFeedParser from email.message import MIMEPart from email.policy import default as default_policy, SMTP +from shutil import copyfileobj from pymodmilter import CustomLogger, Conditions @@ -247,8 +248,8 @@ def _wrap_message(milter): data += encoded_value.encode("ascii", errors="replace") data += b"\r\n" - milter.fp.seek(0) - data += b"\r\n" + milter.fp.read() + milter.body_data.seek(0) + data += b"\r\n" + milter.body_data.read() msg.add_attachment( data, maintype="plain", subtype="text", @@ -275,7 +276,7 @@ def _inject_body(milter, msg): def add_disclaimer(text, html, action, policy, milter, pretend=False, logger=logging.getLogger(__name__)): """Append or prepend a disclaimer to the mail body.""" - milter.fp.seek(0) + milter.body_data.seek(0) fp = BytesFeedParser(policy=default_policy) for field, value in milter.fields: @@ -294,7 +295,7 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, fp.feed(b"\r\n") logger.debug(f"feed body to message object: {field}: {value}") - fp.feed(milter.fp.read()) + fp.feed(milter.body_data.read()) logger.debug("parse message") msg = fp.close() @@ -340,9 +341,9 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, "give up ...") body_pos = data.find(b"\r\n\r\n") + 4 - milter.fp.seek(0) - milter.fp.write(data[body_pos:]) - milter.fp.truncate() + milter.body_data.seek(0) + milter.body_data.write(data[body_pos:]) + milter.body_data.truncate() if pretend: return @@ -390,19 +391,14 @@ def store(directory, milter, pretend=False, timestamp = datetime.now().strftime("%Y%m%d%H%M%S") store_id = f"{timestamp}_{milter.qid}" datafile = os.path.join(directory, store_id) - milter.fp.seek(0) - logger.info("store message in file {datafile}") + milter.fields_data.seek(0) + milter.body_data.seek(0) + logger.info(f"store message in file {datafile}") try: with open(datafile, "wb") as fp: - for field, value in milter.fields: - encoded_value = _replace_illegal_chars( - Header(s=value).encode()) - fp.write(field.encode("ascii", errors="replace")) - fp.write(b": ") - fp.write(encoded_value.encode("ascii", errors="replace")) - fp.write(b"\r\n") + copyfileobj(milter.fields_data, fp) fp.write(b"\r\n") - fp.write(milter.fp.read()) + copyfileobj(milter.body_data, fp) except IOError as e: raise RuntimeError(f"unable to store message: {e}") @@ -414,7 +410,7 @@ class Action: "del_header": ["fields"], "mod_header": ["fields"], "add_disclaimer": ["fields", "body"], - "store": ["fields", "body"]} + "store": ["original_fields", "body"]} def __init__(self, name, local_addrs, conditions, action_type, args, loglevel=logging.INFO, pretend=False): From 55bbc5a2b70092507771abb5ea65c9e0d01d1123 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 17 Jun 2020 16:40:20 +0200 Subject: [PATCH 062/145] add action store to documentation --- README.md | 15 ++++++++++++--- docs/pymodmilter.conf.example | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e103d79..86bdb5e 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ Config options for **action** objects: * **del_header** * **mod_header** * **add_disclaimer** + * **store** * **conditions** (optional) A list of conditions which all have to be true to process the action. * **pretend** (optional) @@ -101,14 +102,22 @@ Config options for **add_disclaimer** actions: Path to a file which contains the text representation of the disclaimer. * **error_policy** (optional) Set the error policy in case the disclaimer cannot be added (e.g. if no body part is present in the e-mail). Possible values are: - * wrap + * **wrap** A new e-mail body is generated with the disclaimer as body and the original e-mail attached. - * ignore + * **ignore** Ignore the error and do nothing. - * reject + * **reject** Reject the e-mail. Default: **wrap** +Config options for **store** actions: + * **storage_type** + Storage type. Possible values are: + * **file** + +Config options for **file** storage: + * **directory** + Directory used to store e-mails. ### Conditions Config options for **conditions** objects: diff --git a/docs/pymodmilter.conf.example b/docs/pymodmilter.conf.example index 9760544..04e887f 100644 --- a/docs/pymodmilter.conf.example +++ b/docs/pymodmilter.conf.example @@ -191,6 +191,22 @@ # Value: [ wrap | ignore | reject ] # "error_policy": "wrap" + }, { + "name": "store_message", + + "type": "store", + + # Option: storage_type + # Type: String + # Notes: The storage type used to store e-mails. + # Value: [ file ] + "storage_type": "file", + + # Option: directory + # Type: String + # Notes: Directory used to store e-mails. + # Value: [ file ] + "directory": "/mnt/messages" } ] } From 4269420633d41a95136abd21085027c2048d26c7 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Mon, 10 Aug 2020 09:29:10 +0200 Subject: [PATCH 063/145] change version to 1.0.9 --- pymodmilter/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/version.py b/pymodmilter/version.py index e13bd59..39e0411 100644 --- a/pymodmilter/version.py +++ b/pymodmilter/version.py @@ -1 +1 @@ -__version__ = "1.0.8" +__version__ = "1.0.9" From 1e228c91ba9bd3567e989baa5f0f2070ea9559f4 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 10 Nov 2020 19:55:18 +0100 Subject: [PATCH 064/145] change header handling in add_disclaimer and store --- pymodmilter/__init__.py | 21 +++++++++------------ pymodmilter/actions.py | 36 ++++++++++++++++-------------------- pymodmilter/run.py | 6 +++--- 3 files changed, 28 insertions(+), 35 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 7905d9b..489a8c2 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -220,8 +220,8 @@ class ModifyMilter(Milter.Base): if not self.rules: self.logger.debug( - f"envelope-to addresses are ignored by all rules, " - f"skip further processing") + "envelope-to addresses are ignored by all rules, " + "skip further processing") return Milter.ACCEPT self.qid = self.getsymval('i') @@ -229,7 +229,7 @@ class ModifyMilter(Milter.Base): self.logger.debug("received queue-id from MTA") self.fields = None - self.fields_data = None + self.fields_bytes = None self.body_data = None needs = [] for rule in self.rules: @@ -238,8 +238,8 @@ class ModifyMilter(Milter.Base): if "fields" in needs: self.fields = [] - if "original_fields" in needs: - self.fields_data = BytesIO() + if "fields_bytes" in needs: + self.fields_bytes = [] if "body" in needs: self.body_data = BytesIO() @@ -253,13 +253,10 @@ class ModifyMilter(Milter.Base): def header(self, name, value): try: - if self.fields_data != None: - self.fields_data.write( - name.encode("ascii", errors="surrogateescape")) - self.fields_data.write(b": ") - self.fields_data.write( - value.encode("ascii", errors="surrogateescape")) - self.fields_data.write(b"\r\n") + if self.fields_data is not None: + self.fields_bytes.append( + (name.encode("ascii", errors="surrogateescape"), + value.encode("ascii", errors="surrogateescape"))) if self.fields is not None: # remove surrogates from value diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index a568bb6..c82b009 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -87,8 +87,8 @@ def mod_header(field, value, milter, search=None, pretend=False, if not new_v: logger.warning( - f"mod_header: resulting value is empty, " - f"skip modification") + "mod_header: resulting value is empty, " + "skip modification") continue header = f"{f}: {v}" @@ -279,19 +279,15 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, milter.body_data.seek(0) fp = BytesFeedParser(policy=default_policy) - for field, value in milter.fields: - field_lower = field.lower() + for field, value in milter.fields_bytes: + field_lower = field.encode("ascii").lower() if not field_lower.startswith("content-") and \ field_lower != "mime-version": continue logger.debug( - f"feed content header to message object: {field}: {value}") - encoded_value = _replace_illegal_chars( - Header(s=value).encode()) - fp.feed(field.encode("ascii", errors="replace")) - fp.feed(b": ") - fp.feed(encoded_value.encode("ascii", errors="replace")) - fp.feed(b"\r\n") + f"feed content header to message object: " + f"{field.encode()}: {value.encode()}") + fp.feed(field + b": " + value + b"\r\n") fp.feed(b"\r\n") logger.debug(f"feed body to message object: {field}: {value}") @@ -318,13 +314,13 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, logger.warning(e) if policy == "ignore": logger.info( - f"unable to add disclaimer to message body, " - f"ignore error according to policy") + "unable to add disclaimer to message body, " + "ignore error according to policy") return elif policy == "reject": logger.info( - f"unable to add disclaimer to message body, " - f"reject message according to policy") + "unable to add disclaimer to message body, " + "reject message according to policy") return [ ("reject", "Message rejected due to error")] @@ -391,13 +387,13 @@ def store(directory, milter, pretend=False, timestamp = datetime.now().strftime("%Y%m%d%H%M%S") store_id = f"{timestamp}_{milter.qid}" datafile = os.path.join(directory, store_id) - milter.fields_data.seek(0) milter.body_data.seek(0) logger.info(f"store message in file {datafile}") try: with open(datafile, "wb") as fp: - copyfileobj(milter.fields_data, fp) - fp.write(b"\r\n") + for field, value in milter.fields_bytes: + fp.write(field + b": " + value + b"\r\n") + copyfileobj(milter.body_data, fp) except IOError as e: raise RuntimeError(f"unable to store message: {e}") @@ -410,7 +406,7 @@ class Action: "del_header": ["fields"], "mod_header": ["fields"], "add_disclaimer": ["fields", "body"], - "store": ["original_fields", "body"]} + "store": ["fields_bytes", "body"]} def __init__(self, name, local_addrs, conditions, action_type, args, loglevel=logging.INFO, pretend=False): @@ -489,7 +485,7 @@ class Action: self._func = store if args["storage_type"] not in ["file"]: raise RuntimeError( - f"invalid storage_type 'args['storage_type']'") + "invalid storage_type 'args['storage_type']'") if args["storage_type"] == "file": self._args["directory"] = args["directory"] diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 87d1ac8..cd4aedb 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -125,8 +125,8 @@ def main(): socket = config["global"]["socket"] else: raise RuntimeError( - f"listening socket is neither specified on the command line " - f"nor in the configuration file") + "listening socket is neither specified on the command line " + "nor in the configuration file") if "local_addrs" in config["global"]: local_addrs = config["global"]["local_addrs"] @@ -140,7 +140,7 @@ def main(): if "rules" not in config: raise RuntimeError( - f"mandatory config section 'rules' not found") + "mandatory config section 'rules' not found") if not config["rules"]: raise RuntimeError("no rules configured") From 7b179d197a6bbda09ac0031795cc401f86a1711c Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 10 Nov 2020 20:08:13 +0100 Subject: [PATCH 065/145] fix header handling --- pymodmilter/__init__.py | 2 +- pymodmilter/actions.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 489a8c2..c284486 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -253,7 +253,7 @@ class ModifyMilter(Milter.Base): def header(self, name, value): try: - if self.fields_data is not None: + if self.fields_bytes is not None: self.fields_bytes.append( (name.encode("ascii", errors="surrogateescape"), value.encode("ascii", errors="surrogateescape"))) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index c82b009..163da45 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -280,17 +280,19 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, fp = BytesFeedParser(policy=default_policy) for field, value in milter.fields_bytes: - field_lower = field.encode("ascii").lower() + decoded_field = field.decode("ascii") + decoded_value = value.decode("ascii") + field_lower = decoded_field.lower() if not field_lower.startswith("content-") and \ field_lower != "mime-version": continue logger.debug( f"feed content header to message object: " - f"{field.encode()}: {value.encode()}") + f"{decoded_field}: {decoded_value}") fp.feed(field + b": " + value + b"\r\n") fp.feed(b"\r\n") - logger.debug(f"feed body to message object: {field}: {value}") + logger.debug(f"feed body to message object") fp.feed(milter.body_data.read()) logger.debug("parse message") From 59486a2e18437e71febc27f0f0f365ba7215ed6d Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 10 Nov 2020 20:10:23 +0100 Subject: [PATCH 066/145] change version to 1.0.10 --- pymodmilter/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/version.py b/pymodmilter/version.py index 39e0411..9fd0f8d 100644 --- a/pymodmilter/version.py +++ b/pymodmilter/version.py @@ -1 +1 @@ -__version__ = "1.0.9" +__version__ = "1.0.10" From d07bb965b3aa0ccf53025e31919d643480287dce Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 11 Nov 2020 00:49:49 +0100 Subject: [PATCH 067/145] change everything to pythons new email lib --- pymodmilter/__init__.py | 154 ++++++++++++++++++++++------------------ pymodmilter/actions.py | 121 ++++++++++++------------------- 2 files changed, 129 insertions(+), 146 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index c284486..41db62e 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -26,35 +26,13 @@ import Milter import logging from Milter.utils import parse_addr -from email.charset import Charset -from email.header import Header, decode_header -from io import BytesIO +from email.message import Message +from email.parser import BytesFeedParser +from email.policy import default as default_policy from pymodmilter.conditions import Conditions -def make_header(decoded_seq, maxlinelen=None, header_name=None, - continuation_ws=' ', errors='strict'): - """Create a Header from a sequence of pairs as returned by decode_header() - - decode_header() takes a header value string and returns a sequence of - pairs of the format (decoded_string, charset) where charset is the string - name of the character set. - - This function takes one of those sequence of pairs and returns a Header - instance. Optional maxlinelen, header_name, and continuation_ws are as in - the Header constructor. - """ - h = Header(maxlinelen=maxlinelen, header_name=header_name, - continuation_ws=continuation_ws) - for s, charset in decoded_seq: - # None means us-ascii but we can simply pass it on to h.append() - if charset is not None and not isinstance(charset, Charset): - charset = Charset(charset) - h.append(s, charset, errors=errors) - return h - - class CustomLogger(logging.LoggerAdapter): def process(self, msg, kwargs): if "name" in self.extra: @@ -91,17 +69,15 @@ class Rule: self.actions = actions self.pretend = pretend - self._needs = [] + self._need_body = False for action in actions: - for need in action.needs(): - if need not in self._needs: - self._needs.append(need) + if action.need_body(): + self._need_body = True + break - self.logger.debug("needs: {}".format(", ".join(self._needs))) - - def needs(self): - """Return the needs of this rule.""" - return self._needs + def need_body(self): + """Return the if this rule needs the message body.""" + return self._need_body def ignores(self, host=None, envfrom=None, envto=None): args = {} @@ -122,17 +98,53 @@ class Rule: return True - def execute(self, milter, pretend=None): + def execute(self, milter, msg, pretend=None): """Execute all actions of this rule.""" if pretend is None: pretend = self.pretend for action in self.actions: - milter_action = action.execute(milter) + milter_action = action.execute(milter, msg, pretend=pretend) if milter_action is not None: return milter_action +class MilterMessage(Message): + def replace_header(self, _name, _value, occ=None): + _name = _name.lower() + counter = 0 + for i, (k, v) in zip(range(len(self._headers)), self._headers): + if k.lower() == _name: + counter += 1 + if not occ or counter == occ: + self._headers[i] = self.policy.header_store_parse( + k, _value) + break + + else: + raise KeyError(_name) + + def remove_header(self, name, occ=None): + name = name.lower() + newheaders = [] + counter = 0 + for k, v in self._headers: + if k.lower() == name: + counter += 1 + if counter != occ: + newheaders.append((k, v)) + else: + newheaders.append((k, v)) + + self._headers = newheaders + + +def remove_surrogates(string): + return string.encode( + "ascii", errors="surrogateescape").decode( + "ascii", errors="replace") + + class ModifyMilter(Milter.Base): """ModifyMilter based on Milter.Base to implement milter communication""" @@ -175,7 +187,7 @@ class ModifyMilter(Milter.Base): return Milter.ACCEPT except Exception as e: self.logger.exception( - f"an exception occured in connect function: {e}") + f"an exception occured in connect method: {e}") return Milter.TEMPFAIL return Milter.CONTINUE @@ -196,7 +208,7 @@ class ModifyMilter(Milter.Base): self.recipients = set() except Exception as e: self.logger.exception( - f"an exception occured in envfrom function: {e}") + f"an exception occured in envfrom method: {e}") return Milter.TEMPFAIL return Milter.CONTINUE @@ -207,7 +219,7 @@ class ModifyMilter(Milter.Base): self.recipients.add("@".join(parse_addr(to)).lower()) except Exception as e: self.logger.exception( - f"an exception occured in envrcpt function: {e}") + f"an exception occured in envrcpt method: {e}") return Milter.TEMPFAIL return Milter.CONTINUE @@ -231,64 +243,64 @@ class ModifyMilter(Milter.Base): self.fields = None self.fields_bytes = None self.body_data = None - needs = [] + + self._fp = BytesFeedParser( + _factory=MilterMessage, policy=default_policy) + self._keep_body = False for rule in self.rules: - needs += rule.needs() - - if "fields" in needs: - self.fields = [] - - if "fields_bytes" in needs: - self.fields_bytes = [] - - if "body" in needs: - self.body_data = BytesIO() + if rule.need_body(): + self._keep_body = True + break except Exception as e: self.logger.exception( - f"an exception occured in data function: {e}") + f"an exception occured in data method: {e}") return Milter.TEMPFAIL return Milter.CONTINUE - def header(self, name, value): + def header(self, field, value): try: - if self.fields_bytes is not None: - self.fields_bytes.append( - (name.encode("ascii", errors="surrogateescape"), - value.encode("ascii", errors="surrogateescape"))) + # feed header line to BytesParser + self._fp.feed(field + b": " + value + b"\r\n") - if self.fields is not None: - # remove surrogates from value - value = value.encode( - errors="surrogateescape").decode(errors="replace") - self.logger.debug(f"received header: {name}: {value}") - header = make_header(decode_header(value), errors="replace") - value = str(header).replace("\x00", "") - self.logger.debug(f"decoded header: {name}: {value}") - self.fields.append((name, value)) + # remove surrogates from field and value + field = remove_surrogates(field) + value = remove_surrogates(value) except Exception as e: self.logger.exception( - f"an exception occured in header function: {e}") + f"an exception occured in header method: {e}") + return Milter.TEMPFAIL + + return Milter.CONTINUE + + def eoh(self): + try: + self._fp.feed(b"\r\n") + except Exception as e: + self.logger.exception( + f"an exception occured in eoh method: {e}") return Milter.TEMPFAIL return Milter.CONTINUE def body(self, chunk): try: - if self.body_data is not None: - self.body_data.write(chunk) + if self._keep_body: + self._fp.feed(chunk) except Exception as e: self.logger.exception( - f"an exception occured in body function: {e}") + f"an exception occured in body method: {e}") return Milter.TEMPFAIL return Milter.CONTINUE def eom(self): try: + msg = self._fp.close() + for rule in self.rules: - milter_action = rule.execute(self) + milter_action = rule.execute(self, msg) if milter_action is not None: if milter_action["action"] == "reject": @@ -303,7 +315,7 @@ class ModifyMilter(Milter.Base): except Exception as e: self.logger.exception( - f"an exception occured in eom function: {e}") + f"an exception occured in eom method: {e}") return Milter.TEMPFAIL return Milter.ACCEPT diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 163da45..2b1c382 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -21,10 +21,8 @@ from collections import defaultdict from copy import copy from datetime import datetime from email.header import Header -from email.parser import BytesFeedParser from email.message import MIMEPart -from email.policy import default as default_policy, SMTP -from shutil import copyfileobj +from email.policy import SMTP from pymodmilter import CustomLogger, Conditions @@ -37,7 +35,7 @@ def _replace_illegal_chars(string): "\n", "") -def add_header(field, value, milter, idx=-1, pretend=False, +def add_header(milter, msg, field, value, pretend=False, logger=logging.getLogger(__name__)): """Add a mail header field.""" header = f"{field}: {value}" @@ -46,21 +44,18 @@ def add_header(field, value, milter, idx=-1, pretend=False, else: logger.info(f"add_header: {header[0:70]}") - if idx == -1: - milter.fields.append((field, value)) - else: - milter.fields.insert(idx, (field, value)) + msg.add_header(field, value) if pretend: return encoded_value = _replace_illegal_chars( Header(s=value).encode()) - milter.logger.debug(f"milter: addheader: {field}[{idx}]: {encoded_value}") - milter.addheader(field, encoded_value, idx) + milter.logger.debug(f"milter: addheader: {field}: {encoded_value}") + milter.addheader(field, encoded_value, -1) -def mod_header(field, value, milter, search=None, pretend=False, +def mod_header(milter, msg, field, value, search=None, pretend=False, logger=logging.getLogger(__name__)): """Change the value of a mail header field.""" if isinstance(field, str): @@ -71,8 +66,9 @@ def mod_header(field, value, milter, search=None, pretend=False, occ = defaultdict(int) - for idx, (f, v) in enumerate(milter.fields): - occ[f] += 1 + for i, (f, v) in enumerate(msg.items()): + f_lower = f.lower() + occ[f_lower] += 1 if not field.match(f): continue @@ -93,12 +89,13 @@ def mod_header(field, value, milter, search=None, pretend=False, header = f"{f}: {v}" new_header = f"{f}: {new_v}" + if logger.getEffectiveLevel() == logging.DEBUG: logger.debug(f"mod_header: {header}: {new_header}") else: logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}") - milter.fields[idx] = (f, new_v) + msg.replace_header(f, new_v, occ=occ[f_lower]) if pretend: continue @@ -106,11 +103,11 @@ def mod_header(field, value, milter, search=None, pretend=False, encoded_value = _replace_illegal_chars( Header(s=new_v).encode()) milter.logger.debug( - f"milter: chgheader: {f}[{occ[f]}]: {encoded_value}") - milter.chgheader(f, occ[f], encoded_value) + f"milter: chgheader: {f}[{occ[f_lower]}]: {encoded_value}") + milter.chgheader(f, occ[f_lower], encoded_value) -def del_header(field, milter, value=None, pretend=False, +def del_header(milter, msg, field, value=None, pretend=False, logger=logging.getLogger(__name__)): """Delete a mail header field.""" if isinstance(field, str): @@ -119,14 +116,13 @@ def del_header(field, milter, value=None, pretend=False, if isinstance(value, str): value = re.compile(value, re.MULTILINE + re.DOTALL + re.IGNORECASE) - idx = -1 occ = defaultdict(int) # iterate a copy of milter.fields because elements may get removed # during iteration - for f, v in milter.fields.copy(): - idx += 1 - occ[f] += 1 + for f, v in msg.items(): + f_lower = f.lower() + occ[f_lower] += 1 if not field.match(f): continue @@ -140,16 +136,14 @@ def del_header(field, milter, value=None, pretend=False, else: logger.info(f"del_header: {header[0:70]}") - del milter.fields[idx] + msg.remove_header(f, occ=occ[f_lower]) + + occ[f_lower] -= 1 if not pretend: - encoded_value = "" milter.logger.debug( - f"milter: chgheader: {f}[{occ[f]}]: {encoded_value}") - milter.chgheader(f, occ[f], encoded_value) - - idx -= 1 - occ[f] -= 1 + f"milter: chgheader: {f}[{occ[f_lower]}]:") + milter.chgheader(f, occ[f_lower], "") def _get_body_content(msg, body_type): @@ -273,31 +267,9 @@ def _inject_body(milter, msg): return new_msg -def add_disclaimer(text, html, action, policy, milter, pretend=False, +def add_disclaimer(milter, msg, text, html, action, policy, pretend=False, logger=logging.getLogger(__name__)): """Append or prepend a disclaimer to the mail body.""" - milter.body_data.seek(0) - fp = BytesFeedParser(policy=default_policy) - - for field, value in milter.fields_bytes: - decoded_field = field.decode("ascii") - decoded_value = value.decode("ascii") - field_lower = decoded_field.lower() - if not field_lower.startswith("content-") and \ - field_lower != "mime-version": - continue - logger.debug( - f"feed content header to message object: " - f"{decoded_field}: {decoded_value}") - fp.feed(field + b": " + value + b"\r\n") - - fp.feed(b"\r\n") - logger.debug(f"feed body to message object") - fp.feed(milter.body_data.read()) - - logger.debug("parse message") - msg = fp.close() - update_headers = False try: @@ -367,48 +339,47 @@ def add_disclaimer(text, html, action, policy, milter, pretend=False, "value": msg.get("Content-Transfer-Encoding"), "modified": False}} - for field, value in milter.fields.copy(): + for field, value in msg.items(): field_lower = field.lower() if field_lower in fields and fields[field_lower]["value"] is not None: - mod_header(field=f"^{field}$", value=fields[field_lower]["value"], - milter=milter, pretend=pretend, logger=logger) + mod_header(milter, msg, field=f"^{field}$", + value=fields[field_lower]["value"], + pretend=pretend, logger=logger) fields[field_lower]["modified"] = True elif field_lower.startswith("content-"): - del_header(field=f"^{field}$", milter=milter, + del_header(milter, msg, field=f"^{field}$", pretend=pretend, logger=logger) for field in fields.values(): if not field["modified"] and field["value"] is not None: - add_header(field=field["field"], value=field["value"], - milter=milter, pretend=pretend, logger=logger) + add_header(milter, msg, field=field["field"], value=field["value"], + pretend=pretend, logger=logger) -def store(directory, milter, pretend=False, +def store(milter, msg, directory, pretend=False, logger=logging.getLogger(__name__)): timestamp = datetime.now().strftime("%Y%m%d%H%M%S") store_id = f"{timestamp}_{milter.qid}" datafile = os.path.join(directory, store_id) + milter.body_data.seek(0) logger.info(f"store message in file {datafile}") try: with open(datafile, "wb") as fp: - for field, value in milter.fields_bytes: - fp.write(field + b": " + value + b"\r\n") - - copyfileobj(milter.body_data, fp) + fp.write(msg.as_bytes()) except IOError as e: raise RuntimeError(f"unable to store message: {e}") class Action: """Action to implement a pre-configured action to perform on e-mails.""" - _types = { - "add_header": ["fields"], - "del_header": ["fields"], - "mod_header": ["fields"], - "add_disclaimer": ["fields", "body"], - "store": ["fields_bytes", "body"]} + _need_body_map = { + "add_header": False, + "del_header": False, + "mod_header": False, + "add_disclaimer": True, + "store": True} def __init__(self, name, local_addrs, conditions, action_type, args, loglevel=logging.INFO, pretend=False): @@ -423,9 +394,9 @@ class Action: self.pretend = pretend self._args = {} - if action_type not in self._types: + if action_type not in self._need_body_map: raise RuntimeError(f"invalid action type '{action_type}'") - self._needs = self._types[action_type] + self._need_body = self._need_body_map[action_type] try: if action_type == "add_header": @@ -498,16 +469,16 @@ class Action: raise RuntimeError( f"mandatory argument not found: {e}") - def needs(self): + def need_body(self): """Return the needs of this action.""" - return self._needs + return self._need_body - def execute(self, milter, pretend=None): + def execute(self, milter, msg, pretend=None): """Execute configured action.""" if pretend is None: pretend = self.pretend logger = CustomLogger(self.logger, {"qid": milter.qid}) - return self._func( - milter=milter, pretend=pretend, logger=logger, **self._args) + return self._func(milter=milter, msg=msg, pretend=pretend, + logger=logger, **self._args) From 1f23c0e40800d5b74a7d5e1c33f77a1dd042da98 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 12 Nov 2020 11:56:20 +0100 Subject: [PATCH 068/145] many fixes for message handling with pythons new email lib --- pymodmilter/__init__.py | 29 ++--- pymodmilter/actions.py | 277 +++++++++++++++++++--------------------- 2 files changed, 139 insertions(+), 167 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 41db62e..f5e038e 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -13,7 +13,6 @@ # __all__ = [ - "make_header", "actions", "conditions", "run", @@ -26,7 +25,7 @@ import Milter import logging from Milter.utils import parse_addr -from email.message import Message +from email.message import MIMEPart from email.parser import BytesFeedParser from email.policy import default as default_policy @@ -98,18 +97,18 @@ class Rule: return True - def execute(self, milter, msg, pretend=None): + def execute(self, milter, pretend=None): """Execute all actions of this rule.""" if pretend is None: pretend = self.pretend for action in self.actions: - milter_action = action.execute(milter, msg, pretend=pretend) + milter_action = action.execute(milter, pretend=pretend) if milter_action is not None: return milter_action -class MilterMessage(Message): +class MilterMessage(MIMEPart): def replace_header(self, _name, _value, occ=None): _name = _name.lower() counter = 0 @@ -139,12 +138,6 @@ class MilterMessage(Message): self._headers = newheaders -def remove_surrogates(string): - return string.encode( - "ascii", errors="surrogateescape").decode( - "ascii", errors="replace") - - class ModifyMilter(Milter.Base): """ModifyMilter based on Milter.Base to implement milter communication""" @@ -261,12 +254,11 @@ class ModifyMilter(Milter.Base): def header(self, field, value): try: - # feed header line to BytesParser - self._fp.feed(field + b": " + value + b"\r\n") + # remove surrogates + field = field.encode("ascii", errors="surrogateescape") + value = value.encode("ascii", errors="surrogateescape") - # remove surrogates from field and value - field = remove_surrogates(field) - value = remove_surrogates(value) + self._fp.feed(field + b": " + value + b"\r\n") except Exception as e: self.logger.exception( f"an exception occured in header method: {e}") @@ -297,10 +289,9 @@ class ModifyMilter(Milter.Base): def eom(self): try: - msg = self._fp.close() - + self.msg = self._fp.close() for rule in self.rules: - milter_action = rule.execute(self, msg) + milter_action = rule.execute(self) if milter_action is not None: if milter_action["action"] == "reject": diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 2b1c382..cad80e1 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -35,16 +35,17 @@ def _replace_illegal_chars(string): "\n", "") -def add_header(milter, msg, field, value, pretend=False, +def add_header(milter, field, value, pretend=False, update_msg=True, logger=logging.getLogger(__name__)): """Add a mail header field.""" - header = f"{field}: {value}" - if logger.getEffectiveLevel() == logging.DEBUG: - logger.debug(f"add_header: {header}") - else: - logger.info(f"add_header: {header[0:70]}") + if update_msg: + header = f"{field}: {value}" + if logger.getEffectiveLevel() == logging.DEBUG: + logger.debug(f"add_header: {header}") + else: + logger.info(f"add_header: {header[0:70]}") - msg.add_header(field, value) + milter.msg.add_header(field, value) if pretend: return @@ -55,8 +56,8 @@ def add_header(milter, msg, field, value, pretend=False, milter.addheader(field, encoded_value, -1) -def mod_header(milter, msg, field, value, search=None, pretend=False, - logger=logging.getLogger(__name__)): +def mod_header(milter, field, value, search=None, pretend=False, + update_msg=True, logger=logging.getLogger(__name__)): """Change the value of a mail header field.""" if isinstance(field, str): field = re.compile(field, re.IGNORECASE) @@ -66,7 +67,7 @@ def mod_header(milter, msg, field, value, search=None, pretend=False, occ = defaultdict(int) - for i, (f, v) in enumerate(msg.items()): + for i, (f, v) in enumerate(milter.msg.items()): f_lower = f.lower() occ[f_lower] += 1 @@ -87,15 +88,16 @@ def mod_header(milter, msg, field, value, search=None, pretend=False, "skip modification") continue - header = f"{f}: {v}" - new_header = f"{f}: {new_v}" + if update_msg: + header = f"{f}: {v}" + new_header = f"{f}: {new_v}" - if logger.getEffectiveLevel() == logging.DEBUG: - logger.debug(f"mod_header: {header}: {new_header}") - else: - logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}") + if logger.getEffectiveLevel() == logging.DEBUG: + logger.debug(f"mod_header: {header}: {new_header}") + else: + logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}") - msg.replace_header(f, new_v, occ=occ[f_lower]) + milter.msg.replace_header(f, new_v, occ=occ[f_lower]) if pretend: continue @@ -107,7 +109,7 @@ def mod_header(milter, msg, field, value, search=None, pretend=False, milter.chgheader(f, occ[f_lower], encoded_value) -def del_header(milter, msg, field, value=None, pretend=False, +def del_header(milter, field, value=None, pretend=False, update_msg=True, logger=logging.getLogger(__name__)): """Delete a mail header field.""" if isinstance(field, str): @@ -118,9 +120,7 @@ def del_header(milter, msg, field, value=None, pretend=False, occ = defaultdict(int) - # iterate a copy of milter.fields because elements may get removed - # during iteration - for f, v in msg.items(): + for f, v in milter.msg.items(): f_lower = f.lower() occ[f_lower] += 1 @@ -130,13 +130,13 @@ def del_header(milter, msg, field, value=None, pretend=False, if value is not None and not value.search(v): continue - header = f"{f}: {v}" - if logger.getEffectiveLevel() == logging.DEBUG: - logger.debug(f"del_header: {header}") - else: - logger.info(f"del_header: {header[0:70]}") - - msg.remove_header(f, occ=occ[f_lower]) + if update_msg: + header = f"{f}: {v}" + if logger.getEffectiveLevel() == logging.DEBUG: + logger.debug(f"del_header: {header}") + else: + logger.info(f"del_header: {header[0:70]}") + milter.msg.remove_header(f, occ=occ[f_lower]) occ[f_lower] -= 1 @@ -146,13 +146,38 @@ def del_header(milter, msg, field, value=None, pretend=False, milter.chgheader(f, occ[f_lower], "") -def _get_body_content(msg, body_type): - content = None - body_part = msg.get_body(preferencelist=(body_type)) - if body_part is not None: - content = body_part.get_content() +def _serialize_msg(msg, logger): + if not msg["MIME-Version"]: + msg.add_header("MIME-Version", "1.0") - return (body_part, content) + try: + logger.debug("serialize message as bytes") + data = msg.as_bytes(policy=SMTP) + except Exception as e: + logger.waring( + f"unable to serialize message as bytes: {e}") + try: + logger.warning("try to serialize message as string") + data = msg.as_string(policy=SMTP) + data = data.encode("ascii", errors="replace") + except Exception as e: + raise e + + return data + + +def _get_body_content(msg, pref): + part = None + content = None + if not msg.is_multipart() and msg.get_content_type() == f"text/{pref}": + part = msg + else: + part = msg.get_body(preferencelist=(pref)) + + if part is not None: + content = part.get_content() + + return (part, content) def _has_content_before_body_tag(soup): @@ -166,9 +191,9 @@ def _has_content_before_body_tag(soup): return False -def _patch_message_body(msg, action, text, html, logger): - text_body, text_content = _get_body_content(msg, "plain") - html_body, html_content = _get_body_content(msg, "html") +def _patch_message_body(milter, action, text, html, logger): + text_body, text_content = _get_body_content(milter.msg, "plain") + html_body, html_content = _get_body_content(milter.msg, "html") if text_content is None and html_content is None: raise RuntimeError("message does not contain any body part") @@ -184,6 +209,7 @@ def _patch_message_body(msg, action, text, html, logger): text_body.set_content( content.encode(), maintype="text", subtype="plain") text_body.set_param("charset", "UTF-8", header="Content-Type") + del text_body["MIME-Version"] if html_content is not None: logger.info(f"{action} html disclaimer") @@ -204,86 +230,84 @@ def _patch_message_body(msg, action, text, html, logger): html_body.set_content( str(soup).encode(), maintype="text", subtype="html") html_body.set_param("charset", "UTF-8", header="Content-Type") + del html_body["MIME-Version"] -def _serialize_msg(msg, logger): - try: - logger.debug("serialize message as bytes") - data = msg.as_bytes(policy=SMTP) - except Exception as e: - logger.waring( - f"unable to serialize message as bytes: {e}") - try: - logger.warning("try to serialize message as string") - data = msg.as_string(policy=SMTP) - data = data.encode("ascii", errors="replace") - except Exception as e: - raise e - - return data +def _update_body(milter, logger): + data = _serialize_msg(milter.msg, logger) + body_pos = data.find(b"\r\n\r\n") + 4 + logger.debug("milter: replacebody") + milter.replacebody(data[body_pos:]) + del data -def _wrap_message(milter): - msg = MIMEPart() - msg.add_header("MIME-Version", "1.0") +def _update_headers(milter, original_headers, logger): + # serialize the message object so it updates its headers internally + milter.msg.as_bytes() + for field, value in original_headers: + if field not in milter.msg: + del_header(milter, field=f"^{field}$", update_msg=False, + logger=logger) - msg.set_content( + for field, value in milter.msg.items(): + field_lower = field.lower() + if not [f for f in original_headers if f[0].lower() == field_lower]: + add_header(milter, field=field, value=value, update_msg=False, + logger=logger) + else: + mod_header(milter, field=f"^{field}$", value=value, + update_msg=False, logger=logger) + + +def _wrap_message(milter, logger): + attachment = MIMEPart() + attachment.set_content(milter.msg.as_bytes(), + maintype="plain", subtype="text", + disposition="attachment", + filename=f"{milter.qid}.eml", + params={"name": f"{milter.qid}.eml"}) + + milter.msg.clear_content() + milter.msg.set_content( "Please see the original email attached.") - msg.add_alternative( - "Please see the original email attached.", + milter.msg.add_alternative( + "Please see the original email attached.", subtype="html") - - data = b"" - for field, value in milter.fields: - encoded_value = _replace_illegal_chars( - Header(s=value).encode()) - data += field.encode("ascii", errors="replace") - data += b": " - data += encoded_value.encode("ascii", errors="replace") - data += b"\r\n" - - milter.body_data.seek(0) - data += b"\r\n" + milter.body_data.read() - - msg.add_attachment( - data, maintype="plain", subtype="text", - filename=f"{milter.qid}.eml") - - return msg + milter.msg.make_mixed() + milter.msg.attach(attachment) -def _inject_body(milter, msg): - if not msg.is_multipart(): - msg.make_mixed() +def _inject_body(milter): + if not milter.msg.is_multipart(): + milter.msg.make_mixed() - new_msg = MIMEPart() - new_msg.add_header("MIME-Version", "1.0") - new_msg.set_content("") - new_msg.add_alternative("", subtype="html") - new_msg.make_mixed() - for attachment in msg.iter_attachments(): - new_msg.attach(attachment) + attachments = [] + for attachment in milter.msg.iter_attachments(): + if "content-disposition" not in attachment: + attachment["Content-Disposition"] = "attachment" + attachments.append(attachment) - return new_msg + milter.msg.clear_content() + milter.msg.set_content("") + milter.msg.add_alternative("", subtype="html") + milter.msg.make_mixed() + + for attachment in attachments: + milter.msg.attach(attachment) -def add_disclaimer(milter, msg, text, html, action, policy, pretend=False, +def add_disclaimer(milter, text, html, action, policy, pretend=False, logger=logging.getLogger(__name__)): """Append or prepend a disclaimer to the mail body.""" - update_headers = False + original_headers = milter.msg.items() try: try: - _patch_message_body(msg, action, text, html, logger) - data = _serialize_msg(msg, logger) - if not msg.is_multipart(): - update_headers = True + _patch_message_body(milter, action, text, html, logger) except RuntimeError as e: logger.info(f"{e}, inject empty plain and html body") - msg = _inject_body(milter, msg) - _patch_message_body(msg, action, text, html, logger) - data = _serialize_msg(msg, logger) - update_headers = True + _inject_body(milter) + _patch_message_body(milter, action, text, html, logger) except Exception as e: logger.warning(e) if policy == "ignore": @@ -300,74 +324,31 @@ def add_disclaimer(milter, msg, text, html, action, policy, pretend=False, logger.info("wrap original message in a new message envelope") try: - msg = _wrap_message(milter) - _patch_message_body(msg, action, text, html, logger) - data = _serialize_msg(msg, logger) - update_headers = True + _wrap_message(milter, logger) + _patch_message_body(milter, action, text, html, logger) except Exception as e: logger.error(e) raise Exception( "unable to wrap message in a new message envelope, " "give up ...") - body_pos = data.find(b"\r\n\r\n") + 4 - milter.body_data.seek(0) - milter.body_data.write(data[body_pos:]) - milter.body_data.truncate() - if pretend: return - logger.debug("milter: replacebody") - milter.replacebody(data[body_pos:]) - del data - - if not update_headers: - return - - fields = { - "mime-version": { - "field": "MIME-Version", - "value": msg.get("MIME-Version"), - "modified": False}, - "content-type": { - "field": "Content-Type", - "value": msg.get("Content-Type"), - "modified": False}, - "content-transfer-encoding": { - "field": "Content-Transfer-Encoding", - "value": msg.get("Content-Transfer-Encoding"), - "modified": False}} - - for field, value in msg.items(): - field_lower = field.lower() - if field_lower in fields and fields[field_lower]["value"] is not None: - mod_header(milter, msg, field=f"^{field}$", - value=fields[field_lower]["value"], - pretend=pretend, logger=logger) - fields[field_lower]["modified"] = True - - elif field_lower.startswith("content-"): - del_header(milter, msg, field=f"^{field}$", - pretend=pretend, logger=logger) - - for field in fields.values(): - if not field["modified"] and field["value"] is not None: - add_header(milter, msg, field=field["field"], value=field["value"], - pretend=pretend, logger=logger) + _update_headers(milter, original_headers, logger) + _update_body(milter, logger) -def store(milter, msg, directory, pretend=False, +def store(milter, directory, pretend=False, logger=logging.getLogger(__name__)): timestamp = datetime.now().strftime("%Y%m%d%H%M%S") store_id = f"{timestamp}_{milter.qid}" datafile = os.path.join(directory, store_id) - milter.body_data.seek(0) logger.info(f"store message in file {datafile}") try: with open(datafile, "wb") as fp: - fp.write(msg.as_bytes()) + fp.write(milter.msg.as_bytes()) except IOError as e: raise RuntimeError(f"unable to store message: {e}") @@ -473,12 +454,12 @@ class Action: """Return the needs of this action.""" return self._need_body - def execute(self, milter, msg, pretend=None): + def execute(self, milter, pretend=None): """Execute configured action.""" if pretend is None: pretend = self.pretend logger = CustomLogger(self.logger, {"qid": milter.qid}) - return self._func(milter=milter, msg=msg, pretend=pretend, + return self._func(milter=milter, pretend=pretend, logger=logger, **self._args) From 3d8c960e05edd46e27e727bd6eb4dfba2ac51f99 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 12 Nov 2020 12:01:31 +0100 Subject: [PATCH 069/145] change version to 1.1.0 --- pymodmilter/__init__.py | 3 ++- pymodmilter/run.py | 2 +- pymodmilter/version.py | 1 - setup.cfg | 2 ++ setup.py | 3 --- 5 files changed, 5 insertions(+), 6 deletions(-) delete mode 100644 pymodmilter/version.py create mode 100644 setup.cfg diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index f5e038e..4a19a3f 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -16,11 +16,12 @@ __all__ = [ "actions", "conditions", "run", - "version", "CustomLogger", "Rule", "ModifyMilter"] +__version__ = "1.1.0" + import Milter import logging diff --git a/pymodmilter/run.py b/pymodmilter/run.py index cd4aedb..6895630 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -22,7 +22,7 @@ from json import loads from re import sub from pymodmilter import Rule, ModifyMilter -from pymodmilter.version import __version__ as version +from pymodmilter import __version__ as version from pymodmilter.actions import Action diff --git a/pymodmilter/version.py b/pymodmilter/version.py deleted file mode 100644 index 9fd0f8d..0000000 --- a/pymodmilter/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "1.0.10" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..f3d2f8a --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +version = attr: pymodmilter.__version__ diff --git a/setup.py b/setup.py index c158fd6..7dde3be 100644 --- a/setup.py +++ b/setup.py @@ -4,11 +4,8 @@ def read_file(fname): with open(fname, 'r') as f: return f.read() -version = {} -exec(read_file("pymodmilter/version.py"), version) setup(name = "pymodmilter", - version = version["__version__"], author = "Thomas Oettli", author_email = "spacefreak@noop.ch", description = "A pymilter based sendmail/postfix pre-queue filter.", From d9f2b515ff3643559914b72d164da0cbc9e1cb0e Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 12 Nov 2020 12:07:04 +0100 Subject: [PATCH 070/145] change version to 1.1.0 --- pymodmilter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 4a19a3f..34f0e08 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -20,7 +20,7 @@ __all__ = [ "Rule", "ModifyMilter"] -__version__ = "1.1.0" +__version__ = "1.1.1" import Milter import logging From 7e33bce6ea1c7d98e0d0c0d7705a0edb92b0081b Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 12 Nov 2020 21:58:00 +0100 Subject: [PATCH 071/145] fix illegal chars in headers and monkey-patch python issues 27257, 30988 --- pymodmilter/__init__.py | 75 +++++++++++++++++++++++++++++++++++++++++ pymodmilter/actions.py | 8 ++--- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 34f0e08..4f933da 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -32,6 +32,81 @@ from email.policy import default as default_policy from pymodmilter.conditions import Conditions +######################################################## +# monkey-patch pythons email library bug 27257,30988 # +######################################################## +# +# https://bugs.python.org/issue27257 +# https://bugs.python.org/issue30988 +# +# fix: https://github.com/python/cpython/pull/15600 + +import email._header_value_parser +from email._header_value_parser import TokenList, NameAddr +from email._header_value_parser import get_display_name, get_angle_addr +from email._header_value_parser import get_cfws, errors +from email._header_value_parser import CFWS_LEADER, PHRASE_ENDS + + +class DisplayName(email._header_value_parser.DisplayName): + @property + def display_name(self): + res = TokenList(self) + if len(res) == 0: + return res.value + if res[0].token_type == 'cfws': + res.pop(0) + else: + if isinstance(res[0], TokenList) and \ + res[0][0].token_type == 'cfws': + res[0] = TokenList(res[0][1:]) + if res[-1].token_type == 'cfws': + res.pop() + else: + if isinstance(res[-1], TokenList) and \ + res[-1][-1].token_type == 'cfws': + res[-1] = TokenList(res[-1][:-1]) + return res.value + + +def get_name_addr(value): + """ name-addr = [display-name] angle-addr + + """ + name_addr = NameAddr() + # Both the optional display name and the angle-addr can start with cfws. + leader = None + if value[0] in CFWS_LEADER: + leader, value = get_cfws(value) + if not value: + raise errors.HeaderParseError( + "expected name-addr but found '{}'".format(leader)) + if value[0] != '<': + if value[0] in PHRASE_ENDS: + raise errors.HeaderParseError( + "expected name-addr but found '{}'".format(value)) + token, value = get_display_name(value) + if not value: + raise errors.HeaderParseError( + "expected name-addr but found '{}'".format(token)) + if leader is not None: + if isinstance(token[0], TokenList): + token[0][:0] = [leader] + else: + token[:0] = [leader] + leader = None + name_addr.append(token) + token, value = get_angle_addr(value) + if leader is not None: + token[:0] = [leader] + name_addr.append(token) + return name_addr, value + + +setattr(email._header_value_parser, "DisplayName", DisplayName) +setattr(email._header_value_parser, "get_name_addr", get_name_addr) + +######################################################## class CustomLogger(logging.LoggerAdapter): def process(self, msg, kwargs): diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index cad80e1..5be5367 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -45,7 +45,7 @@ def add_header(milter, field, value, pretend=False, update_msg=True, else: logger.info(f"add_header: {header[0:70]}") - milter.msg.add_header(field, value) + milter.msg.add_header(field, _replace_illegal_chars(value)) if pretend: return @@ -74,10 +74,9 @@ def mod_header(milter, field, value, search=None, pretend=False, if not field.match(f): continue + new_v = v if search is not None: new_v = search.sub(value, v).strip() - else: - new_v = value.strip() if new_v == v: continue @@ -97,7 +96,8 @@ def mod_header(milter, field, value, search=None, pretend=False, else: logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}") - milter.msg.replace_header(f, new_v, occ=occ[f_lower]) + milter.msg.replace_header( + f, _replace_illegal_chars(new_v), occ=occ[f_lower]) if pretend: continue From 305cad1f87c600f9cab98424e8a5dd8ab054726d Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 12 Nov 2020 22:47:28 +0100 Subject: [PATCH 072/145] add encoding alias for windows-874 to cp874 --- pymodmilter/__init__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 4f933da..558145b 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -24,6 +24,7 @@ __version__ = "1.1.1" import Milter import logging +import encodings from Milter.utils import parse_addr from email.message import MIMEPart @@ -106,7 +107,16 @@ def get_name_addr(value): setattr(email._header_value_parser, "DisplayName", DisplayName) setattr(email._header_value_parser, "get_name_addr", get_name_addr) -######################################################## +################################################ +# add charset alias for windows-874 encoding # +################################################ + +for alias in ["windows-874", "windows_874"]: + if alias not in encodings.aliases.aliases: + encodings.aliases.aliases[alias] = "cp874" + +################################################ + class CustomLogger(logging.LoggerAdapter): def process(self, msg, kwargs): From 267d0dbf1f3f5ee8729e4717bc1bd53684c1f78a Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 13 Nov 2020 09:07:52 +0100 Subject: [PATCH 073/145] remove surrogates from headers --- pymodmilter/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 558145b..818ce15 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -341,8 +341,8 @@ class ModifyMilter(Milter.Base): def header(self, field, value): try: # remove surrogates - field = field.encode("ascii", errors="surrogateescape") - value = value.encode("ascii", errors="surrogateescape") + field = field.encode("ascii", errors="replace") + value = value.encode("ascii", errors="replace") self._fp.feed(field + b": " + value + b"\r\n") except Exception as e: From 6dbc50385f142acf2ce8ce7aca4aa1ccf69b485b Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 13 Nov 2020 17:32:45 +0100 Subject: [PATCH 074/145] fix update logic from msg headers to milter headers --- pymodmilter/actions.py | 130 +++++++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 57 deletions(-) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 5be5367..7db1ac3 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -35,29 +35,40 @@ def _replace_illegal_chars(string): "\n", "") -def add_header(milter, field, value, pretend=False, update_msg=True, +def _add_header(milter, field, value, idx=-1): + value = _replace_illegal_chars( + Header(s=value).encode()) + milter.logger.debug(f"milter: addheader: {field}: {value}") + milter.addheader(field, value, idx) + + +def add_header(milter, field, value, pretend=False, logger=logging.getLogger(__name__)): """Add a mail header field.""" - if update_msg: - header = f"{field}: {value}" - if logger.getEffectiveLevel() == logging.DEBUG: - logger.debug(f"add_header: {header}") - else: - logger.info(f"add_header: {header[0:70]}") + header = f"{field}: {value}" + if logger.getEffectiveLevel() == logging.DEBUG: + logger.debug(f"add_header: {header}") + else: + logger.info(f"add_header: {header[0:70]}") - milter.msg.add_header(field, _replace_illegal_chars(value)) + milter.msg.add_header(field, _replace_illegal_chars(value)) if pretend: return - encoded_value = _replace_illegal_chars( + _add_header(milter, field, value) + + +def _mod_header(milter, field, value, occ=1): + value = _replace_illegal_chars( Header(s=value).encode()) - milter.logger.debug(f"milter: addheader: {field}: {encoded_value}") - milter.addheader(field, encoded_value, -1) + milter.logger.debug( + f"milter: chgheader: {field}[{occ}]: {value}") + milter.chgheader(field, occ, value) def mod_header(milter, field, value, search=None, pretend=False, - update_msg=True, logger=logging.getLogger(__name__)): + logger=logging.getLogger(__name__)): """Change the value of a mail header field.""" if isinstance(field, str): field = re.compile(field, re.IGNORECASE) @@ -74,42 +85,45 @@ def mod_header(milter, field, value, search=None, pretend=False, if not field.match(f): continue - new_v = v + new_value = v if search is not None: - new_v = search.sub(value, v).strip() + new_value = search.sub(value, v).strip() + else: + new_value = value - if new_v == v: - continue - - if not new_v: + if not new_value: logger.warning( "mod_header: resulting value is empty, " "skip modification") continue - if update_msg: - header = f"{f}: {v}" - new_header = f"{f}: {new_v}" + if new_value == v: + continue - if logger.getEffectiveLevel() == logging.DEBUG: - logger.debug(f"mod_header: {header}: {new_header}") - else: - logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}") + header = f"{f}: {v}" + new_header = f"{f}: {new_value}" - milter.msg.replace_header( - f, _replace_illegal_chars(new_v), occ=occ[f_lower]) + if logger.getEffectiveLevel() == logging.DEBUG: + logger.debug(f"mod_header: {header}: {new_header}") + else: + logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}") + + milter.msg.replace_header( + f, _replace_illegal_chars(new_value), occ=occ[f_lower]) if pretend: continue - encoded_value = _replace_illegal_chars( - Header(s=new_v).encode()) - milter.logger.debug( - f"milter: chgheader: {f}[{occ[f_lower]}]: {encoded_value}") - milter.chgheader(f, occ[f_lower], encoded_value) + _mod_header(milter, f, new_value, occ=occ[f_lower]) -def del_header(milter, field, value=None, pretend=False, update_msg=True, +def _del_header(milter, field, occ=1): + milter.logger.debug( + f"milter: delheader: {field}[{occ}]") + milter.chgheader(field, occ, "") + + +def del_header(milter, field, value=None, pretend=False, logger=logging.getLogger(__name__)): """Delete a mail header field.""" if isinstance(field, str): @@ -130,24 +144,21 @@ def del_header(milter, field, value=None, pretend=False, update_msg=True, if value is not None and not value.search(v): continue - if update_msg: - header = f"{f}: {v}" - if logger.getEffectiveLevel() == logging.DEBUG: - logger.debug(f"del_header: {header}") - else: - logger.info(f"del_header: {header[0:70]}") - milter.msg.remove_header(f, occ=occ[f_lower]) + header = f"{f}: {v}" + if logger.getEffectiveLevel() == logging.DEBUG: + logger.debug(f"del_header: {header}") + else: + logger.info(f"del_header: {header[0:70]}") + milter.msg.remove_header(f, occ=occ[f_lower]) + + if not pretend: + _del_header(milter, f, occ=occ[f_lower]) occ[f_lower] -= 1 - if not pretend: - milter.logger.debug( - f"milter: chgheader: {f}[{occ[f_lower]}]:") - milter.chgheader(f, occ[f_lower], "") - def _serialize_msg(msg, logger): - if not msg["MIME-Version"]: + if msg.is_multipart() and not msg["MIME-Version"]: msg.add_header("MIME-Version", "1.0") try: @@ -242,21 +253,26 @@ def _update_body(milter, logger): def _update_headers(milter, original_headers, logger): - # serialize the message object so it updates its headers internally + if milter.msg.is_multipart() and not milter.msg["MIME-Version"]: + milter.msg.add_header("MIME-Version", "1.0") + + # serialize the message object so it updates its internal strucure milter.msg.as_bytes() - for field, value in original_headers: - if field not in milter.msg: - del_header(milter, field=f"^{field}$", update_msg=False, - logger=logger) + + original_headers = [(f, f.lower(), v) for f, v in original_headers] + headers = [(f, f.lower(), v) for f, v in milter.msg.items()] + + occ = defaultdict(int) + for field, field_lower, value in original_headers: + occ[field_lower] += 1 + if (field, field_lower, value) not in headers: + _del_header(milter, field, occ=occ[field_lower]) + occ[field] -= 1 for field, value in milter.msg.items(): field_lower = field.lower() - if not [f for f in original_headers if f[0].lower() == field_lower]: - add_header(milter, field=field, value=value, update_msg=False, - logger=logger) - else: - mod_header(milter, field=f"^{field}$", value=value, - update_msg=False, logger=logger) + if (field, field_lower, value) not in original_headers: + _add_header(milter, field, value) def _wrap_message(milter, logger): From 8381260872e5494929599d891ef65532aca857e6 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 13 Nov 2020 17:39:01 +0100 Subject: [PATCH 075/145] change version to 1.1.2 --- pymodmilter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 818ce15..902ecf1 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -20,7 +20,7 @@ __all__ = [ "Rule", "ModifyMilter"] -__version__ = "1.1.1" +__version__ = "1.1.2" import Milter import logging From 440ee391b19be44319f1018b144cc5e65ba27fd1 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Sun, 15 Nov 2020 15:56:36 +0100 Subject: [PATCH 076/145] fix patch for windows-874 encoding --- pymodmilter/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 902ecf1..7336703 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -111,9 +111,13 @@ setattr(email._header_value_parser, "get_name_addr", get_name_addr) # add charset alias for windows-874 encoding # ################################################ +aliases = encodings.aliases.aliases + for alias in ["windows-874", "windows_874"]: - if alias not in encodings.aliases.aliases: - encodings.aliases.aliases[alias] = "cp874" + if alias not in aliases: + aliases[alias] = "cp874" + +setattr(encodings.aliases, "aliases", aliases) ################################################ From f6513bccd571039ac615a050e2c848579505b3bd Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 17 Nov 2020 10:39:56 +0100 Subject: [PATCH 077/145] fix packaging --- MANIFEST.in | 4 ++-- {docs => pymodmilter/docs}/pymodmilter.conf.example | 0 {docs => pymodmilter/docs}/templates/disclaimer_html.template | 0 {docs => pymodmilter/docs}/templates/disclaimer_text.template | 0 {misc => pymodmilter/misc}/openrc/pymodmilter.confd | 0 {misc => pymodmilter/misc}/openrc/pymodmilter.initd | 0 6 files changed, 2 insertions(+), 2 deletions(-) rename {docs => pymodmilter/docs}/pymodmilter.conf.example (100%) rename {docs => pymodmilter/docs}/templates/disclaimer_html.template (100%) rename {docs => pymodmilter/docs}/templates/disclaimer_text.template (100%) rename {misc => pymodmilter/misc}/openrc/pymodmilter.confd (100%) rename {misc => pymodmilter/misc}/openrc/pymodmilter.initd (100%) diff --git a/MANIFEST.in b/MANIFEST.in index ca4758c..89bd49f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include LICENSE README.md -recursive-include docs * -recursive-include misc * +recursive-include pymodmilter/docs * +recursive-include pymodmilter/misc * diff --git a/docs/pymodmilter.conf.example b/pymodmilter/docs/pymodmilter.conf.example similarity index 100% rename from docs/pymodmilter.conf.example rename to pymodmilter/docs/pymodmilter.conf.example diff --git a/docs/templates/disclaimer_html.template b/pymodmilter/docs/templates/disclaimer_html.template similarity index 100% rename from docs/templates/disclaimer_html.template rename to pymodmilter/docs/templates/disclaimer_html.template diff --git a/docs/templates/disclaimer_text.template b/pymodmilter/docs/templates/disclaimer_text.template similarity index 100% rename from docs/templates/disclaimer_text.template rename to pymodmilter/docs/templates/disclaimer_text.template diff --git a/misc/openrc/pymodmilter.confd b/pymodmilter/misc/openrc/pymodmilter.confd similarity index 100% rename from misc/openrc/pymodmilter.confd rename to pymodmilter/misc/openrc/pymodmilter.confd diff --git a/misc/openrc/pymodmilter.initd b/pymodmilter/misc/openrc/pymodmilter.initd similarity index 100% rename from misc/openrc/pymodmilter.initd rename to pymodmilter/misc/openrc/pymodmilter.initd From 24707b339746c414819c2d456bf92735c1dda102 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 17 Nov 2020 10:40:29 +0100 Subject: [PATCH 078/145] change version to 1.1.3 --- pymodmilter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 7336703..cefdc36 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -20,7 +20,7 @@ __all__ = [ "Rule", "ModifyMilter"] -__version__ = "1.1.2" +__version__ = "1.1.3" import Milter import logging From cf3e433af0d755a6c86c018eb60c06506ccb5c6f Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 17 Nov 2020 10:52:16 +0100 Subject: [PATCH 079/145] fix packaging again --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 7dde3be..b25520c 100644 --- a/setup.py +++ b/setup.py @@ -36,13 +36,13 @@ setup(name = "pymodmilter", ( "/etc/pymodmilter", [ - "docs/pymodmilter.conf.example" + "pymodmilter/docs/pymodmilter.conf.example" ] ), ( "/etc/pymodmilter/templates", [ - "docs/templates/disclaimer_html.template", - "docs/templates/disclaimer_text.template" + "pymodmilter/docs/templates/disclaimer_html.template", + "pymodmilter/docs/templates/disclaimer_text.template" ] ) ], From 65f298dd82eb32a4389e1439cafb3c6d1add0a25 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 17 Nov 2020 10:54:23 +0100 Subject: [PATCH 080/145] change version to 1.1.4 --- pymodmilter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index cefdc36..3753dc5 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -20,7 +20,7 @@ __all__ = [ "Rule", "ModifyMilter"] -__version__ = "1.1.3" +__version__ = "1.1.4" import Milter import logging From 1c949fa6f64d8a677dcf16c629e9f77af6406bf1 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 17 Nov 2020 15:56:55 +0100 Subject: [PATCH 081/145] move runtime patches to separate module --- pymodmilter/__init__.py | 92 +-------------------------- pymodmilter/_runtime_patches.py | 107 ++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 89 deletions(-) create mode 100644 pymodmilter/_runtime_patches.py diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 3753dc5..48d01f1 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -22,105 +22,19 @@ __all__ = [ __version__ = "1.1.4" +import _runtime_patches + import Milter import logging -import encodings from Milter.utils import parse_addr + from email.message import MIMEPart from email.parser import BytesFeedParser from email.policy import default as default_policy from pymodmilter.conditions import Conditions -######################################################## -# monkey-patch pythons email library bug 27257,30988 # -######################################################## -# -# https://bugs.python.org/issue27257 -# https://bugs.python.org/issue30988 -# -# fix: https://github.com/python/cpython/pull/15600 - -import email._header_value_parser -from email._header_value_parser import TokenList, NameAddr -from email._header_value_parser import get_display_name, get_angle_addr -from email._header_value_parser import get_cfws, errors -from email._header_value_parser import CFWS_LEADER, PHRASE_ENDS - - -class DisplayName(email._header_value_parser.DisplayName): - @property - def display_name(self): - res = TokenList(self) - if len(res) == 0: - return res.value - if res[0].token_type == 'cfws': - res.pop(0) - else: - if isinstance(res[0], TokenList) and \ - res[0][0].token_type == 'cfws': - res[0] = TokenList(res[0][1:]) - if res[-1].token_type == 'cfws': - res.pop() - else: - if isinstance(res[-1], TokenList) and \ - res[-1][-1].token_type == 'cfws': - res[-1] = TokenList(res[-1][:-1]) - return res.value - - -def get_name_addr(value): - """ name-addr = [display-name] angle-addr - - """ - name_addr = NameAddr() - # Both the optional display name and the angle-addr can start with cfws. - leader = None - if value[0] in CFWS_LEADER: - leader, value = get_cfws(value) - if not value: - raise errors.HeaderParseError( - "expected name-addr but found '{}'".format(leader)) - if value[0] != '<': - if value[0] in PHRASE_ENDS: - raise errors.HeaderParseError( - "expected name-addr but found '{}'".format(value)) - token, value = get_display_name(value) - if not value: - raise errors.HeaderParseError( - "expected name-addr but found '{}'".format(token)) - if leader is not None: - if isinstance(token[0], TokenList): - token[0][:0] = [leader] - else: - token[:0] = [leader] - leader = None - name_addr.append(token) - token, value = get_angle_addr(value) - if leader is not None: - token[:0] = [leader] - name_addr.append(token) - return name_addr, value - - -setattr(email._header_value_parser, "DisplayName", DisplayName) -setattr(email._header_value_parser, "get_name_addr", get_name_addr) - -################################################ -# add charset alias for windows-874 encoding # -################################################ - -aliases = encodings.aliases.aliases - -for alias in ["windows-874", "windows_874"]: - if alias not in aliases: - aliases[alias] = "cp874" - -setattr(encodings.aliases, "aliases", aliases) - -################################################ - class CustomLogger(logging.LoggerAdapter): def process(self, msg, kwargs): diff --git a/pymodmilter/_runtime_patches.py b/pymodmilter/_runtime_patches.py new file mode 100644 index 0000000..b1a9ad8 --- /dev/null +++ b/pymodmilter/_runtime_patches.py @@ -0,0 +1,107 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +import encodings + + +##################################### +# patch pythons email library # +##################################### +# +# https://bugs.python.org/issue27257 +# https://bugs.python.org/issue30988 +# +# fix: https://github.com/python/cpython/pull/15600 + +import email._header_value_parser +from email._header_value_parser import TokenList, NameAddr +from email._header_value_parser import get_display_name, get_angle_addr +from email._header_value_parser import get_cfws, errors +from email._header_value_parser import CFWS_LEADER, PHRASE_ENDS + + +class DisplayName(email._header_value_parser.DisplayName): + @property + def display_name(self): + res = TokenList(self) + if len(res) == 0: + return res.value + if res[0].token_type == 'cfws': + res.pop(0) + else: + if isinstance(res[0], TokenList) and \ + res[0][0].token_type == 'cfws': + res[0] = TokenList(res[0][1:]) + if res[-1].token_type == 'cfws': + res.pop() + else: + if isinstance(res[-1], TokenList) and \ + res[-1][-1].token_type == 'cfws': + res[-1] = TokenList(res[-1][:-1]) + return res.value + + +def get_name_addr(value): + """ name-addr = [display-name] angle-addr + + """ + name_addr = NameAddr() + # Both the optional display name and the angle-addr can start with cfws. + leader = None + if value[0] in CFWS_LEADER: + leader, value = get_cfws(value) + if not value: + raise errors.HeaderParseError( + "expected name-addr but found '{}'".format(leader)) + if value[0] != '<': + if value[0] in PHRASE_ENDS: + raise errors.HeaderParseError( + "expected name-addr but found '{}'".format(value)) + token, value = get_display_name(value) + if not value: + raise errors.HeaderParseError( + "expected name-addr but found '{}'".format(token)) + if leader is not None: + if isinstance(token[0], TokenList): + token[0][:0] = [leader] + else: + token[:0] = [leader] + leader = None + name_addr.append(token) + token, value = get_angle_addr(value) + if leader is not None: + token[:0] = [leader] + name_addr.append(token) + return name_addr, value + + +setattr(email._header_value_parser, "DisplayName", DisplayName) +setattr(email._header_value_parser, "get_name_addr", get_name_addr) + + +####################################### +# add charset alias for windows-874 # +####################################### +# +# https://bugs.python.org/issue17254 +# +# fix: https://github.com/python/cpython/pull/10237 + +aliases = encodings.aliases.aliases + +for alias in ["windows-874", "windows_874"]: + if alias not in aliases: + aliases[alias] = "cp874" + +setattr(encodings.aliases, "aliases", aliases) From 5a746f56361d5074ff948388eaf701a6782d1bca Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 17 Feb 2021 16:55:08 +0100 Subject: [PATCH 082/145] restructure code and add replace_links --- pymodmilter/__init__.py | 60 ++++++++++++++- pymodmilter/actions.py | 162 ++++++++++++++-------------------------- 2 files changed, 114 insertions(+), 108 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 48d01f1..71a04e4 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -29,9 +29,12 @@ import logging from Milter.utils import parse_addr +from collections import defaultdict + +from email.header import Header from email.message import MIMEPart from email.parser import BytesFeedParser -from email.policy import default as default_policy +from email.policy import default as default_policy, SMTP from pymodmilter.conditions import Conditions @@ -79,7 +82,7 @@ class Rule: break def need_body(self): - """Return the if this rule needs the message body.""" + """Return True if this rule needs the message body.""" return self._need_body def ignores(self, host=None, envfrom=None, envto=None): @@ -142,6 +145,14 @@ class MilterMessage(MIMEPart): self._headers = newheaders +def replace_illegal_chars(string): + """Replace illegal characters in header values.""" + return string.replace( + "\x00", "").replace( + "\r", "").replace( + "\n", "") + + class ModifyMilter(Milter.Base): """ModifyMilter based on Milter.Base to implement milter communication""" @@ -162,6 +173,50 @@ class ModifyMilter(Milter.Base): # save rules, it must not change during runtime self.rules = ModifyMilter._rules.copy() + self.msg = None + + def addheader(self, field, value, idx=-1): + value = replace_illegal_chars(Header(s=value).encode()) + self.logger.debug(f"milter: addheader: {field}: {value}") + super().addheader(field, value, idx) + + def chgheaer(self, field, value, idx=1): + value = replace_illegal_chars(Header(s=value).encode()) + if value: + self.logger.debug(f"milter: chgheader: {field}[{idx}]: {value}") + else: + self.logger.debug(f"milter: delheader: {field}[{idx}]") + super().chgheader(field, idx, value) + + def update_headers(self, old_headers): + if self.msg.is_multipart() and not self.msg["MIME-Version"]: + self.msg.add_header("MIME-Version", "1.0") + + # serialize the message object so it updates its internal strucure + self.msg.as_bytes() + + old_headers = [(f, f.lower(), v) for f, v in old_headers] + headers = [(f, f.lower(), v) for f, v in self.msg.items()] + + idx = defaultdict(int) + for field, field_lower, value in old_headers: + idx[field_lower] += 1 + if (field, field_lower, value) not in headers: + self.chgheader(field, "", idx=idx[field_lower]) + idx[field] -= 1 + + for field, value in self.msg.items(): + field_lower = field.lower() + if (field, field_lower, value) not in old_headers: + self.addheader(field, value) + + def replacebody(self): + data = self.msg.as_bytes(policy=SMTP) + body_pos = data.find(b"\r\n\r\n") + 4 + self.logger.debug("milter: replacebody") + super().replacebody(data[body_pos:]) + del data + def connect(self, IPname, family, hostaddr): try: if hostaddr is None: @@ -210,7 +265,6 @@ class ModifyMilter(Milter.Base): return Milter.CONTINUE - @Milter.noreply def envrcpt(self, to, *str): try: self.recipients.add("@".join(parse_addr(to)).lower()) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 7db1ac3..eae8ae5 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -20,26 +20,9 @@ from bs4 import BeautifulSoup from collections import defaultdict from copy import copy from datetime import datetime -from email.header import Header from email.message import MIMEPart -from email.policy import SMTP -from pymodmilter import CustomLogger, Conditions - - -def _replace_illegal_chars(string): - """Replace illegal characters in header values.""" - return string.replace( - "\x00", "").replace( - "\r", "").replace( - "\n", "") - - -def _add_header(milter, field, value, idx=-1): - value = _replace_illegal_chars( - Header(s=value).encode()) - milter.logger.debug(f"milter: addheader: {field}: {value}") - milter.addheader(field, value, idx) +from pymodmilter import CustomLogger, Conditions, replace_illegal_chars def add_header(milter, field, value, pretend=False, @@ -51,20 +34,10 @@ def add_header(milter, field, value, pretend=False, else: logger.info(f"add_header: {header[0:70]}") - milter.msg.add_header(field, _replace_illegal_chars(value)) + milter.msg.add_header(field, replace_illegal_chars(value)) - if pretend: - return - - _add_header(milter, field, value) - - -def _mod_header(milter, field, value, occ=1): - value = _replace_illegal_chars( - Header(s=value).encode()) - milter.logger.debug( - f"milter: chgheader: {field}[{occ}]: {value}") - milter.chgheader(field, occ, value) + if not pretend: + milter.addheader(field, value) def mod_header(milter, field, value, search=None, pretend=False, @@ -76,11 +49,11 @@ def mod_header(milter, field, value, search=None, pretend=False, if isinstance(search, str): search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE) - occ = defaultdict(int) + idx = defaultdict(int) for i, (f, v) in enumerate(milter.msg.items()): f_lower = f.lower() - occ[f_lower] += 1 + idx[f_lower] += 1 if not field.match(f): continue @@ -109,18 +82,10 @@ def mod_header(milter, field, value, search=None, pretend=False, logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}") milter.msg.replace_header( - f, _replace_illegal_chars(new_value), occ=occ[f_lower]) + f, replace_illegal_chars(new_value), idx=idx[f_lower]) - if pretend: - continue - - _mod_header(milter, f, new_value, occ=occ[f_lower]) - - -def _del_header(milter, field, occ=1): - milter.logger.debug( - f"milter: delheader: {field}[{occ}]") - milter.chgheader(field, occ, "") + if not pretend: + milter.chgheader(f, new_value, idx=idx[f_lower]) def del_header(milter, field, value=None, pretend=False, @@ -132,11 +97,11 @@ def del_header(milter, field, value=None, pretend=False, if isinstance(value, str): value = re.compile(value, re.MULTILINE + re.DOTALL + re.IGNORECASE) - occ = defaultdict(int) + idx = defaultdict(int) for f, v in milter.msg.items(): f_lower = f.lower() - occ[f_lower] += 1 + idx[f_lower] += 1 if not field.match(f): continue @@ -149,32 +114,12 @@ def del_header(milter, field, value=None, pretend=False, logger.debug(f"del_header: {header}") else: logger.info(f"del_header: {header[0:70]}") - milter.msg.remove_header(f, occ=occ[f_lower]) + milter.msg.remove_header(f, idx=idx[f_lower]) if not pretend: - _del_header(milter, f, occ=occ[f_lower]) + milter.chgheader(f, "", idx=idx[f_lower]) - occ[f_lower] -= 1 - - -def _serialize_msg(msg, logger): - if msg.is_multipart() and not msg["MIME-Version"]: - msg.add_header("MIME-Version", "1.0") - - try: - logger.debug("serialize message as bytes") - data = msg.as_bytes(policy=SMTP) - except Exception as e: - logger.waring( - f"unable to serialize message as bytes: {e}") - try: - logger.warning("try to serialize message as string") - data = msg.as_string(policy=SMTP) - data = data.encode("ascii", errors="replace") - except Exception as e: - raise e - - return data + idx[f_lower] -= 1 def _get_body_content(msg, pref): @@ -244,37 +189,6 @@ def _patch_message_body(milter, action, text, html, logger): del html_body["MIME-Version"] -def _update_body(milter, logger): - data = _serialize_msg(milter.msg, logger) - body_pos = data.find(b"\r\n\r\n") + 4 - logger.debug("milter: replacebody") - milter.replacebody(data[body_pos:]) - del data - - -def _update_headers(milter, original_headers, logger): - if milter.msg.is_multipart() and not milter.msg["MIME-Version"]: - milter.msg.add_header("MIME-Version", "1.0") - - # serialize the message object so it updates its internal strucure - milter.msg.as_bytes() - - original_headers = [(f, f.lower(), v) for f, v in original_headers] - headers = [(f, f.lower(), v) for f, v in milter.msg.items()] - - occ = defaultdict(int) - for field, field_lower, value in original_headers: - occ[field_lower] += 1 - if (field, field_lower, value) not in headers: - _del_header(milter, field, occ=occ[field_lower]) - occ[field] -= 1 - - for field, value in milter.msg.items(): - field_lower = field.lower() - if (field, field_lower, value) not in original_headers: - _add_header(milter, field, value) - - def _wrap_message(milter, logger): attachment = MIMEPart() attachment.set_content(milter.msg.as_bytes(), @@ -315,7 +229,7 @@ def _inject_body(milter): def add_disclaimer(milter, text, html, action, policy, pretend=False, logger=logging.getLogger(__name__)): """Append or prepend a disclaimer to the mail body.""" - original_headers = milter.msg.items() + old_headers = milter.msg.items() try: try: @@ -348,11 +262,43 @@ def add_disclaimer(milter, text, html, action, policy, pretend=False, "unable to wrap message in a new message envelope, " "give up ...") - if pretend: - return + if not pretend: + milter.update_headers(old_headers) + milter.replacebody() - _update_headers(milter, original_headers, logger) - _update_body(milter, logger) + +def replace_links(milter, repl, pretend=False, + logger=logging.getLogger(__name__)): + """Replace links in the mail body.""" + + text_body, text_content = _get_body_content(milter.msg, "plain") + html_body, html_content = _get_body_content(milter.msg, "html") + + if text_content is not None: + logger.info("replace links in text body") + + content = text_content + + text_body.set_content( + content.encode(), maintype="text", subtype="plain") + text_body.set_param("charset", "UTF-8", header="Content-Type") + del text_body["MIME-Version"] + + if html_content is not None: + logger.info("replace links in html body") + + soup = BeautifulSoup(html_content, "html.parser") + + for link in soup.find_all("a", href=True): + link["href"] = repl + + html_body.set_content( + str(soup).encode(), maintype="text", subtype="html") + html_body.set_param("charset", "UTF-8", header="Content-Type") + del html_body["MIME-Version"] + + if not pretend: + milter.replacebody() def store(milter, directory, pretend=False, @@ -376,6 +322,7 @@ class Action: "del_header": False, "mod_header": False, "add_disclaimer": True, + "replace_links": True, "store": True} def __init__(self, name, local_addrs, conditions, action_type, args, @@ -451,6 +398,11 @@ class Action: self._args["text"] = f.read() except IOError as e: raise RuntimeError(f"unable to read template: {e}") + + elif action_type == "replace_links": + self._func = replace_links + self._args["repl"] = args["repl"] + elif action_type == "store": self._func = store if args["storage_type"] not in ["file"]: From 0db61ed8335f302798cc98363c0cc48a66958423 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 17 Feb 2021 18:08:39 +0100 Subject: [PATCH 083/145] fix import runtime patches, call milter.replacebody only once per mail --- pymodmilter/__init__.py | 45 ++++++++++++++++++++++++----------------- pymodmilter/actions.py | 2 +- pymodmilter/run.py | 2 ++ 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 71a04e4..719760e 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -22,7 +22,7 @@ __all__ = [ __version__ = "1.1.4" -import _runtime_patches +from pymodmilter import _runtime_patches import Milter import logging @@ -116,13 +116,13 @@ class Rule: class MilterMessage(MIMEPart): - def replace_header(self, _name, _value, occ=None): + def replace_header(self, _name, _value, idx=None): _name = _name.lower() counter = 0 for i, (k, v) in zip(range(len(self._headers)), self._headers): if k.lower() == _name: counter += 1 - if not occ or counter == occ: + if not idx or counter == idx: self._headers[i] = self.policy.header_store_parse( k, _value) break @@ -130,14 +130,14 @@ class MilterMessage(MIMEPart): else: raise KeyError(_name) - def remove_header(self, name, occ=None): + def remove_header(self, name, idx=None): name = name.lower() newheaders = [] counter = 0 for k, v in self._headers: if k.lower() == name: counter += 1 - if counter != occ: + if counter != idx: newheaders.append((k, v)) else: newheaders.append((k, v)) @@ -163,6 +163,7 @@ class ModifyMilter(Milter.Base): def set_rules(rules): ModifyMilter._rules = rules + @staticmethod def set_loglevel(level): ModifyMilter._loglevel = level @@ -174,13 +175,14 @@ class ModifyMilter(Milter.Base): self.rules = ModifyMilter._rules.copy() self.msg = None + self._replace_body = False def addheader(self, field, value, idx=-1): value = replace_illegal_chars(Header(s=value).encode()) self.logger.debug(f"milter: addheader: {field}: {value}") super().addheader(field, value, idx) - def chgheaer(self, field, value, idx=1): + def chgheader(self, field, value, idx=1): value = replace_illegal_chars(Header(s=value).encode()) if value: self.logger.debug(f"milter: chgheader: {field}[{idx}]: {value}") @@ -211,11 +213,7 @@ class ModifyMilter(Milter.Base): self.addheader(field, value) def replacebody(self): - data = self.msg.as_bytes(policy=SMTP) - body_pos = data.find(b"\r\n\r\n") + 4 - self.logger.debug("milter: replacebody") - super().replacebody(data[body_pos:]) - del data + self._replace_body = True def connect(self, IPname, family, hostaddr): try: @@ -348,19 +346,30 @@ class ModifyMilter(Milter.Base): def eom(self): try: self.msg = self._fp.close() + milter_action = None for rule in self.rules: milter_action = rule.execute(self) if milter_action is not None: - if milter_action["action"] == "reject": - self.setreply("554", "5.7.0", milter_action["reason"]) - return Milter.REJECT + break - if milter_action["action"] == "accept": - return Milter.ACCEPT + if self._replace_body: + data = self.msg.as_bytes(policy=SMTP) + body_pos = data.find(b"\r\n\r\n") + 4 + self.logger.debug("milter: replacebody") + super().replacebody(data[body_pos:]) + del data - if milter_action["action"] == "discard": - return Milter.DISCARD + if milter_action is not None: + if milter_action["action"] == "reject": + self.setreply("554", "5.7.0", milter_action["reason"]) + return Milter.REJECT + + if milter_action["action"] == "accept": + return Milter.ACCEPT + + if milter_action["action"] == "discard": + return Milter.DISCARD except Exception as e: self.logger.exception( diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index eae8ae5..c5fac46 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -277,7 +277,7 @@ def replace_links(milter, repl, pretend=False, if text_content is not None: logger.info("replace links in text body") - content = text_content + content = "test content" text_body.set_content( content.encode(), maintype="text", subtype="plain") diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 6895630..4c174ab 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -260,9 +260,11 @@ def main(): rc = 0 try: Milter.runmilter("pymodmilter", socketname=socket, timeout=30) + logger.info("pymodmilter stopped") except Milter.milter.error as e: logger.error(e) rc = 255 + sys.exit(rc) From 6665b1321a2691e819502060c41e584368e66570 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 4 Mar 2021 17:10:44 +0100 Subject: [PATCH 084/145] rename replace_links to rewrite_links and some improvements --- pymodmilter/actions.py | 55 +++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index c5fac46..96283d1 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -16,6 +16,7 @@ import logging import os import re +from base64 import b64encode from bs4 import BeautifulSoup from collections import defaultdict from copy import copy @@ -267,38 +268,38 @@ def add_disclaimer(milter, text, html, action, policy, pretend=False, milter.replacebody() -def replace_links(milter, repl, pretend=False, - logger=logging.getLogger(__name__)): - """Replace links in the mail body.""" +def rewrite_links(milter, repl, pretend=False, + logger=logging.getLogger(__name__)): + """Rewrite link targets in the mail html body.""" - text_body, text_content = _get_body_content(milter.msg, "plain") html_body, html_content = _get_body_content(milter.msg, "html") - - if text_content is not None: - logger.info("replace links in text body") - - content = "test content" - - text_body.set_content( - content.encode(), maintype="text", subtype="plain") - text_body.set_param("charset", "UTF-8", header="Content-Type") - del text_body["MIME-Version"] - if html_content is not None: - logger.info("replace links in html body") - soup = BeautifulSoup(html_content, "html.parser") + rewritten = 0 for link in soup.find_all("a", href=True): - link["href"] = repl + if not link["href"]: + continue - html_body.set_content( - str(soup).encode(), maintype="text", subtype="html") - html_body.set_param("charset", "UTF-8", header="Content-Type") - del html_body["MIME-Version"] + if "{URL_B64}" in repl: + url_b64 = b64encode(link["href"].encode()).decode() + target = repl.replace("{URL_B64}", url_b64) + else: + target = repl - if not pretend: - milter.replacebody() + link["href"] = target + rewritten += 1 + + if rewritten: + logger.info(f"rewrote {rewritten} link(s) in html body") + + html_body.set_content( + str(soup).encode(), maintype="text", subtype="html") + html_body.set_param("charset", "UTF-8", header="Content-Type") + del html_body["MIME-Version"] + + if not pretend: + milter.replacebody() def store(milter, directory, pretend=False, @@ -322,7 +323,7 @@ class Action: "del_header": False, "mod_header": False, "add_disclaimer": True, - "replace_links": True, + "rewrite_links": True, "store": True} def __init__(self, name, local_addrs, conditions, action_type, args, @@ -399,8 +400,8 @@ class Action: except IOError as e: raise RuntimeError(f"unable to read template: {e}") - elif action_type == "replace_links": - self._func = replace_links + elif action_type == "rewrite_links": + self._func = rewrite_links self._args["repl"] = args["repl"] elif action_type == "store": From 915fa509b54de7a2d1c2bf7c483228ab73d4d646 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 5 Mar 2021 17:11:26 +0100 Subject: [PATCH 085/145] add config.py with config classes --- pymodmilter/config.py | 332 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 332 insertions(+) create mode 100644 pymodmilter/config.py diff --git a/pymodmilter/config.py b/pymodmilter/config.py new file mode 100644 index 0000000..2c4a705 --- /dev/null +++ b/pymodmilter/config.py @@ -0,0 +1,332 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +__all__ = [ + "BaseConfig", + "ActionConfig", + "RuleConfig", + "ModifyMilterConfig"] + +import json +import logging +import re + +from bs4 import BeautifulSoup +from netaddr import IPNetwork, AddrFormatError + + +class BaseConfig: + def __init__(self, cfg={}, debug=False): + self._cfg = {} + if "name" in cfg: + assert isinstance(cfg["name"], str), \ + "rule: name: invalid value, should be string" + self["name"] = cfg["name"] + else: + self["name"] = "" + + if debug: + self["loglevel"] = logging.DEBUG + elif "loglevel" in cfg: + level = getattr(logging, cfg["loglevel"].upper(), None) + assert isinstance(level, int), \ + f"{self['name']}: loglevel: invalid value" + self["loglevel"] = level + else: + self["loglevel"] = logging.INFO + + # the keys/values of args are used as parameters + # to functions + self["args"] = {} + + def __setitem__(self, key, value): + self._cfg[key] = value + + def __getitem__(self, key): + return self._cfg[key] + + def __delitem__(self, key): + del self._cfg[key] + + def __contains__(self, key): + return key in self._cfg + + def add_string_arg(self, cfg, args): + if isinstance(args, str): + args = [args] + + for arg in args: + assert arg in cfg, \ + f"{self['name']}: mandatory parameter '{arg}' not found" + assert isinstance(cfg[arg], str), \ + f"{self['name']}: {arg}: invalid value, should be string" + self["args"][arg] = cfg[arg] + + def add_bool_arg(self, cfg, args): + if isinstance(args, str): + args = [args] + + for arg in args: + assert arg in cfg, \ + f"{self['name']}: mandatory parameter '{arg}' not found" + assert isinstance(cfg[arg], bool), \ + f"{self['name']}: {arg}: invalid value, should be bool" + self["args"][arg] = cfg[arg] + + +class ConditionsConfig(BaseConfig): + def __init__(self, parent_cfg, cfg, debug): + if "loglevel" not in cfg: + cfg["loglevel"] = parent_cfg["loglevel"] + + cfg["name"] = f"{parent_cfg['name']}: condition" + + super().__init__(cfg, debug) + + if "local" in cfg: + self.add_bool_arg(cfg, "local") + + if "hosts" in cfg: + hosts = cfg["hosts"] + assert isinstance(hosts, list) and all( + [isinstance(host, str) for host in hosts]), \ + f"{self['name']}: hosts: invalid value, " \ + f"should be list of strings" + + self["args"]["hosts"] = [] + try: + for host in cfg["hosts"]: + self["args"]["hosts"].append(IPNetwork(host)) + except AddrFormatError as e: + raise ValueError(f"{self['name']}: hosts: {e}") + + for arg in ("envfrom", "envto"): + if arg in cfg: + self.add_string_arg(cfg, arg) + try: + self["args"][arg] = re.compile( + self["args"][arg], + re.IGNORECASE) + except re.error as e: + raise ValueError(f"{self['name']}: {arg}: {e}") + + +class ActionConfig(BaseConfig): + def __init__(self, idx, rule_cfg, cfg, debug): + if "name" in cfg: + cfg["name"] = f"{rule_cfg['name']}: {cfg['name']}" + else: + cfg["name"] = f"{rule_cfg['name']}: Action #{idx}" + + if "loglevel" not in cfg: + cfg["loglevel"] = rule_cfg["loglevel"] + + super().__init__(cfg, debug) + + self["pretend"] = rule_cfg["pretend"] + self["conditions"] = None + self["type"] = "" + self["need_body"] = False + + if "pretend" in cfg: + pretend = cfg["pretend"] + assert isinstance(pretend, bool), \ + f"{self['name']}: pretend: invalid value, should be bool" + self["pretend"] = pretend + + assert "type" in cfg, \ + f"{self['name']}: type: invalid value, should be string" + assert cfg["type"] in \ + ("add_header", "del_header", "mod_header", "add_disclaimer", + "rewrite_links", "store"), \ + f"{self['name']}: type: invalid action type" + self["type"] = cfg["type"] + + if self["type"] == "add_header": + self.add_string_arg(cfg, ("field", "value")) + + elif self["type"] == "mod_header": + args = ["field", "value"] + if "search" in cfg: + args.append("search") + + for arg in args: + self.add_string_arg(cfg, arg) + if arg in ("field", "search"): + try: + self["args"][arg] = re.compile( + self["args"][arg], + re.MULTILINE + re.DOTALL + re.IGNORECASE) + except re.error as e: + raise ValueError(f"{self['name']}: {arg}: {e}") + + elif self["type"] == "del_header": + args = ["field"] + if "value" in cfg: + args.append("value") + + for arg in args: + self.add_string_arg(cfg, arg) + try: + self["args"][arg] = re.compile( + self["args"][arg], + re.MULTILINE + re.DOTALL + re.IGNORECASE) + except re.error as e: + raise ValueError(f"{self['name']}: {arg}: {e}") + + elif self["type"] == "add_disclaimer": + if "error_policy" not in cfg: + cfg["error_policy"] = "wrap" + + self.add_string_arg( + cfg, ("action", "html_template", "text_template", + "error_policy")) + assert self["args"]["action"] in ("append", "prepend"), \ + f"{self['name']}: action: invalid value, " \ + f"should be 'append' or 'prepend'" + assert self["args"]["error_policy"] in ("wrap", + "ignore", + "reject"), \ + f"{self['name']}: error_policy: invalid value, " \ + f"should be 'wrap', 'ignore' or 'reject'" + + try: + with open(self["args"]["html_template"], "r") as f: + html = BeautifulSoup(f.read(), "html.parser") + body = html.find('body') + if body: + # just use content within the body tag if present + html = body + self["args"]["html_template"] = html + + with open(self["args"]["text_template"], "r") as f: + self["args"]["text_template"] = f.read() + + except IOError as e: + raise RuntimeError( + f"{self['name']}: unable to open/read template file: {e}") + + self["need_body"] = True + + elif self["type"] == "rewrite_links": + self.add_string_arg(cfg, "repl") + self["need_body"] = True + + elif self["type"] == "store": + self.add_string_arg(cfg, "storage_type") + assert self["storage_type"] in ("file"), \ + f"{self['name']}: storage_type: invalid value, " \ + f"should be 'file'" + + if self["args"]["storage_type"] == "file": + self.add_string_arg(cfg, "directory") + + self["need_body"] = True + + +class RuleConfig(BaseConfig): + def __init__(self, idx, milter_cfg, cfg, debug=False): + if "name" not in cfg: + cfg["name"] = f"Rule #{idx}" + + if "loglevel" not in cfg: + cfg["loglevel"] = milter_cfg["loglevel"] + + super().__init__(cfg, debug) + + self["pretend"] = milter_cfg["pretend"] + self["conditions"] = None + self["actions"] = [] + + if "pretend" in cfg: + pretend = cfg["pretend"] + assert isinstance(pretend, bool), \ + f"{self['name']}: pretend: invalid value, should be bool" + self["pretend"] = pretend + + assert "actions" in cfg, \ + f"{self['name']}: mandatory parameter 'actions' not found" + actions = cfg["actions"] + assert isinstance(actions, list), \ + f"{self['name']}: actions: invalid value, should be list" + + for idx, action_cfg in enumerate(cfg["actions"]): + self["actions"].append( + ActionConfig(idx, self, action_cfg, debug)) + + +class ModifyMilterConfig(BaseConfig): + def __init__(self, cfgfile, debug=False): + try: + with open(cfgfile, "r") as fh: + # remove lines with leading # (comments), they + # are not allowed in json + cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read()) + except IOError as e: + raise RuntimeError(f"unable to open/read config file: {e}") + + try: + cfg = json.loads(cfg) + except json.JSONDecodeError as e: + cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] + msg = "\n".join(cfg_text) + e.msg = f"{msg}\n{e.msg}" + raise e + + if "global" in cfg: + assert isinstance(cfg["global"], dict), \ + "global: invalid type, should be dict" + + super().__init__(cfg["global"], debug) + + if "pretend" in cfg["global"]: + pretend = cfg["global"]["pretend"] + assert isinstance(pretend, bool), \ + "global: pretend: invalid value, should be bool" + self["pretend"] = pretend + else: + self["pretend"] = False + + if "socket" in cfg["global"]: + socket = cfg["global"]["socket"] + assert isinstance(socket, str), \ + "global: socket: invalid value, should be string" + self["socket"] = socket + else: + self["socket"] = None + + if "local_addrs" in cfg["global"]: + local_addrs = cfg["global"]["local_addrs"] + assert isinstance(local_addrs, list) and all( + [isinstance(addr, str) for addr in local_addrs]), \ + "global: local_addrs: invalid value, " \ + "should be list of strings" + self["local_addrs"] = local_addrs + else: + self["local_addrs"] = [ + "::1/128", + "127.0.0.0/8", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16"] + + assert "rules" in cfg, \ + "mandatory parameter 'rules' not found" + assert isinstance(cfg["rules"], list), \ + "rules: invalid value, should be list" + + self["rules"] = [] + for idx, rule_cfg in enumerate(cfg["rules"]): + self["rules"].append( + RuleConfig(idx, self, rule_cfg, debug)) From b4986af1c2b04f09fb5b7717a782ceb688d5398e Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Mar 2021 12:14:48 +0100 Subject: [PATCH 086/145] switch to new config objects --- pymodmilter/__init__.py | 48 +++-- pymodmilter/actions.py | 129 ++++--------- pymodmilter/conditions.py | 56 +----- pymodmilter/config.py | 36 ++-- pymodmilter/docs/pymodmilter.conf.example | 20 +- pymodmilter/run.py | 155 ++-------------- pymodmilter/test.conf | 214 ++++++++++++++++++++++ pymodmilter/test.py | 5 + 8 files changed, 331 insertions(+), 332 deletions(-) create mode 100644 pymodmilter/test.conf create mode 100755 pymodmilter/test.py diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 719760e..03e7e7e 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -36,6 +36,7 @@ from email.message import MIMEPart from email.parser import BytesFeedParser from email.policy import default as default_policy, SMTP +from pymodmilter.actions import Action from pymodmilter.conditions import Conditions @@ -58,28 +59,26 @@ class Rule: Rule to implement multiple actions on emails. """ - def __init__(self, name, local_addrs, conditions, actions, pretend=False, - loglevel=logging.INFO): - logger = logging.getLogger(name) - self.logger = CustomLogger(logger, {"name": name}) - self.logger.setLevel(loglevel) + def __init__(self, milter_cfg, cfg): + logger = logging.getLogger(cfg["name"]) + self.logger = CustomLogger(logger, {"name": cfg["name"]}) + self.logger.setLevel(cfg["loglevel"]) - if logger is None: - logger = logging.getLogger(__name__) - - self.logger = CustomLogger(logger, {"name": name}) - self.conditions = Conditions( - local_addrs=local_addrs, - args=conditions, - logger=self.logger) - self.actions = actions - self.pretend = pretend + if cfg["conditions"] is None: + self.conditions = None + else: + self.conditions = Conditions(milter_cfg, cfg["conditions"]) self._need_body = False - for action in actions: + + self.actions = [] + for action_cfg in cfg["actions"]: + action = Action(milter_cfg, action_cfg) + self.actions.append(action) if action.need_body(): self._need_body = True - break + + self.pretend = cfg["pretend"] def need_body(self): """Return True if this rule needs the message body.""" @@ -97,9 +96,9 @@ class Rule: if envto is not None: args["envto"] = envto - if self.conditions.match(args): + if self.conditions is None or self.conditions.match(args): for action in self.actions: - if action.conditions.match(args): + if action.conditions is None or action.conditions.match(args): return False return True @@ -160,12 +159,11 @@ class ModifyMilter(Milter.Base): _loglevel = logging.INFO @staticmethod - def set_rules(rules): - ModifyMilter._rules = rules - - @staticmethod - def set_loglevel(level): - ModifyMilter._loglevel = level + def set_config(cfg): + ModifyMilter._loglevel = cfg["loglevel"] + for rule_cfg in cfg["rules"]: + ModifyMilter._rules.append( + Rule(cfg, rule_cfg)) def __init__(self): self.logger = logging.getLogger(__name__) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 96283d1..1aa26a5 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -269,7 +269,7 @@ def add_disclaimer(milter, text, html, action, policy, pretend=False, def rewrite_links(milter, repl, pretend=False, - logger=logging.getLogger(__name__)): + logger=logging.getLogger(__name__)): """Rewrite link targets in the mail html body.""" html_body, html_content = _get_body_content(milter.msg, "html") @@ -318,106 +318,41 @@ def store(milter, directory, pretend=False, class Action: """Action to implement a pre-configured action to perform on e-mails.""" - _need_body_map = { - "add_header": False, - "del_header": False, - "mod_header": False, - "add_disclaimer": True, - "rewrite_links": True, - "store": True} - def __init__(self, name, local_addrs, conditions, action_type, args, - loglevel=logging.INFO, pretend=False): - logger = logging.getLogger(name) - self.logger = CustomLogger(logger, {"name": name}) - self.logger.setLevel(loglevel) + def __init__(self, milter_cfg, cfg): + logger = logging.getLogger(cfg["name"]) + self.logger = CustomLogger(logger, {"name": cfg["name"]}) + self.logger.setLevel(cfg["loglevel"]) - self.conditions = Conditions( - local_addrs=local_addrs, - args=conditions, - logger=self.logger) - self.pretend = pretend - self._args = {} + if cfg["conditions"] is None: + self.conditions = None + else: + self.conditions = Conditions(milter_cfg, cfg["conditions"]) - if action_type not in self._need_body_map: - raise RuntimeError(f"invalid action type '{action_type}'") - self._need_body = self._need_body_map[action_type] + self.pretend = cfg["pretend"] + self._args = cfg["args"] - try: - if action_type == "add_header": - self._func = add_header - self._args["field"] = args["header"] - self._args["value"] = args["value"] - if "idx" in args: - self._args["idx"] = args["idx"] - - elif action_type in ["mod_header", "del_header"]: - args["field"] = args["header"] - del args["header"] - regex_args = ["field"] - - if action_type == "mod_header": - self._func = mod_header - self._args["value"] = args["value"] - regex_args.append("search") - elif action_type == "del_header": - self._func = del_header - if "value" in args: - regex_args.append("value") - - for arg in regex_args: - try: - self._args[arg] = re.compile( - args[arg], - re.MULTILINE + re.DOTALL + re.IGNORECASE) - except re.error as e: - raise RuntimeError( - f"unable to parse {arg} regex: {e}") - - elif action_type == "add_disclaimer": - self._func = add_disclaimer - if args["action"] not in ["append", "prepend"]: - raise RuntimeError(f"invalid action '{args['action']}'") - - self._args["action"] = args["action"] - - if args["error_policy"] not in ["wrap", "ignore", "reject"]: - raise RuntimeError(f"invalid policy '{args['policy']}'") - - self._args["policy"] = args["error_policy"] - - try: - with open(args["html_file"], "r") as f: - html = BeautifulSoup( - f.read(), "html.parser") - body = html.find('body') - if body: - # just use content within the body tag if present - html = body - self._args["html"] = html - with open(args["text_file"], "r") as f: - self._args["text"] = f.read() - except IOError as e: - raise RuntimeError(f"unable to read template: {e}") - - elif action_type == "rewrite_links": - self._func = rewrite_links - self._args["repl"] = args["repl"] - - elif action_type == "store": - self._func = store - if args["storage_type"] not in ["file"]: - raise RuntimeError( - "invalid storage_type 'args['storage_type']'") - - if args["storage_type"] == "file": - self._args["directory"] = args["directory"] - else: - raise RuntimeError(f"invalid action type: {action_type}") - - except KeyError as e: - raise RuntimeError( - f"mandatory argument not found: {e}") + action_type = cfg["type"] + if action_type == "add_header": + self._func = add_header + self._need_body = False + elif action_type == "mod_header": + self._func = mod_header + self._need_body = False + elif action_type == "del_header": + self._func = del_header + self._need_body = False + elif action_type == "add_disclaimer": + self._func = add_disclaimer + self._need_body = True + elif action_type == "rewrite_links": + self._func = rewrite_links + self._need_body = True + elif action_type == "store": + self._func = store + self._need_body = True + else: + raise ValueError(f"invalid action type: {action_type}") def need_body(self): """Return the needs of this action.""" diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index c0fbf9a..335baf5 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -13,61 +13,21 @@ # import logging -import re -from netaddr import IPAddress, IPNetwork, AddrFormatError +from netaddr import IPAddress +from pymodmilter import CustomLogger class Conditions: """Conditions to implement conditions for rules and actions.""" - def __init__(self, local_addrs, args, logger=None): - if logger is None: - logger = logging.getLogger(__name__) + def __init__(self, milter_cfg, cfg): + logger = logging.getLogger(cfg["name"]) + self.logger = CustomLogger(logger, {"name": cfg["name"]}) + self.logger.setLevel(cfg["loglevel"]) - self._local_addrs = [] - self.logger = logger - self._args = {} - - try: - for addr in local_addrs: - self._local_addrs.append(IPNetwork(addr)) - except AddrFormatError as e: - raise RuntimeError(f"invalid address in local_addrs: {e}") - - try: - if "local" in args: - logger.debug(f"condition: local = {args['local']}") - self._args["local"] = args["local"] - - if "hosts" in args: - logger.debug(f"condition: hosts = {args['hosts']}") - self._args["hosts"] = [] - try: - for host in args["hosts"]: - self._args["hosts"].append(IPNetwork(host)) - except AddrFormatError as e: - raise RuntimeError(f"invalid address in hosts: {e}") - - if "envfrom" in args: - logger.debug(f"condition: envfrom = {args['envfrom']}") - try: - self._args["envfrom"] = re.compile( - args["envfrom"], re.IGNORECASE) - except re.error as e: - raise RuntimeError(f"unable to parse envfrom regex: {e}") - - if "envto" in args: - logger.debug(f"condition: envto = {args['envto']}") - try: - self._args["envto"] = re.compile( - args["envto"], re.IGNORECASE) - except re.error as e: - raise RuntimeError(f"unable to parse envto regex: {e}") - - except KeyError as e: - raise RuntimeError( - f"mandatory argument not found: {e}") + self._local_addrs = milter_cfg["local_addrs"] + self._args = cfg["args"] def match(self, args): if "host" in args: diff --git a/pymodmilter/config.py b/pymodmilter/config.py index 2c4a705..3d1b651 100644 --- a/pymodmilter/config.py +++ b/pymodmilter/config.py @@ -27,7 +27,12 @@ from netaddr import IPNetwork, AddrFormatError class BaseConfig: - def __init__(self, cfg={}, debug=False): + def __init__(self, cfg={}, debug=False, logger=None): + if logger is None: + logger = logging.getLogger(__name__) + + self.logger = logger + self._cfg = {} if "name" in cfg: assert isinstance(cfg["name"], str), \ @@ -39,10 +44,13 @@ class BaseConfig: if debug: self["loglevel"] = logging.DEBUG elif "loglevel" in cfg: - level = getattr(logging, cfg["loglevel"].upper(), None) - assert isinstance(level, int), \ - f"{self['name']}: loglevel: invalid value" - self["loglevel"] = level + if isinstance(cfg["loglevel"], int): + self["loglevel"] = cfg["loglevel"] + else: + level = getattr(logging, cfg["loglevel"].upper(), None) + assert isinstance(level, int), \ + f"{self['name']}: loglevel: invalid value" + self["loglevel"] = level else: self["loglevel"] = logging.INFO @@ -137,7 +145,6 @@ class ActionConfig(BaseConfig): self["pretend"] = rule_cfg["pretend"] self["conditions"] = None self["type"] = "" - self["need_body"] = False if "pretend" in cfg: pretend = cfg["pretend"] @@ -217,22 +224,23 @@ class ActionConfig(BaseConfig): raise RuntimeError( f"{self['name']}: unable to open/read template file: {e}") - self["need_body"] = True - elif self["type"] == "rewrite_links": self.add_string_arg(cfg, "repl") - self["need_body"] = True elif self["type"] == "store": self.add_string_arg(cfg, "storage_type") - assert self["storage_type"] in ("file"), \ + assert self["args"]["storage_type"] in ("file"), \ f"{self['name']}: storage_type: invalid value, " \ f"should be 'file'" if self["args"]["storage_type"] == "file": self.add_string_arg(cfg, "directory") - self["need_body"] = True + if "conditions" in cfg: + conditions = cfg["conditions"] + assert isinstance(conditions, dict), \ + f"{self['name']}: conditions: invalid value, should be dict" + self["conditions"] = ConditionsConfig(self, conditions, debug) class RuleConfig(BaseConfig): @@ -265,6 +273,12 @@ class RuleConfig(BaseConfig): self["actions"].append( ActionConfig(idx, self, action_cfg, debug)) + if "conditions" in cfg: + conditions = cfg["conditions"] + assert isinstance(conditions, dict), \ + f"{self['name']}: conditions: invalid value, should be dict" + self["conditions"] = ConditionsConfig(self, conditions, debug) + class ModifyMilterConfig(BaseConfig): def __init__(self, cfgfile, debug=False): diff --git a/pymodmilter/docs/pymodmilter.conf.example b/pymodmilter/docs/pymodmilter.conf.example index 04e887f..98967ea 100644 --- a/pymodmilter/docs/pymodmilter.conf.example +++ b/pymodmilter/docs/pymodmilter.conf.example @@ -111,12 +111,12 @@ # "type": "add_header", - # Option: header + # Option: field # Type: String # Notes: Name of the header. # Value: [ NAME ] # - "header": "X-Test-Header", + "field": "X-Test-Header", # Option: value # Type: String @@ -129,12 +129,12 @@ "type": "mod_header", - # Option: header + # Option: field # Type: String # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). # Value: [ REGEX ] # - "header": "^Subject$", + "field": "^Subject$", # Option: search # Type: String @@ -153,12 +153,12 @@ "type": "del_header", - # Option: header + # Option: field # Type: String # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). # Value: [ REGEX ] # - "header": "^Received$" + "field": "^Received$" }, { "name": "add_disclaimer", @@ -171,19 +171,19 @@ # "action": "prepend", - # Option: html_file + # Option: html_template # Type: String # Notes: Path to a file which contains the html representation of the disclaimer. # Value: [ FILE_PATH ] # - "html_file": "/etc/pymodmilter/templates/disclaimer_html.template", + "html_template": "/etc/pymodmilter/templates/disclaimer_html.template", - # Option: text_file + # Option: text_template # Type: String # Notes: Path to a file which contains the text representation of the disclaimer. # Value: [ FILE_PATH ] # - "text_file": "/etc/pymodmilter/templates/disclaimer_text.template", + "text_template": "/etc/pymodmilter/templates/disclaimer_text.template", # Option: error_policy # Type: String diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 4c174ab..b89bebe 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -18,12 +18,9 @@ import logging import logging.handlers import sys -from json import loads -from re import sub - -from pymodmilter import Rule, ModifyMilter +from pymodmilter import ModifyMilter from pymodmilter import __version__ as version -from pymodmilter.actions import Action +from pymodmilter.config import ModifyMilterConfig def main(): @@ -63,13 +60,6 @@ def main(): args = parser.parse_args() - loglevels = { - "error": logging.ERROR, - "warning": logging.WARNING, - "info": logging.INFO, - "debug": logging.DEBUG - } - root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) @@ -92,148 +82,32 @@ def main(): logger.setLevel(logging.INFO) try: - try: - with open(args.config, "r") as fh: - config = sub(r"(?m)^\s*#.*\n?", "", fh.read()) - config = loads(config) - except Exception as e: - for num, line in enumerate(config.splitlines()): - logger.error(f"{num+1}: {line}") - raise RuntimeError( - f"unable to parse config file: {e}") - - if "global" not in config: - config["global"] = {} - - if args.debug: - loglevel = logging.DEBUG - else: - if "loglevel" not in config["global"]: - config["global"]["loglevel"] = "info" - loglevel = loglevels[config["global"]["loglevel"]] - - logger.setLevel(loglevel) - logger.debug("prepar milter configuration") + cfg = ModifyMilterConfig(args.cfgfile, args.debug) - if "pretend" not in config["global"]: - config["global"]["pretend"] = False + if not args.debug: + logger.setLevel(cfg["loglevel"]) if args.socket: socket = args.socket - elif "socket" in config["global"]: - socket = config["global"]["socket"] + elif "socket" in cfg: + socket = cfg["socket"] else: raise RuntimeError( "listening socket is neither specified on the command line " "nor in the configuration file") - if "local_addrs" in config["global"]: - local_addrs = config["global"]["local_addrs"] - else: - local_addrs = [ - "::1/128", - "127.0.0.0/8", - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16"] - - if "rules" not in config: - raise RuntimeError( - "mandatory config section 'rules' not found") - - if not config["rules"]: + if not cfg["rules"]: raise RuntimeError("no rules configured") - logger.debug("initialize rules ...") + logger.debug("initializing rules ...") - rules = [] - for rule_idx, rule in enumerate(config["rules"]): - if "name" in rule: - rule_name = rule["name"] - else: - rule_name = f"Rule #{rule_idx}" - - logger.debug(f"prepare rule {rule_name} ...") - - if "actions" not in rule: + for rule_cfg in cfg["rules"]: + if not rule_cfg["actions"]: raise RuntimeError( - f"{rule_name}: mandatory config " - f"section 'actions' not found") + f"{rule_cfg['name']}: no actions configured") - if not rule["actions"]: - raise RuntimeError("{rule_name}: no actions configured") - - if args.debug: - rule_loglevel = logging.DEBUG - elif "loglevel" in rule: - rule_loglevel = loglevels[rule["loglevel"]] - else: - rule_loglevel = loglevels[config["global"]["loglevel"]] - - if "pretend" in rule: - rule_pretend = rule["pretend"] - else: - rule_pretend = config["global"]["pretend"] - - actions = [] - for action_idx, action in enumerate(rule["actions"]): - if "name" in action: - action_name = f"{rule_name}: {action['name']}" - else: - action_name = f"Action #{action_idx}" - - if args.debug: - action_loglevel = logging.DEBUG - elif "loglevel" in action: - action_loglevel = loglevels[action["loglevel"]] - else: - action_loglevel = rule_loglevel - - if "pretend" in action: - action_pretend = action["pretend"] - else: - action_pretend = rule_pretend - - if "type" not in action: - raise RuntimeError( - f"{rule_name}: {action_name}: mandatory config " - f"section 'actions' not found") - - if "conditions" not in action: - action["conditions"] = {} - - try: - actions.append( - Action( - name=action_name, - local_addrs=local_addrs, - conditions=action["conditions"], - action_type=action["type"], - args=action, - loglevel=action_loglevel, - pretend=action_pretend)) - except RuntimeError as e: - logger.error(f"{action_name}: {e}") - sys.exit(253) - - if "conditions" not in rule: - rule["conditions"] = {} - - try: - rules.append( - Rule( - name=rule_name, - local_addrs=local_addrs, - conditions=rule["conditions"], - actions=actions, - loglevel=rule_loglevel, - pretend=rule_pretend)) - except RuntimeError as e: - logger.error(f"{rule_name}: {e}") - sys.exit(254) - - except RuntimeError as e: + except (RuntimeError, AssertionError) as e: logger.error(e) sys.exit(255) @@ -247,8 +121,7 @@ def main(): stdouthandler.setLevel(logging.DEBUG) logger.info("pymodmilter starting") - ModifyMilter.set_rules(rules) - ModifyMilter.set_loglevel(loglevel) + ModifyMilter.set_config(cfg) # register milter factory class Milter.factory = ModifyMilter diff --git a/pymodmilter/test.conf b/pymodmilter/test.conf new file mode 100644 index 0000000..998e537 --- /dev/null +++ b/pymodmilter/test.conf @@ -0,0 +1,214 @@ +# This is an example /etc/pymodmilter.conf file. +# Copy it into place before use. +# +# The file is in JSON format. +# +# The global option 'log' can be overriden per rule or per modification. +# +{ + # Section: global + # Notes: Global options. + # + "global": { + # Option: socket + # Type: String + # Notes: The socket used to communicate with the MTA. + # + # Examples: + # unix:/path/to/socket a named pipe + # inet:8899 listen on ANY interface + # inet:8899@localhost listen on a specific interface + # inet6:8899 listen on ANY interface + # inet6:8899@[2001:db8:1234::1] listen on a specific interface + # Value: [ SOCKET ] + "socket": "inet:8898@127.0.0.1", + + # Option: local_addrs + # Type: List + # Notes: A list of local hosts and networks. + # Value: [ LIST ] + # + "local_addrs": ["::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], + + # Option: loglevel + # Type: String + # Notes: Set loglevel for rules and actions. + # Value: [ error | warning | info | debug ] + # + "loglevel": "info", + + # Option: pretend + # Type: Bool + # Notes: Just pretend to do the actions, for test purposes. + # Value: [ true | false ] + # + "pretend": true + }, + + # Section: rules + # Notes: Rules and related actions. + # + "rules": [ + { + # Option: name + # Type: String + # Notes: Name of the rule. + # Value: [ NAME ] + # + "name": "myrule", + + # Section: conditions + # Notes: Optional conditions to process the rule. + # If multiple conditions are set, they all + # have to be true to process the rule. + # + "conditions": { + # Option: local + # Type: Bool + # Notes: Condition wheter the senders host address is listed in local_addrs. + # Value: [ true | false ] + # + "local": false, + + # Option: hosts + # Type: String + # Notes: Condition wheter the senders host address is listed in this list. + # Value: [ LIST ] + # + "hosts": [ "127.0.0.1" ], + + # Option: envfrom + # Type: String + # Notes: Condition wheter the envelop-from address matches this regular expression. + # Value: [ REGEX ] + # + "envfrom": "^.+@mypartner\\.com$", + + # Option: envto + # Type: String + # Notes: Condition wheter the envelop-to address matches this regular expression. + # Value: [ REGEX ] + # + "envto": "^postmaster@.+$" + }, + + # Section: actions + # Notes: Actions of the rule. + # + "actions": [ + { + # Option: name + # Type: String + # Notes: Name of the modification. + # Value: [ NAME ] + # + "name": "add_test_header", + + # Option: type + # Type: String + # Notes: Type of the modification. + # Value: [ add_header | del_header | mod_header ] + # + "type": "add_header", + + # Option: field + # Type: String + # Notes: Name of the header. + # Value: [ NAME ] + # + "field": "X-Test-Header", + + # Option: value + # Type: String + # Notes: Value of the header. + # Value: [ VALUE ] + # + "value": "true" + }, { + "name": "modify_subject", + + "type": "mod_header", + + # Option: field + # Type: String + # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). + # Value: [ REGEX ] + # + "field": "^Subject$", + + # Option: search + # Type: String + # Notes: Regular expression to match against the headers value. + # Values: [ VALUE ] + # + "search": "(?P.*)", + + # Option: value + # Type: String + # Notes: New value of the header. + # Values: [ VALUE ] + "value": "[EXTERNAL] \\g" + }, { + "name": "delete_received_header", + + "type": "del_header", + + # Option: field + # Type: String + # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). + # Value: [ REGEX ] + # + "field": "^Received$" + }, { + "name": "add_disclaimer", + + "type": "add_disclaimer", + + # Option: action + # Type: String + # Notes: Action to perform with the disclaimer. + # Value: [ append | prepend ] + # + "action": "prepend", + + # Option: html_template + # Type: String + # Notes: Path to a file which contains the html representation of the disclaimer. + # Value: [ FILE_PATH ] + # + "html_template": "docs/templates/disclaimer_html.template", + + # Option: text_template + # Type: String + # Notes: Path to a file which contains the text representation of the disclaimer. + # Value: [ FILE_PATH ] + # + "text_template": "docs/templates/disclaimer_text.template", + + # Option: error_policy + # Type: String + # Notes: Set what should be done if the modification fails (e.g. no message body present). + # Value: [ wrap | ignore | reject ] + # + "error_policy": "wrap" + }, { + "name": "store_message", + + "type": "store", + + # Option: storage_type + # Type: String + # Notes: The storage type used to store e-mails. + # Value: [ file ] + "storage_type": "file", + + # Option: directory + # Type: String + # Notes: Directory used to store e-mails. + # Value: [ file ] + "directory": "/mnt/messages" + } + ] + } + ] +} diff --git a/pymodmilter/test.py b/pymodmilter/test.py new file mode 100755 index 0000000..dbd784f --- /dev/null +++ b/pymodmilter/test.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +import config + +a = config.ModifyMilterConfig("test.conf", debug=True) From d053851e73c2418241f47ab856728ad907c5c596 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Mar 2021 15:09:56 +0100 Subject: [PATCH 087/145] restructure code and fixes --- pymodmilter/__init__.py | 180 ++++++--------- pymodmilter/actions.py | 153 ++++++++++++- pymodmilter/base.py | 143 ++++++++++++ pymodmilter/conditions.py | 59 ++++- pymodmilter/config.py | 346 ----------------------------- pymodmilter/rules.py | 124 +++++++++++ pymodmilter/run.py | 10 +- pymodmilter/test.py | 5 - pymodmilter/test.conf => test.conf | 4 +- 9 files changed, 548 insertions(+), 476 deletions(-) create mode 100644 pymodmilter/base.py delete mode 100644 pymodmilter/config.py create mode 100644 pymodmilter/rules.py delete mode 100755 pymodmilter/test.py rename pymodmilter/test.conf => test.conf (97%) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 03e7e7e..255b889 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -14,10 +14,11 @@ __all__ = [ "actions", + "base", "conditions", + "rules", "run", - "CustomLogger", - "Rule", + "ModifyMilterConfig", "ModifyMilter"] __version__ = "1.1.4" @@ -26,130 +27,93 @@ from pymodmilter import _runtime_patches import Milter import logging +import re +import json from Milter.utils import parse_addr - from collections import defaultdict - from email.header import Header -from email.message import MIMEPart from email.parser import BytesFeedParser from email.policy import default as default_policy, SMTP -from pymodmilter.actions import Action -from pymodmilter.conditions import Conditions +from pymodmilter.base import CustomLogger, BaseConfig, MilterMessage +from pymodmilter.base import replace_illegal_chars +from pymodmilter.rules import RuleConfig, Rule -class CustomLogger(logging.LoggerAdapter): - def process(self, msg, kwargs): - if "name" in self.extra: - msg = "{}: {}".format(self.extra["name"], msg) +class ModifyMilterConfig(BaseConfig): + def __init__(self, cfgfile, debug=False): + try: + with open(cfgfile, "r") as fh: + # remove lines with leading # (comments), they + # are not allowed in json + cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read()) + except IOError as e: + raise RuntimeError(f"unable to open/read config file: {e}") - if "qid" in self.extra: - msg = "{}: {}".format(self.extra["qid"], msg) + try: + cfg = json.loads(cfg) + except json.JSONDecodeError as e: + cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] + msg = "\n".join(cfg_text) + e.msg = f"{msg}\n{e.msg}" + raise e - if self.logger.getEffectiveLevel() != logging.DEBUG: - msg = msg.replace("\n", "").replace("\r", "") + if "global" in cfg: + assert isinstance(cfg["global"], dict), \ + "global: invalid type, should be dict" - return msg, kwargs + cfg["global"]["name"] = "global" + super().__init__(cfg["global"], debug) + self.logger.debug("initialize config") -class Rule: - """ - Rule to implement multiple actions on emails. - """ - - def __init__(self, milter_cfg, cfg): - logger = logging.getLogger(cfg["name"]) - self.logger = CustomLogger(logger, {"name": cfg["name"]}) - self.logger.setLevel(cfg["loglevel"]) - - if cfg["conditions"] is None: - self.conditions = None - else: - self.conditions = Conditions(milter_cfg, cfg["conditions"]) - - self._need_body = False - - self.actions = [] - for action_cfg in cfg["actions"]: - action = Action(milter_cfg, action_cfg) - self.actions.append(action) - if action.need_body(): - self._need_body = True - - self.pretend = cfg["pretend"] - - def need_body(self): - """Return True if this rule needs the message body.""" - return self._need_body - - def ignores(self, host=None, envfrom=None, envto=None): - args = {} - - if host is not None: - args["host"] = host - - if envfrom is not None: - args["envfrom"] = envfrom - - if envto is not None: - args["envto"] = envto - - if self.conditions is None or self.conditions.match(args): - for action in self.actions: - if action.conditions is None or action.conditions.match(args): - return False - - return True - - def execute(self, milter, pretend=None): - """Execute all actions of this rule.""" - if pretend is None: - pretend = self.pretend - - for action in self.actions: - milter_action = action.execute(milter, pretend=pretend) - if milter_action is not None: - return milter_action - - -class MilterMessage(MIMEPart): - def replace_header(self, _name, _value, idx=None): - _name = _name.lower() - counter = 0 - for i, (k, v) in zip(range(len(self._headers)), self._headers): - if k.lower() == _name: - counter += 1 - if not idx or counter == idx: - self._headers[i] = self.policy.header_store_parse( - k, _value) - break - - else: - raise KeyError(_name) - - def remove_header(self, name, idx=None): - name = name.lower() - newheaders = [] - counter = 0 - for k, v in self._headers: - if k.lower() == name: - counter += 1 - if counter != idx: - newheaders.append((k, v)) + if "pretend" in cfg["global"]: + pretend = cfg["global"]["pretend"] + assert isinstance(pretend, bool), \ + "global: pretend: invalid value, should be bool" + self["pretend"] = pretend else: - newheaders.append((k, v)) + self["pretend"] = False - self._headers = newheaders + if "socket" in cfg["global"]: + socket = cfg["global"]["socket"] + assert isinstance(socket, str), \ + "global: socket: invalid value, should be string" + self["socket"] = socket + else: + self["socket"] = None + if "local_addrs" in cfg["global"]: + local_addrs = cfg["global"]["local_addrs"] + assert isinstance(local_addrs, list) and all( + [isinstance(addr, str) for addr in local_addrs]), \ + "global: local_addrs: invalid value, " \ + "should be list of strings" + self["local_addrs"] = local_addrs + else: + self["local_addrs"] = [ + "::1/128", + "127.0.0.0/8", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16"] -def replace_illegal_chars(string): - """Replace illegal characters in header values.""" - return string.replace( - "\x00", "").replace( - "\r", "").replace( - "\n", "") + self.logger.debug(f"socket={self['socket']}, " + f"local_addrs={self['local_addrs']}, " + f"pretend={self['pretend']}, " + f"loglevel={self['loglevel']}") + + assert "rules" in cfg, \ + "mandatory parameter 'rules' not found" + assert isinstance(cfg["rules"], list), \ + "rules: invalid value, should be list" + + self.logger.debug("initialize rules config") + self["rules"] = [] + for idx, rule_cfg in enumerate(cfg["rules"]): + self["rules"].append( + RuleConfig(idx, self, rule_cfg, debug)) class ModifyMilter(Milter.Base): diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 1aa26a5..1ad4e6f 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -12,6 +12,16 @@ # along with PyMod-Milter. If not, see . # +__all__ = [ + "add_header", + "mod_header", + "del_header", + "add_disclaimer", + "rewrite_links", + "store", + "ActionConfig", + "Action"] + import logging import os import re @@ -23,7 +33,9 @@ from copy import copy from datetime import datetime from email.message import MIMEPart -from pymodmilter import CustomLogger, Conditions, replace_illegal_chars +from pymodmilter import CustomLogger, BaseConfig +from pymodmilter.conditions import ConditionsConfig, Conditions +from pymodmilter import replace_illegal_chars def add_header(milter, field, value, pretend=False, @@ -316,13 +328,145 @@ def store(milter, directory, pretend=False, raise RuntimeError(f"unable to store message: {e}") +class ActionConfig(BaseConfig): + def __init__(self, idx, rule_cfg, cfg, debug): + if "name" in cfg: + cfg["name"] = f"{rule_cfg['name']}: {cfg['name']}" + else: + cfg["name"] = f"{rule_cfg['name']}: Action #{idx}" + + if "loglevel" not in cfg: + cfg["loglevel"] = rule_cfg["loglevel"] + + super().__init__(cfg, debug) + + self["pretend"] = rule_cfg["pretend"] + self["conditions"] = None + self["type"] = "" + + if "pretend" in cfg: + pretend = cfg["pretend"] + assert isinstance(pretend, bool), \ + f"{self['name']}: pretend: invalid value, should be bool" + self["pretend"] = pretend + + assert "type" in cfg, \ + f"{self['name']}: mandatory parameter 'type' not found" + assert isinstance(cfg["type"], str), \ + f"{self['name']}: invalid value, should be string" + self["type"] = cfg["type"] + + if self["type"] == "add_header": + self["func"] = add_header + self["need_body"] = False + self.add_string_arg(cfg, ("field", "value")) + + elif self["type"] == "mod_header": + self["func"] = mod_header + self["need_body"] = False + args = ["field", "value"] + if "search" in cfg: + args.append("search") + + for arg in args: + self.add_string_arg(cfg, arg) + if arg in ("field", "search"): + try: + self["args"][arg] = re.compile( + self["args"][arg], + re.MULTILINE + re.DOTALL + re.IGNORECASE) + except re.error as e: + raise ValueError(f"{self['name']}: {arg}: {e}") + + elif self["type"] == "del_header": + self["func"] = del_header + self["need_body"] = False + args = ["field"] + if "value" in cfg: + args.append("value") + + for arg in args: + self.add_string_arg(cfg, arg) + try: + self["args"][arg] = re.compile( + self["args"][arg], + re.MULTILINE + re.DOTALL + re.IGNORECASE) + except re.error as e: + raise ValueError(f"{self['name']}: {arg}: {e}") + + elif self["type"] == "add_disclaimer": + self["func"] = add_disclaimer + self["need_body"] = True + + if "error_policy" not in cfg: + cfg["error_policy"] = "wrap" + + self.add_string_arg( + cfg, ("action", "html_template", "text_template", + "error_policy")) + assert self["args"]["action"] in ("append", "prepend"), \ + f"{self['name']}: action: invalid value, " \ + f"should be 'append' or 'prepend'" + assert self["args"]["error_policy"] in ("wrap", + "ignore", + "reject"), \ + f"{self['name']}: error_policy: invalid value, " \ + f"should be 'wrap', 'ignore' or 'reject'" + + try: + with open(self["args"]["html_template"], "r") as f: + html = BeautifulSoup(f.read(), "html.parser") + body = html.find('body') + if body: + # just use content within the body tag if present + html = body + self["args"]["html_template"] = html + + with open(self["args"]["text_template"], "r") as f: + self["args"]["text_template"] = f.read() + + except IOError as e: + raise RuntimeError( + f"{self['name']}: unable to open/read template file: {e}") + + elif self["type"] == "rewrite_links": + self["func"] = rewrite_links + self["need_body"] = True + self.add_string_arg(cfg, "repl") + + elif self["type"] == "store": + self["func"] = store + self["need_body"] = True + self.add_string_arg(cfg, "storage_type") + assert self["args"]["storage_type"] in ("file"), \ + f"{self['name']}: storage_type: invalid value, " \ + f"should be 'file'" + + if self["args"]["storage_type"] == "file": + self.add_string_arg(cfg, "directory") + else: + raise RuntimeError(f"{self['name']}: type: invalid action type") + + if "conditions" in cfg: + conditions = cfg["conditions"] + assert isinstance(conditions, dict), \ + f"{self['name']}: conditions: invalid value, should be dict" + self["conditions"] = ConditionsConfig(self, conditions, debug) + + self.logger.debug(f"pretend={self['pretend']}, " + f"loglevel={self['loglevel']}, " + f"type={self['type']}, " + f"args={self['args']}") + + class Action: """Action to implement a pre-configured action to perform on e-mails.""" def __init__(self, milter_cfg, cfg): - logger = logging.getLogger(cfg["name"]) - self.logger = CustomLogger(logger, {"name": cfg["name"]}) - self.logger.setLevel(cfg["loglevel"]) + self.logger = cfg.logger + #logger = logging.getLogger(cfg["name"]) + #self.logger = CustomLogger(logger, {"name": cfg["name"]}) + #self.logger.setLevel(cfg["loglevel"]) if cfg["conditions"] is None: self.conditions = None @@ -330,6 +474,7 @@ class Action: self.conditions = Conditions(milter_cfg, cfg["conditions"]) self.pretend = cfg["pretend"] + self._func = cfg["func"] self._args = cfg["args"] action_type = cfg["type"] diff --git a/pymodmilter/base.py b/pymodmilter/base.py new file mode 100644 index 0000000..3235eae --- /dev/null +++ b/pymodmilter/base.py @@ -0,0 +1,143 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +__all__ = [ + "CustomLogger", + "BaseConfig", + "MilterMessage", + "replace_illegal_chars"] + +import logging + +from email.message import MIMEPart + + +class CustomLogger(logging.LoggerAdapter): + def process(self, msg, kwargs): + if "name" in self.extra: + msg = "{}: {}".format(self.extra["name"], msg) + + if "qid" in self.extra: + msg = "{}: {}".format(self.extra["qid"], msg) + + if self.logger.getEffectiveLevel() != logging.DEBUG: + msg = msg.replace("\n", "").replace("\r", "") + + return msg, kwargs + + +class BaseConfig: + def __init__(self, cfg={}, debug=False, logger=None): + self._cfg = {} + if "name" in cfg: + assert isinstance(cfg["name"], str), \ + "rule: name: invalid value, should be string" + self["name"] = cfg["name"] + else: + self["name"] = __name__ + + if debug: + self["loglevel"] = logging.DEBUG + elif "loglevel" in cfg: + if isinstance(cfg["loglevel"], int): + self["loglevel"] = cfg["loglevel"] + else: + level = getattr(logging, cfg["loglevel"].upper(), None) + assert isinstance(level, int), \ + f"{self['name']}: loglevel: invalid value" + self["loglevel"] = level + else: + self["loglevel"] = logging.INFO + + if logger is None: + logger = logging.getLogger(self["name"]) + logger.setLevel(self["loglevel"]) + + self.logger = CustomLogger(logger, {"name": self["name"]}) + + # the keys/values of args are used as parameters + # to functions + self["args"] = {} + + def __setitem__(self, key, value): + self._cfg[key] = value + + def __getitem__(self, key): + return self._cfg[key] + + def __delitem__(self, key): + del self._cfg[key] + + def __contains__(self, key): + return key in self._cfg + + def add_string_arg(self, cfg, args): + if isinstance(args, str): + args = [args] + + for arg in args: + assert arg in cfg, \ + f"{self['name']}: mandatory parameter '{arg}' not found" + assert isinstance(cfg[arg], str), \ + f"{self['name']}: {arg}: invalid value, should be string" + self["args"][arg] = cfg[arg] + + def add_bool_arg(self, cfg, args): + if isinstance(args, str): + args = [args] + + for arg in args: + assert arg in cfg, \ + f"{self['name']}: mandatory parameter '{arg}' not found" + assert isinstance(cfg[arg], bool), \ + f"{self['name']}: {arg}: invalid value, should be bool" + self["args"][arg] = cfg[arg] + + +class MilterMessage(MIMEPart): + def replace_header(self, _name, _value, idx=None): + _name = _name.lower() + counter = 0 + for i, (k, v) in zip(range(len(self._headers)), self._headers): + if k.lower() == _name: + counter += 1 + if not idx or counter == idx: + self._headers[i] = self.policy.header_store_parse( + k, _value) + break + + else: + raise KeyError(_name) + + def remove_header(self, name, idx=None): + name = name.lower() + newheaders = [] + counter = 0 + for k, v in self._headers: + if k.lower() == name: + counter += 1 + if counter != idx: + newheaders.append((k, v)) + else: + newheaders.append((k, v)) + + self._headers = newheaders + + +def replace_illegal_chars(string): + """Replace illegal characters in header values.""" + return string.replace( + "\x00", "").replace( + "\r", "").replace( + "\n", "") diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index 335baf5..6548862 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -12,19 +12,66 @@ # along with PyMod-Milter. If not, see . # -import logging +__all__ = [ + "ConditionsConfig", + "Conditions"] -from netaddr import IPAddress -from pymodmilter import CustomLogger +import logging +import re + +from netaddr import IPAddress, IPNetwork, AddrFormatError +from pymodmilter import CustomLogger, BaseConfig + + +class ConditionsConfig(BaseConfig): + def __init__(self, parent_cfg, cfg, debug): + if "loglevel" not in cfg: + cfg["loglevel"] = parent_cfg["loglevel"] + + cfg["name"] = f"{parent_cfg['name']}: condition" + + super().__init__(cfg, debug) + + if "local" in cfg: + self.add_bool_arg(cfg, "local") + + if "hosts" in cfg: + hosts = cfg["hosts"] + assert isinstance(hosts, list) and all( + [isinstance(host, str) for host in hosts]), \ + f"{self['name']}: hosts: invalid value, " \ + f"should be list of strings" + + self["args"]["hosts"] = [] + try: + for host in cfg["hosts"]: + self["args"]["hosts"].append(IPNetwork(host)) + except AddrFormatError as e: + raise ValueError(f"{self['name']}: hosts: {e}") + + for arg in ("envfrom", "envto"): + if arg in cfg: + self.add_string_arg(cfg, arg) + try: + self["args"][arg] = re.compile( + self["args"][arg], + re.IGNORECASE) + except re.error as e: + raise ValueError(f"{self['name']}: {arg}: {e}") + + self.logger.debug(f"{self['name']}: " + f"loglevel={self['loglevel']}, " + f"args={self['args']}") class Conditions: """Conditions to implement conditions for rules and actions.""" def __init__(self, milter_cfg, cfg): - logger = logging.getLogger(cfg["name"]) - self.logger = CustomLogger(logger, {"name": cfg["name"]}) - self.logger.setLevel(cfg["loglevel"]) + self.logger = cfg.logger + #logger = logging.getLogger(cfg["name"]) + #self.logger = CustomLogger(logger, {"name": cfg["name"]}) + #self.logger.setLevel(cfg["loglevel"]) self._local_addrs = milter_cfg["local_addrs"] self._args = cfg["args"] diff --git a/pymodmilter/config.py b/pymodmilter/config.py deleted file mode 100644 index 3d1b651..0000000 --- a/pymodmilter/config.py +++ /dev/null @@ -1,346 +0,0 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# PyMod-Milter is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . -# - -__all__ = [ - "BaseConfig", - "ActionConfig", - "RuleConfig", - "ModifyMilterConfig"] - -import json -import logging -import re - -from bs4 import BeautifulSoup -from netaddr import IPNetwork, AddrFormatError - - -class BaseConfig: - def __init__(self, cfg={}, debug=False, logger=None): - if logger is None: - logger = logging.getLogger(__name__) - - self.logger = logger - - self._cfg = {} - if "name" in cfg: - assert isinstance(cfg["name"], str), \ - "rule: name: invalid value, should be string" - self["name"] = cfg["name"] - else: - self["name"] = "" - - if debug: - self["loglevel"] = logging.DEBUG - elif "loglevel" in cfg: - if isinstance(cfg["loglevel"], int): - self["loglevel"] = cfg["loglevel"] - else: - level = getattr(logging, cfg["loglevel"].upper(), None) - assert isinstance(level, int), \ - f"{self['name']}: loglevel: invalid value" - self["loglevel"] = level - else: - self["loglevel"] = logging.INFO - - # the keys/values of args are used as parameters - # to functions - self["args"] = {} - - def __setitem__(self, key, value): - self._cfg[key] = value - - def __getitem__(self, key): - return self._cfg[key] - - def __delitem__(self, key): - del self._cfg[key] - - def __contains__(self, key): - return key in self._cfg - - def add_string_arg(self, cfg, args): - if isinstance(args, str): - args = [args] - - for arg in args: - assert arg in cfg, \ - f"{self['name']}: mandatory parameter '{arg}' not found" - assert isinstance(cfg[arg], str), \ - f"{self['name']}: {arg}: invalid value, should be string" - self["args"][arg] = cfg[arg] - - def add_bool_arg(self, cfg, args): - if isinstance(args, str): - args = [args] - - for arg in args: - assert arg in cfg, \ - f"{self['name']}: mandatory parameter '{arg}' not found" - assert isinstance(cfg[arg], bool), \ - f"{self['name']}: {arg}: invalid value, should be bool" - self["args"][arg] = cfg[arg] - - -class ConditionsConfig(BaseConfig): - def __init__(self, parent_cfg, cfg, debug): - if "loglevel" not in cfg: - cfg["loglevel"] = parent_cfg["loglevel"] - - cfg["name"] = f"{parent_cfg['name']}: condition" - - super().__init__(cfg, debug) - - if "local" in cfg: - self.add_bool_arg(cfg, "local") - - if "hosts" in cfg: - hosts = cfg["hosts"] - assert isinstance(hosts, list) and all( - [isinstance(host, str) for host in hosts]), \ - f"{self['name']}: hosts: invalid value, " \ - f"should be list of strings" - - self["args"]["hosts"] = [] - try: - for host in cfg["hosts"]: - self["args"]["hosts"].append(IPNetwork(host)) - except AddrFormatError as e: - raise ValueError(f"{self['name']}: hosts: {e}") - - for arg in ("envfrom", "envto"): - if arg in cfg: - self.add_string_arg(cfg, arg) - try: - self["args"][arg] = re.compile( - self["args"][arg], - re.IGNORECASE) - except re.error as e: - raise ValueError(f"{self['name']}: {arg}: {e}") - - -class ActionConfig(BaseConfig): - def __init__(self, idx, rule_cfg, cfg, debug): - if "name" in cfg: - cfg["name"] = f"{rule_cfg['name']}: {cfg['name']}" - else: - cfg["name"] = f"{rule_cfg['name']}: Action #{idx}" - - if "loglevel" not in cfg: - cfg["loglevel"] = rule_cfg["loglevel"] - - super().__init__(cfg, debug) - - self["pretend"] = rule_cfg["pretend"] - self["conditions"] = None - self["type"] = "" - - if "pretend" in cfg: - pretend = cfg["pretend"] - assert isinstance(pretend, bool), \ - f"{self['name']}: pretend: invalid value, should be bool" - self["pretend"] = pretend - - assert "type" in cfg, \ - f"{self['name']}: type: invalid value, should be string" - assert cfg["type"] in \ - ("add_header", "del_header", "mod_header", "add_disclaimer", - "rewrite_links", "store"), \ - f"{self['name']}: type: invalid action type" - self["type"] = cfg["type"] - - if self["type"] == "add_header": - self.add_string_arg(cfg, ("field", "value")) - - elif self["type"] == "mod_header": - args = ["field", "value"] - if "search" in cfg: - args.append("search") - - for arg in args: - self.add_string_arg(cfg, arg) - if arg in ("field", "search"): - try: - self["args"][arg] = re.compile( - self["args"][arg], - re.MULTILINE + re.DOTALL + re.IGNORECASE) - except re.error as e: - raise ValueError(f"{self['name']}: {arg}: {e}") - - elif self["type"] == "del_header": - args = ["field"] - if "value" in cfg: - args.append("value") - - for arg in args: - self.add_string_arg(cfg, arg) - try: - self["args"][arg] = re.compile( - self["args"][arg], - re.MULTILINE + re.DOTALL + re.IGNORECASE) - except re.error as e: - raise ValueError(f"{self['name']}: {arg}: {e}") - - elif self["type"] == "add_disclaimer": - if "error_policy" not in cfg: - cfg["error_policy"] = "wrap" - - self.add_string_arg( - cfg, ("action", "html_template", "text_template", - "error_policy")) - assert self["args"]["action"] in ("append", "prepend"), \ - f"{self['name']}: action: invalid value, " \ - f"should be 'append' or 'prepend'" - assert self["args"]["error_policy"] in ("wrap", - "ignore", - "reject"), \ - f"{self['name']}: error_policy: invalid value, " \ - f"should be 'wrap', 'ignore' or 'reject'" - - try: - with open(self["args"]["html_template"], "r") as f: - html = BeautifulSoup(f.read(), "html.parser") - body = html.find('body') - if body: - # just use content within the body tag if present - html = body - self["args"]["html_template"] = html - - with open(self["args"]["text_template"], "r") as f: - self["args"]["text_template"] = f.read() - - except IOError as e: - raise RuntimeError( - f"{self['name']}: unable to open/read template file: {e}") - - elif self["type"] == "rewrite_links": - self.add_string_arg(cfg, "repl") - - elif self["type"] == "store": - self.add_string_arg(cfg, "storage_type") - assert self["args"]["storage_type"] in ("file"), \ - f"{self['name']}: storage_type: invalid value, " \ - f"should be 'file'" - - if self["args"]["storage_type"] == "file": - self.add_string_arg(cfg, "directory") - - if "conditions" in cfg: - conditions = cfg["conditions"] - assert isinstance(conditions, dict), \ - f"{self['name']}: conditions: invalid value, should be dict" - self["conditions"] = ConditionsConfig(self, conditions, debug) - - -class RuleConfig(BaseConfig): - def __init__(self, idx, milter_cfg, cfg, debug=False): - if "name" not in cfg: - cfg["name"] = f"Rule #{idx}" - - if "loglevel" not in cfg: - cfg["loglevel"] = milter_cfg["loglevel"] - - super().__init__(cfg, debug) - - self["pretend"] = milter_cfg["pretend"] - self["conditions"] = None - self["actions"] = [] - - if "pretend" in cfg: - pretend = cfg["pretend"] - assert isinstance(pretend, bool), \ - f"{self['name']}: pretend: invalid value, should be bool" - self["pretend"] = pretend - - assert "actions" in cfg, \ - f"{self['name']}: mandatory parameter 'actions' not found" - actions = cfg["actions"] - assert isinstance(actions, list), \ - f"{self['name']}: actions: invalid value, should be list" - - for idx, action_cfg in enumerate(cfg["actions"]): - self["actions"].append( - ActionConfig(idx, self, action_cfg, debug)) - - if "conditions" in cfg: - conditions = cfg["conditions"] - assert isinstance(conditions, dict), \ - f"{self['name']}: conditions: invalid value, should be dict" - self["conditions"] = ConditionsConfig(self, conditions, debug) - - -class ModifyMilterConfig(BaseConfig): - def __init__(self, cfgfile, debug=False): - try: - with open(cfgfile, "r") as fh: - # remove lines with leading # (comments), they - # are not allowed in json - cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read()) - except IOError as e: - raise RuntimeError(f"unable to open/read config file: {e}") - - try: - cfg = json.loads(cfg) - except json.JSONDecodeError as e: - cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] - msg = "\n".join(cfg_text) - e.msg = f"{msg}\n{e.msg}" - raise e - - if "global" in cfg: - assert isinstance(cfg["global"], dict), \ - "global: invalid type, should be dict" - - super().__init__(cfg["global"], debug) - - if "pretend" in cfg["global"]: - pretend = cfg["global"]["pretend"] - assert isinstance(pretend, bool), \ - "global: pretend: invalid value, should be bool" - self["pretend"] = pretend - else: - self["pretend"] = False - - if "socket" in cfg["global"]: - socket = cfg["global"]["socket"] - assert isinstance(socket, str), \ - "global: socket: invalid value, should be string" - self["socket"] = socket - else: - self["socket"] = None - - if "local_addrs" in cfg["global"]: - local_addrs = cfg["global"]["local_addrs"] - assert isinstance(local_addrs, list) and all( - [isinstance(addr, str) for addr in local_addrs]), \ - "global: local_addrs: invalid value, " \ - "should be list of strings" - self["local_addrs"] = local_addrs - else: - self["local_addrs"] = [ - "::1/128", - "127.0.0.0/8", - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16"] - - assert "rules" in cfg, \ - "mandatory parameter 'rules' not found" - assert isinstance(cfg["rules"], list), \ - "rules: invalid value, should be list" - - self["rules"] = [] - for idx, rule_cfg in enumerate(cfg["rules"]): - self["rules"].append( - RuleConfig(idx, self, rule_cfg, debug)) diff --git a/pymodmilter/rules.py b/pymodmilter/rules.py new file mode 100644 index 0000000..2f71c55 --- /dev/null +++ b/pymodmilter/rules.py @@ -0,0 +1,124 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +__all__ = [ + "RuleConfig", + "Rule"] + +import logging + +from pymodmilter import CustomLogger, BaseConfig +from pymodmilter.actions import ActionConfig, Action +from pymodmilter.conditions import ConditionsConfig, Conditions + + +class RuleConfig(BaseConfig): + def __init__(self, idx, milter_cfg, cfg, debug=False): + if "name" not in cfg: + cfg["name"] = f"Rule #{idx}" + + if "loglevel" not in cfg: + cfg["loglevel"] = milter_cfg["loglevel"] + + super().__init__(cfg, debug) + + self["pretend"] = milter_cfg["pretend"] + self["conditions"] = None + self["actions"] = [] + + if "pretend" in cfg: + pretend = cfg["pretend"] + assert isinstance(pretend, bool), \ + f"{self['name']}: pretend: invalid value, should be bool" + self["pretend"] = pretend + + assert "actions" in cfg, \ + f"{self['name']}: mandatory parameter 'actions' not found" + actions = cfg["actions"] + assert isinstance(actions, list), \ + f"{self['name']}: actions: invalid value, should be list" + + self.logger.debug(f"pretend={self['pretend']}, " + f"loglevel={self['loglevel']}") + + if "conditions" in cfg: + conditions = cfg["conditions"] + assert isinstance(conditions, dict), \ + f"{self['name']}: conditions: invalid value, should be dict" + self["conditions"] = ConditionsConfig(self, conditions, debug) + + for idx, action_cfg in enumerate(cfg["actions"]): + self["actions"].append( + ActionConfig(idx, self, action_cfg, debug)) + + +class Rule: + """ + Rule to implement multiple actions on emails. + """ + + def __init__(self, milter_cfg, cfg): + self.logger = cfg.logger + #logger = logging.getLogger(cfg["name"]) + #self.logger = CustomLogger(logger, {"name": cfg["name"]}) + #self.logger.setLevel(cfg["loglevel"]) + + if cfg["conditions"] is None: + self.conditions = None + else: + self.conditions = Conditions(milter_cfg, cfg["conditions"]) + + self._need_body = False + + self.actions = [] + for action_cfg in cfg["actions"]: + action = Action(milter_cfg, action_cfg) + self.actions.append(action) + if action.need_body(): + self._need_body = True + + self.pretend = cfg["pretend"] + + def need_body(self): + """Return True if this rule needs the message body.""" + return self._need_body + + def ignores(self, host=None, envfrom=None, envto=None): + args = {} + + if host is not None: + args["host"] = host + + if envfrom is not None: + args["envfrom"] = envfrom + + if envto is not None: + args["envto"] = envto + + if self.conditions is None or self.conditions.match(args): + for action in self.actions: + if action.conditions is None or action.conditions.match(args): + return False + + return True + + def execute(self, milter, pretend=None): + """Execute all actions of this rule.""" + if pretend is None: + pretend = self.pretend + + for action in self.actions: + milter_action = action.execute(milter, pretend=pretend) + if milter_action is not None: + return milter_action diff --git a/pymodmilter/run.py b/pymodmilter/run.py index b89bebe..4da8f7a 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -12,6 +12,8 @@ # along with PyMod-Milter. If not, see . # +__all__ = ["main"] + import Milter import argparse import logging @@ -20,7 +22,7 @@ import sys from pymodmilter import ModifyMilter from pymodmilter import __version__ as version -from pymodmilter.config import ModifyMilterConfig +from pymodmilter import ModifyMilterConfig def main(): @@ -83,7 +85,7 @@ def main(): try: logger.debug("prepar milter configuration") - cfg = ModifyMilterConfig(args.cfgfile, args.debug) + cfg = ModifyMilterConfig(args.config, args.debug) if not args.debug: logger.setLevel(cfg["loglevel"]) @@ -100,8 +102,6 @@ def main(): if not cfg["rules"]: raise RuntimeError("no rules configured") - logger.debug("initializing rules ...") - for rule_cfg in cfg["rules"]: if not rule_cfg["actions"]: raise RuntimeError( @@ -112,7 +112,7 @@ def main(): sys.exit(255) if args.test: - print("Configuration ok") + print("Configuration OK") sys.exit(0) # setup console log for runtime diff --git a/pymodmilter/test.py b/pymodmilter/test.py deleted file mode 100755 index dbd784f..0000000 --- a/pymodmilter/test.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 - -import config - -a = config.ModifyMilterConfig("test.conf", debug=True) diff --git a/pymodmilter/test.conf b/test.conf similarity index 97% rename from pymodmilter/test.conf rename to test.conf index 998e537..bcbbfde 100644 --- a/pymodmilter/test.conf +++ b/test.conf @@ -176,14 +176,14 @@ # Notes: Path to a file which contains the html representation of the disclaimer. # Value: [ FILE_PATH ] # - "html_template": "docs/templates/disclaimer_html.template", + "html_template": "pymodmilter/docs/templates/disclaimer_html.template", # Option: text_template # Type: String # Notes: Path to a file which contains the text representation of the disclaimer. # Value: [ FILE_PATH ] # - "text_template": "docs/templates/disclaimer_text.template", + "text_template": "pymodmilter/docs/templates/disclaimer_text.template", # Option: error_policy # Type: String From 16ca8cbbf0a6d79d3b62a1002684c09918e48f9b Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Mar 2021 15:11:30 +0100 Subject: [PATCH 088/145] remote test.conf --- test.conf | 214 ------------------------------------------------------ 1 file changed, 214 deletions(-) delete mode 100644 test.conf diff --git a/test.conf b/test.conf deleted file mode 100644 index bcbbfde..0000000 --- a/test.conf +++ /dev/null @@ -1,214 +0,0 @@ -# This is an example /etc/pymodmilter.conf file. -# Copy it into place before use. -# -# The file is in JSON format. -# -# The global option 'log' can be overriden per rule or per modification. -# -{ - # Section: global - # Notes: Global options. - # - "global": { - # Option: socket - # Type: String - # Notes: The socket used to communicate with the MTA. - # - # Examples: - # unix:/path/to/socket a named pipe - # inet:8899 listen on ANY interface - # inet:8899@localhost listen on a specific interface - # inet6:8899 listen on ANY interface - # inet6:8899@[2001:db8:1234::1] listen on a specific interface - # Value: [ SOCKET ] - "socket": "inet:8898@127.0.0.1", - - # Option: local_addrs - # Type: List - # Notes: A list of local hosts and networks. - # Value: [ LIST ] - # - "local_addrs": ["::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], - - # Option: loglevel - # Type: String - # Notes: Set loglevel for rules and actions. - # Value: [ error | warning | info | debug ] - # - "loglevel": "info", - - # Option: pretend - # Type: Bool - # Notes: Just pretend to do the actions, for test purposes. - # Value: [ true | false ] - # - "pretend": true - }, - - # Section: rules - # Notes: Rules and related actions. - # - "rules": [ - { - # Option: name - # Type: String - # Notes: Name of the rule. - # Value: [ NAME ] - # - "name": "myrule", - - # Section: conditions - # Notes: Optional conditions to process the rule. - # If multiple conditions are set, they all - # have to be true to process the rule. - # - "conditions": { - # Option: local - # Type: Bool - # Notes: Condition wheter the senders host address is listed in local_addrs. - # Value: [ true | false ] - # - "local": false, - - # Option: hosts - # Type: String - # Notes: Condition wheter the senders host address is listed in this list. - # Value: [ LIST ] - # - "hosts": [ "127.0.0.1" ], - - # Option: envfrom - # Type: String - # Notes: Condition wheter the envelop-from address matches this regular expression. - # Value: [ REGEX ] - # - "envfrom": "^.+@mypartner\\.com$", - - # Option: envto - # Type: String - # Notes: Condition wheter the envelop-to address matches this regular expression. - # Value: [ REGEX ] - # - "envto": "^postmaster@.+$" - }, - - # Section: actions - # Notes: Actions of the rule. - # - "actions": [ - { - # Option: name - # Type: String - # Notes: Name of the modification. - # Value: [ NAME ] - # - "name": "add_test_header", - - # Option: type - # Type: String - # Notes: Type of the modification. - # Value: [ add_header | del_header | mod_header ] - # - "type": "add_header", - - # Option: field - # Type: String - # Notes: Name of the header. - # Value: [ NAME ] - # - "field": "X-Test-Header", - - # Option: value - # Type: String - # Notes: Value of the header. - # Value: [ VALUE ] - # - "value": "true" - }, { - "name": "modify_subject", - - "type": "mod_header", - - # Option: field - # Type: String - # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). - # Value: [ REGEX ] - # - "field": "^Subject$", - - # Option: search - # Type: String - # Notes: Regular expression to match against the headers value. - # Values: [ VALUE ] - # - "search": "(?P.*)", - - # Option: value - # Type: String - # Notes: New value of the header. - # Values: [ VALUE ] - "value": "[EXTERNAL] \\g" - }, { - "name": "delete_received_header", - - "type": "del_header", - - # Option: field - # Type: String - # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). - # Value: [ REGEX ] - # - "field": "^Received$" - }, { - "name": "add_disclaimer", - - "type": "add_disclaimer", - - # Option: action - # Type: String - # Notes: Action to perform with the disclaimer. - # Value: [ append | prepend ] - # - "action": "prepend", - - # Option: html_template - # Type: String - # Notes: Path to a file which contains the html representation of the disclaimer. - # Value: [ FILE_PATH ] - # - "html_template": "pymodmilter/docs/templates/disclaimer_html.template", - - # Option: text_template - # Type: String - # Notes: Path to a file which contains the text representation of the disclaimer. - # Value: [ FILE_PATH ] - # - "text_template": "pymodmilter/docs/templates/disclaimer_text.template", - - # Option: error_policy - # Type: String - # Notes: Set what should be done if the modification fails (e.g. no message body present). - # Value: [ wrap | ignore | reject ] - # - "error_policy": "wrap" - }, { - "name": "store_message", - - "type": "store", - - # Option: storage_type - # Type: String - # Notes: The storage type used to store e-mails. - # Value: [ file ] - "storage_type": "file", - - # Option: directory - # Type: String - # Notes: Directory used to store e-mails. - # Value: [ file ] - "directory": "/mnt/messages" - } - ] - } - ] -} From 1349570b87ce034426e003b72bafb788886958e5 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Mar 2021 19:09:21 +0100 Subject: [PATCH 089/145] improve logging and some fixes --- pymodmilter/__init__.py | 14 +++++-- pymodmilter/actions.py | 78 +++++++++++++++++---------------------- pymodmilter/base.py | 6 +-- pymodmilter/conditions.py | 6 +-- pymodmilter/rules.py | 12 +++--- pymodmilter/run.py | 20 +++++----- 6 files changed, 63 insertions(+), 73 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 255b889..50cc08a 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -35,6 +35,7 @@ from collections import defaultdict from email.header import Header from email.parser import BytesFeedParser from email.policy import default as default_policy, SMTP +from netaddr import IPNetwork, AddrFormatError from pymodmilter.base import CustomLogger, BaseConfig, MilterMessage from pymodmilter.base import replace_illegal_chars @@ -56,8 +57,7 @@ class ModifyMilterConfig(BaseConfig): except json.JSONDecodeError as e: cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] msg = "\n".join(cfg_text) - e.msg = f"{msg}\n{e.msg}" - raise e + raise RuntimeError(f"{e}\n{msg}") if "global" in cfg: assert isinstance(cfg["global"], dict), \ @@ -90,15 +90,21 @@ class ModifyMilterConfig(BaseConfig): [isinstance(addr, str) for addr in local_addrs]), \ "global: local_addrs: invalid value, " \ "should be list of strings" - self["local_addrs"] = local_addrs else: - self["local_addrs"] = [ + local_addrs = [ "::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] + self["local_addrs"] = [] + try: + for addr in local_addrs: + self["local_addrs"].append(IPNetwork(addr)) + except AddrFormatError as e: + raise ValueError(f"{self['name']}: local_addrs: {e}") + self.logger.debug(f"socket={self['socket']}, " f"local_addrs={self['local_addrs']}, " f"pretend={self['pretend']}, " diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 1ad4e6f..cad385e 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -160,7 +160,7 @@ def _has_content_before_body_tag(soup): return False -def _patch_message_body(milter, action, text, html, logger): +def _patch_message_body(milter, action, text_template, html_template, logger): text_body, text_content = _get_body_content(milter.msg, "plain") html_body, html_content = _get_body_content(milter.msg, "html") @@ -171,9 +171,9 @@ def _patch_message_body(milter, action, text, html, logger): logger.info(f"{action} text disclaimer") if action == "prepend": - content = f"{text}{text_content}" + content = f"{text_template}{text_content}" else: - content = f"{text_content}{text}" + content = f"{text_content}{text_template}" text_body.set_content( content.encode(), maintype="text", subtype="plain") @@ -192,9 +192,9 @@ def _patch_message_body(milter, action, text, html, logger): body = soup if action == "prepend": - body.insert(0, copy(html)) + body.insert(0, copy(html_template)) else: - body.append(html) + body.append(html_template) html_body.set_content( str(soup).encode(), maintype="text", subtype="html") @@ -239,26 +239,28 @@ def _inject_body(milter): milter.msg.attach(attachment) -def add_disclaimer(milter, text, html, action, policy, pretend=False, - logger=logging.getLogger(__name__)): +def add_disclaimer(milter, text_template, html_template, action, error_policy, + pretend=False, logger=logging.getLogger(__name__)): """Append or prepend a disclaimer to the mail body.""" old_headers = milter.msg.items() try: try: - _patch_message_body(milter, action, text, html, logger) + _patch_message_body( + milter, action, text_template, html_template, logger) except RuntimeError as e: logger.info(f"{e}, inject empty plain and html body") _inject_body(milter) - _patch_message_body(milter, action, text, html, logger) + _patch_message_body( + milter, action, text_template, html_template, logger) except Exception as e: logger.warning(e) - if policy == "ignore": + if error_policy == "ignore": logger.info( "unable to add disclaimer to message body, " "ignore error according to policy") return - elif policy == "reject": + elif error_policy == "reject": logger.info( "unable to add disclaimer to message body, " "reject message according to policy") @@ -268,7 +270,8 @@ def add_disclaimer(milter, text, html, action, policy, pretend=False, logger.info("wrap original message in a new message envelope") try: _wrap_message(milter, logger) - _patch_message_body(milter, action, text, html, logger) + _patch_message_body( + milter, action, text_template, html_template, logger) except Exception as e: logger.error(e) raise Exception( @@ -331,6 +334,9 @@ def store(milter, directory, pretend=False, class ActionConfig(BaseConfig): def __init__(self, idx, rule_cfg, cfg, debug): if "name" in cfg: + assert isinstance(cfg["name"], str), \ + f"{rule_cfg['name']}: Action #{idx}: name: invalid value, " \ + f"should be string" cfg["name"] = f"{rule_cfg['name']}: {cfg['name']}" else: cfg["name"] = f"{rule_cfg['name']}: Action #{idx}" @@ -353,7 +359,7 @@ class ActionConfig(BaseConfig): assert "type" in cfg, \ f"{self['name']}: mandatory parameter 'type' not found" assert isinstance(cfg["type"], str), \ - f"{self['name']}: invalid value, should be string" + f"{self['name']}: type: invalid value, should be string" self["type"] = cfg["type"] if self["type"] == "add_header": @@ -437,13 +443,19 @@ class ActionConfig(BaseConfig): elif self["type"] == "store": self["func"] = store self["need_body"] = True - self.add_string_arg(cfg, "storage_type") - assert self["args"]["storage_type"] in ("file"), \ - f"{self['name']}: storage_type: invalid value, " \ - f"should be 'file'" - if self["args"]["storage_type"] == "file": + assert "storage_type" in cfg, \ + f"{self['name']}: mandatory parameter 'storage_type' not found" + assert isinstance(cfg["type"], str), \ + f"{self['name']}: storage_type: invalid value, " \ + f"should be string" + self["storage_type"] = cfg["storage_type"] + if self["storage_type"] == "file": self.add_string_arg(cfg, "directory") + else: + raise RuntimeError( + f"{self['name']}: storage_type: invalid storage type") + else: raise RuntimeError(f"{self['name']}: type: invalid action type") @@ -464,9 +476,6 @@ class Action: def __init__(self, milter_cfg, cfg): self.logger = cfg.logger - #logger = logging.getLogger(cfg["name"]) - #self.logger = CustomLogger(logger, {"name": cfg["name"]}) - #self.logger.setLevel(cfg["loglevel"]) if cfg["conditions"] is None: self.conditions = None @@ -474,30 +483,10 @@ class Action: self.conditions = Conditions(milter_cfg, cfg["conditions"]) self.pretend = cfg["pretend"] + self._name = cfg["name"] self._func = cfg["func"] self._args = cfg["args"] - - action_type = cfg["type"] - if action_type == "add_header": - self._func = add_header - self._need_body = False - elif action_type == "mod_header": - self._func = mod_header - self._need_body = False - elif action_type == "del_header": - self._func = del_header - self._need_body = False - elif action_type == "add_disclaimer": - self._func = add_disclaimer - self._need_body = True - elif action_type == "rewrite_links": - self._func = rewrite_links - self._need_body = True - elif action_type == "store": - self._func = store - self._need_body = True - else: - raise ValueError(f"invalid action type: {action_type}") + self._need_body = cfg["need_body"] def need_body(self): """Return the needs of this action.""" @@ -508,7 +497,8 @@ class Action: if pretend is None: pretend = self.pretend - logger = CustomLogger(self.logger, {"qid": milter.qid}) + logger = CustomLogger( + self.logger, {"name": self._name, "qid": milter.qid}) return self._func(milter=milter, pretend=pretend, logger=logger, **self._args) diff --git a/pymodmilter/base.py b/pymodmilter/base.py index 3235eae..d099967 100644 --- a/pymodmilter/base.py +++ b/pymodmilter/base.py @@ -26,10 +26,10 @@ from email.message import MIMEPart class CustomLogger(logging.LoggerAdapter): def process(self, msg, kwargs): if "name" in self.extra: - msg = "{}: {}".format(self.extra["name"], msg) + msg = f"{self.extra['name']}: {msg}" if "qid" in self.extra: - msg = "{}: {}".format(self.extra["qid"], msg) + msg = f"{self.extra['qid']}: {msg}" if self.logger.getEffectiveLevel() != logging.DEBUG: msg = msg.replace("\n", "").replace("\r", "") @@ -64,7 +64,7 @@ class BaseConfig: logger = logging.getLogger(self["name"]) logger.setLevel(self["loglevel"]) - self.logger = CustomLogger(logger, {"name": self["name"]}) + self.logger = logger # the keys/values of args are used as parameters # to functions diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index 6548862..0465a14 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -16,11 +16,10 @@ __all__ = [ "ConditionsConfig", "Conditions"] -import logging import re from netaddr import IPAddress, IPNetwork, AddrFormatError -from pymodmilter import CustomLogger, BaseConfig +from pymodmilter import BaseConfig class ConditionsConfig(BaseConfig): @@ -69,9 +68,6 @@ class Conditions: def __init__(self, milter_cfg, cfg): self.logger = cfg.logger - #logger = logging.getLogger(cfg["name"]) - #self.logger = CustomLogger(logger, {"name": cfg["name"]}) - #self.logger.setLevel(cfg["loglevel"]) self._local_addrs = milter_cfg["local_addrs"] self._args = cfg["args"] diff --git a/pymodmilter/rules.py b/pymodmilter/rules.py index 2f71c55..9c0c5fb 100644 --- a/pymodmilter/rules.py +++ b/pymodmilter/rules.py @@ -16,16 +16,17 @@ __all__ = [ "RuleConfig", "Rule"] -import logging - -from pymodmilter import CustomLogger, BaseConfig +from pymodmilter import BaseConfig from pymodmilter.actions import ActionConfig, Action from pymodmilter.conditions import ConditionsConfig, Conditions class RuleConfig(BaseConfig): def __init__(self, idx, milter_cfg, cfg, debug=False): - if "name" not in cfg: + if "name" in cfg: + assert isinstance(cfg["name"], str), \ + f"Rule #{idx}: name: invalid value, should be string" + else: cfg["name"] = f"Rule #{idx}" if "loglevel" not in cfg: @@ -70,9 +71,6 @@ class Rule: def __init__(self, milter_cfg, cfg): self.logger = cfg.logger - #logger = logging.getLogger(cfg["name"]) - #self.logger = CustomLogger(logger, {"name": cfg["name"]}) - #self.logger.setLevel(cfg["loglevel"]) if cfg["conditions"] is None: self.conditions = None diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 4da8f7a..924719a 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -67,17 +67,10 @@ def main(): # setup console log stdouthandler = logging.StreamHandler(sys.stdout) - stdouthandler.setFormatter( - logging.Formatter("%(asctime)s - %(levelname)s: %(message)s")) + formatter = logging.Formatter("%(levelname)s: %(message)s") + stdouthandler.setFormatter(formatter) root_logger.addHandler(stdouthandler) - # setup syslog - sysloghandler = logging.handlers.SysLogHandler( - address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) - sysloghandler.setFormatter( - logging.Formatter("pymodmilter: %(message)s")) - root_logger.addHandler(sysloghandler) - logger = logging.getLogger(__name__) if not args.debug: @@ -120,6 +113,13 @@ def main(): stdouthandler.setFormatter(formatter) stdouthandler.setLevel(logging.DEBUG) + # setup syslog + sysloghandler = logging.handlers.SysLogHandler( + address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) + sysloghandler.setFormatter( + logging.Formatter("pymodmilter: %(message)s")) + root_logger.addHandler(sysloghandler) + logger.info("pymodmilter starting") ModifyMilter.set_config(cfg) @@ -132,7 +132,7 @@ def main(): rc = 0 try: - Milter.runmilter("pymodmilter", socketname=socket, timeout=30) + Milter.runmilter("pymodmilter", socketname=socket, timeout=300) logger.info("pymodmilter stopped") except Milter.milter.error as e: logger.error(e) From ea890591c3c6e806a6e2abaac5be1b78c1ba10d3 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Mar 2021 22:31:50 +0100 Subject: [PATCH 090/145] add additional monkey-patches for python email library --- pymodmilter/_runtime_patches.py | 62 +++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/pymodmilter/_runtime_patches.py b/pymodmilter/_runtime_patches.py index b1a9ad8..11e8c3a 100644 --- a/pymodmilter/_runtime_patches.py +++ b/pymodmilter/_runtime_patches.py @@ -90,6 +90,68 @@ setattr(email._header_value_parser, "DisplayName", DisplayName) setattr(email._header_value_parser, "get_name_addr", get_name_addr) +# https://bugs.python.org/issue30681 +# +# fix: https://github.com/python/cpython/pull/2254 + +import email.errors +from email.errors import HeaderDefect + + +class InvalidDateDefect(HeaderDefect): + """Header has unparseable or invalid date""" + + +setattr(email.errors, "InvalidDateDefect", InvalidDateDefect) + + +import email.utils +from email.utils import _parsedate_tz +import datetime + + +def parsedate_to_datetime(data): + parsed_date_tz = _parsedate_tz(data) + if parsed_date_tz is None: + raise ValueError('Invalid date value or format "%s"' % str(data)) + *dtuple, tz = parsed_date_tz + if tz is None: + return datetime.datetime(*dtuple[:6]) + return datetime.datetime(*dtuple[:6], + tzinfo=datetime.timezone(datetime.timedelta(seconds=tz))) + + +setattr(email.utils, "parsedate_to_datetime", parsedate_to_datetime) + + +import email.headerregistry +from email import utils, _header_value_parser as parser + +@classmethod +def parse(cls, value, kwds): + if not value: + kwds['defects'].append(errors.HeaderMissingRequiredValue()) + kwds['datetime'] = None + kwds['decoded'] = '' + kwds['parse_tree'] = parser.TokenList() + return + if isinstance(value, str): + kwds['decoded'] = value + try: + value = utils.parsedate_to_datetime(value) + except ValueError: + kwds['defects'].append(errors.InvalidDateDefect('Invalid date value or format')) + kwds['datetime'] = None + kwds['parse_tree'] = parser.TokenList() + return + kwds['datetime'] = value + kwds['decoded'] = utils.format_datetime(kwds['datetime']) + kwds['parse_tree'] = cls.value_parser(kwds['decoded']) + + +setattr(email.headerregistry.DateHeader, "parse", parse) + + ####################################### # add charset alias for windows-874 # ####################################### From c854b74f96f0e0a4c0b5578c6586e73a50d0a205 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Mar 2021 00:20:36 +0100 Subject: [PATCH 091/145] add config backward compatibility --- pymodmilter/actions.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index cad385e..805f52f 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -365,11 +365,19 @@ class ActionConfig(BaseConfig): if self["type"] == "add_header": self["func"] = add_header self["need_body"] = False + + if not "field" in cfg and "header" in cfg: + cfg["field"] = cfg["header"] + self.add_string_arg(cfg, ("field", "value")) elif self["type"] == "mod_header": self["func"] = mod_header self["need_body"] = False + + if not "field" in cfg and "header" in cfg: + cfg["field"] = cfg["header"] + args = ["field", "value"] if "search" in cfg: args.append("search") @@ -387,6 +395,10 @@ class ActionConfig(BaseConfig): elif self["type"] == "del_header": self["func"] = del_header self["need_body"] = False + + if not "field" in cfg and "header" in cfg: + cfg["field"] = cfg["header"] + args = ["field"] if "value" in cfg: args.append("value") @@ -404,6 +416,12 @@ class ActionConfig(BaseConfig): self["func"] = add_disclaimer self["need_body"] = True + if not "html_template" in cfg and "html_file" in cfg: + cfg["html_template"] = cfg["html_file"] + + if not "text_template" in cfg and "text_file" in cfg: + cfg["text_template"] = cfg["text_file"] + if "error_policy" not in cfg: cfg["error_policy"] = "wrap" From cb4622df84c005434c9a00447454b65a185c9a5a Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Mar 2021 00:27:38 +0100 Subject: [PATCH 092/145] change README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 86bdb5e..72ce9b9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # pymodmilter A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers. -The project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month. +The project is currently in beta status, but it is already used in a productive enterprise environment that processes about a million e-mails per month. The basic idea is to define rules with conditions and actions which are processed when all conditions are true. @@ -72,19 +72,19 @@ Config options for **action** objects: As described above in the [Global](#Global) section. Config options for **add_header** actions: - * **header** + * **field** Name of the header. * **value** Value of the header. Config options for **del_header** actions: - * **header** + * **field** Regular expression to match against header names. * **value** (optional) Regular expression to match against the headers value. Config options for **mod_header** actions: - * **header** + * **field** Regular expression to match against header names. * **search** (optional) Regular expression to match against header values. You may use subgroups or named subgroups (python syntax) to include parts of the original value in the new value. @@ -96,9 +96,9 @@ Config options for **add_disclaimer** actions: Action to perform with the disclaimer. Possible values are: * append * prepend - * **html_file** + * **html_template** Path to a file which contains the html representation of the disclaimer. - * **text_file** + * **text_template** Path to a file which contains the text representation of the disclaimer. * **error_policy** (optional) Set the error policy in case the disclaimer cannot be added (e.g. if no body part is present in the e-mail). Possible values are: From 5ca0762ac45867d2c1532ee3e664e037817eaecc Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Mar 2021 10:43:32 +0100 Subject: [PATCH 093/145] fix some conditions --- pymodmilter/actions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 805f52f..7539be6 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -366,7 +366,7 @@ class ActionConfig(BaseConfig): self["func"] = add_header self["need_body"] = False - if not "field" in cfg and "header" in cfg: + if "field" not in cfg and "header" in cfg: cfg["field"] = cfg["header"] self.add_string_arg(cfg, ("field", "value")) @@ -375,7 +375,7 @@ class ActionConfig(BaseConfig): self["func"] = mod_header self["need_body"] = False - if not "field" in cfg and "header" in cfg: + if "field" not in cfg and "header" in cfg: cfg["field"] = cfg["header"] args = ["field", "value"] @@ -396,7 +396,7 @@ class ActionConfig(BaseConfig): self["func"] = del_header self["need_body"] = False - if not "field" in cfg and "header" in cfg: + if "field" not in cfg and "header" in cfg: cfg["field"] = cfg["header"] args = ["field"] @@ -416,10 +416,10 @@ class ActionConfig(BaseConfig): self["func"] = add_disclaimer self["need_body"] = True - if not "html_template" in cfg and "html_file" in cfg: + if "html_template" not in cfg and "html_file" in cfg: cfg["html_template"] = cfg["html_file"] - if not "text_template" in cfg and "text_file" in cfg: + if "text_template" not in cfg and "text_file" in cfg: cfg["text_template"] = cfg["text_file"] if "error_policy" not in cfg: From 744641b742506ac570e0fd629ebfc4261419ad20 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Mar 2021 11:32:49 +0100 Subject: [PATCH 094/145] include python version in version output --- pymodmilter/run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 924719a..2f06bbe 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -26,6 +26,9 @@ from pymodmilter import ModifyMilterConfig def main(): + python_version = ".".join([str(v) for v in sys.version_info[0:3]]) + python_version = f"{python_version}-{sys.version_info[3]}" + "Run PyMod-Milter." parser = argparse.ArgumentParser( description="PyMod milter daemon", @@ -58,7 +61,7 @@ def main(): "-v", "--version", help="Print version.", action="version", - version=f"%(prog)s ({version})") + version=f"%(prog)s {version} (python {python_version})") args = parser.parse_args() From d17679a389e15e9baeb09e4466a57ccfffc011eb Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Mar 2021 11:52:16 +0100 Subject: [PATCH 095/145] fix openrc init script --- pymodmilter/misc/openrc/pymodmilter.initd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/misc/openrc/pymodmilter.initd b/pymodmilter/misc/openrc/pymodmilter.initd index f8e8096..b7f3262 100755 --- a/pymodmilter/misc/openrc/pymodmilter.initd +++ b/pymodmilter/misc/openrc/pymodmilter.initd @@ -40,7 +40,7 @@ start_pre() { } stop_pre() { - if [ "${RC_CMD}" != "restart" ]; then + if [ "${RC_CMD}" == "restart" ]; then checkconfig || return $? fi } From fdba57e1e1b8f12b5cc70b88aa1802c8056edc7e Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Mar 2021 12:12:57 +0100 Subject: [PATCH 096/145] further improve openrc init script --- pymodmilter/misc/openrc/pymodmilter.initd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymodmilter/misc/openrc/pymodmilter.initd b/pymodmilter/misc/openrc/pymodmilter.initd index b7f3262..70160aa 100755 --- a/pymodmilter/misc/openrc/pymodmilter.initd +++ b/pymodmilter/misc/openrc/pymodmilter.initd @@ -7,9 +7,9 @@ pidfile="/run/${RC_SVCNAME}.pid" command="/usr/bin/pymodmilter" command_args="${milter_opts}" command_background=true -start_stop_daemon_args="--user ${user}" - +command_user="${user}" extra_commands="configtest" +start_stop_daemon_args="--wait 500" depend() { need net From ed66c090d5a813a6fc91491bf4842ef449bede31 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Mar 2021 17:34:28 +0100 Subject: [PATCH 097/145] raise milter timeout to 600s --- pymodmilter/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 2f06bbe..5998a50 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -135,7 +135,7 @@ def main(): rc = 0 try: - Milter.runmilter("pymodmilter", socketname=socket, timeout=300) + Milter.runmilter("pymodmilter", socketname=socket, timeout=600) logger.info("pymodmilter stopped") except Milter.milter.error as e: logger.error(e) From b670aa3eecbcb8705db8057aced2da2db6f7acb8 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Mar 2021 19:43:47 +0100 Subject: [PATCH 098/145] change version to 1.1.5 --- pymodmilter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 50cc08a..d427eb5 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -21,7 +21,7 @@ __all__ = [ "ModifyMilterConfig", "ModifyMilter"] -__version__ = "1.1.4" +__version__ = "1.1.5" from pymodmilter import _runtime_patches From 314796f593f6086c9d863081d959fa965ae44eb1 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Mar 2021 19:45:58 +0100 Subject: [PATCH 099/145] add BeautifulSoup to dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b25520c..2214bdc 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,6 @@ setup(name = "pymodmilter", ] ) ], - install_requires = ["pymilter", "netaddr"], + install_requires = ["pymilter", "netaddr", "beautifulsoup4[lxml]"], python_requires = ">=3.6" ) From 401d8a36bf8b94c26c7481f9087caa18ce5ba4ee Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Mar 2021 19:47:52 +0100 Subject: [PATCH 100/145] change README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 72ce9b9..10027b4 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ The basic idea is to define rules with conditions and actions which are processe Pymodmilter is depending on these python packages, but they are installed automatically if you are working with pip. * [pymilter](https://pythonhosted.org/pymilter/) * [netaddr](https://github.com/drkjam/netaddr/) +* [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) ## Installation * Install pymodmilter with pip and copy the example config file. From 7adbd8d76b86084fb9779136fc7ca8e873802b0f Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 10 Mar 2021 19:49:17 +0100 Subject: [PATCH 101/145] change version to 1.1.6 --- pymodmilter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index d427eb5..42d68c2 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -21,7 +21,7 @@ __all__ = [ "ModifyMilterConfig", "ModifyMilter"] -__version__ = "1.1.5" +__version__ = "1.1.6" from pymodmilter import _runtime_patches From c05593bfaed03cd029bf6a25e38ea9b6ae6c02d7 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 12 Mar 2021 14:47:28 +0100 Subject: [PATCH 102/145] fix handling of illegal characters in header values --- pymodmilter/__init__.py | 4 ++-- pymodmilter/base.py | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 42d68c2..a6f9128 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -279,8 +279,8 @@ class ModifyMilter(Milter.Base): def header(self, field, value): try: # remove surrogates - field = field.encode("ascii", errors="replace") - value = value.encode("ascii", errors="replace") + field = field.encode("ascii", errors="surrogateescape") + value = value.encode("ascii", errors="surrogateescape") self._fp.feed(field + b": " + value + b"\r\n") except Exception as e: diff --git a/pymodmilter/base.py b/pymodmilter/base.py index d099967..4eb91ae 100644 --- a/pymodmilter/base.py +++ b/pymodmilter/base.py @@ -136,8 +136,5 @@ class MilterMessage(MIMEPart): def replace_illegal_chars(string): - """Replace illegal characters in header values.""" - return string.replace( - "\x00", "").replace( - "\r", "").replace( - "\n", "") + """Remove illegal characters from header values.""" + return "".join(string.replace("\x00", "").splitlines()) From 1977851262c68a9419c79f6fec0d63fca7cf9146 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 12 Mar 2021 21:35:19 +0100 Subject: [PATCH 103/145] remove surrogates from headers again, the cause troubles --- pymodmilter/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index a6f9128..42d68c2 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -279,8 +279,8 @@ class ModifyMilter(Milter.Base): def header(self, field, value): try: # remove surrogates - field = field.encode("ascii", errors="surrogateescape") - value = value.encode("ascii", errors="surrogateescape") + field = field.encode("ascii", errors="replace") + value = value.encode("ascii", errors="replace") self._fp.feed(field + b": " + value + b"\r\n") except Exception as e: From 6caf7049a6df9b0996cd597c670131a0fe0b58a9 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 6 May 2021 16:47:18 +0200 Subject: [PATCH 104/145] change version to 1.1.7 --- pymodmilter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 42d68c2..58aeb38 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -21,7 +21,7 @@ __all__ = [ "ModifyMilterConfig", "ModifyMilter"] -__version__ = "1.1.6" +__version__ = "1.1.7" from pymodmilter import _runtime_patches From 6b8ad1b07831c467027500abe83d0d2d8a695332 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 1 Sep 2021 14:45:21 +0200 Subject: [PATCH 105/145] change link to PR to fix handling of invalid Date fields --- pymodmilter/_runtime_patches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/_runtime_patches.py b/pymodmilter/_runtime_patches.py index 11e8c3a..d96e4b5 100644 --- a/pymodmilter/_runtime_patches.py +++ b/pymodmilter/_runtime_patches.py @@ -92,7 +92,7 @@ setattr(email._header_value_parser, "get_name_addr", get_name_addr) # https://bugs.python.org/issue30681 # -# fix: https://github.com/python/cpython/pull/2254 +# fix: https://github.com/python/cpython/pull/22090 import email.errors from email.errors import HeaderDefect From d20e8684524805d1b341ac53933f04e99a4b95df Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Mon, 6 Sep 2021 15:11:21 +0200 Subject: [PATCH 106/145] add option to store unparse msg and rework rules logic --- README.md | 5 +- pymodmilter/__init__.py | 116 ++++++++++------------ pymodmilter/actions.py | 15 ++- pymodmilter/docs/pymodmilter.conf.example | 11 +- 4 files changed, 80 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 10027b4..bbce560 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Config options in **global** section: The socket used to communicate with the MTA. If it is not specified in the config, it has to be set as command line option. * **local_addrs** (optional) A list of hosts and network addresses which are considered local. It is used to for the condition option [local](#Conditions). - Default: **::1/128, 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16** + Default: **fe80::/64, ::1/128, 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16** * **loglevel** (optional) Set the log level. This option may be overriden by any rule or action object. Possible values are: * **error** @@ -115,6 +115,9 @@ Config options for **store** actions: * **storage_type** Storage type. Possible values are: * **file** + * **original** (optional) + Default: **false** + If set to true, store the message as received by the MTA instead of storing the current state of the message, that may was modified already by other actions. Config options for **file** storage: * **directory** diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 58aeb38..8430f39 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -32,9 +32,10 @@ import json from Milter.utils import parse_addr from collections import defaultdict +from email import message_from_binary_file from email.header import Header -from email.parser import BytesFeedParser from email.policy import default as default_policy, SMTP +from io import BytesIO from netaddr import IPNetwork, AddrFormatError from pymodmilter.base import CustomLogger, BaseConfig, MilterMessage @@ -92,6 +93,7 @@ class ModifyMilterConfig(BaseConfig): "should be list of strings" else: local_addrs = [ + "fe80::/64", "::1/128", "127.0.0.0/8", "10.0.0.0/8", @@ -139,12 +141,6 @@ class ModifyMilter(Milter.Base): self.logger = logging.getLogger(__name__) self.logger.setLevel(ModifyMilter._loglevel) - # save rules, it must not change during runtime - self.rules = ModifyMilter._rules.copy() - - self.msg = None - self._replace_body = False - def addheader(self, field, value, idx=-1): value = replace_illegal_chars(Header(s=value).encode()) self.logger.debug(f"milter: addheader: {field}: {value}") @@ -181,28 +177,20 @@ class ModifyMilter(Milter.Base): self.addheader(field, value) def replacebody(self): - self._replace_body = True + self._replacebody = True def connect(self, IPname, family, hostaddr): try: if hostaddr is None: - self.logger.error("unable to proceed, host address is None") + self.logger.error(f"received invalid host address {hostaddr}, " + f"unable to proceed") return Milter.TEMPFAIL + self.IP = hostaddr[0] + self.port = hostaddr[1] self.logger.debug( - f"accepted milter connection from {hostaddr[0]} " - f"port {hostaddr[1]}") - - # remove rules which ignore this host - for rule in self.rules.copy(): - if rule.ignores(host=hostaddr[0]): - self.rules.remove(rule) - - if not self.rules: - self.logger.debug( - f"host {hostaddr[0]} is ignored by all rules, " - f"skip further processing") - return Milter.ACCEPT + f"accepted milter connection from {self.IP} " + f"port {self.port}") except Exception as e: self.logger.exception( f"an exception occured in connect method: {e}") @@ -210,20 +198,21 @@ class ModifyMilter(Milter.Base): return Milter.CONTINUE + def hello(self, heloname): + try: + self.heloname = heloname + self.logger.debug(f"received HELO name: {heloname}") + except Exception as e: + self.logger.exception( + f"an exception occured in hello method: {e}") + return Milter.TEMPFAIL + + return Milter.CONTINUE + def envfrom(self, mailfrom, *str): try: - mailfrom = "@".join(parse_addr(mailfrom)).lower() - for rule in self.rules.copy(): - if rule.ignores(envfrom=mailfrom): - self.rules.remove(rule) - - if not self.rules: - self.logger.debug( - f"envelope-from address {mailfrom} is ignored by " - f"all rules, skip further processing") - return Milter.ACCEPT - - self.recipients = set() + self.mailfrom = "@".join(parse_addr(mailfrom)).lower() + self.rcpts = set() except Exception as e: self.logger.exception( f"an exception occured in envfrom method: {e}") @@ -233,7 +222,7 @@ class ModifyMilter(Milter.Base): def envrcpt(self, to, *str): try: - self.recipients.add("@".join(parse_addr(to)).lower()) + self.rcpts.add("@".join(parse_addr(to)).lower()) except Exception as e: self.logger.exception( f"an exception occured in envrcpt method: {e}") @@ -243,32 +232,25 @@ class ModifyMilter(Milter.Base): def data(self): try: - for rule in self.rules.copy(): - if rule.ignores(envto=[*self.recipients]): - self.rules.remove(rule) - - if not self.rules: - self.logger.debug( - "envelope-to addresses are ignored by all rules, " - "skip further processing") - return Milter.ACCEPT - self.qid = self.getsymval('i') self.logger = CustomLogger(self.logger, {"qid": self.qid}) self.logger.debug("received queue-id from MTA") - self.fields = None - self.fields_bytes = None - self.body_data = None + self.rules = [] + self._headersonly = True + for rule in ModifyMilter._rules: + if not rule.ignores(host=self.IP, envfrom=self.mailfrom, + envto=[*self.rcpts]): + self.rules.append(rule) + if rule.need_body(): + self._headersonly = False - self._fp = BytesFeedParser( - _factory=MilterMessage, policy=default_policy) - self._keep_body = False - for rule in self.rules: - if rule.need_body(): - self._keep_body = True - break + if not self.rules: + self.logger.debug( + "message is ignored by all rules, skip further processing") + return Milter.ACCEPT + self.fp = BytesIO() except Exception as e: self.logger.exception( f"an exception occured in data method: {e}") @@ -282,7 +264,7 @@ class ModifyMilter(Milter.Base): field = field.encode("ascii", errors="replace") value = value.encode("ascii", errors="replace") - self._fp.feed(field + b": " + value + b"\r\n") + self.fp.write(field + b": " + value + b"\r\n") except Exception as e: self.logger.exception( f"an exception occured in header method: {e}") @@ -292,7 +274,7 @@ class ModifyMilter(Milter.Base): def eoh(self): try: - self._fp.feed(b"\r\n") + self.fp.write(b"\r\n") except Exception as e: self.logger.exception( f"an exception occured in eoh method: {e}") @@ -302,8 +284,8 @@ class ModifyMilter(Milter.Base): def body(self, chunk): try: - if self._keep_body: - self._fp.feed(chunk) + if not self._headersonly: + self.fp.write(chunk) except Exception as e: self.logger.exception( f"an exception occured in body method: {e}") @@ -313,7 +295,19 @@ class ModifyMilter(Milter.Base): def eom(self): try: - self.msg = self._fp.close() + self.fp.seek(0) + self.msg = message_from_binary_file( + self.fp, _class=MilterMessage, policy=default_policy) + + self.msg_info = defaultdict(str) + self.msg_info["ip"] = self.IP + self.msg_info["port"] = self.port + self.msg_info["heloname"] = self.heloname + self.msg_info["envfrom"] = self.mailfrom + self.msg_info["rcpts"] = self.rcpts + self.msg_info["qid"] = self.qid + + self._replacebody = False milter_action = None for rule in self.rules: milter_action = rule.execute(self) @@ -321,7 +315,7 @@ class ModifyMilter(Milter.Base): if milter_action is not None: break - if self._replace_body: + if self._replacebody: data = self.msg.as_bytes(policy=SMTP) body_pos = data.find(b"\r\n\r\n") + 4 self.logger.debug("milter: replacebody") diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 7539be6..78d10c0 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -32,6 +32,7 @@ from collections import defaultdict from copy import copy from datetime import datetime from email.message import MIMEPart +from email.policy import SMTP from pymodmilter import CustomLogger, BaseConfig from pymodmilter.conditions import ConditionsConfig, Conditions @@ -204,7 +205,7 @@ def _patch_message_body(milter, action, text_template, html_template, logger): def _wrap_message(milter, logger): attachment = MIMEPart() - attachment.set_content(milter.msg.as_bytes(), + attachment.set_content(milter.msg.as_bytes(policy=SMTP), maintype="plain", subtype="text", disposition="attachment", filename=f"{milter.qid}.eml", @@ -317,7 +318,7 @@ def rewrite_links(milter, repl, pretend=False, milter.replacebody() -def store(milter, directory, pretend=False, +def store(milter, directory, original=False, pretend=False, logger=logging.getLogger(__name__)): timestamp = datetime.now().strftime("%Y%m%d%H%M%S") store_id = f"{timestamp}_{milter.qid}" @@ -326,7 +327,11 @@ def store(milter, directory, pretend=False, logger.info(f"store message in file {datafile}") try: with open(datafile, "wb") as fp: - fp.write(milter.msg.as_bytes()) + if original: + milter.fp.seek(0) + fp.write(milter.fp.read()) + else: + fp.write(milter.msg.as_bytes(policy=SMTP)) except IOError as e: raise RuntimeError(f"unable to store message: {e}") @@ -468,6 +473,10 @@ class ActionConfig(BaseConfig): f"{self['name']}: storage_type: invalid value, " \ f"should be string" self["storage_type"] = cfg["storage_type"] + + if "original" in cfg: + self.add_bool_arg(cfg, "original") + if self["storage_type"] == "file": self.add_string_arg(cfg, "directory") else: diff --git a/pymodmilter/docs/pymodmilter.conf.example b/pymodmilter/docs/pymodmilter.conf.example index 98967ea..e7e5d02 100644 --- a/pymodmilter/docs/pymodmilter.conf.example +++ b/pymodmilter/docs/pymodmilter.conf.example @@ -28,7 +28,7 @@ # Notes: A list of local hosts and networks. # Value: [ LIST ] # - "local_addrs": ["::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], + "local_addrs": ["fe80::/64", "::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], # Option: loglevel # Type: String @@ -206,7 +206,14 @@ # Type: String # Notes: Directory used to store e-mails. # Value: [ file ] - "directory": "/mnt/messages" + "directory": "/mnt/messages", + + # Option: original + # Type: Bool + # Notes: If set to true, store the message as received by the MTA instead of storing the current state + # of the message, that may was modified already by other actions. + # Value: [ true | false ] + "original": true } ] } From e27b5a77f6a2b1076d3103c777d1fbde1b0190c3 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Mon, 6 Sep 2021 22:34:00 +0200 Subject: [PATCH 107/145] fix encoding issues --- pymodmilter/__init__.py | 6 ++-- pymodmilter/_runtime_patches.py | 60 +++++++++++++++++++++++++++++++++ pymodmilter/actions.py | 5 ++- 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 8430f39..f3f2066 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -34,7 +34,7 @@ from Milter.utils import parse_addr from collections import defaultdict from email import message_from_binary_file from email.header import Header -from email.policy import default as default_policy, SMTP +from email.policy import SMTPUTF8 from io import BytesIO from netaddr import IPNetwork, AddrFormatError @@ -297,7 +297,7 @@ class ModifyMilter(Milter.Base): try: self.fp.seek(0) self.msg = message_from_binary_file( - self.fp, _class=MilterMessage, policy=default_policy) + self.fp, _class=MilterMessage, policy=SMTPUTF8) self.msg_info = defaultdict(str) self.msg_info["ip"] = self.IP @@ -316,7 +316,7 @@ class ModifyMilter(Milter.Base): break if self._replacebody: - data = self.msg.as_bytes(policy=SMTP) + data = self.msg.as_bytes() body_pos = data.find(b"\r\n\r\n") + 4 self.logger.debug("milter: replacebody") super().replacebody(data[body_pos:]) diff --git a/pymodmilter/_runtime_patches.py b/pymodmilter/_runtime_patches.py index d96e4b5..4d868ea 100644 --- a/pymodmilter/_runtime_patches.py +++ b/pymodmilter/_runtime_patches.py @@ -90,6 +90,66 @@ setattr(email._header_value_parser, "DisplayName", DisplayName) setattr(email._header_value_parser, "get_name_addr", get_name_addr) +# https://bugs.python.org/issue42484 +# +# fix: https://github.com/python/cpython/pull/24669 + +from email._header_value_parser import DOT, ObsLocalPart, ValueTerminal, get_word + + +def get_obs_local_part(value): + """ obs-local-part = word *("." word) + """ + obs_local_part = ObsLocalPart() + last_non_ws_was_dot = False + while value and (value[0]=='\\' or value[0] not in PHRASE_ENDS): + if value[0] == '.': + if last_non_ws_was_dot: + obs_local_part.defects.append(errors.InvalidHeaderDefect( + "invalid repeated '.'")) + obs_local_part.append(DOT) + last_non_ws_was_dot = True + value = value[1:] + continue + elif value[0]=='\\': + obs_local_part.append(ValueTerminal(value[0], + 'misplaced-special')) + value = value[1:] + obs_local_part.defects.append(errors.InvalidHeaderDefect( + "'\\' character outside of quoted-string/ccontent")) + last_non_ws_was_dot = False + continue + if obs_local_part and obs_local_part[-1].token_type != 'dot': + obs_local_part.defects.append(errors.InvalidHeaderDefect( + "missing '.' between words")) + try: + token, value = get_word(value) + last_non_ws_was_dot = False + except errors.HeaderParseError: + if value[0] not in CFWS_LEADER: + raise + token, value = get_cfws(value) + obs_local_part.append(token) + if not obs_local_part: + return obs_local_part, value + if (obs_local_part[0].token_type == 'dot' or + obs_local_part[0].token_type=='cfws' and + obs_local_part[1].token_type=='dot'): + obs_local_part.defects.append(errors.InvalidHeaderDefect( + "Invalid leading '.' in local part")) + if (obs_local_part[-1].token_type == 'dot' or + obs_local_part[-1].token_type=='cfws' and + obs_local_part[-2].token_type=='dot'): + obs_local_part.defects.append(errors.InvalidHeaderDefect( + "Invalid trailing '.' in local part")) + if obs_local_part.defects: + obs_local_part.token_type = 'invalid-obs-local-part' + return obs_local_part, value + + +setattr(email._header_value_parser, "get_obs_local_part", get_obs_local_part) + + # https://bugs.python.org/issue30681 # # fix: https://github.com/python/cpython/pull/22090 diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 78d10c0..3568cb5 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -32,7 +32,6 @@ from collections import defaultdict from copy import copy from datetime import datetime from email.message import MIMEPart -from email.policy import SMTP from pymodmilter import CustomLogger, BaseConfig from pymodmilter.conditions import ConditionsConfig, Conditions @@ -205,7 +204,7 @@ def _patch_message_body(milter, action, text_template, html_template, logger): def _wrap_message(milter, logger): attachment = MIMEPart() - attachment.set_content(milter.msg.as_bytes(policy=SMTP), + attachment.set_content(milter.msg.as_bytes(), maintype="plain", subtype="text", disposition="attachment", filename=f"{milter.qid}.eml", @@ -331,7 +330,7 @@ def store(milter, directory, original=False, pretend=False, milter.fp.seek(0) fp.write(milter.fp.read()) else: - fp.write(milter.msg.as_bytes(policy=SMTP)) + fp.write(milter.msg.as_bytes()) except IOError as e: raise RuntimeError(f"unable to store message: {e}") From 0ba60c45bcdad805062a0ae20ca9fbe529ca4f39 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 7 Sep 2021 16:29:10 +0200 Subject: [PATCH 108/145] fix handling of CR/LF in address parts --- pymodmilter/__init__.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index f3f2066..7eccecf 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -33,7 +33,8 @@ import json from Milter.utils import parse_addr from collections import defaultdict from email import message_from_binary_file -from email.header import Header +from email.header import Header, decode_header, make_header +from email.headerregistry import AddressHeader, _default_header_map from email.policy import SMTPUTF8 from io import BytesIO from netaddr import IPNetwork, AddrFormatError @@ -129,6 +130,8 @@ class ModifyMilter(Milter.Base): _rules = [] _loglevel = logging.INFO + _addr_fields = [f for f, v in _default_header_map.items() + if issubclass(v, AddressHeader)] @staticmethod def set_config(cfg): @@ -260,6 +263,19 @@ class ModifyMilter(Milter.Base): def header(self, field, value): try: + # remove CR and LF from address fields, otherwise pythons + # email library throws an exception + if field.lower() in ModifyMilter._addr_fields: + try: + v = str(make_header(decode_header(value))) + except Exception as e: + self.logger.error( + f"unable to decode field '{field}': {e}") + else: + if any(c in v for c in ["\r", "\n"]): + v = v.replace("\r", "").replace("\n", "") + value = Header(s=v).encode() + # remove surrogates field = field.encode("ascii", errors="replace") value = value.encode("ascii", errors="replace") @@ -297,7 +313,8 @@ class ModifyMilter(Milter.Base): try: self.fp.seek(0) self.msg = message_from_binary_file( - self.fp, _class=MilterMessage, policy=SMTPUTF8) + self.fp, _class=MilterMessage, policy=SMTPUTF8.clone( + refold_source='none')) self.msg_info = defaultdict(str) self.msg_info["ip"] = self.IP From f25909b34b020e2e9c25c866dcd7aca4c0c38bd0 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 7 Sep 2021 22:44:12 +0200 Subject: [PATCH 109/145] add header condition, rework conditions logic and fix logging --- README.md | 4 +- pymodmilter/__init__.py | 29 ++++++++++---- pymodmilter/actions.py | 13 ++++--- pymodmilter/conditions.py | 82 ++++++++++++++++++++++++++------------- pymodmilter/rules.py | 47 +++++----------------- 5 files changed, 96 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index bbce560..a349d98 100644 --- a/README.md +++ b/README.md @@ -67,10 +67,10 @@ Config options for **action** objects: * **store** * **conditions** (optional) A list of conditions which all have to be true to process the action. -* **pretend** (optional) - Just pretend all actions of this rule, for test purposes. * **loglevel** (optional) As described above in the [Global](#Global) section. +* **pretend** (optional) + As described above in the [Global](#Global) section. Config options for **add_header** actions: * **field** diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 7eccecf..2eeab12 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -32,6 +32,7 @@ import json from Milter.utils import parse_addr from collections import defaultdict +from copy import copy from email import message_from_binary_file from email.header import Header, decode_header, make_header from email.headerregistry import AddressHeader, _default_header_map @@ -239,14 +240,29 @@ class ModifyMilter(Milter.Base): self.logger = CustomLogger(self.logger, {"qid": self.qid}) self.logger.debug("received queue-id from MTA") + # pre-filter rules and actions by the host condition, other + # conditions (headers, envelope-from, envelope-to) may get + # changed by executed actions later on. + # rules are copied to preserve the originally configured actions. + # also check if the mail body is needed by any upcoming action. + self.rules = [] self._headersonly = True for rule in ModifyMilter._rules: - if not rule.ignores(host=self.IP, envfrom=self.mailfrom, - envto=[*self.rcpts]): - self.rules.append(rule) - if rule.need_body(): - self._headersonly = False + if rule.conditions is None or \ + rule.conditions.match(host=self.IP): + actions = [] + for action in rule.actions: + if action.conditions is None or \ + action.conditions.match(host=self.IP): + actions.append(action) + if action.need_body(): + self._headersonly = False + + if actions: + rule = copy(rule) + rule.actions = actions + self.rules.append(rule) if not self.rules: self.logger.debug( @@ -311,11 +327,11 @@ class ModifyMilter(Milter.Base): def eom(self): try: + # setup msg and msg_info to be read/modified by rules and actions self.fp.seek(0) self.msg = message_from_binary_file( self.fp, _class=MilterMessage, policy=SMTPUTF8.clone( refold_source='none')) - self.msg_info = defaultdict(str) self.msg_info["ip"] = self.IP self.msg_info["port"] = self.port @@ -328,7 +344,6 @@ class ModifyMilter(Milter.Base): milter_action = None for rule in self.rules: milter_action = rule.execute(self) - if milter_action is not None: break diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 3568cb5..a985434 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -518,13 +518,14 @@ class Action: """Return the needs of this action.""" return self._need_body - def execute(self, milter, pretend=None): + def execute(self, milter): """Execute configured action.""" - if pretend is None: - pretend = self.pretend - logger = CustomLogger( self.logger, {"name": self._name, "qid": milter.qid}) - return self._func(milter=milter, pretend=pretend, - logger=logger, **self._args) + if self.conditions is None or \ + self.conditions.match(envfrom=milter.mailfrom, + envto=[*milter.rcpts], + headers=milter.msg.items()): + return self._func(milter=milter, pretend=self.pretend, + logger=logger, **self._args) diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index 0465a14..b47d78c 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -19,7 +19,7 @@ __all__ = [ import re from netaddr import IPAddress, IPNetwork, AddrFormatError -from pymodmilter import BaseConfig +from pymodmilter import CustomLogger, BaseConfig class ConditionsConfig(BaseConfig): @@ -58,6 +58,15 @@ class ConditionsConfig(BaseConfig): except re.error as e: raise ValueError(f"{self['name']}: {arg}: {e}") + if "header" in cfg: + self.add_string_arg(cfg, "header") + try: + self["args"]["header"] = re.compile( + self["args"]["header"], + re.IGNORECASE + re.DOTALL + re.MULTILINE) + except re.error as e: + raise ValueError(f"{self['name']}: header: {e}") + self.logger.debug(f"{self['name']}: " f"loglevel={self['loglevel']}, " f"args={self['args']}") @@ -70,11 +79,13 @@ class Conditions: self.logger = cfg.logger self._local_addrs = milter_cfg["local_addrs"] + self._name = cfg["name"] self._args = cfg["args"] - def match(self, args): - if "host" in args: - ip = IPAddress(args["host"]) + def match(self, host=None, envfrom=None, envto=None, headers=None): + logger = CustomLogger(self.logger, {"name": self._name}) + if host: + ip = IPAddress(host) if "local" in self._args: is_local = False @@ -84,13 +95,13 @@ class Conditions: break if is_local != self._args["local"]: - self.logger.debug( - f"ignore host {args['host']}, " + logger.debug( + f"ignore host {host}, " f"condition local does not match") return False - self.logger.debug( - f"condition local matches for host {args['host']}") + logger.debug( + f"condition local matches for host {host}") if "hosts" in self._args: found = False @@ -100,38 +111,55 @@ class Conditions: break if not found: - self.logger.debug( - f"ignore host {args['host']}, " + logger.debug( + f"ignore host {host}, " f"condition hosts does not match") return False - self.logger.debug( - f"condition hosts matches for host {args['host']}") + logger.debug( + f"condition hosts matches for host {host}") - if "envfrom" in args and "envfrom" in self._args: - if not self._args["envfrom"].match(args["envfrom"]): - self.logger.debug( - f"ignore envelope-from address {args['envfrom']}, " + if envfrom and "envfrom" in self._args: + if not self._args["envfrom"].match(envfrom): + logger.debug( + f"ignore envelope-from address {envfrom}, " f"condition envfrom does not match") return False - self.logger.debug( + logger.debug( f"condition envfrom matches for " - f"envelope-from address {args['envfrom']}") + f"envelope-from address {envfrom}") - if "envto" in args and "envto" in self._args: - if not isinstance(args["envto"], list): - args["envto"] = [args["envto"]] + if envto and "envto" in self._args: + if not isinstance(envto, list): + envto = [envto] - for envto in args["envto"]: - if not self._args["envto"].match(envto): - self.logger.debug( - f"ignore envelope-to address {args['envto']}, " + for to in envto: + if not self._args["envto"].match(to): + logger.debug( + f"ignore envelope-to address {envto}, " f"condition envto does not match") return False - self.logger.debug( + logger.debug( f"condition envto matches for " - f"envelope-to address {args['envto']}") + f"envelope-to address {envto}") + + if headers and "header" in self._args: + match = None + for field, value in headers: + header = f"{field}: {value}" + match = self._args["header"].search(header) + if match: + logger.debug( + f"condition header matches for " + f"header: {header}") + break + + if not match: + logger.debug( + "ignore message, " + "condition header does not match") + return False return True diff --git a/pymodmilter/rules.py b/pymodmilter/rules.py index 9c0c5fb..ef925eb 100644 --- a/pymodmilter/rules.py +++ b/pymodmilter/rules.py @@ -77,46 +77,19 @@ class Rule: else: self.conditions = Conditions(milter_cfg, cfg["conditions"]) - self._need_body = False - self.actions = [] for action_cfg in cfg["actions"]: - action = Action(milter_cfg, action_cfg) - self.actions.append(action) - if action.need_body(): - self._need_body = True + self.actions.append(Action(milter_cfg, action_cfg)) self.pretend = cfg["pretend"] - def need_body(self): - """Return True if this rule needs the message body.""" - return self._need_body - - def ignores(self, host=None, envfrom=None, envto=None): - args = {} - - if host is not None: - args["host"] = host - - if envfrom is not None: - args["envfrom"] = envfrom - - if envto is not None: - args["envto"] = envto - - if self.conditions is None or self.conditions.match(args): - for action in self.actions: - if action.conditions is None or action.conditions.match(args): - return False - - return True - - def execute(self, milter, pretend=None): + def execute(self, milter): """Execute all actions of this rule.""" - if pretend is None: - pretend = self.pretend - - for action in self.actions: - milter_action = action.execute(milter, pretend=pretend) - if milter_action is not None: - return milter_action + if self.conditions is None or \ + self.conditions.match(envfrom=milter.mailfrom, + envto=[*milter.rcpts], + headers=milter.msg.items()): + for action in self.actions: + milter_action = action.execute(milter) + if milter_action is not None: + return milter_action From be715d8b01f8756cb0466b04e24ac21bad348aa0 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 7 Sep 2021 23:03:50 +0200 Subject: [PATCH 110/145] make store procedure aware of pretend option --- pymodmilter/__init__.py | 2 +- pymodmilter/actions.py | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 2eeab12..f24f459 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -243,7 +243,6 @@ class ModifyMilter(Milter.Base): # pre-filter rules and actions by the host condition, other # conditions (headers, envelope-from, envelope-to) may get # changed by executed actions later on. - # rules are copied to preserve the originally configured actions. # also check if the mail body is needed by any upcoming action. self.rules = [] @@ -260,6 +259,7 @@ class ModifyMilter(Milter.Base): self._headersonly = False if actions: + # copy needed rules to preserve configured actions rule = copy(rule) rule.actions = actions self.rules.append(rule) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index a985434..3a0b625 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -324,15 +324,17 @@ def store(milter, directory, original=False, pretend=False, datafile = os.path.join(directory, store_id) logger.info(f"store message in file {datafile}") - try: - with open(datafile, "wb") as fp: - if original: - milter.fp.seek(0) - fp.write(milter.fp.read()) - else: - fp.write(milter.msg.as_bytes()) - except IOError as e: - raise RuntimeError(f"unable to store message: {e}") + + if not pretend: + try: + with open(datafile, "wb") as fp: + if original: + milter.fp.seek(0) + fp.write(milter.fp.read()) + else: + fp.write(milter.msg.as_bytes()) + except IOError as e: + raise RuntimeError(f"unable to store message: {e}") class ActionConfig(BaseConfig): From 9b30bb68c4c891d2ee2d182f62b4c95c302da9ac Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 9 Sep 2021 11:34:47 +0200 Subject: [PATCH 111/145] rename need_body to headersonly --- pymodmilter/__init__.py | 2 +- pymodmilter/actions.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index f24f459..5223a7a 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -255,7 +255,7 @@ class ModifyMilter(Milter.Base): if action.conditions is None or \ action.conditions.match(host=self.IP): actions.append(action) - if action.need_body(): + if not action.headersonly(): self._headersonly = False if actions: diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 3a0b625..09733ef 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -370,7 +370,7 @@ class ActionConfig(BaseConfig): if self["type"] == "add_header": self["func"] = add_header - self["need_body"] = False + self["headersonly"] = True if "field" not in cfg and "header" in cfg: cfg["field"] = cfg["header"] @@ -379,7 +379,7 @@ class ActionConfig(BaseConfig): elif self["type"] == "mod_header": self["func"] = mod_header - self["need_body"] = False + self["headersonly"] = True if "field" not in cfg and "header" in cfg: cfg["field"] = cfg["header"] @@ -400,7 +400,7 @@ class ActionConfig(BaseConfig): elif self["type"] == "del_header": self["func"] = del_header - self["need_body"] = False + self["headersonly"] = True if "field" not in cfg and "header" in cfg: cfg["field"] = cfg["header"] @@ -420,7 +420,7 @@ class ActionConfig(BaseConfig): elif self["type"] == "add_disclaimer": self["func"] = add_disclaimer - self["need_body"] = True + self["headersonly"] = False if "html_template" not in cfg and "html_file" in cfg: cfg["html_template"] = cfg["html_file"] @@ -461,12 +461,12 @@ class ActionConfig(BaseConfig): elif self["type"] == "rewrite_links": self["func"] = rewrite_links - self["need_body"] = True + self["headersonly"] = False self.add_string_arg(cfg, "repl") elif self["type"] == "store": self["func"] = store - self["need_body"] = True + self["headersonly"] = False assert "storage_type" in cfg, \ f"{self['name']}: mandatory parameter 'storage_type' not found" @@ -514,11 +514,11 @@ class Action: self._name = cfg["name"] self._func = cfg["func"] self._args = cfg["args"] - self._need_body = cfg["need_body"] + self._headersonly = cfg["headersonly"] - def need_body(self): + def headersonly(self): """Return the needs of this action.""" - return self._need_body + return self._headersonly def execute(self, milter): """Execute configured action.""" From 83df637792ebb1f01077e2f98b2733c6abf07a32 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 10 Sep 2021 01:18:00 +0200 Subject: [PATCH 112/145] add new file storage/quarantine and change version to 1.2.0 --- pymodmilter/__init__.py | 16 +- pymodmilter/actions.py | 339 ++--------------------------------- pymodmilter/modifications.py | 320 +++++++++++++++++++++++++++++++++ pymodmilter/storages.py | 226 +++++++++++++++++++++++ 4 files changed, 570 insertions(+), 331 deletions(-) create mode 100644 pymodmilter/modifications.py create mode 100644 pymodmilter/storages.py diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 5223a7a..f54d680 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -16,12 +16,13 @@ __all__ = [ "actions", "base", "conditions", + "modifications", "rules", "run", "ModifyMilterConfig", "ModifyMilter"] -__version__ = "1.1.7" +__version__ = "1.2.0" from pymodmilter import _runtime_patches @@ -327,18 +328,15 @@ class ModifyMilter(Milter.Base): def eom(self): try: - # setup msg and msg_info to be read/modified by rules and actions + # msg and msginfo contain the runtime data that + # is read/modified by actions self.fp.seek(0) self.msg = message_from_binary_file( self.fp, _class=MilterMessage, policy=SMTPUTF8.clone( refold_source='none')) - self.msg_info = defaultdict(str) - self.msg_info["ip"] = self.IP - self.msg_info["port"] = self.port - self.msg_info["heloname"] = self.heloname - self.msg_info["envfrom"] = self.mailfrom - self.msg_info["rcpts"] = self.rcpts - self.msg_info["qid"] = self.qid + self.msginfo = { + "mailfrom": self.mailfrom, + "rcpts": self.rcpts} self._replacebody = False milter_action = None diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 09733ef..ebe7d78 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -13,328 +13,17 @@ # __all__ = [ - "add_header", - "mod_header", - "del_header", - "add_disclaimer", - "rewrite_links", - "store", "ActionConfig", "Action"] -import logging import os import re -from base64 import b64encode from bs4 import BeautifulSoup -from collections import defaultdict -from copy import copy -from datetime import datetime -from email.message import MIMEPart from pymodmilter import CustomLogger, BaseConfig from pymodmilter.conditions import ConditionsConfig, Conditions -from pymodmilter import replace_illegal_chars - - -def add_header(milter, field, value, pretend=False, - logger=logging.getLogger(__name__)): - """Add a mail header field.""" - header = f"{field}: {value}" - if logger.getEffectiveLevel() == logging.DEBUG: - logger.debug(f"add_header: {header}") - else: - logger.info(f"add_header: {header[0:70]}") - - milter.msg.add_header(field, replace_illegal_chars(value)) - - if not pretend: - milter.addheader(field, value) - - -def mod_header(milter, field, value, search=None, pretend=False, - logger=logging.getLogger(__name__)): - """Change the value of a mail header field.""" - if isinstance(field, str): - field = re.compile(field, re.IGNORECASE) - - if isinstance(search, str): - search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE) - - idx = defaultdict(int) - - for i, (f, v) in enumerate(milter.msg.items()): - f_lower = f.lower() - idx[f_lower] += 1 - - if not field.match(f): - continue - - new_value = v - if search is not None: - new_value = search.sub(value, v).strip() - else: - new_value = value - - if not new_value: - logger.warning( - "mod_header: resulting value is empty, " - "skip modification") - continue - - if new_value == v: - continue - - header = f"{f}: {v}" - new_header = f"{f}: {new_value}" - - if logger.getEffectiveLevel() == logging.DEBUG: - logger.debug(f"mod_header: {header}: {new_header}") - else: - logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}") - - milter.msg.replace_header( - f, replace_illegal_chars(new_value), idx=idx[f_lower]) - - if not pretend: - milter.chgheader(f, new_value, idx=idx[f_lower]) - - -def del_header(milter, field, value=None, pretend=False, - logger=logging.getLogger(__name__)): - """Delete a mail header field.""" - if isinstance(field, str): - field = re.compile(field, re.IGNORECASE) - - if isinstance(value, str): - value = re.compile(value, re.MULTILINE + re.DOTALL + re.IGNORECASE) - - idx = defaultdict(int) - - for f, v in milter.msg.items(): - f_lower = f.lower() - idx[f_lower] += 1 - - if not field.match(f): - continue - - if value is not None and not value.search(v): - continue - - header = f"{f}: {v}" - if logger.getEffectiveLevel() == logging.DEBUG: - logger.debug(f"del_header: {header}") - else: - logger.info(f"del_header: {header[0:70]}") - milter.msg.remove_header(f, idx=idx[f_lower]) - - if not pretend: - milter.chgheader(f, "", idx=idx[f_lower]) - - idx[f_lower] -= 1 - - -def _get_body_content(msg, pref): - part = None - content = None - if not msg.is_multipart() and msg.get_content_type() == f"text/{pref}": - part = msg - else: - part = msg.get_body(preferencelist=(pref)) - - if part is not None: - content = part.get_content() - - return (part, content) - - -def _has_content_before_body_tag(soup): - s = copy(soup) - for element in s.find_all("head") + s.find_all("body"): - element.extract() - - if len(s.text.strip()) > 0: - return True - - return False - - -def _patch_message_body(milter, action, text_template, html_template, logger): - text_body, text_content = _get_body_content(milter.msg, "plain") - html_body, html_content = _get_body_content(milter.msg, "html") - - if text_content is None and html_content is None: - raise RuntimeError("message does not contain any body part") - - if text_content is not None: - logger.info(f"{action} text disclaimer") - - if action == "prepend": - content = f"{text_template}{text_content}" - else: - content = f"{text_content}{text_template}" - - text_body.set_content( - content.encode(), maintype="text", subtype="plain") - text_body.set_param("charset", "UTF-8", header="Content-Type") - del text_body["MIME-Version"] - - if html_content is not None: - logger.info(f"{action} html disclaimer") - - soup = BeautifulSoup(html_content, "html.parser") - - body = soup.find('body') - if not body: - body = soup - elif _has_content_before_body_tag(soup): - body = soup - - if action == "prepend": - body.insert(0, copy(html_template)) - else: - body.append(html_template) - - html_body.set_content( - str(soup).encode(), maintype="text", subtype="html") - html_body.set_param("charset", "UTF-8", header="Content-Type") - del html_body["MIME-Version"] - - -def _wrap_message(milter, logger): - attachment = MIMEPart() - attachment.set_content(milter.msg.as_bytes(), - maintype="plain", subtype="text", - disposition="attachment", - filename=f"{milter.qid}.eml", - params={"name": f"{milter.qid}.eml"}) - - milter.msg.clear_content() - milter.msg.set_content( - "Please see the original email attached.") - milter.msg.add_alternative( - "Please see the original email attached.", - subtype="html") - milter.msg.make_mixed() - milter.msg.attach(attachment) - - -def _inject_body(milter): - if not milter.msg.is_multipart(): - milter.msg.make_mixed() - - attachments = [] - for attachment in milter.msg.iter_attachments(): - if "content-disposition" not in attachment: - attachment["Content-Disposition"] = "attachment" - attachments.append(attachment) - - milter.msg.clear_content() - milter.msg.set_content("") - milter.msg.add_alternative("", subtype="html") - milter.msg.make_mixed() - - for attachment in attachments: - milter.msg.attach(attachment) - - -def add_disclaimer(milter, text_template, html_template, action, error_policy, - pretend=False, logger=logging.getLogger(__name__)): - """Append or prepend a disclaimer to the mail body.""" - old_headers = milter.msg.items() - - try: - try: - _patch_message_body( - milter, action, text_template, html_template, logger) - except RuntimeError as e: - logger.info(f"{e}, inject empty plain and html body") - _inject_body(milter) - _patch_message_body( - milter, action, text_template, html_template, logger) - except Exception as e: - logger.warning(e) - if error_policy == "ignore": - logger.info( - "unable to add disclaimer to message body, " - "ignore error according to policy") - return - elif error_policy == "reject": - logger.info( - "unable to add disclaimer to message body, " - "reject message according to policy") - return [ - ("reject", "Message rejected due to error")] - - logger.info("wrap original message in a new message envelope") - try: - _wrap_message(milter, logger) - _patch_message_body( - milter, action, text_template, html_template, logger) - except Exception as e: - logger.error(e) - raise Exception( - "unable to wrap message in a new message envelope, " - "give up ...") - - if not pretend: - milter.update_headers(old_headers) - milter.replacebody() - - -def rewrite_links(milter, repl, pretend=False, - logger=logging.getLogger(__name__)): - """Rewrite link targets in the mail html body.""" - - html_body, html_content = _get_body_content(milter.msg, "html") - if html_content is not None: - soup = BeautifulSoup(html_content, "html.parser") - - rewritten = 0 - for link in soup.find_all("a", href=True): - if not link["href"]: - continue - - if "{URL_B64}" in repl: - url_b64 = b64encode(link["href"].encode()).decode() - target = repl.replace("{URL_B64}", url_b64) - else: - target = repl - - link["href"] = target - rewritten += 1 - - if rewritten: - logger.info(f"rewrote {rewritten} link(s) in html body") - - html_body.set_content( - str(soup).encode(), maintype="text", subtype="html") - html_body.set_param("charset", "UTF-8", header="Content-Type") - del html_body["MIME-Version"] - - if not pretend: - milter.replacebody() - - -def store(milter, directory, original=False, pretend=False, - logger=logging.getLogger(__name__)): - timestamp = datetime.now().strftime("%Y%m%d%H%M%S") - store_id = f"{timestamp}_{milter.qid}" - datafile = os.path.join(directory, store_id) - - logger.info(f"store message in file {datafile}") - - if not pretend: - try: - with open(datafile, "wb") as fp: - if original: - milter.fp.seek(0) - fp.write(milter.fp.read()) - else: - fp.write(milter.msg.as_bytes()) - except IOError as e: - raise RuntimeError(f"unable to store message: {e}") +from pymodmilter import modifications, storages class ActionConfig(BaseConfig): @@ -369,7 +58,7 @@ class ActionConfig(BaseConfig): self["type"] = cfg["type"] if self["type"] == "add_header": - self["func"] = add_header + self["class"] = modifications.AddHeader self["headersonly"] = True if "field" not in cfg and "header" in cfg: @@ -378,7 +67,7 @@ class ActionConfig(BaseConfig): self.add_string_arg(cfg, ("field", "value")) elif self["type"] == "mod_header": - self["func"] = mod_header + self["class"] = modifications.ModHeader self["headersonly"] = True if "field" not in cfg and "header" in cfg: @@ -399,7 +88,7 @@ class ActionConfig(BaseConfig): raise ValueError(f"{self['name']}: {arg}: {e}") elif self["type"] == "del_header": - self["func"] = del_header + self["class"] = modifications.DelHeader self["headersonly"] = True if "field" not in cfg and "header" in cfg: @@ -419,7 +108,7 @@ class ActionConfig(BaseConfig): raise ValueError(f"{self['name']}: {arg}: {e}") elif self["type"] == "add_disclaimer": - self["func"] = add_disclaimer + self["class"] = modifications.AddDisclaimer self["headersonly"] = False if "html_template" not in cfg and "html_file" in cfg: @@ -460,12 +149,11 @@ class ActionConfig(BaseConfig): f"{self['name']}: unable to open/read template file: {e}") elif self["type"] == "rewrite_links": - self["func"] = rewrite_links + self["class"] = modifications.RewriteLinks self["headersonly"] = False self.add_string_arg(cfg, "repl") elif self["type"] == "store": - self["func"] = store self["headersonly"] = False assert "storage_type" in cfg, \ @@ -479,7 +167,15 @@ class ActionConfig(BaseConfig): self.add_bool_arg(cfg, "original") if self["storage_type"] == "file": + self["class"] = storages.FileMailStorage self.add_string_arg(cfg, "directory") + # check if directory exists and is writable + if not os.path.isdir(self["args"]["directory"]) or \ + not os.access(self["args"]["directory"], os.W_OK): + raise RuntimeError( + f"{self['name']}: file quarantine directory " + f"'{self['directory']}' does not exist or is " + f"not writable") else: raise RuntimeError( f"{self['name']}: storage_type: invalid storage type") @@ -512,8 +208,7 @@ class Action: self.pretend = cfg["pretend"] self._name = cfg["name"] - self._func = cfg["func"] - self._args = cfg["args"] + self._class = cfg["class"](**cfg["args"]) self._headersonly = cfg["headersonly"] def headersonly(self): @@ -529,5 +224,5 @@ class Action: self.conditions.match(envfrom=milter.mailfrom, envto=[*milter.rcpts], headers=milter.msg.items()): - return self._func(milter=milter, pretend=self.pretend, - logger=logger, **self._args) + return self._class.execute( + milter=milter, pretend=self.pretend, logger=logger) diff --git a/pymodmilter/modifications.py b/pymodmilter/modifications.py new file mode 100644 index 0000000..0f398af --- /dev/null +++ b/pymodmilter/modifications.py @@ -0,0 +1,320 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +__all__ = [ + "AddHeader", + "ModHeader", + "DelHeader", + "AddDisclaimer", + "RewriteLinks"] + +import logging + +from base64 import b64encode +from bs4 import BeautifulSoup +from collections import defaultdict +from copy import copy +from email.message import MIMEPart + +from pymodmilter import replace_illegal_chars + + +class AddHeader: + """Add a mail header field.""" + def __init__(self, field, value): + self.field = field + self.value = value + + def execute(self, milter, pretend=False, + logger=logging.getLogger(__name__)): + header = f"{self.field}: {self.value}" + if logger.getEffectiveLevel() == logging.DEBUG: + logger.debug(f"add_header: {header}") + else: + logger.info(f"add_header: {header[0:70]}") + + milter.msg.add_header(self.field, self.value) + + if not pretend: + milter.addheader(self.field, self.value) + + +class ModHeader: + """Change the value of a mail header field.""" + def __init__(self, field, value, search=None): + self.field = field + self.value = value + self.search = search + + def execute(self, milter, pretend=False, + logger=logging.getLogger(__name__)): + idx = defaultdict(int) + + for i, (field, value) in enumerate(milter.msg.items()): + field_lower = field.lower() + idx[field_lower] += 1 + + if not self.field.match(field): + continue + + new_value = value + if self.search is not None: + new_value = self.search.sub(self.value, value).strip() + else: + new_value = self.value + + if not new_value: + logger.warning( + "mod_header: resulting value is empty, " + "skip modification") + continue + + if new_value == value: + continue + + header = f"{field}: {value}" + new_header = f"{field}: {new_value}" + + if logger.getEffectiveLevel() == logging.DEBUG: + logger.debug(f"mod_header: {header}: {new_header}") + else: + logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}") + + milter.msg.replace_header( + field, replace_illegal_chars(new_value), idx=idx[field_lower]) + + if not pretend: + milter.chgheader(field, new_value, idx=idx[field_lower]) + + +class DelHeader: + """Delete a mail header field.""" + def __init__(self, field, value=None): + self.field = field + self.value = value + + def execute(self, milter, pretend=False, + logger=logging.getLogger(__name__)): + idx = defaultdict(int) + + for field, value in milter.msg.items(): + field_lower = field.lower() + idx[field_lower] += 1 + + if not self.field.match(field): + continue + + if self.value is not None and not self.value.search(value): + continue + + header = f"{field}: {value}" + if logger.getEffectiveLevel() == logging.DEBUG: + logger.debug(f"del_header: {header}") + else: + logger.info(f"del_header: {header[0:70]}") + milter.msg.remove_header(field, idx=idx[field_lower]) + + if not pretend: + milter.chgheader(field, "", idx=idx[field_lower]) + + idx[field_lower] -= 1 + + +def _get_body_content(msg, pref): + part = None + content = None + if not msg.is_multipart() and msg.get_content_type() == f"text/{pref}": + part = msg + else: + part = msg.get_body(preferencelist=(pref)) + + if part is not None: + content = part.get_content() + + return (part, content) + + +def _has_content_before_body_tag(soup): + s = copy(soup) + for element in s.find_all("head") + s.find_all("body"): + element.extract() + + if len(s.text.strip()) > 0: + return True + + return False + + +def _inject_body(milter): + if not milter.msg.is_multipart(): + milter.msg.make_mixed() + + attachments = [] + for attachment in milter.msg.iter_attachments(): + if "content-disposition" not in attachment: + attachment["Content-Disposition"] = "attachment" + attachments.append(attachment) + + milter.msg.clear_content() + milter.msg.set_content("") + milter.msg.add_alternative("", subtype="html") + milter.msg.make_mixed() + + for attachment in attachments: + milter.msg.attach(attachment) + + +def _wrap_message(milter): + attachment = MIMEPart() + attachment.set_content(milter.msg.as_bytes(), + maintype="plain", subtype="text", + disposition="attachment", + filename=f"{milter.qid}.eml", + params={"name": f"{milter.qid}.eml"}) + + milter.msg.clear_content() + milter.msg.set_content( + "Please see the original email attached.") + milter.msg.add_alternative( + "Please see the original email attached.", + subtype="html") + milter.msg.make_mixed() + milter.msg.attach(attachment) + + +class AddDisclaimer: + """Append or prepend a disclaimer to the mail body.""" + def __init__(self, text_template, html_template, action, error_policy): + self.text_template = text_template + self.html_template = html_template + self.action = action + self.error_policy = error_policy + + def patch_message_body(self, milter, logger): + text_body, text_content = _get_body_content(milter.msg, "plain") + html_body, html_content = _get_body_content(milter.msg, "html") + + if text_content is None and html_content is None: + raise RuntimeError("message does not contain any body part") + + if text_content is not None: + logger.info(f"{self.action} text disclaimer") + + if self.action == "prepend": + content = f"{self.text_template}{text_content}" + else: + content = f"{text_content}{self.text_template}" + + text_body.set_content( + content.encode(), maintype="text", subtype="plain") + text_body.set_param("charset", "UTF-8", header="Content-Type") + del text_body["MIME-Version"] + + if html_content is not None: + logger.info(f"{self.action} html disclaimer") + + soup = BeautifulSoup(html_content, "html.parser") + + body = soup.find('body') + if not body: + body = soup + elif _has_content_before_body_tag(soup): + body = soup + + if self.action == "prepend": + body.insert(0, copy(self.html_template)) + else: + body.append(self.html_template) + + html_body.set_content( + str(soup).encode(), maintype="text", subtype="html") + html_body.set_param("charset", "UTF-8", header="Content-Type") + del html_body["MIME-Version"] + + def execute(self, milter, pretend=False, + logger=logging.getLogger(__name__)): + old_headers = milter.msg.items() + + try: + try: + self.patch_message_body(milter, logger) + except RuntimeError as e: + logger.info(f"{e}, inject empty plain and html body") + _inject_body(milter) + self.patch_message_body(milter, logger) + + except Exception as e: + logger.warning(e) + if self.error_policy == "ignore": + logger.info( + "unable to add disclaimer to message body, " + "ignore error according to policy") + return + elif self.error_policy == "reject": + logger.info( + "unable to add disclaimer to message body, " + "reject message according to policy") + return [ + ("reject", "Message rejected due to error")] + + logger.info("wrap original message in a new message envelope") + try: + _wrap_message(milter) + self.patch_message_body(milter, logger) + except Exception as e: + logger.error(e) + raise Exception( + "unable to wrap message in a new message envelope, " + "give up ...") + + if not pretend: + milter.update_headers(old_headers) + milter.replacebody() + + +class RewriteLinks: + """Rewrite link targets in the mail html body.""" + def __init__(self, repl): + self.repl = repl + + def execute(self, milter, pretend=False, + logger=logging.getLogger(__name__)): + html_body, html_content = _get_body_content(milter.msg, "html") + if html_content is not None: + soup = BeautifulSoup(html_content, "html.parser") + + rewritten = 0 + for link in soup.find_all("a", href=True): + if not link["href"]: + continue + + if "{URL_B64}" in self.repl: + url_b64 = b64encode(link["href"].encode()).decode() + target = self.repl.replace("{URL_B64}", url_b64) + else: + target = self.repl + + link["href"] = target + rewritten += 1 + + if rewritten: + logger.info(f"rewrote {rewritten} link(s) in html body") + + html_body.set_content( + str(soup).encode(), maintype="text", subtype="html") + html_body.set_param("charset", "UTF-8", header="Content-Type") + del html_body["MIME-Version"] + + if not pretend: + milter.replacebody() diff --git a/pymodmilter/storages.py b/pymodmilter/storages.py new file mode 100644 index 0000000..8376cbd --- /dev/null +++ b/pymodmilter/storages.py @@ -0,0 +1,226 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +import json +import logging +import os + +from calendar import timegm +from datetime import datetime +from glob import glob +from time import gmtime + + +class BaseMailStorage(object): + "Mail storage base class" + def __init__(self): + return + + def add(self, data, qid, mailfrom="", recipients=[]): + "Add email to storage." + return ("", "") + + def find(self, mailfrom=None, recipients=None, older_than=None): + "Find emails in storage." + return + + def get_metadata(self, storage_id): + "Return metadata of email in storage." + return + + def delete(self, storage_id, recipients=None): + "Delete email from storage." + return + + def get_mail(self, storage_id): + "Return email and metadata." + return + + +class FileMailStorage(BaseMailStorage): + "Storage class to store mails on filesystem." + def __init__(self, directory, original=False): + super().__init__() + self.directory = directory + self.original = original + self._metadata_suffix = ".metadata" + + def _save_datafile(self, storage_id, data): + datafile = os.path.join(self.directory, storage_id) + try: + with open(datafile, "wb") as f: + f.write(data) + except IOError as e: + raise RuntimeError(f"unable save data file: {e}") + + return datafile + + def _save_metafile(self, storage_id, metadata): + metafile = os.path.join( + self.directory, f"{storage_id}{self._metadata_suffix}") + try: + with open(metafile, "w") as f: + json.dump(metadata, f, indent=2) + except IOError as e: + raise RuntimeError(f"unable to save metadata file: {e}") + + def _remove(self, storage_id): + datafile = os.path.join(self.directory, storage_id) + metafile = f"{datafile}{self._metadata_suffix}" + + try: + os.remove(metafile) + except IOError as e: + raise RuntimeError(f"unable to remove metadata file: {e}") + + try: + os.remove(datafile) + except IOError as e: + raise RuntimeError(f"unable to remove data file: {e}") + + def add(self, data, qid, mailfrom="", recipients=[], subject=""): + "Add email to file storage and return storage id." + super().add(data, qid, mailfrom, recipients) + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + storage_id = f"{timestamp}_{qid}" + + # save mail + datafile = self._save_datafile(storage_id, data) + + # save metadata + metadata = { + "mailfrom": mailfrom, + "recipients": recipients, + "subject": subject, + "timestamp": timegm(gmtime()), + "queue_id": qid} + + try: + self._save_metafile(storage_id, metadata) + except RuntimeError as e: + os.remove(datafile) + raise e + + return (storage_id, datafile) + + def execute(self, milter, pretend=False, + logger=logging.getLogger(__name__)): + if self.original: + milter.fp.seek(0) + data = milter.fp.read + mailfrom = milter.mailfrom + recipients = list(milter.rcpts) + subject = "" + else: + data = milter.msg.as_bytes + mailfrom = milter.msginfo["mailfrom"] + recipients = list(milter.msginfo["rcpts"]) + subject = milter.msg["subject"] or "" + + storage_id, datafile = self.add( + data(), milter.qid, mailfrom, recipients, subject) + logger.info(f"stored message in file {datafile}") + + def get_metadata(self, storage_id): + "Return metadata of email in storage." + super(FileMailStorage, self).get_metadata(storage_id) + + metafile = os.path.join( + self.directory, f"{storage_id}{self._metadata_suffix}") + if not os.path.isfile(metafile): + raise RuntimeError( + f"invalid storage id '{storage_id}'") + + try: + with open(metafile, "r") as f: + metadata = json.load(f) + except IOError as e: + raise RuntimeError(f"unable to read metadata file: {e}") + except json.JSONDecodeError as e: + raise RuntimeError( + f"invalid metafile '{metafile}': {e}") + + return metadata + + def find(self, mailfrom=None, recipients=None, older_than=None): + "Find emails in storage." + super(FileMailStorage, self).find(mailfrom, recipients, older_than) + if isinstance(mailfrom, str): + mailfrom = [mailfrom] + if isinstance(recipients, str): + recipients = [recipients] + + emails = {} + metafiles = glob(os.path.join( + self.directory, f"*{self._metadata_suffix}")) + for metafile in metafiles: + if not os.path.isfile(metafile): + continue + + storage_id = os.path.basename( + metafile[:-len(self._metadata_suffix)]) + metadata = self.get_metadata(storage_id) + if older_than is not None: + if timegm(gmtime()) - metadata["date"] < (older_than * 86400): + continue + + if mailfrom is not None: + if metadata["mailfrom"] not in mailfrom: + continue + + if recipients is not None: + if len(recipients) == 1 and \ + recipients[0] not in metadata["recipients"]: + continue + elif len(set(recipients + metadata["recipients"])) == \ + len(recipients + metadata["recipients"]): + continue + + emails[storage_id] = metadata + + return emails + + def delete(self, storage_id, recipients=None): + "Delete email from storage." + super(FileMailStorage, self).delete(storage_id, recipients) + + try: + metadata = self.get_metadata(storage_id) + except RuntimeError as e: + raise RuntimeError(f"unable to delete email: {e}") + + if not recipients: + self._remove(storage_id) + else: + if type(recipients) == str: + recipients = [recipients] + for recipient in recipients: + if recipient not in metadata["recipients"]: + raise RuntimeError(f"invalid recipient '{recipient}'") + metadata["recipients"].remove(recipient) + if not metadata["recipients"]: + self._remove(storage_id) + else: + self._save_metafile(storage_id, metadata) + + def get_mail(self, storage_id): + super(FileMailStorage, self).get_mail(storage_id) + + metadata = self.get_metadata(storage_id) + datafile = os.path.join(self.directory, storage_id) + try: + fp = open(datafile, "rb") + except IOError as e: + raise RuntimeError(f"unable to open email data file: {e}") + return (fp, metadata) From 45f5a80d85d968878d7ce519ae0ec0029bf2ae1c Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 10 Sep 2021 01:42:33 +0200 Subject: [PATCH 113/145] add option to skip metadata and save storage_id during runtime --- pymodmilter/__init__.py | 3 ++- pymodmilter/actions.py | 9 ++++++--- pymodmilter/storages.py | 29 ++++++++++++++++------------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index f54d680..725d291 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -336,7 +336,8 @@ class ModifyMilter(Milter.Base): refold_source='none')) self.msginfo = { "mailfrom": self.mailfrom, - "rcpts": self.rcpts} + "rcpts": self.rcpts, + "storage_id": None} self._replacebody = False milter_action = None diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index ebe7d78..b082876 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -176,6 +176,10 @@ class ActionConfig(BaseConfig): f"{self['name']}: file quarantine directory " f"'{self['directory']}' does not exist or is " f"not writable") + + if "skip_metadata" in cfg: + self.add_bool_arg(cfg, "skip_metadata") + else: raise RuntimeError( f"{self['name']}: storage_type: invalid storage type") @@ -217,12 +221,11 @@ class Action: def execute(self, milter): """Execute configured action.""" - logger = CustomLogger( - self.logger, {"name": self._name, "qid": milter.qid}) - if self.conditions is None or \ self.conditions.match(envfrom=milter.mailfrom, envto=[*milter.rcpts], headers=milter.msg.items()): + logger = CustomLogger( + self.logger, {"name": self._name, "qid": milter.qid}) return self._class.execute( milter=milter, pretend=self.pretend, logger=logger) diff --git a/pymodmilter/storages.py b/pymodmilter/storages.py index 8376cbd..07fcd50 100644 --- a/pymodmilter/storages.py +++ b/pymodmilter/storages.py @@ -50,10 +50,11 @@ class BaseMailStorage(object): class FileMailStorage(BaseMailStorage): "Storage class to store mails on filesystem." - def __init__(self, directory, original=False): + def __init__(self, directory, original=False, skip_metadata=False): super().__init__() self.directory = directory self.original = original + self.skip_metadata = skip_metadata self._metadata_suffix = ".metadata" def _save_datafile(self, storage_id, data): @@ -98,19 +99,20 @@ class FileMailStorage(BaseMailStorage): # save mail datafile = self._save_datafile(storage_id, data) - # save metadata - metadata = { - "mailfrom": mailfrom, - "recipients": recipients, - "subject": subject, - "timestamp": timegm(gmtime()), - "queue_id": qid} + if not self.skip_metadata: + # save metadata + metadata = { + "mailfrom": mailfrom, + "recipients": recipients, + "subject": subject, + "timestamp": timegm(gmtime()), + "queue_id": qid} - try: - self._save_metafile(storage_id, metadata) - except RuntimeError as e: - os.remove(datafile) - raise e + try: + self._save_metafile(storage_id, metadata) + except RuntimeError as e: + os.remove(datafile) + raise e return (storage_id, datafile) @@ -131,6 +133,7 @@ class FileMailStorage(BaseMailStorage): storage_id, datafile = self.add( data(), milter.qid, mailfrom, recipients, subject) logger.info(f"stored message in file {datafile}") + milter.msginfo["storage_id"] = storage_id def get_metadata(self, storage_id): "Return metadata of email in storage." From 78df57ab9aa0a07632b2de5fc2707a1933042438 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 10 Sep 2021 01:57:48 +0200 Subject: [PATCH 114/145] add mailer.py --- pymodmilter/__init__.py | 2 + pymodmilter/mailer.py | 86 +++++++++++++++++++++++++++++++++++++++++ pymodmilter/run.py | 3 +- 3 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 pymodmilter/mailer.py diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 725d291..050690e 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -16,9 +16,11 @@ __all__ = [ "actions", "base", "conditions", + "mailer", "modifications", "rules", "run", + "storages", "ModifyMilterConfig", "ModifyMilter"] diff --git a/pymodmilter/mailer.py b/pymodmilter/mailer.py new file mode 100644 index 0000000..51a913c --- /dev/null +++ b/pymodmilter/mailer.py @@ -0,0 +1,86 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +import logging +import smtplib + +from multiprocessing import Process, Queue + + +logger = logging.getLogger(__name__) +queue = Queue(maxsize=50) +process = None + + +def smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail): + s = smtplib.SMTP(host=smtp_host, port=smtp_port) + s.ehlo() + if s.has_extn("STARTTLS"): + s.starttls() + s.ehlo() + s.sendmail(mailfrom, [recipient], mail) + s.quit() + + +def mailprocess(): + "Mailer process to send emails asynchronously." + global logger + global queue + + try: + while True: + m = queue.get() + if not m: + break + + smtp_host, smtp_port, qid, mailfrom, recipient, mail, emailtype = m + try: + smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail) + except Exception as e: + logger.error( + f"{qid}: error while sending {emailtype} " + f"to '{recipient}': {e}") + else: + logger.info( + f"{qid}: successfully sent {emailtype} to: {recipient}") + except KeyboardInterrupt: + pass + logger.debug("mailer process terminated") + + +def sendmail(smtp_host, smtp_port, qid, mailfrom, recipients, mail, + emailtype="email"): + "Send an email." + global logger + global process + global queue + + if isinstance(recipients, str): + recipients = [recipients] + + # start mailprocess if it is not started yet + if process is None: + process = Process(target=mailprocess) + process.daemon = True + logger.debug("starting mailer process") + process.start() + + for recipient in recipients: + try: + queue.put( + (smtp_host, smtp_port, qid, mailfrom, recipient, mail, + emailtype), + timeout=30) + except Queue.Full: + raise RuntimeError("email queue is full") diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 5998a50..00baf6e 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -20,9 +20,8 @@ import logging import logging.handlers import sys -from pymodmilter import ModifyMilter +from pymodmilter import ModifyMilterConfig, ModifyMilter from pymodmilter import __version__ as version -from pymodmilter import ModifyMilterConfig def main(): From 6dcfeb05f610048ef0bfce6d1d7faa5f60039bd9 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 10 Sep 2021 03:20:02 +0200 Subject: [PATCH 115/145] add notifications.py --- pymodmilter/notifications.py | 309 +++++++++++++++++++++++++++++++++++ pymodmilter/storages.py | 4 + 2 files changed, 313 insertions(+) create mode 100644 pymodmilter/notifications.py diff --git a/pymodmilter/notifications.py b/pymodmilter/notifications.py new file mode 100644 index 0000000..6211187 --- /dev/null +++ b/pymodmilter/notifications.py @@ -0,0 +1,309 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +import email +import logging +import re + +from bs4 import BeautifulSoup +from collections import defaultdict +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.image import MIMEImage +from html import escape +from os.path import basename +from urllib.parse import quote + +from pymodmilter import mailer + + +class BaseNotification(object): + "Notification base class" + + def __init__(self): + return + + def notify(self, msg, qid, mailfrom, recipients, + template_vars=defaultdict(str), synchronous=False): + return + + def execute(self, milter, pretend=False, + logger=logging.getLogger(__name__)): + return + + +class EMailNotification(BaseNotification): + "Notification class to send notifications via mail." + notification_type = "email" + _bad_tags = [ + "applet", + "embed", + "frame", + "frameset", + "head", + "iframe", + "script" + ] + _good_tags = [ + "a", + "b", + "br", + "center", + "div", + "font", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "i", + "img", + "li", + "p", + "pre", + "span", + "table", + "td", + "th", + "tr", + "tt", + "u", + "ul" + ] + _good_attributes = [ + "align", + "alt", + "bgcolor", + "border", + "cellpadding", + "cellspacing", + "class", + "color", + "colspan", + "dir", + "face", + "headers", + "height", + "id", + "name", + "rowspan", + "size", + "src", + "style", + "title", + "type", + "valign", + "value", + "width" + ] + + def __init__(self, smtp_host, smtp_port, envelope_from, from_header, + subject, template, embed_imgs=[], repl_img=None, + strip_imgs=False, parser_lib="lxml"): + super().__init__() + + self.smtp_host = smtp_host + self.smtp_port = smtp_port + self.mailfrom = envelope_from + self.from_header = from_header + self.subject = subject + self.template = open(template, "r").read() + self.embedded_imgs = [] + for img_path in embed_imgs: + img = MIMEImage(open(img_path, "rb").read()) + filename = basename(img_path) + img.add_header("Content-ID", f"<{filename}>") + self.embedded_imgs.append(img) + + self.replacement_img = repl_img + self.strip_images = strip_imgs + + if not strip_imgs and repl_img: + self.replacement_img = MIMEImage( + open(repl_img, "rb").read()) + self.replacement_img.add_header( + "Content-ID", "") + + self.parser_lib = parser_lib + + def get_email_body_soup(self, msg, logger=None): + "Extract and decode email body and return it as BeautifulSoup object." + if logger is None: + logger = self.logger + + # try to find the body part + logger.debug("trying to find email body") + try: + body = msg.get_body(preferencelist=("html", "plain")) + except Exception as e: + logger.error( + f"an error occured in email.message.EmailMessage.get_body: " + f"{e}") + body = None + + if body: + charset = body.get_content_charset() or "utf-8" + content = body.get_payload(decode=True) + try: + content = content.decode(encoding=charset, errors="replace") + except LookupError: + logger.info( + f"unknown encoding '{charset}', falling back to UTF-8") + content = content.decode("utf-8", errors="replace") + content_type = body.get_content_type() + if content_type == "text/plain": + # convert text/plain to text/html + logger.debug( + f"content type is {content_type}, " + f"converting to text/html") + content = re.sub(r"^(.*)$", r"\1
", + escape(content, quote=False), + flags=re.MULTILINE) + else: + logger.debug(f"content type is {content_type}") + else: + logger.error("unable to find email body") + content = "ERROR: unable to find email body" + + # create BeautifulSoup object + length = len(content) + logger.debug( + f"trying to create BeatufilSoup object with " + f"parser lib {self.parser_lib}, " + f"text length is {length} bytes") + soup = BeautifulSoup(content, self.parser_lib) + logger.debug("sucessfully created BeautifulSoup object") + + return soup + + def sanitize(self, soup, logger=None): + "Sanitize mail html text." + if logger is None: + logger = self.logger + + logger.debug("sanitizing email text") + + # completly remove bad elements + for element in soup(EMailNotification._bad_tags): + logger.debug( + f"removing dangerous tag '{element.name}' " + f"and its content") + element.extract() + + # remove not whitelisted elements, but keep their content + for element in soup.find_all(True): + if element.name not in EMailNotification._good_tags: + logger.debug( + f"removing tag '{element.name}', keep its content") + element.replaceWithChildren() + + # remove not whitelisted attributes + for element in soup.find_all(True): + for attribute in list(element.attrs.keys()): + if attribute not in EMailNotification._good_attributes: + if element.name == "a" and attribute == "href": + logger.debug( + f"setting attribute href to '#' " + f"on tag '{element.name}'") + element["href"] = "#" + else: + logger.debug( + f"removing attribute '{attribute}' " + f"from tag '{element.name}'") + del(element.attrs[attribute]) + return soup + + def notify(self, msg, qid, mailfrom, recipients, + template_vars=defaultdict(str), synchronous=False, + logger=None): + "Notify recipients via email." + super().notify( + msg, qid, mailfrom, recipients, template_vars, synchronous, logger) + + if logger is None: + logger = self.logger + + # extract body from email + soup = self.get_email_body_soup(msg, logger) + + # replace picture sources + image_replaced = False + if self.strip_images: + logger.debug("looking for images to strip") + for element in soup("img"): + if "src" in element.attrs.keys(): + logger.debug(f"strip image: {element['src']}") + element.extract() + elif self.replacement_img: + logger.debug("looking for images to replace") + for element in soup("img"): + if "src" in element.attrs.keys(): + logger.debug(f"replacing image: {element['src']}") + element["src"] = "cid:removed_for_security_reasons" + image_replaced = True + + # sanitizing email text of original email + sanitized_text = self.sanitize(soup, logger) + del soup + + # sending email notifications + for recipient in recipients: + logger.debug(f"generating notification email for '{recipient}'") + logger.debug("parsing email template") + + # generate dict containing all template variables + template_vars.update({ + "HTML_TEXT": sanitized_text, + "FROM": escape(msg["from"], quote=False), + "ENVELOPE_FROM": escape(mailfrom, quote=False), + "EMAIL_ENVELOPE_FROM_URL": escape(quote(mailfrom), + quote=False), + "TO": escape(msg["to"], quote=False), + "ENVELOPE_TO": escape(recipient, quote=False), + "ENVELOPE_TO_URL": escape(quote(recipient)), + "SUBJECT": escape(msg["subject"], quote=False)}) + + # parse template + htmltext = self.template.format_map(template_vars) + + msg = MIMEMultipart('related') + msg["From"] = self.from_header.format_map( + defaultdict(str, EMAIL_FROM=msg["from"])) + msg["To"] = msg["to"] + msg["Subject"] = self.subject.format_map( + defaultdict(str, EMAIL_SUBJECT=msg["subject"])) + msg["Date"] = email.utils.formatdate() + msg.attach(MIMEText(htmltext, "html", 'UTF-8')) + + if image_replaced: + logger.debug("attaching notification_replacement_img") + msg.attach(self.replacement_img) + + for img in self.embedded_imgs: + logger.debug("attaching imgage") + msg.attach(img) + + logger.debug(f"sending notification email to: {recipient}") + if synchronous: + try: + mailer.smtp_send(self.smtp_host, self.smtp_port, + self.mailfrom, recipient, msg.as_string()) + except Exception as e: + raise RuntimeError( + f"error while sending email to '{recipient}': {e}") + else: + mailer.sendmail(self.smtp_host, self.smtp_port, qid, + self.mailfrom, recipient, msg.as_string(), + "notification email") diff --git a/pymodmilter/storages.py b/pymodmilter/storages.py index 07fcd50..f36cb6e 100644 --- a/pymodmilter/storages.py +++ b/pymodmilter/storages.py @@ -31,6 +31,10 @@ class BaseMailStorage(object): "Add email to storage." return ("", "") + def execute(self, milter, pretend=False, + logger=logging.getLogger(__name__)): + return + def find(self, mailfrom=None, recipients=None, older_than=None): "Find emails in storage." return From 4725dc9784feb827017ec528108287f7b82c02f9 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 10 Sep 2021 04:12:14 +0200 Subject: [PATCH 116/145] shorten file names --- pymodmilter/__init__.py | 15 +++++++++------ pymodmilter/{actions.py => action.py} | 14 +++++++------- pymodmilter/{modifications.py => modify.py} | 0 pymodmilter/{notifications.py => notify.py} | 14 ++++++++++++-- pymodmilter/{rules.py => rule.py} | 2 +- pymodmilter/{storages.py => storage.py} | 10 ++++++---- 6 files changed, 35 insertions(+), 20 deletions(-) rename pymodmilter/{actions.py => action.py} (95%) rename pymodmilter/{modifications.py => modify.py} (100%) rename pymodmilter/{notifications.py => notify.py} (95%) rename pymodmilter/{rules.py => rule.py} (98%) rename pymodmilter/{storages.py => storage.py} (96%) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 050690e..86ad245 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -13,14 +13,15 @@ # __all__ = [ - "actions", + "action", "base", "conditions", "mailer", - "modifications", - "rules", + "modify", + "notify", + "rule", "run", - "storages", + "storage", "ModifyMilterConfig", "ModifyMilter"] @@ -45,7 +46,7 @@ from netaddr import IPNetwork, AddrFormatError from pymodmilter.base import CustomLogger, BaseConfig, MilterMessage from pymodmilter.base import replace_illegal_chars -from pymodmilter.rules import RuleConfig, Rule +from pymodmilter.rule import RuleConfig, Rule class ModifyMilterConfig(BaseConfig): @@ -339,12 +340,14 @@ class ModifyMilter(Milter.Base): self.msginfo = { "mailfrom": self.mailfrom, "rcpts": self.rcpts, - "storage_id": None} + "vars": {}} self._replacebody = False milter_action = None for rule in self.rules: milter_action = rule.execute(self) + self.logger.debug( + f"current template variables: {self.msginfo['vars']}") if milter_action is not None: break diff --git a/pymodmilter/actions.py b/pymodmilter/action.py similarity index 95% rename from pymodmilter/actions.py rename to pymodmilter/action.py index b082876..c295053 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/action.py @@ -23,7 +23,7 @@ from bs4 import BeautifulSoup from pymodmilter import CustomLogger, BaseConfig from pymodmilter.conditions import ConditionsConfig, Conditions -from pymodmilter import modifications, storages +from pymodmilter import modify, storage class ActionConfig(BaseConfig): @@ -58,7 +58,7 @@ class ActionConfig(BaseConfig): self["type"] = cfg["type"] if self["type"] == "add_header": - self["class"] = modifications.AddHeader + self["class"] = modify.AddHeader self["headersonly"] = True if "field" not in cfg and "header" in cfg: @@ -67,7 +67,7 @@ class ActionConfig(BaseConfig): self.add_string_arg(cfg, ("field", "value")) elif self["type"] == "mod_header": - self["class"] = modifications.ModHeader + self["class"] = modify.ModHeader self["headersonly"] = True if "field" not in cfg and "header" in cfg: @@ -88,7 +88,7 @@ class ActionConfig(BaseConfig): raise ValueError(f"{self['name']}: {arg}: {e}") elif self["type"] == "del_header": - self["class"] = modifications.DelHeader + self["class"] = modify.DelHeader self["headersonly"] = True if "field" not in cfg and "header" in cfg: @@ -108,7 +108,7 @@ class ActionConfig(BaseConfig): raise ValueError(f"{self['name']}: {arg}: {e}") elif self["type"] == "add_disclaimer": - self["class"] = modifications.AddDisclaimer + self["class"] = modify.AddDisclaimer self["headersonly"] = False if "html_template" not in cfg and "html_file" in cfg: @@ -149,7 +149,7 @@ class ActionConfig(BaseConfig): f"{self['name']}: unable to open/read template file: {e}") elif self["type"] == "rewrite_links": - self["class"] = modifications.RewriteLinks + self["class"] = modify.RewriteLinks self["headersonly"] = False self.add_string_arg(cfg, "repl") @@ -167,7 +167,7 @@ class ActionConfig(BaseConfig): self.add_bool_arg(cfg, "original") if self["storage_type"] == "file": - self["class"] = storages.FileMailStorage + self["class"] = storage.FileMailStorage self.add_string_arg(cfg, "directory") # check if directory exists and is writable if not os.path.isdir(self["args"]["directory"]) or \ diff --git a/pymodmilter/modifications.py b/pymodmilter/modify.py similarity index 100% rename from pymodmilter/modifications.py rename to pymodmilter/modify.py diff --git a/pymodmilter/notifications.py b/pymodmilter/notify.py similarity index 95% rename from pymodmilter/notifications.py rename to pymodmilter/notify.py index 6211187..3252a4e 100644 --- a/pymodmilter/notifications.py +++ b/pymodmilter/notify.py @@ -264,7 +264,9 @@ class EMailNotification(BaseNotification): logger.debug("parsing email template") # generate dict containing all template variables - template_vars.update({ + + variables = defaultdict(str, template_vars) + variables.update({ "HTML_TEXT": sanitized_text, "FROM": escape(msg["from"], quote=False), "ENVELOPE_FROM": escape(mailfrom, quote=False), @@ -276,7 +278,7 @@ class EMailNotification(BaseNotification): "SUBJECT": escape(msg["subject"], quote=False)}) # parse template - htmltext = self.template.format_map(template_vars) + htmltext = self.template.format_map(variables) msg = MIMEMultipart('related') msg["From"] = self.from_header.format_map( @@ -307,3 +309,11 @@ class EMailNotification(BaseNotification): mailer.sendmail(self.smtp_host, self.smtp_port, qid, self.mailfrom, recipient, msg.as_string(), "notification email") + + def execute(self, milter, pretend=False, + logger=logging.getLogger(__name__)): + self.notify(msg=milter.msg, qid=milter.qid, + mailfrom=milter.msginfo["mailfrom"], + recipients=milter.msginfo["rcpts"], + template_vars=milter["msginfo"]["vars"], + logger=logger) diff --git a/pymodmilter/rules.py b/pymodmilter/rule.py similarity index 98% rename from pymodmilter/rules.py rename to pymodmilter/rule.py index ef925eb..3b7ebce 100644 --- a/pymodmilter/rules.py +++ b/pymodmilter/rule.py @@ -17,7 +17,7 @@ __all__ = [ "Rule"] from pymodmilter import BaseConfig -from pymodmilter.actions import ActionConfig, Action +from pymodmilter.action import ActionConfig, Action from pymodmilter.conditions import ConditionsConfig, Conditions diff --git a/pymodmilter/storages.py b/pymodmilter/storage.py similarity index 96% rename from pymodmilter/storages.py rename to pymodmilter/storage.py index f36cb6e..a3a23c6 100644 --- a/pymodmilter/storages.py +++ b/pymodmilter/storage.py @@ -134,10 +134,12 @@ class FileMailStorage(BaseMailStorage): recipients = list(milter.msginfo["rcpts"]) subject = milter.msg["subject"] or "" - storage_id, datafile = self.add( - data(), milter.qid, mailfrom, recipients, subject) - logger.info(f"stored message in file {datafile}") - milter.msginfo["storage_id"] = storage_id + if not pretend: + storage_id, datafile = self.add( + data(), milter.qid, mailfrom, recipients, subject) + logger.info(f"stored message in file {datafile}") + milter.msginfo["vars"]["STORAGEID"] = storage_id + milter.msginfo["vars"]["DATAFILE"] = datafile def get_metadata(self, storage_id): "Return metadata of email in storage." From e632f0d511a6451ce60b2d48e04a340b9a6f335f Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 10 Sep 2021 11:13:36 +0200 Subject: [PATCH 117/145] add metavar option and rework storage logic --- pymodmilter/action.py | 5 ++- pymodmilter/storage.py | 85 ++++++++++++++++++++++++------------------ 2 files changed, 52 insertions(+), 38 deletions(-) diff --git a/pymodmilter/action.py b/pymodmilter/action.py index c295053..daedd7e 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -176,10 +176,13 @@ class ActionConfig(BaseConfig): f"{self['name']}: file quarantine directory " f"'{self['directory']}' does not exist or is " f"not writable") - + if "skip_metadata" in cfg: self.add_bool_arg(cfg, "skip_metadata") + if "metavar" in cfg: + self.add_string_arg(cfg, "metavar") + else: raise RuntimeError( f"{self['name']}: storage_type: invalid storage type") diff --git a/pymodmilter/storage.py b/pymodmilter/storage.py index a3a23c6..f4817fc 100644 --- a/pymodmilter/storage.py +++ b/pymodmilter/storage.py @@ -54,26 +54,32 @@ class BaseMailStorage(object): class FileMailStorage(BaseMailStorage): "Storage class to store mails on filesystem." - def __init__(self, directory, original=False, skip_metadata=False): + def __init__(self, directory, original=False, skip_metadata=False, + metavar=None): super().__init__() self.directory = directory self.original = original self.skip_metadata = skip_metadata + self.metavar = metavar self._metadata_suffix = ".metadata" - def _save_datafile(self, storage_id, data): + def get_storageid(self, qid): + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + return f"{timestamp}_{qid}" + + def _get_file_paths(self, storage_id): datafile = os.path.join(self.directory, storage_id) + metafile = f"{datafile}${self._metadata_suffix}" + return metafile, datafile + + def _save_datafile(self, datafile, data): try: with open(datafile, "wb") as f: f.write(data) except IOError as e: raise RuntimeError(f"unable save data file: {e}") - return datafile - - def _save_metafile(self, storage_id, metadata): - metafile = os.path.join( - self.directory, f"{storage_id}{self._metadata_suffix}") + def _save_metafile(self, metafile, metadata): try: with open(metafile, "w") as f: json.dump(metadata, f, indent=2) @@ -81,27 +87,23 @@ class FileMailStorage(BaseMailStorage): raise RuntimeError(f"unable to save metadata file: {e}") def _remove(self, storage_id): - datafile = os.path.join(self.directory, storage_id) - metafile = f"{datafile}{self._metadata_suffix}" + metafile, datafile = self._get_file_paths(storage_id) try: os.remove(metafile) - except IOError as e: - raise RuntimeError(f"unable to remove metadata file: {e}") - - try: os.remove(datafile) except IOError as e: - raise RuntimeError(f"unable to remove data file: {e}") + raise RuntimeError(f"unable to remove file: {e}") def add(self, data, qid, mailfrom="", recipients=[], subject=""): "Add email to file storage and return storage id." super().add(data, qid, mailfrom, recipients) - timestamp = datetime.now().strftime("%Y%m%d%H%M%S") - storage_id = f"{timestamp}_{qid}" + + storage_id = self.get_storageid(qid) + metafile, datafile = self._get_file_paths(storage_id) # save mail - datafile = self._save_datafile(storage_id, data) + self._save_datafile(datafile, data) if not self.skip_metadata: # save metadata @@ -113,12 +115,12 @@ class FileMailStorage(BaseMailStorage): "queue_id": qid} try: - self._save_metafile(storage_id, metadata) + self._save_metafile(metafile, metadata) except RuntimeError as e: os.remove(datafile) raise e - return (storage_id, datafile) + return storage_id, metafile, datafile def execute(self, milter, pretend=False, logger=logging.getLogger(__name__)): @@ -135,18 +137,23 @@ class FileMailStorage(BaseMailStorage): subject = milter.msg["subject"] or "" if not pretend: - storage_id, datafile = self.add( + storage_id, metafile, datafile = self.add( data(), milter.qid, mailfrom, recipients, subject) logger.info(f"stored message in file {datafile}") - milter.msginfo["vars"]["STORAGEID"] = storage_id - milter.msginfo["vars"]["DATAFILE"] = datafile + else: + storage_id = self.get_storageid(milter.qid) + metafile, datafile = self._get_file_paths(storage_id) + + if self.metavar: + milter.msginfo["vars"][f"{self.metavar}_ID"] = storage_id + milter.msginfo["vars"][f"{self.metavar}_METAFILE"] = metafile + milter.msginfo["vars"][f"{self.metavar}_DATAFILE"] = datafile def get_metadata(self, storage_id): "Return metadata of email in storage." super(FileMailStorage, self).get_metadata(storage_id) - metafile = os.path.join( - self.directory, f"{storage_id}{self._metadata_suffix}") + metafile, _ = self._get_file_paths(storage_id) if not os.path.isfile(metafile): raise RuntimeError( f"invalid storage id '{storage_id}'") @@ -204,24 +211,28 @@ class FileMailStorage(BaseMailStorage): "Delete email from storage." super(FileMailStorage, self).delete(storage_id, recipients) + if not recipients: + self._remove(storage_id) + return + try: metadata = self.get_metadata(storage_id) except RuntimeError as e: raise RuntimeError(f"unable to delete email: {e}") - if not recipients: - self._remove(storage_id) - else: - if type(recipients) == str: - recipients = [recipients] - for recipient in recipients: - if recipient not in metadata["recipients"]: - raise RuntimeError(f"invalid recipient '{recipient}'") - metadata["recipients"].remove(recipient) - if not metadata["recipients"]: - self._remove(storage_id) - else: - self._save_metafile(storage_id, metadata) + metafile, _ = self._get_file_paths(storage_id) + + if type(recipients) == str: + recipients = [recipients] + + for recipient in recipients: + if recipient not in metadata["recipients"]: + raise RuntimeError(f"invalid recipient '{recipient}'") + metadata["recipients"].remove(recipient) + if not metadata["recipients"]: + self._remove(storage_id) + else: + self._save_metafile(metafile, metadata) def get_mail(self, storage_id): super(FileMailStorage, self).get_mail(storage_id) From 2b73e43d2fc0c9c3a796c4b8b19cd9bc935c0c3a Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 10 Sep 2021 15:34:58 +0200 Subject: [PATCH 118/145] improve error handling --- pymodmilter/action.py | 70 +++++-------------------------------------- pymodmilter/modify.py | 38 +++++++++++++++++++---- pymodmilter/notify.py | 32 +++++++++++--------- pymodmilter/run.py | 9 ++++-- 4 files changed, 64 insertions(+), 85 deletions(-) diff --git a/pymodmilter/action.py b/pymodmilter/action.py index daedd7e..10022bd 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -17,9 +17,6 @@ __all__ = [ "Action"] import os -import re - -from bs4 import BeautifulSoup from pymodmilter import CustomLogger, BaseConfig from pymodmilter.conditions import ConditionsConfig, Conditions @@ -60,70 +57,33 @@ class ActionConfig(BaseConfig): if self["type"] == "add_header": self["class"] = modify.AddHeader self["headersonly"] = True - - if "field" not in cfg and "header" in cfg: - cfg["field"] = cfg["header"] - - self.add_string_arg(cfg, ("field", "value")) - + self.add_string_arg(cfg, ["field", "value"]) elif self["type"] == "mod_header": self["class"] = modify.ModHeader self["headersonly"] = True - - if "field" not in cfg and "header" in cfg: - cfg["field"] = cfg["header"] - args = ["field", "value"] if "search" in cfg: args.append("search") - for arg in args: - self.add_string_arg(cfg, arg) - if arg in ("field", "search"): - try: - self["args"][arg] = re.compile( - self["args"][arg], - re.MULTILINE + re.DOTALL + re.IGNORECASE) - except re.error as e: - raise ValueError(f"{self['name']}: {arg}: {e}") - + self.add_string_arg(cfg, args) elif self["type"] == "del_header": self["class"] = modify.DelHeader self["headersonly"] = True - - if "field" not in cfg and "header" in cfg: - cfg["field"] = cfg["header"] - args = ["field"] if "value" in cfg: args.append("value") - for arg in args: - self.add_string_arg(cfg, arg) - try: - self["args"][arg] = re.compile( - self["args"][arg], - re.MULTILINE + re.DOTALL + re.IGNORECASE) - except re.error as e: - raise ValueError(f"{self['name']}: {arg}: {e}") - + self.add_string_arg(cfg, args) elif self["type"] == "add_disclaimer": self["class"] = modify.AddDisclaimer self["headersonly"] = False - - if "html_template" not in cfg and "html_file" in cfg: - cfg["html_template"] = cfg["html_file"] - - if "text_template" not in cfg and "text_file" in cfg: - cfg["text_template"] = cfg["text_file"] - if "error_policy" not in cfg: cfg["error_policy"] = "wrap" self.add_string_arg( - cfg, ("action", "html_template", "text_template", - "error_policy")) - assert self["args"]["action"] in ("append", "prepend"), \ + cfg, ["action", "html_template", "text_template", + "error_policy"]) + assert self["args"]["action"] in ["append", "prepend"], \ f"{self['name']}: action: invalid value, " \ f"should be 'append' or 'prepend'" assert self["args"]["error_policy"] in ("wrap", @@ -132,22 +92,6 @@ class ActionConfig(BaseConfig): f"{self['name']}: error_policy: invalid value, " \ f"should be 'wrap', 'ignore' or 'reject'" - try: - with open(self["args"]["html_template"], "r") as f: - html = BeautifulSoup(f.read(), "html.parser") - body = html.find('body') - if body: - # just use content within the body tag if present - html = body - self["args"]["html_template"] = html - - with open(self["args"]["text_template"], "r") as f: - self["args"]["text_template"] = f.read() - - except IOError as e: - raise RuntimeError( - f"{self['name']}: unable to open/read template file: {e}") - elif self["type"] == "rewrite_links": self["class"] = modify.RewriteLinks self["headersonly"] = False @@ -176,7 +120,7 @@ class ActionConfig(BaseConfig): f"{self['name']}: file quarantine directory " f"'{self['directory']}' does not exist or is " f"not writable") - + if "skip_metadata" in cfg: self.add_bool_arg(cfg, "skip_metadata") diff --git a/pymodmilter/modify.py b/pymodmilter/modify.py index 0f398af..4715a04 100644 --- a/pymodmilter/modify.py +++ b/pymodmilter/modify.py @@ -20,6 +20,7 @@ __all__ = [ "RewriteLinks"] import logging +import re from base64 import b64encode from bs4 import BeautifulSoup @@ -53,9 +54,17 @@ class AddHeader: class ModHeader: """Change the value of a mail header field.""" def __init__(self, field, value, search=None): - self.field = field self.value = value - self.search = search + + try: + self.field = re.compile(field, re.IGNORECASE) + if search is not None: + self.search = re.compile( + search, re.MULTILINE + re.DOTALL + re.IGNORECASE) + else: + self.search = search + except re.error as e: + raise RuntimeError(e) def execute(self, milter, pretend=False, logger=logging.getLogger(__name__)): @@ -101,8 +110,15 @@ class ModHeader: class DelHeader: """Delete a mail header field.""" def __init__(self, field, value=None): - self.field = field - self.value = value + try: + self.field = re.compile(field, re.IGNORECASE) + if value is not None: + self.value = re.compile( + value, re.MULTILINE + re.DOTALL + re.IGNORECASE) + else: + self.value = value + except re.error as e: + raise RuntimeError(e) def execute(self, milter, pretend=False, logger=logging.getLogger(__name__)): @@ -196,8 +212,18 @@ def _wrap_message(milter): class AddDisclaimer: """Append or prepend a disclaimer to the mail body.""" def __init__(self, text_template, html_template, action, error_policy): - self.text_template = text_template - self.html_template = html_template + try: + with open(text_template, "r") as f: + self.text_template = f.read() + + with open(html_template, "r") as f: + html = BeautifulSoup(f.read(), "html.parser") + + except IOError as e: + raise RuntimeError(e) + + body = html.find('body') + self.html_template = body or html self.action = action self.error_policy = error_policy diff --git a/pymodmilter/notify.py b/pymodmilter/notify.py index 3252a4e..0aefecc 100644 --- a/pymodmilter/notify.py +++ b/pymodmilter/notify.py @@ -119,22 +119,26 @@ class EMailNotification(BaseNotification): self.mailfrom = envelope_from self.from_header = from_header self.subject = subject - self.template = open(template, "r").read() - self.embedded_imgs = [] - for img_path in embed_imgs: - img = MIMEImage(open(img_path, "rb").read()) - filename = basename(img_path) - img.add_header("Content-ID", f"<{filename}>") - self.embedded_imgs.append(img) + try: + self.template = open(template, "r").read() + self.embedded_imgs = [] + for img_path in embed_imgs: + img = MIMEImage(open(img_path, "rb").read()) + filename = basename(img_path) + img.add_header("Content-ID", f"<{filename}>") + self.embedded_imgs.append(img) - self.replacement_img = repl_img - self.strip_images = strip_imgs + self.replacement_img = repl_img + self.strip_images = strip_imgs - if not strip_imgs and repl_img: - self.replacement_img = MIMEImage( - open(repl_img, "rb").read()) - self.replacement_img.add_header( - "Content-ID", "") + if not strip_imgs and repl_img: + self.replacement_img = MIMEImage( + open(repl_img, "rb").read()) + self.replacement_img.add_header( + "Content-ID", "") + + except IOError as e: + raise RuntimeError(e) self.parser_lib = parser_lib diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 00baf6e..3d4c462 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -79,7 +79,7 @@ def main(): logger.setLevel(logging.INFO) try: - logger.debug("prepar milter configuration") + logger.debug("prepare milter configuration") cfg = ModifyMilterConfig(args.config, args.debug) if not args.debug: @@ -106,6 +106,12 @@ def main(): logger.error(e) sys.exit(255) + try: + ModifyMilter.set_config(cfg) + except (RuntimeError, ValueError) as e: + logger.error(e) + sys.exit(254) + if args.test: print("Configuration OK") sys.exit(0) @@ -123,7 +129,6 @@ def main(): root_logger.addHandler(sysloghandler) logger.info("pymodmilter starting") - ModifyMilter.set_config(cfg) # register milter factory class Milter.factory = ModifyMilter From 24d9eb35023709c84a9eca2d351bf404b80c781f Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 10 Sep 2021 15:59:58 +0200 Subject: [PATCH 119/145] add action notify to action config --- pymodmilter/action.py | 19 +++++++++++++++++++ pymodmilter/base.py | 11 +++++++++++ pymodmilter/notify.py | 6 +++--- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/pymodmilter/action.py b/pymodmilter/action.py index 10022bd..5daabde 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -131,6 +131,25 @@ class ActionConfig(BaseConfig): raise RuntimeError( f"{self['name']}: storage_type: invalid storage type") + elif self["type"] == "notify": + self["headersonly"] = False + + self.add_string_arg( + cfg, ["smtp_host", "envelope_from", "from", "subject", + "template"]) + self.add_int_arg(cfg, "smtp_port") + + if "embedded_imgs" in cfg: + assert isinstance(cfg["embedded_imgs"], list), \ + f"{self['name']}: embedded_imgs: invalid value, " \ + f"should be list" + for img in cfg["embedded_imgs"]: + assert isinstance(img, str), \ + f"{self['name']}: embedded_imgs: invalid entry, " \ + f"should be string" + + self["args"]["embedded_imgs"] = cfg["embedded_imgs"] + else: raise RuntimeError(f"{self['name']}: type: invalid action type") diff --git a/pymodmilter/base.py b/pymodmilter/base.py index 4eb91ae..2ec6a6f 100644 --- a/pymodmilter/base.py +++ b/pymodmilter/base.py @@ -104,6 +104,17 @@ class BaseConfig: f"{self['name']}: {arg}: invalid value, should be bool" self["args"][arg] = cfg[arg] + def add_int_arg(self, cfg, args): + if isinstance(args, str): + args = [args] + + for arg in args: + assert arg in cfg, \ + f"{self['name']}: mandatory parameter '{arg}' not found" + assert isinstance(cfg[arg], int), \ + f"{self['name']}: {arg}: invalid value, should be integer" + self["args"][arg] = cfg[arg] + class MilterMessage(MIMEPart): def replace_header(self, _name, _value, idx=None): diff --git a/pymodmilter/notify.py b/pymodmilter/notify.py index 0aefecc..2602d61 100644 --- a/pymodmilter/notify.py +++ b/pymodmilter/notify.py @@ -121,12 +121,12 @@ class EMailNotification(BaseNotification): self.subject = subject try: self.template = open(template, "r").read() - self.embedded_imgs = [] + self.embed_imgs = [] for img_path in embed_imgs: img = MIMEImage(open(img_path, "rb").read()) filename = basename(img_path) img.add_header("Content-ID", f"<{filename}>") - self.embedded_imgs.append(img) + self.embed_imgs.append(img) self.replacement_img = repl_img self.strip_images = strip_imgs @@ -297,7 +297,7 @@ class EMailNotification(BaseNotification): logger.debug("attaching notification_replacement_img") msg.attach(self.replacement_img) - for img in self.embedded_imgs: + for img in self.embed_imgs: logger.debug("attaching imgage") msg.attach(img) From 737a7b555b6cfba6bd3f5b9ec2f6ac27c5eda2bc Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Sat, 11 Sep 2021 18:35:31 +0200 Subject: [PATCH 120/145] fix encoding error when wrapping message --- pymodmilter/modify.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pymodmilter/modify.py b/pymodmilter/modify.py index 4715a04..8e992e5 100644 --- a/pymodmilter/modify.py +++ b/pymodmilter/modify.py @@ -27,6 +27,7 @@ from bs4 import BeautifulSoup from collections import defaultdict from copy import copy from email.message import MIMEPart +from email.policy import SMTPUTF8 from pymodmilter import replace_illegal_chars @@ -192,7 +193,7 @@ def _inject_body(milter): def _wrap_message(milter): - attachment = MIMEPart() + attachment = MIMEPart(policy=SMTPUTF8) attachment.set_content(milter.msg.as_bytes(), maintype="plain", subtype="text", disposition="attachment", From e1709f763f23420e29204e6e7f5d5994e591f98b Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Mon, 13 Sep 2021 14:47:13 +0200 Subject: [PATCH 121/145] fix actions, storages and mailer shutdown --- pymodmilter/action.py | 24 ++++++++++++--------- pymodmilter/notify.py | 49 +++++++++++++++++++++--------------------- pymodmilter/run.py | 4 +++- pymodmilter/storage.py | 26 ++++++++++++++-------- 4 files changed, 58 insertions(+), 45 deletions(-) diff --git a/pymodmilter/action.py b/pymodmilter/action.py index 5daabde..b3c09e3 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -20,7 +20,7 @@ import os from pymodmilter import CustomLogger, BaseConfig from pymodmilter.conditions import ConditionsConfig, Conditions -from pymodmilter import modify, storage +from pymodmilter import modify, notify, storage class ActionConfig(BaseConfig): @@ -133,22 +133,26 @@ class ActionConfig(BaseConfig): elif self["type"] == "notify": self["headersonly"] = False + self["class"] = notify.EMailNotification - self.add_string_arg( - cfg, ["smtp_host", "envelope_from", "from", "subject", - "template"]) + args = ["smtp_host", "envelope_from", "from_header", "subject", + "template"] + if "repl_img" in cfg: + args.append("repl_img") + + self.add_string_arg(cfg, args) self.add_int_arg(cfg, "smtp_port") - if "embedded_imgs" in cfg: - assert isinstance(cfg["embedded_imgs"], list), \ - f"{self['name']}: embedded_imgs: invalid value, " \ + if "embed_imgs" in cfg: + assert isinstance(cfg["embed_imgs"], list), \ + f"{self['name']}: embed_imgs: invalid value, " \ f"should be list" - for img in cfg["embedded_imgs"]: + for img in cfg["embed_imgs"]: assert isinstance(img, str), \ - f"{self['name']}: embedded_imgs: invalid entry, " \ + f"{self['name']}: embed_imgs: invalid entry, " \ f"should be string" - self["args"]["embedded_imgs"] = cfg["embedded_imgs"] + self["args"]["embed_imgs"] = cfg["embed_imgs"] else: raise RuntimeError(f"{self['name']}: type: invalid action type") diff --git a/pymodmilter/notify.py b/pymodmilter/notify.py index 2602d61..e517b6d 100644 --- a/pymodmilter/notify.py +++ b/pymodmilter/notify.py @@ -28,18 +28,14 @@ from urllib.parse import quote from pymodmilter import mailer -class BaseNotification(object): +class BaseNotification: "Notification base class" def __init__(self): + self.logger = logging.getLogger(__name__) return - def notify(self, msg, qid, mailfrom, recipients, - template_vars=defaultdict(str), synchronous=False): - return - - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + def execute(self, milter, pretend=False, logger=None): return @@ -233,9 +229,6 @@ class EMailNotification(BaseNotification): template_vars=defaultdict(str), synchronous=False, logger=None): "Notify recipients via email." - super().notify( - msg, qid, mailfrom, recipients, template_vars, synchronous, logger) - if logger is None: logger = self.logger @@ -274,7 +267,7 @@ class EMailNotification(BaseNotification): "HTML_TEXT": sanitized_text, "FROM": escape(msg["from"], quote=False), "ENVELOPE_FROM": escape(mailfrom, quote=False), - "EMAIL_ENVELOPE_FROM_URL": escape(quote(mailfrom), + "ENVELOPE_FROM_URL": escape(quote(mailfrom), quote=False), "TO": escape(msg["to"], quote=False), "ENVELOPE_TO": escape(recipient, quote=False), @@ -284,40 +277,46 @@ class EMailNotification(BaseNotification): # parse template htmltext = self.template.format_map(variables) - msg = MIMEMultipart('related') - msg["From"] = self.from_header.format_map( - defaultdict(str, EMAIL_FROM=msg["from"])) - msg["To"] = msg["to"] - msg["Subject"] = self.subject.format_map( - defaultdict(str, EMAIL_SUBJECT=msg["subject"])) - msg["Date"] = email.utils.formatdate() - msg.attach(MIMEText(htmltext, "html", 'UTF-8')) + newmsg = MIMEMultipart('related') + newmsg["From"] = self.from_header.format_map( + defaultdict(str, FROM=msg["from"])) + newmsg["To"] = msg["to"] + newmsg["Subject"] = self.subject.format_map( + defaultdict(str, SUBJECT=msg["subject"])) + newmsg["Date"] = email.utils.formatdate() + newmsg.attach(MIMEText(htmltext, "html", 'UTF-8')) if image_replaced: logger.debug("attaching notification_replacement_img") - msg.attach(self.replacement_img) + newmsg.attach(self.replacement_img) for img in self.embed_imgs: logger.debug("attaching imgage") - msg.attach(img) + newmsg.attach(img) logger.debug(f"sending notification email to: {recipient}") if synchronous: try: mailer.smtp_send(self.smtp_host, self.smtp_port, - self.mailfrom, recipient, msg.as_string()) + self.mailfrom, recipient, + newmsg.as_string()) except Exception as e: raise RuntimeError( f"error while sending email to '{recipient}': {e}") else: mailer.sendmail(self.smtp_host, self.smtp_port, qid, - self.mailfrom, recipient, msg.as_string(), + self.mailfrom, recipient, newmsg.as_string(), "notification email") def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + logger=None): + super().execute(milter, pretend, logger) + + if logger is None: + logger = self.logger + self.notify(msg=milter.msg, qid=milter.qid, mailfrom=milter.msginfo["mailfrom"], recipients=milter.msginfo["rcpts"], - template_vars=milter["msginfo"]["vars"], + template_vars=milter.msginfo["vars"], logger=logger) diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 3d4c462..a3a64d8 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -20,6 +20,7 @@ import logging import logging.handlers import sys +from pymodmilter import mailer from pymodmilter import ModifyMilterConfig, ModifyMilter from pymodmilter import __version__ as version @@ -140,11 +141,12 @@ def main(): rc = 0 try: Milter.runmilter("pymodmilter", socketname=socket, timeout=600) - logger.info("pymodmilter stopped") except Milter.milter.error as e: logger.error(e) rc = 255 + mailer.queue.put(None) + logger.info("pymodmilter stopped") sys.exit(rc) diff --git a/pymodmilter/storage.py b/pymodmilter/storage.py index f4817fc..93fb518 100644 --- a/pymodmilter/storage.py +++ b/pymodmilter/storage.py @@ -22,7 +22,7 @@ from glob import glob from time import gmtime -class BaseMailStorage(object): +class BaseMailStorage: "Mail storage base class" def __init__(self): return @@ -69,7 +69,7 @@ class FileMailStorage(BaseMailStorage): def _get_file_paths(self, storage_id): datafile = os.path.join(self.directory, storage_id) - metafile = f"{datafile}${self._metadata_suffix}" + metafile = f"{datafile}{self._metadata_suffix}" return metafile, datafile def _save_datafile(self, datafile, data): @@ -90,7 +90,9 @@ class FileMailStorage(BaseMailStorage): metafile, datafile = self._get_file_paths(storage_id) try: - os.remove(metafile) + if not self.skip_metadata: + os.remove(metafile) + os.remove(datafile) except IOError as e: raise RuntimeError(f"unable to remove file: {e}") @@ -105,7 +107,9 @@ class FileMailStorage(BaseMailStorage): # save mail self._save_datafile(datafile, data) - if not self.skip_metadata: + if self.skip_metadata: + metafile = None + else: # save metadata metadata = { "mailfrom": mailfrom, @@ -146,13 +150,17 @@ class FileMailStorage(BaseMailStorage): if self.metavar: milter.msginfo["vars"][f"{self.metavar}_ID"] = storage_id - milter.msginfo["vars"][f"{self.metavar}_METAFILE"] = metafile milter.msginfo["vars"][f"{self.metavar}_DATAFILE"] = datafile + if not self.skip_metadata: + milter.msginfo["vars"][f"{self.metavar}_METAFILE"] = metafile def get_metadata(self, storage_id): "Return metadata of email in storage." super(FileMailStorage, self).get_metadata(storage_id) + if self.skip_metadata: + return None + metafile, _ = self._get_file_paths(storage_id) if not os.path.isfile(metafile): raise RuntimeError( @@ -211,7 +219,7 @@ class FileMailStorage(BaseMailStorage): "Delete email from storage." super(FileMailStorage, self).delete(storage_id, recipients) - if not recipients: + if not recipients or self.skip_metadata: self._remove(storage_id) return @@ -238,9 +246,9 @@ class FileMailStorage(BaseMailStorage): super(FileMailStorage, self).get_mail(storage_id) metadata = self.get_metadata(storage_id) - datafile = os.path.join(self.directory, storage_id) + _, datafile = self._get_file_paths(storage_id) try: - fp = open(datafile, "rb") + data = open(datafile, "rb").read() except IOError as e: raise RuntimeError(f"unable to open email data file: {e}") - return (fp, metadata) + return (data, metadata) From cc297fb70dc2e2026e71e0a11fcac515af767c00 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Mon, 13 Sep 2021 15:52:36 +0200 Subject: [PATCH 122/145] fix logging --- pymodmilter/__init__.py | 5 +++-- pymodmilter/action.py | 15 ++++++++------- pymodmilter/conditions.py | 15 ++++++++++----- pymodmilter/notify.py | 2 +- pymodmilter/rule.py | 3 ++- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 86ad245..fcca40f 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -253,11 +253,12 @@ class ModifyMilter(Milter.Base): self._headersonly = True for rule in ModifyMilter._rules: if rule.conditions is None or \ - rule.conditions.match(host=self.IP): + rule.conditions.match(host=self.IP, qid=self.qid): actions = [] for action in rule.actions: if action.conditions is None or \ - action.conditions.match(host=self.IP): + action.conditions.match(host=self.IP, + qid=self.qid): actions.append(action) if not action.headersonly(): self._headersonly = False diff --git a/pymodmilter/action.py b/pymodmilter/action.py index b3c09e3..ecbd880 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -18,9 +18,10 @@ __all__ = [ import os -from pymodmilter import CustomLogger, BaseConfig -from pymodmilter.conditions import ConditionsConfig, Conditions +from pymodmilter import BaseConfig from pymodmilter import modify, notify, storage +from pymodmilter.base import CustomLogger +from pymodmilter.conditions import ConditionsConfig, Conditions class ActionConfig(BaseConfig): @@ -173,8 +174,6 @@ class Action: """Action to implement a pre-configured action to perform on e-mails.""" def __init__(self, milter_cfg, cfg): - self.logger = cfg.logger - if cfg["conditions"] is None: self.conditions = None else: @@ -184,6 +183,7 @@ class Action: self._name = cfg["name"] self._class = cfg["class"](**cfg["args"]) self._headersonly = cfg["headersonly"] + self.logger = cfg.logger def headersonly(self): """Return the needs of this action.""" @@ -191,11 +191,12 @@ class Action: def execute(self, milter): """Execute configured action.""" + logger = CustomLogger( + self.logger, {"qid": milter.qid, "name": self._name}) if self.conditions is None or \ self.conditions.match(envfrom=milter.mailfrom, envto=[*milter.rcpts], - headers=milter.msg.items()): - logger = CustomLogger( - self.logger, {"name": self._name, "qid": milter.qid}) + headers=milter.msg.items(), + qid=milter.qid): return self._class.execute( milter=milter, pretend=self.pretend, logger=logger) diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index b47d78c..99bad32 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -19,7 +19,7 @@ __all__ = [ import re from netaddr import IPAddress, IPNetwork, AddrFormatError -from pymodmilter import CustomLogger, BaseConfig +from pymodmilter import BaseConfig, CustomLogger class ConditionsConfig(BaseConfig): @@ -76,14 +76,19 @@ class Conditions: """Conditions to implement conditions for rules and actions.""" def __init__(self, milter_cfg, cfg): - self.logger = cfg.logger - self._local_addrs = milter_cfg["local_addrs"] self._name = cfg["name"] self._args = cfg["args"] + self.logger = cfg.logger + + def match(self, host=None, envfrom=None, envto=None, headers=None, + qid=None): + if qid is None: + logger = self.logger + else: + logger = CustomLogger( + self.logger, {"qid": qid, "name": self._name}) - def match(self, host=None, envfrom=None, envto=None, headers=None): - logger = CustomLogger(self.logger, {"name": self._name}) if host: ip = IPAddress(host) diff --git a/pymodmilter/notify.py b/pymodmilter/notify.py index e517b6d..72fc685 100644 --- a/pymodmilter/notify.py +++ b/pymodmilter/notify.py @@ -268,7 +268,7 @@ class EMailNotification(BaseNotification): "FROM": escape(msg["from"], quote=False), "ENVELOPE_FROM": escape(mailfrom, quote=False), "ENVELOPE_FROM_URL": escape(quote(mailfrom), - quote=False), + quote=False), "TO": escape(msg["to"], quote=False), "ENVELOPE_TO": escape(recipient, quote=False), "ENVELOPE_TO_URL": escape(quote(recipient)), diff --git a/pymodmilter/rule.py b/pymodmilter/rule.py index 3b7ebce..fae266d 100644 --- a/pymodmilter/rule.py +++ b/pymodmilter/rule.py @@ -88,7 +88,8 @@ class Rule: if self.conditions is None or \ self.conditions.match(envfrom=milter.mailfrom, envto=[*milter.rcpts], - headers=milter.msg.items()): + headers=milter.msg.items(), + qid=milter.qid): for action in self.actions: milter_action = action.execute(milter) if milter_action is not None: From ef025d758cb5994f25511f222c3f32b6ce1631a3 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Mon, 13 Sep 2021 16:00:19 +0200 Subject: [PATCH 123/145] improve logging again --- pymodmilter/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index fcca40f..38c78ba 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -151,15 +151,15 @@ class ModifyMilter(Milter.Base): def addheader(self, field, value, idx=-1): value = replace_illegal_chars(Header(s=value).encode()) - self.logger.debug(f"milter: addheader: {field}: {value}") + self.logger.debug(f"addheader: {field}: {value}") super().addheader(field, value, idx) def chgheader(self, field, value, idx=1): value = replace_illegal_chars(Header(s=value).encode()) if value: - self.logger.debug(f"milter: chgheader: {field}[{idx}]: {value}") + self.logger.debug(f"chgheader: {field}[{idx}]: {value}") else: - self.logger.debug(f"milter: delheader: {field}[{idx}]") + self.logger.debug(f"delheader: {field}[{idx}]") super().chgheader(field, idx, value) def update_headers(self, old_headers): @@ -241,7 +241,8 @@ class ModifyMilter(Milter.Base): def data(self): try: self.qid = self.getsymval('i') - self.logger = CustomLogger(self.logger, {"qid": self.qid}) + self.logger = CustomLogger( + self.logger, {"qid": self.qid, "name": "milter"}) self.logger.debug("received queue-id from MTA") # pre-filter rules and actions by the host condition, other @@ -355,7 +356,7 @@ class ModifyMilter(Milter.Base): if self._replacebody: data = self.msg.as_bytes() body_pos = data.find(b"\r\n\r\n") + 4 - self.logger.debug("milter: replacebody") + self.logger.debug("replacebody") super().replacebody(data[body_pos:]) del data From e34e85af6b70892c37b1e49e0ef3f9bf7daed255 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 14 Sep 2021 01:36:32 +0200 Subject: [PATCH 124/145] improve conditions handling --- pymodmilter/__init__.py | 62 +++++++++++++++----------------- pymodmilter/action.py | 5 +-- pymodmilter/conditions.py | 75 ++++++++++++++++++++------------------- pymodmilter/rule.py | 5 +-- 4 files changed, 70 insertions(+), 77 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 38c78ba..bbc3e08 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -199,6 +199,33 @@ class ModifyMilter(Milter.Base): self.logger.debug( f"accepted milter connection from {self.IP} " f"port {self.port}") + + # pre-filter rules and actions by the host condition + # also check if the mail body is needed by any upcoming action. + self.rules = [] + self._headersonly = True + for rule in ModifyMilter._rules: + if rule.conditions is None or \ + rule.conditions.match_host(self.IP): + actions = [] + for action in rule.actions: + if action.conditions is None or \ + action.conditions.match_host(self.IP): + actions.append(action) + if not action.headersonly(): + self._headersonly = False + + if actions: + # copy needed rules to preserve configured actions + rule = copy(rule) + rule.actions = actions + self.rules.append(rule) + + if not self.rules: + self.logger.debug( + "host is ignored by all rules, skip further processing") + return Milter.ACCEPT + except Exception as e: self.logger.exception( f"an exception occured in connect method: {e}") @@ -244,37 +271,6 @@ class ModifyMilter(Milter.Base): self.logger = CustomLogger( self.logger, {"qid": self.qid, "name": "milter"}) self.logger.debug("received queue-id from MTA") - - # pre-filter rules and actions by the host condition, other - # conditions (headers, envelope-from, envelope-to) may get - # changed by executed actions later on. - # also check if the mail body is needed by any upcoming action. - - self.rules = [] - self._headersonly = True - for rule in ModifyMilter._rules: - if rule.conditions is None or \ - rule.conditions.match(host=self.IP, qid=self.qid): - actions = [] - for action in rule.actions: - if action.conditions is None or \ - action.conditions.match(host=self.IP, - qid=self.qid): - actions.append(action) - if not action.headersonly(): - self._headersonly = False - - if actions: - # copy needed rules to preserve configured actions - rule = copy(rule) - rule.actions = actions - self.rules.append(rule) - - if not self.rules: - self.logger.debug( - "message is ignored by all rules, skip further processing") - return Milter.ACCEPT - self.fp = BytesIO() except Exception as e: self.logger.exception( @@ -341,7 +337,7 @@ class ModifyMilter(Milter.Base): refold_source='none')) self.msginfo = { "mailfrom": self.mailfrom, - "rcpts": self.rcpts, + "rcpts": [*self.rcpts], "vars": {}} self._replacebody = False @@ -356,7 +352,7 @@ class ModifyMilter(Milter.Base): if self._replacebody: data = self.msg.as_bytes() body_pos = data.find(b"\r\n\r\n") + 4 - self.logger.debug("replacebody") + self.logger.debug("replace body") super().replacebody(data[body_pos:]) del data diff --git a/pymodmilter/action.py b/pymodmilter/action.py index ecbd880..7287478 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -194,9 +194,6 @@ class Action: logger = CustomLogger( self.logger, {"qid": milter.qid, "name": self._name}) if self.conditions is None or \ - self.conditions.match(envfrom=milter.mailfrom, - envto=[*milter.rcpts], - headers=milter.msg.items(), - qid=milter.qid): + self.conditions.match(milter): return self._class.execute( milter=milter, pretend=self.pretend, logger=logger) diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index 99bad32..fb438a0 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -81,49 +81,51 @@ class Conditions: self._args = cfg["args"] self.logger = cfg.logger - def match(self, host=None, envfrom=None, envto=None, headers=None, - qid=None): - if qid is None: - logger = self.logger - else: - logger = CustomLogger( - self.logger, {"qid": qid, "name": self._name}) + def match_host(self, host): + logger = CustomLogger( + self.logger, {"name": self._name}) - if host: - ip = IPAddress(host) + ip = IPAddress(host) - if "local" in self._args: - is_local = False - for addr in self._local_addrs: - if ip in addr: - is_local = True - break - - if is_local != self._args["local"]: - logger.debug( - f"ignore host {host}, " - f"condition local does not match") - return False + if "local" in self._args: + is_local = False + for addr in self._local_addrs: + if ip in addr: + is_local = True + break + if is_local != self._args["local"]: logger.debug( - f"condition local matches for host {host}") + f"ignore host {host}, " + f"condition local does not match") + return False - if "hosts" in self._args: - found = False - for addr in self._args["hosts"]: - if ip in addr: - found = True - break + logger.debug( + f"condition local matches for host {host}") - if not found: - logger.debug( - f"ignore host {host}, " - f"condition hosts does not match") - return False + if "hosts" in self._args: + found = False + for addr in self._args["hosts"]: + if ip in addr: + found = True + break + if not found: logger.debug( - f"condition hosts matches for host {host}") + f"ignore host {host}, " + f"condition hosts does not match") + return False + logger.debug( + f"condition hosts matches for host {host}") + + return True + + def match(self, milter): + logger = CustomLogger( + self.logger, {"qid": milter.qid, "name": self._name}) + + envfrom = milter.msginfo["mailfrom"] if envfrom and "envfrom" in self._args: if not self._args["envfrom"].match(envfrom): logger.debug( @@ -135,6 +137,7 @@ class Conditions: f"condition envfrom matches for " f"envelope-from address {envfrom}") + envto = milter.msginfo["rcpts"] if envto and "envto" in self._args: if not isinstance(envto, list): envto = [envto] @@ -150,9 +153,9 @@ class Conditions: f"condition envto matches for " f"envelope-to address {envto}") - if headers and "header" in self._args: + if "header" in self._args: match = None - for field, value in headers: + for field, value in milter.msg.items(): header = f"{field}: {value}" match = self._args["header"].search(header) if match: diff --git a/pymodmilter/rule.py b/pymodmilter/rule.py index fae266d..70cfe3b 100644 --- a/pymodmilter/rule.py +++ b/pymodmilter/rule.py @@ -86,10 +86,7 @@ class Rule: def execute(self, milter): """Execute all actions of this rule.""" if self.conditions is None or \ - self.conditions.match(envfrom=milter.mailfrom, - envto=[*milter.rcpts], - headers=milter.msg.items(), - qid=milter.qid): + self.conditions.match(milter): for action in self.actions: milter_action = action.execute(milter) if milter_action is not None: From b91460b629d0b5cedb0ea731f6885e59417efa53 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 14 Sep 2021 14:39:40 +0200 Subject: [PATCH 125/145] improve conditions and logging, add metavar to conditions --- pymodmilter/action.py | 2 +- pymodmilter/conditions.py | 86 +++++++++++++++++++++++---------------- pymodmilter/rule.py | 2 +- 3 files changed, 54 insertions(+), 36 deletions(-) diff --git a/pymodmilter/action.py b/pymodmilter/action.py index 7287478..7aa789a 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -164,7 +164,7 @@ class ActionConfig(BaseConfig): f"{self['name']}: conditions: invalid value, should be dict" self["conditions"] = ConditionsConfig(self, conditions, debug) - self.logger.debug(f"pretend={self['pretend']}, " + self.logger.debug(f"{self['name']}: pretend={self['pretend']}, " f"loglevel={self['loglevel']}, " f"type={self['type']}, " f"args={self['args']}") diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index fb438a0..953889d 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -35,37 +35,22 @@ class ConditionsConfig(BaseConfig): self.add_bool_arg(cfg, "local") if "hosts" in cfg: - hosts = cfg["hosts"] - assert isinstance(hosts, list) and all( - [isinstance(host, str) for host in hosts]), \ + assert isinstance(cfg["hosts"], list) and all( + [isinstance(host, str) for host in cfg["hosts"]]), \ f"{self['name']}: hosts: invalid value, " \ f"should be list of strings" - self["args"]["hosts"] = [] - try: - for host in cfg["hosts"]: - self["args"]["hosts"].append(IPNetwork(host)) - except AddrFormatError as e: - raise ValueError(f"{self['name']}: hosts: {e}") + self["args"]["hosts"] = cfg["hosts"] for arg in ("envfrom", "envto"): if arg in cfg: self.add_string_arg(cfg, arg) - try: - self["args"][arg] = re.compile( - self["args"][arg], - re.IGNORECASE) - except re.error as e: - raise ValueError(f"{self['name']}: {arg}: {e}") if "header" in cfg: self.add_string_arg(cfg, "header") - try: - self["args"]["header"] = re.compile( - self["args"]["header"], - re.IGNORECASE + re.DOTALL + re.MULTILINE) - except re.error as e: - raise ValueError(f"{self['name']}: header: {e}") + + if "metavar" in cfg: + self.add_string_arg(cfg, "metavar") self.logger.debug(f"{self['name']}: " f"loglevel={self['loglevel']}, " @@ -78,7 +63,35 @@ class Conditions: def __init__(self, milter_cfg, cfg): self._local_addrs = milter_cfg["local_addrs"] self._name = cfg["name"] - self._args = cfg["args"] + + for arg in ("local", "hosts", "envfrom", "envto", "header", "metavar"): + value = cfg["args"][arg] if arg in cfg["args"] else None + setattr(self, arg, value) + if value is None: + continue + elif arg == "hosts": + try: + hosts = [] + for host in self.hosts: + hosts.append(IPNetwork(host)) + except AddrFormatError as e: + raise RuntimeError(e) + + self.hosts = hosts + elif arg in ("envfrom", "envto"): + try: + setattr(self, arg, re.compile( + getattr(self, arg), re.IGNORECASE)) + except re.error as e: + raise RuntimeError(e) + + elif arg == "header": + try: + self.header = re.compile( + self.header, re.IGNORECASE + re.DOTALL + re.MULTILINE) + except re.error as e: + raise RuntimeError(e) + self.logger = cfg.logger def match_host(self, host): @@ -87,14 +100,14 @@ class Conditions: ip = IPAddress(host) - if "local" in self._args: + if self.local is not None: is_local = False for addr in self._local_addrs: if ip in addr: is_local = True break - if is_local != self._args["local"]: + if is_local != self.local: logger.debug( f"ignore host {host}, " f"condition local does not match") @@ -103,9 +116,9 @@ class Conditions: logger.debug( f"condition local matches for host {host}") - if "hosts" in self._args: + if self.hosts is not None: found = False - for addr in self._args["hosts"]: + for addr in self.hosts: if ip in addr: found = True break @@ -125,9 +138,9 @@ class Conditions: logger = CustomLogger( self.logger, {"qid": milter.qid, "name": self._name}) - envfrom = milter.msginfo["mailfrom"] - if envfrom and "envfrom" in self._args: - if not self._args["envfrom"].match(envfrom): + if self.envfrom is not None: + envfrom = milter.msginfo["mailfrom"] + if not self.envfrom.match(envfrom): logger.debug( f"ignore envelope-from address {envfrom}, " f"condition envfrom does not match") @@ -137,13 +150,13 @@ class Conditions: f"condition envfrom matches for " f"envelope-from address {envfrom}") - envto = milter.msginfo["rcpts"] - if envto and "envto" in self._args: + if self.envto is not None: + envto = milter.msginfo["rcpts"] if not isinstance(envto, list): envto = [envto] for to in envto: - if not self._args["envto"].match(to): + if not self.envto.match(to): logger.debug( f"ignore envelope-to address {envto}, " f"condition envto does not match") @@ -153,15 +166,20 @@ class Conditions: f"condition envto matches for " f"envelope-to address {envto}") - if "header" in self._args: + if self.header is not None: match = None for field, value in milter.msg.items(): header = f"{field}: {value}" - match = self._args["header"].search(header) + match = self.header.search(header) if match: logger.debug( f"condition header matches for " f"header: {header}") + if self.metavar is not None: + named_subgroups = match.groupdict(default="") + for group, value in named_subgroups.items(): + name = f"{self.metavar}_{group}" + milter.msginfo["vars"][name] = value break if not match: diff --git a/pymodmilter/rule.py b/pymodmilter/rule.py index 70cfe3b..c6a342e 100644 --- a/pymodmilter/rule.py +++ b/pymodmilter/rule.py @@ -50,7 +50,7 @@ class RuleConfig(BaseConfig): assert isinstance(actions, list), \ f"{self['name']}: actions: invalid value, should be list" - self.logger.debug(f"pretend={self['pretend']}, " + self.logger.debug(f"{self['name']}: pretend={self['pretend']}, " f"loglevel={self['loglevel']}") if "conditions" in cfg: From 46a710390089aada90d7cac4255903a440a71a21 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Sun, 19 Sep 2021 18:47:02 +0200 Subject: [PATCH 126/145] add condition on variables --- pymodmilter/action.py | 20 +++++++++----------- pymodmilter/conditions.py | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/pymodmilter/action.py b/pymodmilter/action.py index 7aa789a..c45620d 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -41,7 +41,6 @@ class ActionConfig(BaseConfig): self["pretend"] = rule_cfg["pretend"] self["conditions"] = None - self["type"] = "" if "pretend" in cfg: pretend = cfg["pretend"] @@ -53,13 +52,12 @@ class ActionConfig(BaseConfig): f"{self['name']}: mandatory parameter 'type' not found" assert isinstance(cfg["type"], str), \ f"{self['name']}: type: invalid value, should be string" - self["type"] = cfg["type"] - if self["type"] == "add_header": + if cfg["type"] == "add_header": self["class"] = modify.AddHeader self["headersonly"] = True self.add_string_arg(cfg, ["field", "value"]) - elif self["type"] == "mod_header": + elif cfg["type"] == "mod_header": self["class"] = modify.ModHeader self["headersonly"] = True args = ["field", "value"] @@ -67,7 +65,7 @@ class ActionConfig(BaseConfig): args.append("search") self.add_string_arg(cfg, args) - elif self["type"] == "del_header": + elif cfg["type"] == "del_header": self["class"] = modify.DelHeader self["headersonly"] = True args = ["field"] @@ -75,7 +73,7 @@ class ActionConfig(BaseConfig): args.append("value") self.add_string_arg(cfg, args) - elif self["type"] == "add_disclaimer": + elif cfg["type"] == "add_disclaimer": self["class"] = modify.AddDisclaimer self["headersonly"] = False if "error_policy" not in cfg: @@ -93,17 +91,17 @@ class ActionConfig(BaseConfig): f"{self['name']}: error_policy: invalid value, " \ f"should be 'wrap', 'ignore' or 'reject'" - elif self["type"] == "rewrite_links": + elif cfg["type"] == "rewrite_links": self["class"] = modify.RewriteLinks self["headersonly"] = False self.add_string_arg(cfg, "repl") - elif self["type"] == "store": + elif cfg["type"] == "store": self["headersonly"] = False assert "storage_type" in cfg, \ f"{self['name']}: mandatory parameter 'storage_type' not found" - assert isinstance(cfg["type"], str), \ + assert isinstance(cfg["storage_type"], str), \ f"{self['name']}: storage_type: invalid value, " \ f"should be string" self["storage_type"] = cfg["storage_type"] @@ -132,7 +130,7 @@ class ActionConfig(BaseConfig): raise RuntimeError( f"{self['name']}: storage_type: invalid storage type") - elif self["type"] == "notify": + elif cfg["type"] == "notify": self["headersonly"] = False self["class"] = notify.EMailNotification @@ -166,7 +164,7 @@ class ActionConfig(BaseConfig): self.logger.debug(f"{self['name']}: pretend={self['pretend']}, " f"loglevel={self['loglevel']}, " - f"type={self['type']}, " + f"type={cfg['type']}, " f"args={self['args']}") diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index 953889d..10b2ff8 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -49,6 +49,9 @@ class ConditionsConfig(BaseConfig): if "header" in cfg: self.add_string_arg(cfg, "header") + if "var" in cfg: + self.add_string_arg(cfg, "var") + if "metavar" in cfg: self.add_string_arg(cfg, "metavar") @@ -64,7 +67,8 @@ class Conditions: self._local_addrs = milter_cfg["local_addrs"] self._name = cfg["name"] - for arg in ("local", "hosts", "envfrom", "envto", "header", "metavar"): + for arg in ("local", "hosts", "envfrom", "envto", "header", "metavar", + "var"): value = cfg["args"][arg] if arg in cfg["args"] else None setattr(self, arg, value) if value is None: @@ -176,8 +180,10 @@ class Conditions: f"condition header matches for " f"header: {header}") if self.metavar is not None: - named_subgroups = match.groupdict(default="") + named_subgroups = match.groupdict(default=None) for group, value in named_subgroups.items(): + if value is None: + continue name = f"{self.metavar}_{group}" milter.msginfo["vars"][name] = value break @@ -188,4 +194,8 @@ class Conditions: "condition header does not match") return False + if self.var is not None: + if self.var not in milter.msginfo["vars"]: + return False + return True From 42e65848c4d213271b530e595594e47e078b127c Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Mon, 20 Sep 2021 18:08:56 +0200 Subject: [PATCH 127/145] refactor config structure --- pymodmilter/__init__.py | 39 +++--- pymodmilter/action.py | 270 ++++++++++++++++++-------------------- pymodmilter/base.py | 62 ++++----- pymodmilter/conditions.py | 34 ++--- pymodmilter/rule.py | 72 +++++----- pymodmilter/run.py | 14 +- pymodmilter/storage.py | 6 + 7 files changed, 240 insertions(+), 257 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index bbc3e08..cc5f5d2 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -79,17 +79,17 @@ class ModifyMilterConfig(BaseConfig): pretend = cfg["global"]["pretend"] assert isinstance(pretend, bool), \ "global: pretend: invalid value, should be bool" - self["pretend"] = pretend + self.pretend = pretend else: - self["pretend"] = False + self.pretend = False if "socket" in cfg["global"]: socket = cfg["global"]["socket"] assert isinstance(socket, str), \ "global: socket: invalid value, should be string" - self["socket"] = socket + self.socket = socket else: - self["socket"] = None + self.socket = None if "local_addrs" in cfg["global"]: local_addrs = cfg["global"]["local_addrs"] @@ -106,17 +106,17 @@ class ModifyMilterConfig(BaseConfig): "172.16.0.0/12", "192.168.0.0/16"] - self["local_addrs"] = [] + self.local_addrs = [] try: for addr in local_addrs: - self["local_addrs"].append(IPNetwork(addr)) + self.local_addrs.append(IPNetwork(addr)) except AddrFormatError as e: - raise ValueError(f"{self['name']}: local_addrs: {e}") + raise ValueError(f"{self.name}: local_addrs: {e}") - self.logger.debug(f"socket={self['socket']}, " - f"local_addrs={self['local_addrs']}, " - f"pretend={self['pretend']}, " - f"loglevel={self['loglevel']}") + self.logger.debug(f"socket={self.socket}, " + f"local_addrs={self.local_addrs}, " + f"pretend={self.pretend}, " + f"loglevel={self.loglevel}") assert "rules" in cfg, \ "mandatory parameter 'rules' not found" @@ -124,10 +124,15 @@ class ModifyMilterConfig(BaseConfig): "rules: invalid value, should be list" self.logger.debug("initialize rules config") - self["rules"] = [] + self.rules = [] for idx, rule_cfg in enumerate(cfg["rules"]): - self["rules"].append( - RuleConfig(idx, self, rule_cfg, debug)) + if "name" not in rule_cfg: + rule_cfg["name"] = "Rule #{idx}" + if "loglevel" not in rule_cfg: + rule_cfg["loglevel"] = self.loglevel + if "pretend" not in rule_cfg: + rule_cfg["pretend"] = self.pretend + self.rules.append(RuleConfig(rule_cfg, debug)) class ModifyMilter(Milter.Base): @@ -140,10 +145,10 @@ class ModifyMilter(Milter.Base): @staticmethod def set_config(cfg): - ModifyMilter._loglevel = cfg["loglevel"] - for rule_cfg in cfg["rules"]: + ModifyMilter._loglevel = cfg.loglevel + for rule_cfg in cfg.rules: ModifyMilter._rules.append( - Rule(cfg, rule_cfg)) + Rule(rule_cfg, cfg.local_addrs)) def __init__(self): self.logger = logging.getLogger(__name__) diff --git a/pymodmilter/action.py b/pymodmilter/action.py index c45620d..569acdf 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -16,8 +16,6 @@ __all__ = [ "ActionConfig", "Action"] -import os - from pymodmilter import BaseConfig from pymodmilter import modify, notify, storage from pymodmilter.base import CustomLogger @@ -25,163 +23,153 @@ from pymodmilter.conditions import ConditionsConfig, Conditions class ActionConfig(BaseConfig): - def __init__(self, idx, rule_cfg, cfg, debug): - if "name" in cfg: - assert isinstance(cfg["name"], str), \ - f"{rule_cfg['name']}: Action #{idx}: name: invalid value, " \ - f"should be string" - cfg["name"] = f"{rule_cfg['name']}: {cfg['name']}" - else: - cfg["name"] = f"{rule_cfg['name']}: Action #{idx}" - - if "loglevel" not in cfg: - cfg["loglevel"] = rule_cfg["loglevel"] + TYPES = {"add_header": "_add_header", + "mod_header": "_mod_header", + "del_header": "_del_header", + "add_disclaimer": "_add_disclaimer", + "rewrite_links": "_rewrite_links", + "store": "_store", + "notify": "_notify"} + def __init__(self, cfg, debug): super().__init__(cfg, debug) - self["pretend"] = rule_cfg["pretend"] - self["conditions"] = None - + self.pretend = False if "pretend" in cfg: - pretend = cfg["pretend"] - assert isinstance(pretend, bool), \ - f"{self['name']}: pretend: invalid value, should be bool" - self["pretend"] = pretend + assert isinstance(cfg["pretend"], bool), \ + f"{self.name}: pretend: invalid value, should be bool" + self.pretend = cfg["pretend"] assert "type" in cfg, \ - f"{self['name']}: mandatory parameter 'type' not found" + f"{self.name}: mandatory parameter 'type' not found" assert isinstance(cfg["type"], str), \ - f"{self['name']}: type: invalid value, should be string" + f"{self.name}: type: invalid value, should be string" + assert cfg["type"] in ActionConfig.TYPES, \ + f"{self.name}: type: invalid action type" - if cfg["type"] == "add_header": - self["class"] = modify.AddHeader - self["headersonly"] = True - self.add_string_arg(cfg, ["field", "value"]) - elif cfg["type"] == "mod_header": - self["class"] = modify.ModHeader - self["headersonly"] = True - args = ["field", "value"] - if "search" in cfg: - args.append("search") - - self.add_string_arg(cfg, args) - elif cfg["type"] == "del_header": - self["class"] = modify.DelHeader - self["headersonly"] = True - args = ["field"] - if "value" in cfg: - args.append("value") - - self.add_string_arg(cfg, args) - elif cfg["type"] == "add_disclaimer": - self["class"] = modify.AddDisclaimer - self["headersonly"] = False - if "error_policy" not in cfg: - cfg["error_policy"] = "wrap" - - self.add_string_arg( - cfg, ["action", "html_template", "text_template", - "error_policy"]) - assert self["args"]["action"] in ["append", "prepend"], \ - f"{self['name']}: action: invalid value, " \ - f"should be 'append' or 'prepend'" - assert self["args"]["error_policy"] in ("wrap", - "ignore", - "reject"), \ - f"{self['name']}: error_policy: invalid value, " \ - f"should be 'wrap', 'ignore' or 'reject'" - - elif cfg["type"] == "rewrite_links": - self["class"] = modify.RewriteLinks - self["headersonly"] = False - self.add_string_arg(cfg, "repl") - - elif cfg["type"] == "store": - self["headersonly"] = False - - assert "storage_type" in cfg, \ - f"{self['name']}: mandatory parameter 'storage_type' not found" - assert isinstance(cfg["storage_type"], str), \ - f"{self['name']}: storage_type: invalid value, " \ - f"should be string" - self["storage_type"] = cfg["storage_type"] - - if "original" in cfg: - self.add_bool_arg(cfg, "original") - - if self["storage_type"] == "file": - self["class"] = storage.FileMailStorage - self.add_string_arg(cfg, "directory") - # check if directory exists and is writable - if not os.path.isdir(self["args"]["directory"]) or \ - not os.access(self["args"]["directory"], os.W_OK): - raise RuntimeError( - f"{self['name']}: file quarantine directory " - f"'{self['directory']}' does not exist or is " - f"not writable") - - if "skip_metadata" in cfg: - self.add_bool_arg(cfg, "skip_metadata") - - if "metavar" in cfg: - self.add_string_arg(cfg, "metavar") - - else: - raise RuntimeError( - f"{self['name']}: storage_type: invalid storage type") - - elif cfg["type"] == "notify": - self["headersonly"] = False - self["class"] = notify.EMailNotification - - args = ["smtp_host", "envelope_from", "from_header", "subject", - "template"] - if "repl_img" in cfg: - args.append("repl_img") - - self.add_string_arg(cfg, args) - self.add_int_arg(cfg, "smtp_port") - - if "embed_imgs" in cfg: - assert isinstance(cfg["embed_imgs"], list), \ - f"{self['name']}: embed_imgs: invalid value, " \ - f"should be list" - for img in cfg["embed_imgs"]: - assert isinstance(img, str), \ - f"{self['name']}: embed_imgs: invalid entry, " \ - f"should be string" - - self["args"]["embed_imgs"] = cfg["embed_imgs"] - - else: - raise RuntimeError(f"{self['name']}: type: invalid action type") + getattr(self, cfg["type"])(cfg) if "conditions" in cfg: - conditions = cfg["conditions"] - assert isinstance(conditions, dict), \ - f"{self['name']}: conditions: invalid value, should be dict" - self["conditions"] = ConditionsConfig(self, conditions, debug) + assert isinstance(cfg["conditions"], dict), \ + f"{self.name}: conditions: invalid value, should be dict" + cfg["conditions"]["name"] = f"{self.name}: condition" + if "loglevel" not in cfg["conditions"]: + cfg["conditions"]["loglevel"] = self.loglevel + self.conditions = ConditionsConfig(cfg["conditions"], debug) + else: + self.conditions = None - self.logger.debug(f"{self['name']}: pretend={self['pretend']}, " - f"loglevel={self['loglevel']}, " + self.logger.debug(f"{self.name}: pretend={self.pretend}, " + f"loglevel={self.loglevel}, " f"type={cfg['type']}, " - f"args={self['args']}") + f"args={self.args}") + + def add_header(self, cfg): + self.action = modify.AddHeader + self.headersonly = True + self.add_string_arg(cfg, ["field", "value"]) + + def mod_header(self, cfg): + self.action = modify.ModHeader + self.headersonly = True + args = ["field", "value"] + if "search" in cfg: + args.append("search") + + self.add_string_arg(cfg, args) + + def del_header(self, cfg): + self.action = modify.DelHeader + self.headersonly = True + args = ["field"] + if "value" in cfg: + args.append("value") + + self.add_string_arg(cfg, args) + + def add_disclaimer(self, cfg): + self.action = modify.AddDisclaimer + self.headersonly = False + if "error_policy" not in cfg: + cfg["error_policy"] = "wrap" + + self.add_string_arg( + cfg, ["action", "html_template", "text_template", + "error_policy"]) + assert self.args["action"] in ["append", "prepend"], \ + f"{self.name}: action: invalid value, " \ + f"should be 'append' or 'prepend'" + assert self.args["error_policy"] in ("wrap", + "ignore", + "reject"), \ + f"{self.name}: error_policy: invalid value, " \ + f"should be 'wrap', 'ignore' or 'reject'" + + def rewrite_links(self, cfg): + self.action = modify.RewriteLinks + self.headersonly = False + self.add_string_arg(cfg, "repl") + + def store(self, cfg): + self.headersonly = False + + assert "storage_type" in cfg, \ + f"{self.name}: mandatory parameter 'storage_type' not found" + assert isinstance(cfg["storage_type"], str), \ + f"{self.name}: storage_type: invalid value, " \ + f"should be string" + + if "original" in cfg: + self.add_bool_arg(cfg, "original") + + if cfg["storage_type"] == "file": + self.action = storage.FileMailStorage + self.add_string_arg(cfg, "directory") + + if "skip_metadata" in cfg: + self.add_bool_arg(cfg, "skip_metadata") + + if "metavar" in cfg: + self.add_string_arg(cfg, "metavar") + + else: + raise RuntimeError( + f"{self.name}: storage_type: invalid storage type") + + def notify(self, cfg): + self.headersonly = False + self.action = notify.EMailNotification + + args = ["smtp_host", "envelope_from", "from_header", "subject", + "template"] + if "repl_img" in cfg: + args.append("repl_img") + + self.add_string_arg(cfg, args) + self.add_int_arg(cfg, "smtp_port") + + if "embed_imgs" in cfg: + assert isinstance(cfg["embed_imgs"], list) and all( + [isinstance(img, str) for img in cfg["embed_imgs"]]), \ + f"{self.name}: embed_imgs: invalid value, " \ + f"should be list of strings" + self.args["embed_imgs"] = cfg["embed_imgs"] class Action: """Action to implement a pre-configured action to perform on e-mails.""" - def __init__(self, milter_cfg, cfg): - if cfg["conditions"] is None: + def __init__(self, cfg, local_addrs): + self.logger = cfg.logger + if cfg.conditions is None: self.conditions = None else: - self.conditions = Conditions(milter_cfg, cfg["conditions"]) + self.conditions = Conditions(cfg.conditions, local_addrs) - self.pretend = cfg["pretend"] - self._name = cfg["name"] - self._class = cfg["class"](**cfg["args"]) - self._headersonly = cfg["headersonly"] - self.logger = cfg.logger + self.pretend = cfg.pretend + self.name = cfg.name + self.action = cfg.action(**cfg.args) + self._headersonly = cfg.headersonly def headersonly(self): """Return the needs of this action.""" @@ -190,8 +178,8 @@ class Action: def execute(self, milter): """Execute configured action.""" logger = CustomLogger( - self.logger, {"qid": milter.qid, "name": self._name}) + self.logger, {"qid": milter.qid, "name": self.name}) if self.conditions is None or \ self.conditions.match(milter): - return self._class.execute( + return self.action.execute( milter=milter, pretend=self.pretend, logger=logger) diff --git a/pymodmilter/base.py b/pymodmilter/base.py index 2ec6a6f..004826b 100644 --- a/pymodmilter/base.py +++ b/pymodmilter/base.py @@ -38,49 +38,33 @@ class CustomLogger(logging.LoggerAdapter): class BaseConfig: - def __init__(self, cfg={}, debug=False, logger=None): - self._cfg = {} + def __init__(self, cfg={}, debug=False): if "name" in cfg: assert isinstance(cfg["name"], str), \ - "rule: name: invalid value, should be string" - self["name"] = cfg["name"] + "name: invalid value, should be string" + self.name = cfg["name"] else: - self["name"] = __name__ + self.name = __name__ + self.logger = logging.getLogger(self.name) if debug: - self["loglevel"] = logging.DEBUG + self.loglevel = logging.DEBUG elif "loglevel" in cfg: if isinstance(cfg["loglevel"], int): - self["loglevel"] = cfg["loglevel"] + self.loglevel = cfg["loglevel"] else: level = getattr(logging, cfg["loglevel"].upper(), None) assert isinstance(level, int), \ - f"{self['name']}: loglevel: invalid value" - self["loglevel"] = level + f"{self.name}: loglevel: invalid value" + self.loglevel = level else: - self["loglevel"] = logging.INFO + self.loglevel = logging.INFO - if logger is None: - logger = logging.getLogger(self["name"]) - logger.setLevel(self["loglevel"]) + self.logger.setLevel(self.loglevel) - self.logger = logger - - # the keys/values of args are used as parameters - # to functions - self["args"] = {} - - def __setitem__(self, key, value): - self._cfg[key] = value - - def __getitem__(self, key): - return self._cfg[key] - - def __delitem__(self, key): - del self._cfg[key] - - def __contains__(self, key): - return key in self._cfg + # the keys/values in args are used as parameters + # to initialize action classes + self.args = {} def add_string_arg(self, cfg, args): if isinstance(args, str): @@ -88,10 +72,10 @@ class BaseConfig: for arg in args: assert arg in cfg, \ - f"{self['name']}: mandatory parameter '{arg}' not found" + f"{self.name}: mandatory parameter '{arg}' not found" assert isinstance(cfg[arg], str), \ - f"{self['name']}: {arg}: invalid value, should be string" - self["args"][arg] = cfg[arg] + f"{self.name}: {arg}: invalid value, should be string" + self.args[arg] = cfg[arg] def add_bool_arg(self, cfg, args): if isinstance(args, str): @@ -99,10 +83,10 @@ class BaseConfig: for arg in args: assert arg in cfg, \ - f"{self['name']}: mandatory parameter '{arg}' not found" + f"{self.name}: mandatory parameter '{arg}' not found" assert isinstance(cfg[arg], bool), \ - f"{self['name']}: {arg}: invalid value, should be bool" - self["args"][arg] = cfg[arg] + f"{self.name}: {arg}: invalid value, should be bool" + self.args[arg] = cfg[arg] def add_int_arg(self, cfg, args): if isinstance(args, str): @@ -110,10 +94,10 @@ class BaseConfig: for arg in args: assert arg in cfg, \ - f"{self['name']}: mandatory parameter '{arg}' not found" + f"{self.name}: mandatory parameter '{arg}' not found" assert isinstance(cfg[arg], int), \ - f"{self['name']}: {arg}: invalid value, should be integer" - self["args"][arg] = cfg[arg] + f"{self.name}: {arg}: invalid value, should be integer" + self.args[arg] = cfg[arg] class MilterMessage(MIMEPart): diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index 10b2ff8..7df41b5 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -23,12 +23,7 @@ from pymodmilter import BaseConfig, CustomLogger class ConditionsConfig(BaseConfig): - def __init__(self, parent_cfg, cfg, debug): - if "loglevel" not in cfg: - cfg["loglevel"] = parent_cfg["loglevel"] - - cfg["name"] = f"{parent_cfg['name']}: condition" - + def __init__(self, cfg, debug): super().__init__(cfg, debug) if "local" in cfg: @@ -37,10 +32,10 @@ class ConditionsConfig(BaseConfig): if "hosts" in cfg: assert isinstance(cfg["hosts"], list) and all( [isinstance(host, str) for host in cfg["hosts"]]), \ - f"{self['name']}: hosts: invalid value, " \ + f"{self.name}: hosts: invalid value, " \ f"should be list of strings" - self["args"]["hosts"] = cfg["hosts"] + self.args["hosts"] = cfg["hosts"] for arg in ("envfrom", "envto"): if arg in cfg: @@ -55,21 +50,22 @@ class ConditionsConfig(BaseConfig): if "metavar" in cfg: self.add_string_arg(cfg, "metavar") - self.logger.debug(f"{self['name']}: " - f"loglevel={self['loglevel']}, " - f"args={self['args']}") + self.logger.debug(f"{self.name}: " + f"loglevel={self.loglevel}, " + f"args={self.args}") class Conditions: """Conditions to implement conditions for rules and actions.""" - def __init__(self, milter_cfg, cfg): - self._local_addrs = milter_cfg["local_addrs"] - self._name = cfg["name"] + def __init__(self, cfg, local_addrs): + self.logger = cfg.logger + self.name = cfg.name + self.local_addrs = local_addrs for arg in ("local", "hosts", "envfrom", "envto", "header", "metavar", "var"): - value = cfg["args"][arg] if arg in cfg["args"] else None + value = cfg.args[arg] if arg in cfg.args else None setattr(self, arg, value) if value is None: continue @@ -96,17 +92,15 @@ class Conditions: except re.error as e: raise RuntimeError(e) - self.logger = cfg.logger - def match_host(self, host): logger = CustomLogger( - self.logger, {"name": self._name}) + self.logger, {"name": self.name}) ip = IPAddress(host) if self.local is not None: is_local = False - for addr in self._local_addrs: + for addr in self.local_addrs: if ip in addr: is_local = True break @@ -140,7 +134,7 @@ class Conditions: def match(self, milter): logger = CustomLogger( - self.logger, {"qid": milter.qid, "name": self._name}) + self.logger, {"qid": milter.qid, "name": self.name}) if self.envfrom is not None: envfrom = milter.msginfo["mailfrom"] diff --git a/pymodmilter/rule.py b/pymodmilter/rule.py index c6a342e..3341ae2 100644 --- a/pymodmilter/rule.py +++ b/pymodmilter/rule.py @@ -22,46 +22,52 @@ from pymodmilter.conditions import ConditionsConfig, Conditions class RuleConfig(BaseConfig): - def __init__(self, idx, milter_cfg, cfg, debug=False): - if "name" in cfg: - assert isinstance(cfg["name"], str), \ - f"Rule #{idx}: name: invalid value, should be string" - else: - cfg["name"] = f"Rule #{idx}" - - if "loglevel" not in cfg: - cfg["loglevel"] = milter_cfg["loglevel"] - + def __init__(self, cfg, debug=False): super().__init__(cfg, debug) - self["pretend"] = milter_cfg["pretend"] - self["conditions"] = None - self["actions"] = [] + self.conditions = None + self.actions = [] + self.pretend = False if "pretend" in cfg: - pretend = cfg["pretend"] - assert isinstance(pretend, bool), \ - f"{self['name']}: pretend: invalid value, should be bool" - self["pretend"] = pretend + assert isinstance(cfg["pretend"], bool), \ + f"{self.name}: pretend: invalid value, should be bool" + self.pretend = cfg["pretend"] assert "actions" in cfg, \ - f"{self['name']}: mandatory parameter 'actions' not found" + f"{self.name}: mandatory parameter 'actions' not found" actions = cfg["actions"] assert isinstance(actions, list), \ - f"{self['name']}: actions: invalid value, should be list" + f"{self.name}: actions: invalid value, should be list" - self.logger.debug(f"{self['name']}: pretend={self['pretend']}, " - f"loglevel={self['loglevel']}") + self.logger.debug(f"{self.name}: pretend={self.pretend}, " + f"loglevel={self.loglevel}") if "conditions" in cfg: - conditions = cfg["conditions"] - assert isinstance(conditions, dict), \ - f"{self['name']}: conditions: invalid value, should be dict" - self["conditions"] = ConditionsConfig(self, conditions, debug) + assert isinstance(cfg["conditions"], dict), \ + f"{self.name}: conditions: invalid value, should be dict" + cfg["conditions"]["name"] = f"{self.name}: condition" + if "loglevel" not in cfg["conditions"]: + cfg["conditions"]["loglevel"] = self.loglevel + self.conditions = ConditionsConfig(cfg["conditions"], debug) + else: + self.conditions = None for idx, action_cfg in enumerate(cfg["actions"]): - self["actions"].append( - ActionConfig(idx, self, action_cfg, debug)) + if "name" in action_cfg: + assert isinstance(action_cfg["name"], str), \ + f"{self.name}: Action #{idx}: name: invalid value, " \ + f"should be string" + action_cfg["name"] = f"{self.name}: {action_cfg['name']}" + else: + action_cfg["name"] = f"{self.name}: Action #{idx}" + + if "loglevel" not in action_cfg: + action_cfg["loglevel"] = self.loglevel + if "pretend" not in action_cfg: + action_cfg["pretend"] = self.pretend + self.actions.append( + ActionConfig(action_cfg, debug)) class Rule: @@ -69,19 +75,19 @@ class Rule: Rule to implement multiple actions on emails. """ - def __init__(self, milter_cfg, cfg): + def __init__(self, cfg, local_addrs): self.logger = cfg.logger - if cfg["conditions"] is None: + if cfg.conditions is None: self.conditions = None else: - self.conditions = Conditions(milter_cfg, cfg["conditions"]) + self.conditions = Conditions(cfg.conditions, local_addrs) self.actions = [] - for action_cfg in cfg["actions"]: - self.actions.append(Action(milter_cfg, action_cfg)) + for action_cfg in cfg.actions: + self.actions.append(Action(action_cfg, local_addrs)) - self.pretend = cfg["pretend"] + self.pretend = cfg.pretend def execute(self, milter): """Execute all actions of this rule.""" diff --git a/pymodmilter/run.py b/pymodmilter/run.py index a3a64d8..6bd6304 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -84,24 +84,24 @@ def main(): cfg = ModifyMilterConfig(args.config, args.debug) if not args.debug: - logger.setLevel(cfg["loglevel"]) + logger.setLevel(cfg.loglevel) if args.socket: socket = args.socket - elif "socket" in cfg: - socket = cfg["socket"] + elif cfg.socket: + socket = cfg.socket else: raise RuntimeError( "listening socket is neither specified on the command line " "nor in the configuration file") - if not cfg["rules"]: + if not cfg.rules: raise RuntimeError("no rules configured") - for rule_cfg in cfg["rules"]: - if not rule_cfg["actions"]: + for rule_cfg in cfg.rules: + if not rule_cfg.actions: raise RuntimeError( - f"{rule_cfg['name']}: no actions configured") + f"{rule_cfg.name}: no actions configured") except (RuntimeError, AssertionError) as e: logger.error(e) diff --git a/pymodmilter/storage.py b/pymodmilter/storage.py index 93fb518..7a058f3 100644 --- a/pymodmilter/storage.py +++ b/pymodmilter/storage.py @@ -57,6 +57,12 @@ class FileMailStorage(BaseMailStorage): def __init__(self, directory, original=False, skip_metadata=False, metavar=None): super().__init__() + # check if directory exists and is writable + if not os.path.isdir(directory) or \ + not os.access(directory, os.W_OK): + raise RuntimeError( + f"directory '{directory}' does not exist or is " + f"not writable") self.directory = directory self.original = original self.skip_metadata = skip_metadata From cd470e89470d316350d988b8c11dad79b36ed0f0 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 21 Sep 2021 00:06:15 +0200 Subject: [PATCH 128/145] add quarantine functionality and refactor --- pymodmilter/__init__.py | 13 +++--- pymodmilter/action.py | 86 ++++++++++++++++++++++++++------------ pymodmilter/base.py | 1 + pymodmilter/conditions.py | 8 ++-- pymodmilter/modify.py | 10 +++++ pymodmilter/notify.py | 7 +++- pymodmilter/rule.py | 13 +++--- pymodmilter/storage.py | 88 +++++++++++++++++++++++++++------------ 8 files changed, 157 insertions(+), 69 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index cc5f5d2..8d38044 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -132,7 +132,7 @@ class ModifyMilterConfig(BaseConfig): rule_cfg["loglevel"] = self.loglevel if "pretend" not in rule_cfg: rule_cfg["pretend"] = self.pretend - self.rules.append(RuleConfig(rule_cfg, debug)) + self.rules.append(RuleConfig(rule_cfg, self.local_addrs, debug)) class ModifyMilter(Milter.Base): @@ -148,7 +148,7 @@ class ModifyMilter(Milter.Base): ModifyMilter._loglevel = cfg.loglevel for rule_cfg in cfg.rules: ModifyMilter._rules.append( - Rule(rule_cfg, cfg.local_addrs)) + Rule(rule_cfg)) def __init__(self): self.logger = logging.getLogger(__name__) @@ -362,14 +362,15 @@ class ModifyMilter(Milter.Base): del data if milter_action is not None: - if milter_action["action"] == "reject": - self.setreply("554", "5.7.0", milter_action["reason"]) + action, reason = milter_action + if action == "REJECT": + self.setreply("554", "5.7.0", reason) return Milter.REJECT - if milter_action["action"] == "accept": + if action == "ACCEPT": return Milter.ACCEPT - if milter_action["action"] == "discard": + if action == "DISCARD": return Milter.DISCARD except Exception as e: diff --git a/pymodmilter/action.py b/pymodmilter/action.py index 569acdf..69866c7 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -29,11 +29,15 @@ class ActionConfig(BaseConfig): "add_disclaimer": "_add_disclaimer", "rewrite_links": "_rewrite_links", "store": "_store", - "notify": "_notify"} + "notify": "_notify", + "quarantine": "_quarantine"} - def __init__(self, cfg, debug): + def __init__(self, cfg, local_addrs, debug): super().__init__(cfg, debug) + self.local_addrs = local_addrs + self.debug = debug + self.pretend = False if "pretend" in cfg: assert isinstance(cfg["pretend"], bool), \ @@ -47,7 +51,7 @@ class ActionConfig(BaseConfig): assert cfg["type"] in ActionConfig.TYPES, \ f"{self.name}: type: invalid action type" - getattr(self, cfg["type"])(cfg) + getattr(self, ActionConfig.TYPES[cfg["type"]])(cfg) if "conditions" in cfg: assert isinstance(cfg["conditions"], dict), \ @@ -55,7 +59,8 @@ class ActionConfig(BaseConfig): cfg["conditions"]["name"] = f"{self.name}: condition" if "loglevel" not in cfg["conditions"]: cfg["conditions"]["loglevel"] = self.loglevel - self.conditions = ConditionsConfig(cfg["conditions"], debug) + self.conditions = ConditionsConfig( + cfg["conditions"], local_addrs, debug) else: self.conditions = None @@ -64,32 +69,28 @@ class ActionConfig(BaseConfig): f"type={cfg['type']}, " f"args={self.args}") - def add_header(self, cfg): + def _add_header(self, cfg): self.action = modify.AddHeader - self.headersonly = True self.add_string_arg(cfg, ["field", "value"]) - def mod_header(self, cfg): + def _mod_header(self, cfg): self.action = modify.ModHeader - self.headersonly = True args = ["field", "value"] if "search" in cfg: args.append("search") self.add_string_arg(cfg, args) - def del_header(self, cfg): + def _del_header(self, cfg): self.action = modify.DelHeader - self.headersonly = True args = ["field"] if "value" in cfg: args.append("value") self.add_string_arg(cfg, args) - def add_disclaimer(self, cfg): + def _add_disclaimer(self, cfg): self.action = modify.AddDisclaimer - self.headersonly = False if "error_policy" not in cfg: cfg["error_policy"] = "wrap" @@ -105,14 +106,11 @@ class ActionConfig(BaseConfig): f"{self.name}: error_policy: invalid value, " \ f"should be 'wrap', 'ignore' or 'reject'" - def rewrite_links(self, cfg): + def _rewrite_links(self, cfg): self.action = modify.RewriteLinks - self.headersonly = False self.add_string_arg(cfg, "repl") - def store(self, cfg): - self.headersonly = False - + def _store(self, cfg): assert "storage_type" in cfg, \ f"{self.name}: mandatory parameter 'storage_type' not found" assert isinstance(cfg["storage_type"], str), \ @@ -126,9 +124,6 @@ class ActionConfig(BaseConfig): self.action = storage.FileMailStorage self.add_string_arg(cfg, "directory") - if "skip_metadata" in cfg: - self.add_bool_arg(cfg, "skip_metadata") - if "metavar" in cfg: self.add_string_arg(cfg, "metavar") @@ -136,16 +131,15 @@ class ActionConfig(BaseConfig): raise RuntimeError( f"{self.name}: storage_type: invalid storage type") - def notify(self, cfg): - self.headersonly = False + def _notify(self, cfg): self.action = notify.EMailNotification args = ["smtp_host", "envelope_from", "from_header", "subject", "template"] if "repl_img" in cfg: args.append("repl_img") - self.add_string_arg(cfg, args) + self.add_int_arg(cfg, "smtp_port") if "embed_imgs" in cfg: @@ -155,25 +149,63 @@ class ActionConfig(BaseConfig): f"should be list of strings" self.args["embed_imgs"] = cfg["embed_imgs"] + def _quarantine(self, cfg): + self.action = storage.Quarantine + assert "storage" in cfg, \ + f"{self.name}: mandatory parameter 'storage' not found" + assert isinstance(cfg["storage"], dict), \ + f"{self.name}: storage: invalid value, " \ + f"should be dict" + cfg["storage"]["type"] = "store" + cfg["storage"]["name"] = f"{self.name}: storage" + + args = ["storage"] + if "notification" in cfg: + assert isinstance(cfg["notification"], dict), \ + f"{self.name}: notification: invalid value, " \ + f"should be dict" + cfg["notification"]["type"] = "notify" + cfg["notification"]["name"] = f"{self.name}: notification" + args.append("notification") + + for arg in args: + if "loglevel" not in cfg[arg]: + cfg[arg]["loglevel"] = self.loglevel + if "pretend" not in cfg[arg]: + cfg[arg]["pretend"] = self.pretend + + self.args[arg] = ActionConfig( + cfg[arg], self.local_addrs, self.debug) + + if "milter_action" in cfg: + self.add_string_arg(cfg, "milter_action") + self.args["milter_action"] = self.args["milter_action"].upper() + assert self.args["milter_action"] in ["REJECT", "DISCARD", + "ACCEPT"], \ + f"{self.name}: milter_action: invalid value, " \ + f"should be 'ACCEPT', 'REJECT' or 'DISCARD'" + if self.args["milter_action"] == "REJECT": + if "reject_reason" in cfg: + self.add_string_arg(cfg, "reject_reason") + class Action: """Action to implement a pre-configured action to perform on e-mails.""" - def __init__(self, cfg, local_addrs): + def __init__(self, cfg): self.logger = cfg.logger if cfg.conditions is None: self.conditions = None else: - self.conditions = Conditions(cfg.conditions, local_addrs) + self.conditions = Conditions(cfg.conditions) self.pretend = cfg.pretend self.name = cfg.name self.action = cfg.action(**cfg.args) - self._headersonly = cfg.headersonly def headersonly(self): """Return the needs of this action.""" - return self._headersonly + return self.action._headersonly def execute(self, milter): """Execute configured action.""" diff --git a/pymodmilter/base.py b/pymodmilter/base.py index 004826b..aa5cea8 100644 --- a/pymodmilter/base.py +++ b/pymodmilter/base.py @@ -61,6 +61,7 @@ class BaseConfig: self.loglevel = logging.INFO self.logger.setLevel(self.loglevel) + self.debug = debug # the keys/values in args are used as parameters # to initialize action classes diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index 7df41b5..1a4c462 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -23,9 +23,11 @@ from pymodmilter import BaseConfig, CustomLogger class ConditionsConfig(BaseConfig): - def __init__(self, cfg, debug): + def __init__(self, cfg, local_addrs, debug): super().__init__(cfg, debug) + self.local_addrs = local_addrs + if "local" in cfg: self.add_bool_arg(cfg, "local") @@ -58,10 +60,10 @@ class ConditionsConfig(BaseConfig): class Conditions: """Conditions to implement conditions for rules and actions.""" - def __init__(self, cfg, local_addrs): + def __init__(self, cfg): self.logger = cfg.logger self.name = cfg.name - self.local_addrs = local_addrs + self.local_addrs = cfg.local_addrs for arg in ("local", "hosts", "envfrom", "envto", "header", "metavar", "var"): diff --git a/pymodmilter/modify.py b/pymodmilter/modify.py index 8e992e5..e1d947f 100644 --- a/pymodmilter/modify.py +++ b/pymodmilter/modify.py @@ -34,6 +34,8 @@ from pymodmilter import replace_illegal_chars class AddHeader: """Add a mail header field.""" + _headersonly = True + def __init__(self, field, value): self.field = field self.value = value @@ -54,6 +56,8 @@ class AddHeader: class ModHeader: """Change the value of a mail header field.""" + _headersonly = True + def __init__(self, field, value, search=None): self.value = value @@ -110,6 +114,8 @@ class ModHeader: class DelHeader: """Delete a mail header field.""" + _headersonly = True + def __init__(self, field, value=None): try: self.field = re.compile(field, re.IGNORECASE) @@ -212,6 +218,8 @@ def _wrap_message(milter): class AddDisclaimer: """Append or prepend a disclaimer to the mail body.""" + _headersonly = False + def __init__(self, text_template, html_template, action, error_policy): try: with open(text_template, "r") as f: @@ -312,6 +320,8 @@ class AddDisclaimer: class RewriteLinks: """Rewrite link targets in the mail html body.""" + _headersonly = False + def __init__(self, repl): self.repl = repl diff --git a/pymodmilter/notify.py b/pymodmilter/notify.py index 72fc685..0c436d5 100644 --- a/pymodmilter/notify.py +++ b/pymodmilter/notify.py @@ -12,6 +12,10 @@ # along with PyMod-Milter. If not, see . # +__all__ = [ + "BaseNotification", + "EMailNotification"] + import email import logging import re @@ -30,6 +34,7 @@ from pymodmilter import mailer class BaseNotification: "Notification base class" + _headersonly = True def __init__(self): self.logger = logging.getLogger(__name__) @@ -41,7 +46,7 @@ class BaseNotification: class EMailNotification(BaseNotification): "Notification class to send notifications via mail." - notification_type = "email" + _headersonly = False _bad_tags = [ "applet", "embed", diff --git a/pymodmilter/rule.py b/pymodmilter/rule.py index 3341ae2..5052d2e 100644 --- a/pymodmilter/rule.py +++ b/pymodmilter/rule.py @@ -22,7 +22,7 @@ from pymodmilter.conditions import ConditionsConfig, Conditions class RuleConfig(BaseConfig): - def __init__(self, cfg, debug=False): + def __init__(self, cfg, local_addrs, debug=False): super().__init__(cfg, debug) self.conditions = None @@ -49,7 +49,8 @@ class RuleConfig(BaseConfig): cfg["conditions"]["name"] = f"{self.name}: condition" if "loglevel" not in cfg["conditions"]: cfg["conditions"]["loglevel"] = self.loglevel - self.conditions = ConditionsConfig(cfg["conditions"], debug) + self.conditions = ConditionsConfig( + cfg["conditions"], local_addrs, debug) else: self.conditions = None @@ -67,7 +68,7 @@ class RuleConfig(BaseConfig): if "pretend" not in action_cfg: action_cfg["pretend"] = self.pretend self.actions.append( - ActionConfig(action_cfg, debug)) + ActionConfig(action_cfg, local_addrs, debug)) class Rule: @@ -75,17 +76,17 @@ class Rule: Rule to implement multiple actions on emails. """ - def __init__(self, cfg, local_addrs): + def __init__(self, cfg): self.logger = cfg.logger if cfg.conditions is None: self.conditions = None else: - self.conditions = Conditions(cfg.conditions, local_addrs) + self.conditions = Conditions(cfg.conditions) self.actions = [] for action_cfg in cfg.actions: - self.actions.append(Action(action_cfg, local_addrs)) + self.actions.append(Action(action_cfg)) self.pretend = cfg.pretend diff --git a/pymodmilter/storage.py b/pymodmilter/storage.py index 7a058f3..54886f4 100644 --- a/pymodmilter/storage.py +++ b/pymodmilter/storage.py @@ -12,6 +12,11 @@ # along with PyMod-Milter. If not, see . # +__all__ = [ + "BaseMailStorage", + "FileMailStorage", + "Quarantine"] + import json import logging import os @@ -24,7 +29,12 @@ from time import gmtime class BaseMailStorage: "Mail storage base class" - def __init__(self): + _headersonly = True + + def __init__(self, original=False, metadata=False, metavar=None): + self.original = original + self.metadata = metadata + self.metavar = metavar return def add(self, data, qid, mailfrom="", recipients=[]): @@ -54,9 +64,11 @@ class BaseMailStorage: class FileMailStorage(BaseMailStorage): "Storage class to store mails on filesystem." - def __init__(self, directory, original=False, skip_metadata=False, + _headersonly = False + + def __init__(self, directory, original=False, metadata=False, metavar=None): - super().__init__() + super().__init__(original, metadata, metavar) # check if directory exists and is writable if not os.path.isdir(directory) or \ not os.access(directory, os.W_OK): @@ -64,9 +76,6 @@ class FileMailStorage(BaseMailStorage): f"directory '{directory}' does not exist or is " f"not writable") self.directory = directory - self.original = original - self.skip_metadata = skip_metadata - self.metavar = metavar self._metadata_suffix = ".metadata" def get_storageid(self, qid): @@ -96,7 +105,7 @@ class FileMailStorage(BaseMailStorage): metafile, datafile = self._get_file_paths(storage_id) try: - if not self.skip_metadata: + if self.metadata: os.remove(metafile) os.remove(datafile) @@ -113,22 +122,22 @@ class FileMailStorage(BaseMailStorage): # save mail self._save_datafile(datafile, data) - if self.skip_metadata: - metafile = None - else: - # save metadata - metadata = { - "mailfrom": mailfrom, - "recipients": recipients, - "subject": subject, - "timestamp": timegm(gmtime()), - "queue_id": qid} + if not self.metadata: + return storage_id, None, datafile - try: - self._save_metafile(metafile, metadata) - except RuntimeError as e: - os.remove(datafile) - raise e + # save metadata + metadata = { + "mailfrom": mailfrom, + "recipients": recipients, + "subject": subject, + "timestamp": timegm(gmtime()), + "queue_id": qid} + + try: + self._save_metafile(metafile, metadata) + except RuntimeError as e: + os.remove(datafile) + raise e return storage_id, metafile, datafile @@ -157,14 +166,14 @@ class FileMailStorage(BaseMailStorage): if self.metavar: milter.msginfo["vars"][f"{self.metavar}_ID"] = storage_id milter.msginfo["vars"][f"{self.metavar}_DATAFILE"] = datafile - if not self.skip_metadata: + if self.metadata: milter.msginfo["vars"][f"{self.metavar}_METAFILE"] = metafile def get_metadata(self, storage_id): "Return metadata of email in storage." super(FileMailStorage, self).get_metadata(storage_id) - if self.skip_metadata: + if not self.metadata: return None metafile, _ = self._get_file_paths(storage_id) @@ -191,6 +200,9 @@ class FileMailStorage(BaseMailStorage): if isinstance(recipients, str): recipients = [recipients] + if not self.metadata: + return {} + emails = {} metafiles = glob(os.path.join( self.directory, f"*{self._metadata_suffix}")) @@ -225,7 +237,7 @@ class FileMailStorage(BaseMailStorage): "Delete email from storage." super(FileMailStorage, self).delete(storage_id, recipients) - if not recipients or self.skip_metadata: + if not recipients or not self.metadata: self._remove(storage_id) return @@ -257,4 +269,28 @@ class FileMailStorage(BaseMailStorage): data = open(datafile, "rb").read() except IOError as e: raise RuntimeError(f"unable to open email data file: {e}") - return (data, metadata) + return (metadata, data) + + +class Quarantine: + "Quarantine class." + _headersonly = False + + def __init__(self, storage, notification=None, milter_action=None, + reject_reason="Message rejected"): + self.storage = storage.action(**storage.args, metadata=True) + self.notification = notification + if self.notification is not None: + self.notification = notification.action(**notification.args) + self.milter_action = milter_action + self.reject_reason = reject_reason + + def execute(self, milter, pretend=False, + logger=logging.getLogger(__name__)): + self.storage.execute(milter, pretend, logger) + if self.notification is not None: + self.notification.execute(milter, pretend, logger) + milter.msginfo["rcpts"] = [] + + if self.milter_action is not None: + return (self.milter_action, self.reject_reason) From f4f26f08fd434873a7d1b1667894c1d37c87d61f Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 21 Sep 2021 00:35:12 +0200 Subject: [PATCH 129/145] fix logging --- pymodmilter/storage.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pymodmilter/storage.py b/pymodmilter/storage.py index 54886f4..3dd52f9 100644 --- a/pymodmilter/storage.py +++ b/pymodmilter/storage.py @@ -26,6 +26,8 @@ from datetime import datetime from glob import glob from time import gmtime +from pymodmilter.base import CustomLogger + class BaseMailStorage: "Mail storage base class" @@ -279,17 +281,26 @@ class Quarantine: def __init__(self, storage, notification=None, milter_action=None, reject_reason="Message rejected"): self.storage = storage.action(**storage.args, metadata=True) + self.storage_name = storage.name + self.storage_logger = storage.logger + self.notification = notification if self.notification is not None: self.notification = notification.action(**notification.args) + self.notification_name = notification.name + self.notification_logger = notification.logger self.milter_action = milter_action self.reject_reason = reject_reason def execute(self, milter, pretend=False, logger=logging.getLogger(__name__)): - self.storage.execute(milter, pretend, logger) + custom_logger = CustomLogger( + self.storage_logger, {"name": self.storage_name}) + self.storage.execute(milter, pretend, custom_logger) if self.notification is not None: - self.notification.execute(milter, pretend, logger) + custom_logger = CustomLogger( + self.notification_logger, {"name": self.notification_name}) + self.notification.execute(milter, pretend, custom_logger) milter.msginfo["rcpts"] = [] if self.milter_action is not None: From 6333a0913ddf01f298a725e691a184b27e9d2031 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 21 Sep 2021 00:55:54 +0200 Subject: [PATCH 130/145] add delrcpt --- pymodmilter/__init__.py | 9 +++++++++ pymodmilter/storage.py | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 8d38044..c0596ca 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -192,6 +192,15 @@ class ModifyMilter(Milter.Base): def replacebody(self): self._replacebody = True + def delrcpt(self, rcpts): + "Remove recipient. May be called from eom callback only." + if not isinstance(rcpts, list): + rcpts = [rcpts] + for rcpt in rcpts: + self.logger.debug(f"delrcpt: {rcpt}") + self.msginfo["rcpts"].remove(rcpt) + super().delrcpt(rcpt) + def connect(self, IPname, family, hostaddr): try: if hostaddr is None: diff --git a/pymodmilter/storage.py b/pymodmilter/storage.py index 3dd52f9..fc6d0f0 100644 --- a/pymodmilter/storage.py +++ b/pymodmilter/storage.py @@ -301,7 +301,8 @@ class Quarantine: custom_logger = CustomLogger( self.notification_logger, {"name": self.notification_name}) self.notification.execute(milter, pretend, custom_logger) - milter.msginfo["rcpts"] = [] + + milter.delrcpt(milter.msginfo["rcpts"].copy()) if self.milter_action is not None: return (self.milter_action, self.reject_reason) From f4bc545f9bc9b3158477adddf577b73c905a4a0f Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 21 Sep 2021 01:35:47 +0200 Subject: [PATCH 131/145] fix replacebody logic --- pymodmilter/__init__.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index c0596ca..591e2a7 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -190,7 +190,16 @@ class ModifyMilter(Milter.Base): self.addheader(field, value) def replacebody(self): - self._replacebody = True + self._body_changed = True + + def _replacebody(self): + if not self._body_changed: + return + data = self.msg.as_bytes() + body_pos = data.find(b"\r\n\r\n") + 4 + self.logger.debug("replace body") + super().replacebody(data[body_pos:]) + del data def delrcpt(self, rcpts): "Remove recipient. May be called from eom callback only." @@ -354,7 +363,7 @@ class ModifyMilter(Milter.Base): "rcpts": [*self.rcpts], "vars": {}} - self._replacebody = False + self._body_changed = False milter_action = None for rule in self.rules: milter_action = rule.execute(self) @@ -363,23 +372,16 @@ class ModifyMilter(Milter.Base): if milter_action is not None: break - if self._replacebody: - data = self.msg.as_bytes() - body_pos = data.find(b"\r\n\r\n") + 4 - self.logger.debug("replace body") - super().replacebody(data[body_pos:]) - del data - - if milter_action is not None: + if milter_action is None: + self._replacebody() + else: action, reason = milter_action - if action == "REJECT": + if action == "ACCEPT": + self._replacebody() + elif action == "REJECT": self.setreply("554", "5.7.0", reason) return Milter.REJECT - - if action == "ACCEPT": - return Milter.ACCEPT - - if action == "DISCARD": + elif action == "DISCARD": return Milter.DISCARD except Exception as e: From f4bb0d38ebb18a9e8efc544f1e3eae75b1475175 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 21 Sep 2021 05:20:47 +0200 Subject: [PATCH 132/145] add whitelist functionality to quarantine --- pymodmilter/__init__.py | 1 + pymodmilter/action.py | 8 ++ pymodmilter/conditions.py | 46 +++++++ pymodmilter/storage.py | 26 +++- pymodmilter/whitelist.py | 276 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 354 insertions(+), 3 deletions(-) create mode 100644 pymodmilter/whitelist.py diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 591e2a7..d1f032f 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -22,6 +22,7 @@ __all__ = [ "rule", "run", "storage", + "whitelist", "ModifyMilterConfig", "ModifyMilter"] diff --git a/pymodmilter/action.py b/pymodmilter/action.py index 69866c7..d5ef9ee 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -188,6 +188,14 @@ class ActionConfig(BaseConfig): if "reject_reason" in cfg: self.add_string_arg(cfg, "reject_reason") + if "whitelist" in cfg: + wl = {"whitelist": cfg["whitelist"]} + wl["name"] = f"{self.name}: whitelist" + if "loglevel" not in wl: + wl["loglevel"] = self.loglevel + self.args["whitelist"] = ConditionsConfig( + wl, self.local_addrs, self.debug) + class Action: """Action to implement a pre-configured action to perform on e-mails.""" diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index 1a4c462..f097ff6 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -20,6 +20,7 @@ import re from netaddr import IPAddress, IPNetwork, AddrFormatError from pymodmilter import BaseConfig, CustomLogger +from pymodmilter.whitelist import DatabaseWhitelist class ConditionsConfig(BaseConfig): @@ -52,6 +53,33 @@ class ConditionsConfig(BaseConfig): if "metavar" in cfg: self.add_string_arg(cfg, "metavar") + if "whitelist" in cfg: + assert isinstance(cfg["whitelist"], dict), \ + f"{self.name}: whitelist: invalid value, " \ + f"should be dict" + whitelist = cfg["whitelist"] + assert "type" in whitelist, \ + f"{self.name}: whitelist: mandatory parameter 'type' not found" + assert isinstance(whitelist["type"], str), \ + f"{self.name}: whitelist: type: invalid value, " \ + f"should be string" + self.args["whitelist"] = { + "type": whitelist["type"], + "name": f"{self.name}: whitelist"} + if whitelist["type"] == "db": + for arg in ["connection", "table"]: + assert arg in whitelist, \ + f"{self.name}: whitelist: mandatory parameter " \ + f"'{arg}' not found" + assert isinstance(whitelist[arg], str), \ + f"{self.name}: whitelist: {arg}: invalid value, " \ + f"should be string" + self.args["whitelist"][arg] = whitelist[arg] + + else: + raise RuntimeError( + f"{self.name}: whitelist: type: invalid type") + self.logger.debug(f"{self.name}: " f"loglevel={self.loglevel}, " f"args={self.args}") @@ -94,6 +122,13 @@ class Conditions: except re.error as e: raise RuntimeError(e) + if "whitelist" in cfg.args: + wl_cfg = cfg.args["whitelist"] + if wl_cfg["type"] == "db": + self.whitelist = DatabaseWhitelist(wl_cfg) + else: + raise RuntimeError("invalid storage type") + def match_host(self, host): logger = CustomLogger( self.logger, {"name": self.name}) @@ -134,6 +169,17 @@ class Conditions: return True + def get_wl_rcpts(self, mailfrom, rcpts): + if not self.whitelist: + return {} + + wl_rcpts = [] + for rcpt in rcpts: + if self.whitelist.check(mailfrom, rcpt): + wl_rcpts.append(rcpt) + + return wl_rcpts + def match(self, milter): logger = CustomLogger( self.logger, {"qid": milter.qid, "name": self.name}) diff --git a/pymodmilter/storage.py b/pymodmilter/storage.py index fc6d0f0..9666ca7 100644 --- a/pymodmilter/storage.py +++ b/pymodmilter/storage.py @@ -27,6 +27,7 @@ from glob import glob from time import gmtime from pymodmilter.base import CustomLogger +from pymodmilter.conditions import Conditions class BaseMailStorage: @@ -278,8 +279,8 @@ class Quarantine: "Quarantine class." _headersonly = False - def __init__(self, storage, notification=None, milter_action=None, - reject_reason="Message rejected"): + def __init__(self, storage, notification=None, whitelist=None, + milter_action=None, reject_reason="Message rejected"): self.storage = storage.action(**storage.args, metadata=True) self.storage_name = storage.name self.storage_logger = storage.logger @@ -289,20 +290,39 @@ class Quarantine: self.notification = notification.action(**notification.args) self.notification_name = notification.name self.notification_logger = notification.logger + self.whitelist = Conditions(whitelist) self.milter_action = milter_action self.reject_reason = reject_reason def execute(self, milter, pretend=False, logger=logging.getLogger(__name__)): + wl_rcpts = [] + if self.whitelist: + wl_rcpts = self.whitelist.get_wl_rcpts( + milter.msginfo["mailfrom"], milter.msginfo["rcpts"]) + logger.info(f"whitelisted recipients: {wl_rcpts}") + + rcpts = [ + rcpt for rcpt in milter.msginfo["rcpts"] if rcpt not in wl_rcpts] + + if not rcpts: + # all recipients whitelisted + return + + logger.info(f"add to quarantine for recipients: {rcpts}") + milter.msginfo["rcpts"] = rcpts + custom_logger = CustomLogger( self.storage_logger, {"name": self.storage_name}) self.storage.execute(milter, pretend, custom_logger) + if self.notification is not None: custom_logger = CustomLogger( self.notification_logger, {"name": self.notification_name}) self.notification.execute(milter, pretend, custom_logger) - milter.delrcpt(milter.msginfo["rcpts"].copy()) + milter.msginfo["rcpts"].extend(wl_rcpts) + milter.delrcpt(rcpts) if self.milter_action is not None: return (self.milter_action, self.reject_reason) diff --git a/pymodmilter/whitelist.py b/pymodmilter/whitelist.py new file mode 100644 index 0000000..9c0fcd9 --- /dev/null +++ b/pymodmilter/whitelist.py @@ -0,0 +1,276 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +__all__ = [ + "WhitelistBase", + "DatabaseWhitelist"] + +import logging +import peewee +import re + +from datetime import datetime +from playhouse.db_url import connect + + +class WhitelistBase: + "Whitelist base class" + def __init__(self, cfg): + self.name = cfg["name"] + self.logger = logging.getLogger(__name__) + self.valid_entry_regex = re.compile( + r"^[a-zA-Z0-9_.=+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$") + + def check(self, mailfrom, recipient): + "Check if mailfrom/recipient combination is whitelisted." + return + + def find(self, mailfrom=None, recipients=None, older_than=None): + "Find whitelist entries." + return + + def add(self, mailfrom, recipient, comment, permanent): + "Add entry to whitelist." + # check if mailfrom and recipient are valid + if not self.valid_entry_regex.match(mailfrom): + raise RuntimeError("invalid from address") + if not self.valid_entry_regex.match(recipient): + raise RuntimeError("invalid recipient") + return + + def delete(self, whitelist_id): + "Delete entry from whitelist." + return + + +class WhitelistModel(peewee.Model): + mailfrom = peewee.CharField() + recipient = peewee.CharField() + created = peewee.DateTimeField(default=datetime.now) + last_used = peewee.DateTimeField(default=datetime.now) + comment = peewee.TextField(default="") + permanent = peewee.BooleanField(default=False) + + +class Meta: + indexes = ( + # trailing comma is mandatory if only one index should be created + (('mailfrom', 'recipient'), True), + ) + + +class DatabaseWhitelist(WhitelistBase): + "Whitelist class to store whitelist in a database" + whitelist_type = "db" + _db_connections = {} + _db_tables = {} + + def __init__(self, cfg): + super().__init__(cfg) + + tablename = cfg["table"] + connection_string = cfg["connection"] + + if connection_string in DatabaseWhitelist._db_connections.keys(): + db = DatabaseWhitelist._db_connections[connection_string] + else: + try: + # connect to database + conn = re.sub( + r"(.*?://.*?):.*?(@.*)", + r"\1:\2", + connection_string) + self.logger.debug( + f"connecting to database '{conn}'") + db = connect(connection_string) + except Exception as e: + raise RuntimeError( + f"unable to connect to database: {e}") + + DatabaseWhitelist._db_connections[connection_string] = db + + # generate model meta class + self.meta = Meta + self.meta.database = db + self.meta.table_name = tablename + self.model = type(f"WhitelistModel_{self.name}", (WhitelistModel,), { + "Meta": self.meta + }) + + if connection_string not in DatabaseWhitelist._db_tables.keys(): + DatabaseWhitelist._db_tables[connection_string] = [] + + if tablename not in DatabaseWhitelist._db_tables[connection_string]: + DatabaseWhitelist._db_tables[connection_string].append(tablename) + try: + db.create_tables([self.model]) + except Exception as e: + raise RuntimeError( + f"unable to initialize table '{tablename}': {e}") + + def _entry_to_dict(self, entry): + result = {} + result[entry.id] = { + "id": entry.id, + "mailfrom": entry.mailfrom, + "recipient": entry.recipient, + "created": entry.created, + "last_used": entry.last_used, + "comment": entry.comment, + "permanent": entry.permanent + } + return result + + def get_weight(self, entry): + value = 0 + for address in [entry.mailfrom, entry.recipient]: + if address == "": + value += 2 + elif address[0] == "@": + value += 1 + return value + + def check(self, mailfrom, recipient): + # check if mailfrom/recipient combination is whitelisted + super().check(mailfrom, recipient) + + # generate list of possible mailfroms + self.logger.debug( + f"query database for whitelist entries from <{mailfrom}> " + f"to <{recipient}>") + mailfroms = [""] + if "@" in mailfrom and not mailfrom.startswith("@"): + domain = mailfrom.split("@")[1] + mailfroms.append(f"@{domain}") + mailfroms.append(mailfrom) + + # generate list of possible recipients + recipients = [""] + if "@" in recipient and not recipient.startswith("@"): + domain = recipient.split("@")[1] + recipients.append(f"@{domain}") + recipients.append(recipient) + + # query the database + try: + entries = list( + self.model.select().where( + self.model.mailfrom.in_(mailfroms), + self.model.recipient.in_(recipients))) + except Exception as e: + raise RuntimeError(f"unable to query database: {e}") + + if not entries: + # no whitelist entry found + return {} + + if len(entries) > 1: + entries.sort(key=lambda x: self.get_weight(x), reverse=True) + + # use entry with the highest weight + entry = entries[0] + entry.last_used = datetime.now() + entry.save() + result = {} + for entry in entries: + result.update(self._entry_to_dict(entry)) + + return result + + def find(self, mailfrom=None, recipients=None, older_than=None): + "Find whitelist entries." + super().find(mailfrom, recipients, older_than) + + if isinstance(mailfrom, str): + mailfrom = [mailfrom] + if isinstance(recipients, str): + recipients = [recipients] + + entries = {} + try: + for entry in list(self.model.select()): + if older_than is not None: + delta = (datetime.now() - entry.last_used).total_seconds() + if delta < (older_than * 86400): + continue + + if mailfrom is not None: + if entry.mailfrom not in mailfrom: + continue + + if recipients is not None: + if entry.recipient not in recipients: + continue + + entries.update(self._entry_to_dict(entry)) + except Exception as e: + raise RuntimeError(f"unable to query database: {e}") + + return entries + + def add(self, mailfrom, recipient, comment, permanent): + "Add entry to whitelist." + super().add( + mailfrom, + recipient, + comment, + permanent) + + try: + self.model.create( + mailfrom=mailfrom, + recipient=recipient, + comment=comment, + permanent=permanent) + except Exception as e: + raise RuntimeError(f"unable to add entry to database: {e}") + + def delete(self, whitelist_id): + "Delete entry from whitelist." + super().delete(whitelist_id) + + try: + query = self.model.delete().where(self.model.id == whitelist_id) + deleted = query.execute() + except Exception as e: + raise RuntimeError( + f"unable to delete entry from database: {e}") + + if deleted == 0: + raise RuntimeError("invalid whitelist id") + + +class WhitelistCache: + def __init__(self): + self.cache = {} + + def load(self, whitelist, mailfrom, recipients): + for recipient in recipients: + self.check(whitelist, mailfrom, recipient) + + def check(self, whitelist, mailfrom, recipient): + if whitelist not in self.cache.keys(): + self.cache[whitelist] = {} + if recipient not in self.cache[whitelist].keys(): + self.cache[whitelist][recipient] = None + if self.cache[whitelist][recipient] is None: + self.cache[whitelist][recipient] = whitelist.check( + mailfrom, recipient) + return self.cache[whitelist][recipient] + + def get_recipients(self, whitelist, mailfrom, recipients): + self.load(whitelist, mailfrom, recipients) + return list(filter( + lambda x: self.cache[whitelist][x], + self.cache[whitelist].keys())) From 677b6ccb453385199ce6c6a18194adc260a2de86 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 21 Sep 2021 10:09:37 +0200 Subject: [PATCH 133/145] fix milter_action on quarantine action --- pymodmilter/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodmilter/storage.py b/pymodmilter/storage.py index 9666ca7..de4a2a9 100644 --- a/pymodmilter/storage.py +++ b/pymodmilter/storage.py @@ -324,5 +324,5 @@ class Quarantine: milter.msginfo["rcpts"].extend(wl_rcpts) milter.delrcpt(rcpts) - if self.milter_action is not None: + if self.milter_action is not None and not milter.msginfo["rcpts"]: return (self.milter_action, self.reject_reason) From 1e53393238952feba44bae33aca4d4993f9936d3 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 21 Sep 2021 10:39:23 +0200 Subject: [PATCH 134/145] remove WhitelistCache --- pymodmilter/whitelist.py | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/pymodmilter/whitelist.py b/pymodmilter/whitelist.py index 9c0fcd9..5595857 100644 --- a/pymodmilter/whitelist.py +++ b/pymodmilter/whitelist.py @@ -13,8 +13,8 @@ # __all__ = [ - "WhitelistBase", - "DatabaseWhitelist"] + "DatabaseWhitelist", + "WhitelistBase"] import logging import peewee @@ -249,28 +249,3 @@ class DatabaseWhitelist(WhitelistBase): if deleted == 0: raise RuntimeError("invalid whitelist id") - - -class WhitelistCache: - def __init__(self): - self.cache = {} - - def load(self, whitelist, mailfrom, recipients): - for recipient in recipients: - self.check(whitelist, mailfrom, recipient) - - def check(self, whitelist, mailfrom, recipient): - if whitelist not in self.cache.keys(): - self.cache[whitelist] = {} - if recipient not in self.cache[whitelist].keys(): - self.cache[whitelist][recipient] = None - if self.cache[whitelist][recipient] is None: - self.cache[whitelist][recipient] = whitelist.check( - mailfrom, recipient) - return self.cache[whitelist][recipient] - - def get_recipients(self, whitelist, mailfrom, recipients): - self.load(whitelist, mailfrom, recipients) - return list(filter( - lambda x: self.cache[whitelist][x], - self.cache[whitelist].keys())) From 54e73273fb20b353d512fd845b6d2c8ade0efbfc Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 21 Sep 2021 11:10:57 +0200 Subject: [PATCH 135/145] handle BATV (Bounce Address Tag Validation) in whitelist --- pymodmilter/whitelist.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pymodmilter/whitelist.py b/pymodmilter/whitelist.py index 5595857..c744eb5 100644 --- a/pymodmilter/whitelist.py +++ b/pymodmilter/whitelist.py @@ -31,6 +31,8 @@ class WhitelistBase: self.logger = logging.getLogger(__name__) self.valid_entry_regex = re.compile( r"^[a-zA-Z0-9_.=+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$") + self.batv_regex = re.compile( + r"^prvs=[0-9]{4}[0-9A-Fa-f]{6}=") def check(self, mailfrom, recipient): "Check if mailfrom/recipient combination is whitelisted." @@ -144,6 +146,8 @@ class DatabaseWhitelist(WhitelistBase): def check(self, mailfrom, recipient): # check if mailfrom/recipient combination is whitelisted super().check(mailfrom, recipient) + mailfrom = self.batv_regex.sub("", mailfrom, count=1) + recipient = self.batv_regex.sub("", recipient, count=1) # generate list of possible mailfroms self.logger.debug( @@ -227,6 +231,9 @@ class DatabaseWhitelist(WhitelistBase): comment, permanent) + mailfrom = self.batv_regex.sub("", mailfrom, count=1) + recipient = self.batv_regex.sub("", recipient, count=1) + try: self.model.create( mailfrom=mailfrom, From 2a7b08fc06f3647d458815ed1346454075e6830a Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 21 Sep 2021 11:18:40 +0200 Subject: [PATCH 136/145] fix setup.py --- setup.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 2214bdc..5fae1b5 100644 --- a/setup.py +++ b/setup.py @@ -4,13 +4,12 @@ def read_file(fname): with open(fname, 'r') as f: return f.read() - setup(name = "pymodmilter", author = "Thomas Oettli", author_email = "spacefreak@noop.ch", description = "A pymilter based sendmail/postfix pre-queue filter.", license = "GPL 3", - keywords = "header milter", + keywords = "header quarantine milter", url = "https://github.com/spacefreak86/pymodmilter", packages = ["pymodmilter"], long_description = read_file("README.md"), @@ -46,6 +45,6 @@ setup(name = "pymodmilter", ] ) ], - install_requires = ["pymilter", "netaddr", "beautifulsoup4[lxml]"], - python_requires = ">=3.6" + install_requires = ["pymilter", "netaddr", "beautifulsoup4[lxml]", "peewee"], + python_requires = ">=3.8" ) From c7168c6671f5bcd22fbb7fb32d44a249f2209b9d Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Sep 2021 01:00:02 +0200 Subject: [PATCH 137/145] improve BATV handling --- pymodmilter/whitelist.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pymodmilter/whitelist.py b/pymodmilter/whitelist.py index c744eb5..4a5ac8a 100644 --- a/pymodmilter/whitelist.py +++ b/pymodmilter/whitelist.py @@ -32,7 +32,11 @@ class WhitelistBase: self.valid_entry_regex = re.compile( r"^[a-zA-Z0-9_.=+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$") self.batv_regex = re.compile( - r"^prvs=[0-9]{4}[0-9A-Fa-f]{6}=") + r"^prvs=[0-9]{4}[0-9A-Fa-f]{6}=(?P.+?)@") + + def remove_batv(self, addr): + return self.batv_regex.sub("\g", addr, count=1) + def check(self, mailfrom, recipient): "Check if mailfrom/recipient combination is whitelisted." @@ -146,8 +150,8 @@ class DatabaseWhitelist(WhitelistBase): def check(self, mailfrom, recipient): # check if mailfrom/recipient combination is whitelisted super().check(mailfrom, recipient) - mailfrom = self.batv_regex.sub("", mailfrom, count=1) - recipient = self.batv_regex.sub("", recipient, count=1) + mailfrom = self.remove_batv(mailfrom) + recipient = self.remove_batv(recipient) # generate list of possible mailfroms self.logger.debug( @@ -231,8 +235,8 @@ class DatabaseWhitelist(WhitelistBase): comment, permanent) - mailfrom = self.batv_regex.sub("", mailfrom, count=1) - recipient = self.batv_regex.sub("", recipient, count=1) + mailfrom = self.remove_batv(mailfrom) + recipient = self.remove_batv(recipient) try: self.model.create( From 4d9baa79f7aaa131edcf68993ac5eaafdcebe61a Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Sep 2021 01:00:28 +0200 Subject: [PATCH 138/145] add JSON config schema --- pymodmilter/base.py | 462 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 461 insertions(+), 1 deletion(-) diff --git a/pymodmilter/base.py b/pymodmilter/base.py index aa5cea8..eb1363d 100644 --- a/pymodmilter/base.py +++ b/pymodmilter/base.py @@ -16,7 +16,8 @@ __all__ = [ "CustomLogger", "BaseConfig", "MilterMessage", - "replace_illegal_chars"] + "replace_illegal_chars", + "config_schema"] import logging @@ -134,3 +135,462 @@ class MilterMessage(MIMEPart): def replace_illegal_chars(string): """Remove illegal characters from header values.""" return "".join(string.replace("\x00", "").splitlines()) + + +JSON_CONFIG_SCHEMA = """ +{ + "$id": "https://example.com/schemas/config", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Root", + "type": "object", + "required": ["rules"], + "additionalProperties": false, + "properties": { + "global": { + "title": "Section global", + "type": "object", + "additionalProperties": false, + "properties": { + "local_addrs": { "$ref": "/schemas/config/hosts" }, + "loglevel": { "$ref": "/schemas/config/loglevel" }, + "socket": { + "title": "Socket", + "type": "string", + "pattern": "^((unix|local):.+|inet6?:[0-9]{1,5}(@.+)?)$" + } + } + }, + "rules": { + "title": "Section rules", + "type": "array", + "items": { + "title": "Rules", + "type": "object", + "required": [ + "actions" + ], + "additionalProperties": false, + "properties": { + "name": { "$ref": "/schemas/config/name" }, + "pretend": { "$ref": "/schemas/config/pretend" }, + "conditions": { "$ref": "/schemas/config/conditions" }, + "loglevel": { "$ref": "/schemas/config/loglevel" }, + "actions": { + "title": "Section actions", + "type": "array", + "items": { + "title": "Actions", + "type": "object", + "required": ["type"], + "properties": { + "type": { "$ref": "/schemas/config/actiontype" } + }, + "if": { "properties": { "type": { "const": "add_header" } } }, + "then": { "$ref": "/schemas/config/add_header" }, + "else": { + "if": { "properties": { "type": { "const": "mod_header" } } }, + "then": { "$ref": "/schemas/config/mod_header" }, + "else": { + "if": { "properties": { "type": { "const": "del_header" } } }, + "then": { "$ref": "/schemas/config/del_header" }, + "else": { + "if": { "properties": { "type": { "const": "add_disclaimer" } } }, + "then": { "$ref": "/schemas/config/add_disclaimer" }, + "else": { + "if": { "properties": { "type": { "const": "rewrite_links" } } }, + "then": { "$ref": "/schemas/config/rewrite_links" }, + "else": { + "if": { "properties": { "type": { "const": "store" } } }, + "then": { "$ref": "/schemas/config/store" }, + "else": { + "if": { "properties": { "type": { "const": "notify" } } }, + "then": { "$ref": "/schemas/config/notify" }, + "else": { + "if": { "properties": { "type": { "const": "quarantine" } } }, + "then": { "$ref": "/schemas/config/quarantine" }, + "else": { + "additionalProperties": false + } + } + } + } + } + } + } + } + } + } + } + } + } + }, + "$defs": { + "name": { + "$id": "/schemas/config/name", + "title": "Name", + "type": "string", + "pattern": "^.+$" + }, + "hosts": { + "$id": "/schemas/config/hosts", + "title": "Hosts/networks", + "type": "array", + "items": { + "title": "Hosts/Networks", + "type": "string", + "pattern": "^.+$" + } + }, + "pretend": { + "$id": "/schemas/config/pretend", + "title": "Pretend", + "type": "boolean" + }, + "loglevel": { + "$id": "/schemas/config/loglevel", + "title": "Loglevel", + "type": "string", + "pattern": "^(critical|error|warning|info|debug)$" + }, + "actiontype": { + "$id": "/schemas/config/actiontype", + "title": "Action type", + "enum": [ + "add_header", "mod_header", "del_header", "add_disclaimer", + "rewrite_links", "store", "notify", "quarantine"] + }, + "storagetype": { + "$id": "/schemas/config/storagetype", + "title": "Storage type", + "enum": ["file"] + }, + "whitelisttype": { + "$id": "/schemas/config/whitelisttype", + "title": "Whitelist type", + "enum": ["db"] + }, + "field": { + "$id": "/schemas/config/field", + "title": "Field", + "type": "string", + "pattern": "^.+$" + }, + "value": { + "$id": "/schemas/config/value", + "title": "Value", + "type": "string", + "pattern": "^.+$" + }, + "original": { + "$id": "/schemas/config/original", + "title": "Original", + "type": "boolean" + }, + "metavar": { + "$id": "/schemas/config/metavar", + "title": "Meta variable", + "type": "string", + "pattern": "^.+$" + }, + "conditions": { + "$id": "/schemas/config/conditions", + "title": "Conditions", + "type": "object", + "properties": { + "metavar": { "$ref": "/schemas/config/metavar" }, + "local": { + "title": "Local", + "type": "boolean" + }, + "hosts": { + "title": "Hosts/Networks", + "type": "array", + "items":{ + "title": "Host/Network", + "type": "string", + "pattern": "^.+$" + } + }, + "envfrom": { + "title": "Envelope from", + "type": "string", + "pattern": "^.+$" + }, + "envto": { + "title": "Envelope to", + "type": "string", + "pattern": "^.+$" + }, + "header": { + "title": "Header", + "type": "string", + "pattern": "^.+$" + }, + "var": { + "title": "Variable", + "type": "string", + "pattern": "^.+$" + } + }, + "additionalProperties": false, + "anyOf": [ + {"required": ["local"]}, + {"required": ["hosts"]}, + {"required": ["envfrom"]}, + {"required": ["envto"]}, + {"required": ["header"]}, + {"required": ["var"]} + ] + }, + "add_header": { + "$id": "/schemas/config/add_header", + "title": "Add header", + "type": "object", + "required": ["type", "field", "value"], + "properties": { + "type": { "$ref": "/schemas/config/actiontype" }, + "name": { "$ref": "/schemas/config/name" }, + "pretend": { "$ref": "/schemas/config/pretend" }, + "conditions": { "$ref": "/schemas/config/conditions" }, + "loglevel": { "$ref": "/schemas/config/loglevel" }, + "field": { "$ref": "/schemas/config/field" }, + "value": { "$ref": "/schemas/config/value" } + }, + "additionalProperties": false + }, + "mod_header": { + "$id": "/schemas/config/mod_header", + "title": "Modify header", + "type": "object", + "required": ["type", "field", "value"], + "properties": { + "type": { "$ref": "/schemas/config/actiontype" }, + "name": { "$ref": "/schemas/config/name" }, + "pretend": { "$ref": "/schemas/config/pretend" }, + "conditions": { "$ref": "/schemas/config/conditions" }, + "loglevel": { "$ref": "/schemas/config/loglevel" }, + "field": { "$ref": "/schemas/config/field" }, + "value": { "$ref": "/schemas/config/value" }, + "search": { + "title": "Search", + "type": "string", + "pattern": "^.+$" + } + }, + "additionalProperties": false + }, + "del_header": { + "$id": "/schemas/config/del_header", + "title": "Delete header", + "type": "object", + "required": ["type", "field"], + "properties": { + "type": { "$ref": "/schemas/config/actiontype" }, + "name": { "$ref": "/schemas/config/name" }, + "pretend": { "$ref": "/schemas/config/pretend" }, + "conditions": { "$ref": "/schemas/config/conditions" }, + "loglevel": { "$ref": "/schemas/config/loglevel" }, + "field": { "$ref": "/schemas/config/field" }, + "value": { "$ref": "/schemas/config/value" } + }, + "additionalProperties": false + }, + "add_disclaimer": { + "$id": "/schemas/config/add_disclaimer", + "title": "Add disclaimer", + "type": "object", + "required": ["type", "action", "html_template", "text_template"], + "properties": { + "type": { "$ref": "/schemas/config/actiontype" }, + "name": { "$ref": "/schemas/config/name" }, + "pretend": { "$ref": "/schemas/config/pretend" }, + "conditions": { "$ref": "/schemas/config/conditions" }, + "loglevel": { "$ref": "/schemas/config/loglevel" }, + "action": { + "title": "Action", + "enum": ["append", "prepend"] + }, + "html_template": { + "title": "HTML template", + "type": "string", + "pattern": "^.+$" + }, + "text_template": { + "title": "Text template", + "type": "string", + "pattern": "^.+$" + }, + "error_policy": { + "title": "Action", + "enum": [ + "wrap", "ignore", "reject", + "WRAP", "IGNORE", "REJECT"] + } + }, + "additionalProperties": false + }, + "rewrite_links": { + "$id": "/schemas/config/rewrite_links", + "title": "Rewrite links", + "type": "object", + "required": ["type", "repl"], + "properties": { + "type": { "$ref": "/schemas/config/actiontype" }, + "name": { "$ref": "/schemas/config/name" }, + "pretend": { "$ref": "/schemas/config/pretend" }, + "conditions": { "$ref": "/schemas/config/conditions" }, + "loglevel": { "$ref": "/schemas/config/loglevel" }, + "repl": { + "title": "Replacement", + "type": "string", + "pattern": "^.+$" + } + }, + "additionalProperties": false + }, + "store": { + "$id": "/schemas/config/store", + "title": "Store", + "type": "object", + "required": ["storage_type"], + "properties": { + "storage_type": { "$ref": "/schemas/config/storagetype" } + }, + "if": { "properties": { "storage_type": { "const": "file" } } }, + "then": { + "properties": { + "type": { "$ref": "/schemas/config/actiontype" }, + "storage_type": { "$ref": "/schemas/config/storagetype" }, + "name": { "$ref": "/schemas/config/name" }, + "pretend": { "$ref": "/schemas/config/pretend" }, + "conditions": { "$ref": "/schemas/config/conditions" }, + "loglevel": { "$ref": "/schemas/config/loglevel" }, + "original": { "$ref": "/schemas/config/original" }, + "metavar": { "$ref": "/schemas/config/metavar" }, + "directory": { + "title": "Directory", + "type": "string", + "pattern": "^.+$" + } + }, + "additionalProperties": false + }, + "else": { + "additionalProperties": false + } + }, + "notify": { + "$id": "/schemas/config/notify", + "title": "Notify", + "type": "object", + "required": ["smtp_host", "smtp_port", "envelope_from", "from_header", "subject", "template"], + "properties": { + "type": { "$ref": "/schemas/config/actiontype" }, + "name": { "$ref": "/schemas/config/name" }, + "pretend": { "$ref": "/schemas/config/pretend" }, + "conditions": { "$ref": "/schemas/config/conditions" }, + "loglevel": { "$ref": "/schemas/config/loglevel" }, + "smtp_host": { + "title": "SMTP host", + "type": "string", + "pattern": "^.+$" + }, + "smtp_port": { + "title": "SMTP port", + "type": "number" + }, + "envelope_from": { + "title": "Envelope from", + "type": "string", + "pattern": "^.+$" + }, + "from_header": { + "title": "From-Header", + "type": "string", + "pattern": "^.+$" + }, + "subject": { + "title": "Subject", + "type": "string", + "pattern": "^.+$" + }, + "template": { + "title": "Template", + "type": "string", + "pattern": "^.+$" + }, + "repl_img": { + "title": "Replacement image", + "type": "string", + "pattern": "^.+$" + }, + "embed_imgs": { + "title": "Embedded images", + "type": "array", + "items": { + "title": "Embedded image", + "type": "string", + "pattern": "^.+$" + } + } + }, + "additionalProperties": false + }, + "quarantine": { + "$id": "/schemas/config/quarantine", + "title": "Quarantine", + "type": "object", + "required": ["storage"], + "properties": { + "type": { "$ref": "/schemas/config/actiontype" }, + "name": { "$ref": "/schemas/config/name" }, + "pretend": { "$ref": "/schemas/config/pretend" }, + "conditions": { "$ref": "/schemas/config/conditions" }, + "loglevel": { "$ref": "/schemas/config/loglevel" }, + "storage": { "$ref": "/schemas/config/store" }, + "notification": { "$ref": "/schemas/config/notify" }, + "milter_action": { + "title": "Milter action", + "enum": [ + "reject", "discard", "accept", + "REJECT", "DISCARD", "ACCEPT"] + }, + "reject_reason": { + "title": "Reject reason", + "type": "string", + "pattern": "^.+$" + }, + "whitelist": { + "title": "Whitelist", + "type": "object", + "required": ["type"], + "properties": { + "type": { "$ref": "/schemas/config/whitelisttype" } + }, + "if": { "properties": { "type": { "const": "db" } } }, + "then": { + "required": ["connection", "table"], + "properties": { + "type": { "$ref": "/schemas/config/whitelisttype" }, + "connection": { + "title": "DB connection", + "type": "string", + "pattern": "^.+$" + }, + "table": { + "title": "DB table", + "type": "string", + "pattern": "^.+$" + } + }, + "additionalProperties": false + }, + "else": { + "additionalProperties": false + } + } + } + } + } +} +""" From 08b6ae6377c734e041bcf0a9ec836e83b6734cca Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 30 Sep 2021 01:07:21 +0200 Subject: [PATCH 139/145] use jsonschema to validate config and massive refactor --- pymodmilter/__init__.py | 120 ++------- pymodmilter/action.py | 230 +++-------------- pymodmilter/base.py | 527 +------------------------------------- pymodmilter/conditions.py | 156 ++++------- pymodmilter/config.py | 330 ++++++++++++++++++++++++ pymodmilter/modify.py | 97 ++++--- pymodmilter/notify.py | 71 +++-- pymodmilter/rule.py | 100 +++----- pymodmilter/run.py | 47 +++- pymodmilter/storage.py | 150 ++++++++--- pymodmilter/whitelist.py | 36 ++- setup.py | 2 +- 12 files changed, 762 insertions(+), 1104 deletions(-) create mode 100644 pymodmilter/config.py diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index d1f032f..aac9594 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -32,8 +32,6 @@ from pymodmilter import _runtime_patches import Milter import logging -import re -import json from Milter.utils import parse_addr from collections import defaultdict @@ -45,95 +43,9 @@ from email.policy import SMTPUTF8 from io import BytesIO from netaddr import IPNetwork, AddrFormatError -from pymodmilter.base import CustomLogger, BaseConfig, MilterMessage +from pymodmilter.base import CustomLogger, MilterMessage from pymodmilter.base import replace_illegal_chars -from pymodmilter.rule import RuleConfig, Rule - - -class ModifyMilterConfig(BaseConfig): - def __init__(self, cfgfile, debug=False): - try: - with open(cfgfile, "r") as fh: - # remove lines with leading # (comments), they - # are not allowed in json - cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read()) - except IOError as e: - raise RuntimeError(f"unable to open/read config file: {e}") - - try: - cfg = json.loads(cfg) - except json.JSONDecodeError as e: - cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] - msg = "\n".join(cfg_text) - raise RuntimeError(f"{e}\n{msg}") - - if "global" in cfg: - assert isinstance(cfg["global"], dict), \ - "global: invalid type, should be dict" - - cfg["global"]["name"] = "global" - super().__init__(cfg["global"], debug) - - self.logger.debug("initialize config") - - if "pretend" in cfg["global"]: - pretend = cfg["global"]["pretend"] - assert isinstance(pretend, bool), \ - "global: pretend: invalid value, should be bool" - self.pretend = pretend - else: - self.pretend = False - - if "socket" in cfg["global"]: - socket = cfg["global"]["socket"] - assert isinstance(socket, str), \ - "global: socket: invalid value, should be string" - self.socket = socket - else: - self.socket = None - - if "local_addrs" in cfg["global"]: - local_addrs = cfg["global"]["local_addrs"] - assert isinstance(local_addrs, list) and all( - [isinstance(addr, str) for addr in local_addrs]), \ - "global: local_addrs: invalid value, " \ - "should be list of strings" - else: - local_addrs = [ - "fe80::/64", - "::1/128", - "127.0.0.0/8", - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16"] - - self.local_addrs = [] - try: - for addr in local_addrs: - self.local_addrs.append(IPNetwork(addr)) - except AddrFormatError as e: - raise ValueError(f"{self.name}: local_addrs: {e}") - - self.logger.debug(f"socket={self.socket}, " - f"local_addrs={self.local_addrs}, " - f"pretend={self.pretend}, " - f"loglevel={self.loglevel}") - - assert "rules" in cfg, \ - "mandatory parameter 'rules' not found" - assert isinstance(cfg["rules"], list), \ - "rules: invalid value, should be list" - - self.logger.debug("initialize rules config") - self.rules = [] - for idx, rule_cfg in enumerate(cfg["rules"]): - if "name" not in rule_cfg: - rule_cfg["name"] = "Rule #{idx}" - if "loglevel" not in rule_cfg: - rule_cfg["loglevel"] = self.loglevel - if "pretend" not in rule_cfg: - rule_cfg["pretend"] = self.pretend - self.rules.append(RuleConfig(rule_cfg, self.local_addrs, debug)) +from pymodmilter.rule import Rule class ModifyMilter(Milter.Base): @@ -145,11 +57,29 @@ class ModifyMilter(Milter.Base): if issubclass(v, AddressHeader)] @staticmethod - def set_config(cfg): - ModifyMilter._loglevel = cfg.loglevel - for rule_cfg in cfg.rules: - ModifyMilter._rules.append( - Rule(rule_cfg)) + def set_config(cfg, debug): + ModifyMilter._loglevel = cfg.get_loglevel(debug) + + try: + local_addrs = [] + for addr in cfg["local_addrs"]: + local_addrs.append(IPNetwork(addr)) + except AddrFormatError as e: + raise RuntimeError(e) + + logger = logging.getLogger(__name__) + logger.setLevel(ModifyMilter._loglevel) + for idx, rule_cfg in enumerate(cfg["rules"]): + if "name" not in rule_cfg: + rule_cfg["name"] = f"rule#{idx}" + if "loglevel" not in rule_cfg: + rule_cfg["loglevel"] = cfg["loglevel"] + if "pretend" not in rule_cfg: + rule_cfg["pretend"] = cfg["pretend"] + rule = Rule(rule_cfg, local_addrs, debug) + + logger.debug(rule) + ModifyMilter._rules.append(rule) def __init__(self): self.logger = logging.getLogger(__name__) diff --git a/pymodmilter/action.py b/pymodmilter/action.py index d5ef9ee..a2bc31a 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -12,204 +12,51 @@ # along with PyMod-Milter. If not, see . # -__all__ = [ - "ActionConfig", - "Action"] +__all__ = ["Action"] + +import logging -from pymodmilter import BaseConfig from pymodmilter import modify, notify, storage from pymodmilter.base import CustomLogger -from pymodmilter.conditions import ConditionsConfig, Conditions - - -class ActionConfig(BaseConfig): - TYPES = {"add_header": "_add_header", - "mod_header": "_mod_header", - "del_header": "_del_header", - "add_disclaimer": "_add_disclaimer", - "rewrite_links": "_rewrite_links", - "store": "_store", - "notify": "_notify", - "quarantine": "_quarantine"} - - def __init__(self, cfg, local_addrs, debug): - super().__init__(cfg, debug) - - self.local_addrs = local_addrs - self.debug = debug - - self.pretend = False - if "pretend" in cfg: - assert isinstance(cfg["pretend"], bool), \ - f"{self.name}: pretend: invalid value, should be bool" - self.pretend = cfg["pretend"] - - assert "type" in cfg, \ - f"{self.name}: mandatory parameter 'type' not found" - assert isinstance(cfg["type"], str), \ - f"{self.name}: type: invalid value, should be string" - assert cfg["type"] in ActionConfig.TYPES, \ - f"{self.name}: type: invalid action type" - - getattr(self, ActionConfig.TYPES[cfg["type"]])(cfg) - - if "conditions" in cfg: - assert isinstance(cfg["conditions"], dict), \ - f"{self.name}: conditions: invalid value, should be dict" - cfg["conditions"]["name"] = f"{self.name}: condition" - if "loglevel" not in cfg["conditions"]: - cfg["conditions"]["loglevel"] = self.loglevel - self.conditions = ConditionsConfig( - cfg["conditions"], local_addrs, debug) - else: - self.conditions = None - - self.logger.debug(f"{self.name}: pretend={self.pretend}, " - f"loglevel={self.loglevel}, " - f"type={cfg['type']}, " - f"args={self.args}") - - def _add_header(self, cfg): - self.action = modify.AddHeader - self.add_string_arg(cfg, ["field", "value"]) - - def _mod_header(self, cfg): - self.action = modify.ModHeader - args = ["field", "value"] - if "search" in cfg: - args.append("search") - - self.add_string_arg(cfg, args) - - def _del_header(self, cfg): - self.action = modify.DelHeader - args = ["field"] - if "value" in cfg: - args.append("value") - - self.add_string_arg(cfg, args) - - def _add_disclaimer(self, cfg): - self.action = modify.AddDisclaimer - if "error_policy" not in cfg: - cfg["error_policy"] = "wrap" - - self.add_string_arg( - cfg, ["action", "html_template", "text_template", - "error_policy"]) - assert self.args["action"] in ["append", "prepend"], \ - f"{self.name}: action: invalid value, " \ - f"should be 'append' or 'prepend'" - assert self.args["error_policy"] in ("wrap", - "ignore", - "reject"), \ - f"{self.name}: error_policy: invalid value, " \ - f"should be 'wrap', 'ignore' or 'reject'" - - def _rewrite_links(self, cfg): - self.action = modify.RewriteLinks - self.add_string_arg(cfg, "repl") - - def _store(self, cfg): - assert "storage_type" in cfg, \ - f"{self.name}: mandatory parameter 'storage_type' not found" - assert isinstance(cfg["storage_type"], str), \ - f"{self.name}: storage_type: invalid value, " \ - f"should be string" - - if "original" in cfg: - self.add_bool_arg(cfg, "original") - - if cfg["storage_type"] == "file": - self.action = storage.FileMailStorage - self.add_string_arg(cfg, "directory") - - if "metavar" in cfg: - self.add_string_arg(cfg, "metavar") - - else: - raise RuntimeError( - f"{self.name}: storage_type: invalid storage type") - - def _notify(self, cfg): - self.action = notify.EMailNotification - - args = ["smtp_host", "envelope_from", "from_header", "subject", - "template"] - if "repl_img" in cfg: - args.append("repl_img") - self.add_string_arg(cfg, args) - - self.add_int_arg(cfg, "smtp_port") - - if "embed_imgs" in cfg: - assert isinstance(cfg["embed_imgs"], list) and all( - [isinstance(img, str) for img in cfg["embed_imgs"]]), \ - f"{self.name}: embed_imgs: invalid value, " \ - f"should be list of strings" - self.args["embed_imgs"] = cfg["embed_imgs"] - - def _quarantine(self, cfg): - self.action = storage.Quarantine - assert "storage" in cfg, \ - f"{self.name}: mandatory parameter 'storage' not found" - assert isinstance(cfg["storage"], dict), \ - f"{self.name}: storage: invalid value, " \ - f"should be dict" - cfg["storage"]["type"] = "store" - cfg["storage"]["name"] = f"{self.name}: storage" - - args = ["storage"] - if "notification" in cfg: - assert isinstance(cfg["notification"], dict), \ - f"{self.name}: notification: invalid value, " \ - f"should be dict" - cfg["notification"]["type"] = "notify" - cfg["notification"]["name"] = f"{self.name}: notification" - args.append("notification") - - for arg in args: - if "loglevel" not in cfg[arg]: - cfg[arg]["loglevel"] = self.loglevel - if "pretend" not in cfg[arg]: - cfg[arg]["pretend"] = self.pretend - - self.args[arg] = ActionConfig( - cfg[arg], self.local_addrs, self.debug) - - if "milter_action" in cfg: - self.add_string_arg(cfg, "milter_action") - self.args["milter_action"] = self.args["milter_action"].upper() - assert self.args["milter_action"] in ["REJECT", "DISCARD", - "ACCEPT"], \ - f"{self.name}: milter_action: invalid value, " \ - f"should be 'ACCEPT', 'REJECT' or 'DISCARD'" - if self.args["milter_action"] == "REJECT": - if "reject_reason" in cfg: - self.add_string_arg(cfg, "reject_reason") - - if "whitelist" in cfg: - wl = {"whitelist": cfg["whitelist"]} - wl["name"] = f"{self.name}: whitelist" - if "loglevel" not in wl: - wl["loglevel"] = self.loglevel - self.args["whitelist"] = ConditionsConfig( - wl, self.local_addrs, self.debug) +from pymodmilter.conditions import Conditions class Action: """Action to implement a pre-configured action to perform on e-mails.""" + ACTION_TYPES = { + "add_header": modify.Modify, + "mod_header": modify.Modify, + "del_header": modify.Modify, + "add_disclaimer": modify.Modify, + "rewrite_links": modify.Modify, + "store": storage.Store, + "notify": notify.Notify, + "quarantine": storage.Quarantine} - def __init__(self, cfg): - self.logger = cfg.logger - if cfg.conditions is None: - self.conditions = None - else: - self.conditions = Conditions(cfg.conditions) + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) - self.pretend = cfg.pretend - self.name = cfg.name - self.action = cfg.action(**cfg.args) + self.conditions = cfg["conditions"] if "conditions" in cfg else None + if self.conditions is not None: + self.conditions["name"] = f"{cfg['name']}: conditions" + self.conditions["loglevel"] = cfg["loglevel"] + self.conditions = Conditions(self.conditions, local_addrs, debug) + + action_type = cfg["type"] + self.action = self.ACTION_TYPES[action_type]( + cfg, local_addrs, debug) + + def __str__(self): + cfg = [] + for key in ["name", "loglevel", "pretend", "type"]: + value = self.cfg[key] + cfg.append(f"{key}={value}") + if self.conditions is not None: + cfg.append(f"conditions={self.conditions}") + cfg.append(f"action={self.action}") + return "Action(" + ", ".join(cfg) + ")" def headersonly(self): """Return the needs of this action.""" @@ -218,8 +65,7 @@ class Action: def execute(self, milter): """Execute configured action.""" logger = CustomLogger( - self.logger, {"qid": milter.qid, "name": self.name}) + self.logger, {"qid": milter.qid, "name": self.cfg["name"]}) if self.conditions is None or \ self.conditions.match(milter): - return self.action.execute( - milter=milter, pretend=self.pretend, logger=logger) + return self.action.execute(milter) diff --git a/pymodmilter/base.py b/pymodmilter/base.py index eb1363d..a88671a 100644 --- a/pymodmilter/base.py +++ b/pymodmilter/base.py @@ -14,10 +14,8 @@ __all__ = [ "CustomLogger", - "BaseConfig", "MilterMessage", - "replace_illegal_chars", - "config_schema"] + "replace_illegal_chars"] import logging @@ -38,70 +36,6 @@ class CustomLogger(logging.LoggerAdapter): return msg, kwargs -class BaseConfig: - def __init__(self, cfg={}, debug=False): - if "name" in cfg: - assert isinstance(cfg["name"], str), \ - "name: invalid value, should be string" - self.name = cfg["name"] - else: - self.name = __name__ - - self.logger = logging.getLogger(self.name) - if debug: - self.loglevel = logging.DEBUG - elif "loglevel" in cfg: - if isinstance(cfg["loglevel"], int): - self.loglevel = cfg["loglevel"] - else: - level = getattr(logging, cfg["loglevel"].upper(), None) - assert isinstance(level, int), \ - f"{self.name}: loglevel: invalid value" - self.loglevel = level - else: - self.loglevel = logging.INFO - - self.logger.setLevel(self.loglevel) - self.debug = debug - - # the keys/values in args are used as parameters - # to initialize action classes - self.args = {} - - def add_string_arg(self, cfg, args): - if isinstance(args, str): - args = [args] - - for arg in args: - assert arg in cfg, \ - f"{self.name}: mandatory parameter '{arg}' not found" - assert isinstance(cfg[arg], str), \ - f"{self.name}: {arg}: invalid value, should be string" - self.args[arg] = cfg[arg] - - def add_bool_arg(self, cfg, args): - if isinstance(args, str): - args = [args] - - for arg in args: - assert arg in cfg, \ - f"{self.name}: mandatory parameter '{arg}' not found" - assert isinstance(cfg[arg], bool), \ - f"{self.name}: {arg}: invalid value, should be bool" - self.args[arg] = cfg[arg] - - def add_int_arg(self, cfg, args): - if isinstance(args, str): - args = [args] - - for arg in args: - assert arg in cfg, \ - f"{self.name}: mandatory parameter '{arg}' not found" - assert isinstance(cfg[arg], int), \ - f"{self.name}: {arg}: invalid value, should be integer" - self.args[arg] = cfg[arg] - - class MilterMessage(MIMEPart): def replace_header(self, _name, _value, idx=None): _name = _name.lower() @@ -135,462 +69,3 @@ class MilterMessage(MIMEPart): def replace_illegal_chars(string): """Remove illegal characters from header values.""" return "".join(string.replace("\x00", "").splitlines()) - - -JSON_CONFIG_SCHEMA = """ -{ - "$id": "https://example.com/schemas/config", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Root", - "type": "object", - "required": ["rules"], - "additionalProperties": false, - "properties": { - "global": { - "title": "Section global", - "type": "object", - "additionalProperties": false, - "properties": { - "local_addrs": { "$ref": "/schemas/config/hosts" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "socket": { - "title": "Socket", - "type": "string", - "pattern": "^((unix|local):.+|inet6?:[0-9]{1,5}(@.+)?)$" - } - } - }, - "rules": { - "title": "Section rules", - "type": "array", - "items": { - "title": "Rules", - "type": "object", - "required": [ - "actions" - ], - "additionalProperties": false, - "properties": { - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "actions": { - "title": "Section actions", - "type": "array", - "items": { - "title": "Actions", - "type": "object", - "required": ["type"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" } - }, - "if": { "properties": { "type": { "const": "add_header" } } }, - "then": { "$ref": "/schemas/config/add_header" }, - "else": { - "if": { "properties": { "type": { "const": "mod_header" } } }, - "then": { "$ref": "/schemas/config/mod_header" }, - "else": { - "if": { "properties": { "type": { "const": "del_header" } } }, - "then": { "$ref": "/schemas/config/del_header" }, - "else": { - "if": { "properties": { "type": { "const": "add_disclaimer" } } }, - "then": { "$ref": "/schemas/config/add_disclaimer" }, - "else": { - "if": { "properties": { "type": { "const": "rewrite_links" } } }, - "then": { "$ref": "/schemas/config/rewrite_links" }, - "else": { - "if": { "properties": { "type": { "const": "store" } } }, - "then": { "$ref": "/schemas/config/store" }, - "else": { - "if": { "properties": { "type": { "const": "notify" } } }, - "then": { "$ref": "/schemas/config/notify" }, - "else": { - "if": { "properties": { "type": { "const": "quarantine" } } }, - "then": { "$ref": "/schemas/config/quarantine" }, - "else": { - "additionalProperties": false - } - } - } - } - } - } - } - } - } - } - } - } - } - }, - "$defs": { - "name": { - "$id": "/schemas/config/name", - "title": "Name", - "type": "string", - "pattern": "^.+$" - }, - "hosts": { - "$id": "/schemas/config/hosts", - "title": "Hosts/networks", - "type": "array", - "items": { - "title": "Hosts/Networks", - "type": "string", - "pattern": "^.+$" - } - }, - "pretend": { - "$id": "/schemas/config/pretend", - "title": "Pretend", - "type": "boolean" - }, - "loglevel": { - "$id": "/schemas/config/loglevel", - "title": "Loglevel", - "type": "string", - "pattern": "^(critical|error|warning|info|debug)$" - }, - "actiontype": { - "$id": "/schemas/config/actiontype", - "title": "Action type", - "enum": [ - "add_header", "mod_header", "del_header", "add_disclaimer", - "rewrite_links", "store", "notify", "quarantine"] - }, - "storagetype": { - "$id": "/schemas/config/storagetype", - "title": "Storage type", - "enum": ["file"] - }, - "whitelisttype": { - "$id": "/schemas/config/whitelisttype", - "title": "Whitelist type", - "enum": ["db"] - }, - "field": { - "$id": "/schemas/config/field", - "title": "Field", - "type": "string", - "pattern": "^.+$" - }, - "value": { - "$id": "/schemas/config/value", - "title": "Value", - "type": "string", - "pattern": "^.+$" - }, - "original": { - "$id": "/schemas/config/original", - "title": "Original", - "type": "boolean" - }, - "metavar": { - "$id": "/schemas/config/metavar", - "title": "Meta variable", - "type": "string", - "pattern": "^.+$" - }, - "conditions": { - "$id": "/schemas/config/conditions", - "title": "Conditions", - "type": "object", - "properties": { - "metavar": { "$ref": "/schemas/config/metavar" }, - "local": { - "title": "Local", - "type": "boolean" - }, - "hosts": { - "title": "Hosts/Networks", - "type": "array", - "items":{ - "title": "Host/Network", - "type": "string", - "pattern": "^.+$" - } - }, - "envfrom": { - "title": "Envelope from", - "type": "string", - "pattern": "^.+$" - }, - "envto": { - "title": "Envelope to", - "type": "string", - "pattern": "^.+$" - }, - "header": { - "title": "Header", - "type": "string", - "pattern": "^.+$" - }, - "var": { - "title": "Variable", - "type": "string", - "pattern": "^.+$" - } - }, - "additionalProperties": false, - "anyOf": [ - {"required": ["local"]}, - {"required": ["hosts"]}, - {"required": ["envfrom"]}, - {"required": ["envto"]}, - {"required": ["header"]}, - {"required": ["var"]} - ] - }, - "add_header": { - "$id": "/schemas/config/add_header", - "title": "Add header", - "type": "object", - "required": ["type", "field", "value"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "field": { "$ref": "/schemas/config/field" }, - "value": { "$ref": "/schemas/config/value" } - }, - "additionalProperties": false - }, - "mod_header": { - "$id": "/schemas/config/mod_header", - "title": "Modify header", - "type": "object", - "required": ["type", "field", "value"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "field": { "$ref": "/schemas/config/field" }, - "value": { "$ref": "/schemas/config/value" }, - "search": { - "title": "Search", - "type": "string", - "pattern": "^.+$" - } - }, - "additionalProperties": false - }, - "del_header": { - "$id": "/schemas/config/del_header", - "title": "Delete header", - "type": "object", - "required": ["type", "field"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "field": { "$ref": "/schemas/config/field" }, - "value": { "$ref": "/schemas/config/value" } - }, - "additionalProperties": false - }, - "add_disclaimer": { - "$id": "/schemas/config/add_disclaimer", - "title": "Add disclaimer", - "type": "object", - "required": ["type", "action", "html_template", "text_template"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "action": { - "title": "Action", - "enum": ["append", "prepend"] - }, - "html_template": { - "title": "HTML template", - "type": "string", - "pattern": "^.+$" - }, - "text_template": { - "title": "Text template", - "type": "string", - "pattern": "^.+$" - }, - "error_policy": { - "title": "Action", - "enum": [ - "wrap", "ignore", "reject", - "WRAP", "IGNORE", "REJECT"] - } - }, - "additionalProperties": false - }, - "rewrite_links": { - "$id": "/schemas/config/rewrite_links", - "title": "Rewrite links", - "type": "object", - "required": ["type", "repl"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "repl": { - "title": "Replacement", - "type": "string", - "pattern": "^.+$" - } - }, - "additionalProperties": false - }, - "store": { - "$id": "/schemas/config/store", - "title": "Store", - "type": "object", - "required": ["storage_type"], - "properties": { - "storage_type": { "$ref": "/schemas/config/storagetype" } - }, - "if": { "properties": { "storage_type": { "const": "file" } } }, - "then": { - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "storage_type": { "$ref": "/schemas/config/storagetype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "original": { "$ref": "/schemas/config/original" }, - "metavar": { "$ref": "/schemas/config/metavar" }, - "directory": { - "title": "Directory", - "type": "string", - "pattern": "^.+$" - } - }, - "additionalProperties": false - }, - "else": { - "additionalProperties": false - } - }, - "notify": { - "$id": "/schemas/config/notify", - "title": "Notify", - "type": "object", - "required": ["smtp_host", "smtp_port", "envelope_from", "from_header", "subject", "template"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "smtp_host": { - "title": "SMTP host", - "type": "string", - "pattern": "^.+$" - }, - "smtp_port": { - "title": "SMTP port", - "type": "number" - }, - "envelope_from": { - "title": "Envelope from", - "type": "string", - "pattern": "^.+$" - }, - "from_header": { - "title": "From-Header", - "type": "string", - "pattern": "^.+$" - }, - "subject": { - "title": "Subject", - "type": "string", - "pattern": "^.+$" - }, - "template": { - "title": "Template", - "type": "string", - "pattern": "^.+$" - }, - "repl_img": { - "title": "Replacement image", - "type": "string", - "pattern": "^.+$" - }, - "embed_imgs": { - "title": "Embedded images", - "type": "array", - "items": { - "title": "Embedded image", - "type": "string", - "pattern": "^.+$" - } - } - }, - "additionalProperties": false - }, - "quarantine": { - "$id": "/schemas/config/quarantine", - "title": "Quarantine", - "type": "object", - "required": ["storage"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "storage": { "$ref": "/schemas/config/store" }, - "notification": { "$ref": "/schemas/config/notify" }, - "milter_action": { - "title": "Milter action", - "enum": [ - "reject", "discard", "accept", - "REJECT", "DISCARD", "ACCEPT"] - }, - "reject_reason": { - "title": "Reject reason", - "type": "string", - "pattern": "^.+$" - }, - "whitelist": { - "title": "Whitelist", - "type": "object", - "required": ["type"], - "properties": { - "type": { "$ref": "/schemas/config/whitelisttype" } - }, - "if": { "properties": { "type": { "const": "db" } } }, - "then": { - "required": ["connection", "table"], - "properties": { - "type": { "$ref": "/schemas/config/whitelisttype" }, - "connection": { - "title": "DB connection", - "type": "string", - "pattern": "^.+$" - }, - "table": { - "title": "DB table", - "type": "string", - "pattern": "^.+$" - } - }, - "additionalProperties": false - }, - "else": { - "additionalProperties": false - } - } - } - } - } -} -""" diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index f097ff6..9761c2f 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -12,127 +12,77 @@ # along with PyMod-Milter. If not, see . # -__all__ = [ - "ConditionsConfig", - "Conditions"] +__all__ = ["Conditions"] +import logging import re from netaddr import IPAddress, IPNetwork, AddrFormatError -from pymodmilter import BaseConfig, CustomLogger +from pymodmilter import CustomLogger from pymodmilter.whitelist import DatabaseWhitelist -class ConditionsConfig(BaseConfig): - def __init__(self, cfg, local_addrs, debug): - super().__init__(cfg, debug) - - self.local_addrs = local_addrs - - if "local" in cfg: - self.add_bool_arg(cfg, "local") - - if "hosts" in cfg: - assert isinstance(cfg["hosts"], list) and all( - [isinstance(host, str) for host in cfg["hosts"]]), \ - f"{self.name}: hosts: invalid value, " \ - f"should be list of strings" - - self.args["hosts"] = cfg["hosts"] - - for arg in ("envfrom", "envto"): - if arg in cfg: - self.add_string_arg(cfg, arg) - - if "header" in cfg: - self.add_string_arg(cfg, "header") - - if "var" in cfg: - self.add_string_arg(cfg, "var") - - if "metavar" in cfg: - self.add_string_arg(cfg, "metavar") - - if "whitelist" in cfg: - assert isinstance(cfg["whitelist"], dict), \ - f"{self.name}: whitelist: invalid value, " \ - f"should be dict" - whitelist = cfg["whitelist"] - assert "type" in whitelist, \ - f"{self.name}: whitelist: mandatory parameter 'type' not found" - assert isinstance(whitelist["type"], str), \ - f"{self.name}: whitelist: type: invalid value, " \ - f"should be string" - self.args["whitelist"] = { - "type": whitelist["type"], - "name": f"{self.name}: whitelist"} - if whitelist["type"] == "db": - for arg in ["connection", "table"]: - assert arg in whitelist, \ - f"{self.name}: whitelist: mandatory parameter " \ - f"'{arg}' not found" - assert isinstance(whitelist[arg], str), \ - f"{self.name}: whitelist: {arg}: invalid value, " \ - f"should be string" - self.args["whitelist"][arg] = whitelist[arg] - - else: - raise RuntimeError( - f"{self.name}: whitelist: type: invalid type") - - self.logger.debug(f"{self.name}: " - f"loglevel={self.loglevel}, " - f"args={self.args}") - - class Conditions: """Conditions to implement conditions for rules and actions.""" - def __init__(self, cfg): - self.logger = cfg.logger - self.name = cfg.name - self.local_addrs = cfg.local_addrs + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.local_addrs = local_addrs + + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) for arg in ("local", "hosts", "envfrom", "envto", "header", "metavar", "var"): - value = cfg.args[arg] if arg in cfg.args else None - setattr(self, arg, value) - if value is None: + if arg not in cfg: + setattr(self, arg, None) continue - elif arg == "hosts": + + if arg == "hosts": try: - hosts = [] - for host in self.hosts: - hosts.append(IPNetwork(host)) + self.hosts = [] + for host in cfg["hosts"]: + self.hosts.append(IPNetwork(host)) except AddrFormatError as e: raise RuntimeError(e) - - self.hosts = hosts elif arg in ("envfrom", "envto"): try: setattr(self, arg, re.compile( - getattr(self, arg), re.IGNORECASE)) + cfg[arg], re.IGNORECASE)) except re.error as e: raise RuntimeError(e) - elif arg == "header": try: self.header = re.compile( - self.header, re.IGNORECASE + re.DOTALL + re.MULTILINE) + cfg["header"], + re.IGNORECASE + re.DOTALL + re.MULTILINE) except re.error as e: raise RuntimeError(e) - - if "whitelist" in cfg.args: - wl_cfg = cfg.args["whitelist"] - if wl_cfg["type"] == "db": - self.whitelist = DatabaseWhitelist(wl_cfg) else: - raise RuntimeError("invalid storage type") + setattr(self, arg, cfg[arg]) + + self.whitelist = cfg["whitelist"] if "whitelist" in cfg else None + if self.whitelist is not None: + self.whitelist["name"] = f"{cfg['name']}: whitelist" + self.whitelist["loglevel"] = cfg["loglevel"] + if self.whitelist["type"] == "db": + self.whitelist = DatabaseWhitelist(self.whitelist, debug) + else: + raise RuntimeError("invalid whitelist type") + + def __str__(self): + cfg = [] + for arg in ("local", "hosts", "envfrom", "envto", "header", + "var", "metavar"): + if arg in self.cfg: + cfg.append(f"{arg}={self.cfg[arg]}") + if self.whitelist is not None: + cfg.append(f"whitelist={self.whitelist}") + return "Conditions(" + ", ".join(cfg) + ")" def match_host(self, host): logger = CustomLogger( - self.logger, {"name": self.name}) - + self.logger, {"name": self.cfg["name"]}) ip = IPAddress(host) if self.local is not None: @@ -145,11 +95,11 @@ class Conditions: if is_local != self.local: logger.debug( f"ignore host {host}, " - f"condition local does not match") + f"local does not match") return False logger.debug( - f"condition local matches for host {host}") + f"local matches for host {host}") if self.hosts is not None: found = False @@ -161,39 +111,39 @@ class Conditions: if not found: logger.debug( f"ignore host {host}, " - f"condition hosts does not match") + f"hosts does not match") return False logger.debug( - f"condition hosts matches for host {host}") + f"hosts matches for host {host}") return True - def get_wl_rcpts(self, mailfrom, rcpts): + def get_wl_rcpts(self, mailfrom, rcpts, logger): if not self.whitelist: return {} wl_rcpts = [] for rcpt in rcpts: - if self.whitelist.check(mailfrom, rcpt): + if self.whitelist.check(mailfrom, rcpt, logger): wl_rcpts.append(rcpt) return wl_rcpts def match(self, milter): logger = CustomLogger( - self.logger, {"qid": milter.qid, "name": self.name}) + self.logger, {"qid": milter.qid, "name": self.cfg["name"]}) if self.envfrom is not None: envfrom = milter.msginfo["mailfrom"] if not self.envfrom.match(envfrom): logger.debug( f"ignore envelope-from address {envfrom}, " - f"condition envfrom does not match") + f"envfrom does not match") return False logger.debug( - f"condition envfrom matches for " + f"envfrom matches for " f"envelope-from address {envfrom}") if self.envto is not None: @@ -205,11 +155,11 @@ class Conditions: if not self.envto.match(to): logger.debug( f"ignore envelope-to address {envto}, " - f"condition envto does not match") + f"envto does not match") return False logger.debug( - f"condition envto matches for " + f"envto matches for " f"envelope-to address {envto}") if self.header is not None: @@ -219,7 +169,7 @@ class Conditions: match = self.header.search(header) if match: logger.debug( - f"condition header matches for " + f"header matches for " f"header: {header}") if self.metavar is not None: named_subgroups = match.groupdict(default=None) @@ -233,7 +183,7 @@ class Conditions: if not match: logger.debug( "ignore message, " - "condition header does not match") + "header does not match") return False if self.var is not None: diff --git a/pymodmilter/config.py b/pymodmilter/config.py new file mode 100644 index 0000000..48020f5 --- /dev/null +++ b/pymodmilter/config.py @@ -0,0 +1,330 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +__all__ = [ + "BaseConfig", + "ConditionsConfig", + "AddHeaderConfig", + "ModHeaderConfig", + "DelHeaderConfig", + "AddDisclaimerConfig", + "RewriteLinksConfig", + "StoreConfig", + "NotifyConfig", + "WhitelistConfig", + "QuarantineConfig", + "ActionConfig", + "RuleConfig", + "MilterConfig"] + +import jsonschema +import logging + + +class BaseConfig: + JSON_SCHEMA = { + "type": "object", + "required": [], + "additionalProperties": True, + "properties": { + "loglevel": {"type": "string", "default": "info"}}} + + def __init__(self, config): + required = self.JSON_SCHEMA["required"] + properties = self.JSON_SCHEMA["properties"] + for p in properties.keys(): + if p in required: + continue + elif p not in config and "default" in properties[p]: + config[p] = properties[p]["default"] + try: + jsonschema.validate(config, self.JSON_SCHEMA) + except jsonschema.exceptions.ValidationError as e: + raise RuntimeError(e) + + self._config = config + + def __getitem__(self, key): + return self._config[key] + + def __setitem__(self, key, value): + self._config[key] = value + + def __delitem__(self, key): + del self._config[key] + + def __contains__(self, key): + return key in self._config + + def keys(self): + return self._config.keys() + + def items(self): + return self._config.items() + + def get_loglevel(self, debug): + if debug: + level = logging.DEBUG + else: + level = getattr(logging, self["loglevel"].upper(), None) + assert isinstance(level, int), \ + "loglevel: invalid value" + return level + + def get_config(self): + return self._config + +class WhitelistConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["type"], + "additionalProperties": True, + "properties": { + "type": {"enum": ["db"]}}, + "if": {"properties": {"type": {"const": "db"}}}, + "then": { + "required": ["connection", "table"], + "additionalProperties": False, + "properties": { + "type": {"type": "string"}, + "connection": {"type": "string"}, + "table": {"type": "string"}}}} + + +class ConditionsConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": [], + "additionalProperties": False, + "properties": { + "metavar": {"type": "string"}, + "local": {"type": "boolean"}, + "hosts": {"type": "array", + "items": {"type": "string"}}, + "envfrom": {"type": "string"}, + "envto": {"type": "string"}, + "header": {"type": "string"}, + "var": {"type": "string"}, + "whitelist": {"type": "object"}}} + + def __init__(self, config, rec=True): + super().__init__(config) + if rec: + if "whitelist" in self: + self["whitelist"] = WhitelistConfig(self["whitelist"]) + + +class AddHeaderConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["field", "value"], + "additionalProperties": False, + "properties": { + "field": {"type": "string"}, + "value": {"type": "string"}}} + + +class ModHeaderConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["field", "value"], + "additionalProperties": False, + "properties": { + "field": {"type": "string"}, + "value": {"type": "string"}, + "search": {"type": "string"}}} + + +class DelHeaderConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["field"], + "additionalProperties": False, + "properties": { + "field": {"type": "string"}, + "value": {"type": "string"}}} + + +class AddDisclaimerConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["action", "html_template", "text_template"], + "additionalProperties": False, + "properties": { + "action": {"type": "string"}, + "html_template": {"type": "string"}, + "text_template": {"type": "string"}, + "error_policy": {"type": "string"}}} + + +class RewriteLinksConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["repl"], + "additionalProperties": False, + "properties": { + "repl": {"type": "string"}}} + + +class StoreConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["type"], + "additionalProperties": True, + "properties": { + "type": {"enum": ["file"]}}, + "if": {"properties": {"type": {"const": "file"}}}, + "then": { + "required": ["directory"], + "additionalProperties": False, + "properties": { + "type": {"type": "string"}, + "directory": {"type": "string"}, + "metavar": {"type": "string"}, + "original": {"type": "boolean", "default": True}}}} + + +class NotifyConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["type"], + "additionalProperties": True, + "properties": { + "type": {"enum": ["email"]}}, + "if": {"properties": {"type": {"const": "email"}}}, + "then": { + "required": ["smtp_host", "smtp_port", "envelope_from", + "from_header", "subject", "template"], + "additionalProperties": False, + "properties": { + "type": {"type": "string"}, + "smtp_host": {"type": "string"}, + "smtp_port": {"type": "number"}, + "envelope_from": {"type": "string"}, + "from_header": {"type": "string"}, + "subject": {"type": "string"}, + "template": {"type": "string"}, + "repl_img": {"type": "string"}, + "embed_imgs": { + "type": "array", + "items": {"type": "string"}, + "default": True}}}} + + +class QuarantineConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["store"], + "additionalProperties": False, + "properties": { + "name": {"type": "string"}, + "notify": {"type": "object"}, + "milter_action": {"type": "string"}, + "reject_reason": {"type": "string"}, + "whitelist": {"type": "object"}, + "store": {"type": "object"}}} + + def __init__(self, config, rec=True): + super().__init__(config) + if rec: + self["store"] = StoreConfig(self["store"]) + if "notify" in self: + self["notify"] = NotifyConfig(self["notify"]) + if "whitelist" in self: + self["whitelist"] = ConditionsConfig( + {"whitelist": self["whitelist"]}, rec) + + +class ActionConfig(BaseConfig): + ACTION_TYPES = { + "add_header": AddHeaderConfig, + "mod_header": ModHeaderConfig, + "del_header": DelHeaderConfig, + "add_disclaimer": AddDisclaimerConfig, + "rewrite_links": RewriteLinksConfig, + "store": StoreConfig, + "notify": NotifyConfig, + "quarantine": QuarantineConfig} + + JSON_SCHEMA = { + "type": "object", + "required": ["type", "args"], + "additionalProperties": False, + "properties": { + "name": {"type": "string", "default": "action"}, + "loglevel": {"type": "string", "default": "info"}, + "pretend": {"type": "boolean", "default": False}, + "conditions": {"type": "object"}, + "type": {"enum": list(ACTION_TYPES.keys())}, + "args": {"type": "object"}}} + + def __init__(self, config, rec=True): + super().__init__(config) + if rec: + if "conditions" in self: + self["conditions"] = ConditionsConfig(self["conditions"]) + self["action"] = self.ACTION_TYPES[self["type"]](self["args"]) + + +class RuleConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["actions"], + "additionalProperties": False, + "properties": { + "name": {"type": "string", "default": "rule"}, + "loglevel": {"type": "string", "default": "info"}, + "pretend": {"type": "boolean", "default": False}, + "conditions": {"type": "object"}, + "actions": {"type": "array"}}} + + def __init__(self, config, rec=True): + super().__init__(config) + if rec: + if "conditions" in self: + self["conditions"] = ConditionsConfig(self["conditions"]) + + actions = [] + for idx, action in enumerate(self["actions"]): + actions.append(ActionConfig(action, rec)) + self["actions"] = actions + + +class MilterConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["rules"], + "additionalProperties": False, + "properties": { + "socket": {"type": "string"}, + "local_addrs": {"type": "array", + "items": {"type": "string"}, + "default": [ + "fe80::/64", + "::1/128", + "127.0.0.0/8", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16"]}, + "loglevel": {"type": "string", "default": "info"}, + "pretend": {"type": "boolean", "default": False}, + "rules": {"type": "array"}}} + + def __init__(self, config, rec=True): + super().__init__(config) + if rec: + rules = [] + for idx, rule in enumerate(self["rules"]): + rules.append(RuleConfig(rule, rec)) + self["rules"] = rules diff --git a/pymodmilter/modify.py b/pymodmilter/modify.py index e1d947f..8222a81 100644 --- a/pymodmilter/modify.py +++ b/pymodmilter/modify.py @@ -30,18 +30,19 @@ from email.message import MIMEPart from email.policy import SMTPUTF8 from pymodmilter import replace_illegal_chars +from pymodmilter.base import CustomLogger class AddHeader: """Add a mail header field.""" _headersonly = True - def __init__(self, field, value): + def __init__(self, field, value, pretend=False): self.field = field self.value = value + self.pretend = pretend - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + def execute(self, milter, logger): header = f"{self.field}: {self.value}" if logger.getEffectiveLevel() == logging.DEBUG: logger.debug(f"add_header: {header}") @@ -49,8 +50,7 @@ class AddHeader: logger.info(f"add_header: {header[0:70]}") milter.msg.add_header(self.field, self.value) - - if not pretend: + if not self.pretend: milter.addheader(self.field, self.value) @@ -58,21 +58,21 @@ class ModHeader: """Change the value of a mail header field.""" _headersonly = True - def __init__(self, field, value, search=None): - self.value = value - + def __init__(self, field, value, search=None, pretend=False): try: self.field = re.compile(field, re.IGNORECASE) - if search is not None: + self.search = search + if self.search is not None: self.search = re.compile( - search, re.MULTILINE + re.DOTALL + re.IGNORECASE) - else: - self.search = search + self.search, re.MULTILINE + re.DOTALL + re.IGNORECASE) + except re.error as e: raise RuntimeError(e) - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + self.value = value + self.pretend = pretend + + def execute(self, milter, logger): idx = defaultdict(int) for i, (field, value) in enumerate(milter.msg.items()): @@ -103,12 +103,13 @@ class ModHeader: if logger.getEffectiveLevel() == logging.DEBUG: logger.debug(f"mod_header: {header}: {new_header}") else: - logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}") + logger.info( + f"mod_header: {header[0:70]}: {new_header[0:70]}") milter.msg.replace_header( field, replace_illegal_chars(new_value), idx=idx[field_lower]) - if not pretend: + if not self.pretend: milter.chgheader(field, new_value, idx=idx[field_lower]) @@ -116,19 +117,19 @@ class DelHeader: """Delete a mail header field.""" _headersonly = True - def __init__(self, field, value=None): + def __init__(self, field, value=None, pretend=False): try: self.field = re.compile(field, re.IGNORECASE) - if value is not None: + self.value = value + if self.value is not None: self.value = re.compile( value, re.MULTILINE + re.DOTALL + re.IGNORECASE) - else: - self.value = value except re.error as e: raise RuntimeError(e) - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + self.pretend = pretend + + def execute(self, milter, logger): idx = defaultdict(int) for field, value in milter.msg.items(): @@ -148,7 +149,7 @@ class DelHeader: logger.info(f"del_header: {header[0:70]}") milter.msg.remove_header(field, idx=idx[field_lower]) - if not pretend: + if not self.pretend: milter.chgheader(field, "", idx=idx[field_lower]) idx[field_lower] -= 1 @@ -220,7 +221,10 @@ class AddDisclaimer: """Append or prepend a disclaimer to the mail body.""" _headersonly = False - def __init__(self, text_template, html_template, action, error_policy): + def __init__(self, text_template, html_template, action, error_policy, + pretend=False): + self.text_template_path = text_template + self.html_template_path = html_template try: with open(text_template, "r") as f: self.text_template = f.read() @@ -230,11 +234,11 @@ class AddDisclaimer: except IOError as e: raise RuntimeError(e) - body = html.find('body') self.html_template = body or html self.action = action self.error_policy = error_policy + self.pretend = pretend def patch_message_body(self, milter, logger): text_body, text_content = _get_body_content(milter.msg, "plain") @@ -277,8 +281,7 @@ class AddDisclaimer: html_body.set_param("charset", "UTF-8", header="Content-Type") del html_body["MIME-Version"] - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + def execute(self, milter, logger): old_headers = milter.msg.items() try: @@ -313,7 +316,7 @@ class AddDisclaimer: "unable to wrap message in a new message envelope, " "give up ...") - if not pretend: + if not self.pretend: milter.update_headers(old_headers) milter.replacebody() @@ -322,11 +325,11 @@ class RewriteLinks: """Rewrite link targets in the mail html body.""" _headersonly = False - def __init__(self, repl): + def __init__(self, repl, pretend=False): self.repl = repl + self.pretend = pretend - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + def execute(self, milter, logger): html_body, html_content = _get_body_content(milter.msg, "html") if html_content is not None: soup = BeautifulSoup(html_content, "html.parser") @@ -353,5 +356,35 @@ class RewriteLinks: html_body.set_param("charset", "UTF-8", header="Content-Type") del html_body["MIME-Version"] - if not pretend: + if not self.pretend: milter.replacebody() + + +class Modify: + MODIFICATION_TYPES = { + "add_header": AddHeader, + "mod_header": ModHeader, + "del_header": DelHeader, + "add_disclaimer": AddDisclaimer, + "rewrite_links": RewriteLinks} + + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) + cfg["args"]["pretend"] = cfg["pretend"] + self._modification = self.MODIFICATION_TYPES[cfg["type"]]( + **cfg["args"]) + self._headersonly = self._modification._headersonly + + def __str__(self): + cfg = [] + for key, value in self.cfg["args"].items(): + cfg.append(f"{key}={value}") + class_name = type(self._modification).__name__ + return f"{class_name}(" + ", ".join(cfg) + ")" + + def execute(self, milter): + logger = CustomLogger( + self.logger, {"name": self.cfg["name"], "qid": milter.qid}) + self._modification.execute(milter, logger) diff --git a/pymodmilter/notify.py b/pymodmilter/notify.py index 0c436d5..32d3270 100644 --- a/pymodmilter/notify.py +++ b/pymodmilter/notify.py @@ -29,6 +29,7 @@ from html import escape from os.path import basename from urllib.parse import quote +from pymodmilter.base import CustomLogger from pymodmilter import mailer @@ -36,11 +37,10 @@ class BaseNotification: "Notification base class" _headersonly = True - def __init__(self): - self.logger = logging.getLogger(__name__) - return + def __init__(self, pretend=False): + self.pretend = pretend - def execute(self, milter, pretend=False, logger=None): + def execute(self, milter, logger): return @@ -112,9 +112,8 @@ class EMailNotification(BaseNotification): def __init__(self, smtp_host, smtp_port, envelope_from, from_header, subject, template, embed_imgs=[], repl_img=None, - strip_imgs=False, parser_lib="lxml"): - super().__init__() - + strip_imgs=False, parser_lib="lxml", pretend=False): + super().__init__(pretend) self.smtp_host = smtp_host self.smtp_port = smtp_port self.mailfrom = envelope_from @@ -143,11 +142,8 @@ class EMailNotification(BaseNotification): self.parser_lib = parser_lib - def get_email_body_soup(self, msg, logger=None): + def get_email_body_soup(self, msg, logger): "Extract and decode email body and return it as BeautifulSoup object." - if logger is None: - logger = self.logger - # try to find the body part logger.debug("trying to find email body") try: @@ -193,11 +189,8 @@ class EMailNotification(BaseNotification): return soup - def sanitize(self, soup, logger=None): + def sanitize(self, soup, logger): "Sanitize mail html text." - if logger is None: - logger = self.logger - logger.debug("sanitizing email text") # completly remove bad elements @@ -230,13 +223,9 @@ class EMailNotification(BaseNotification): del(element.attrs[attribute]) return soup - def notify(self, msg, qid, mailfrom, recipients, - template_vars=defaultdict(str), synchronous=False, - logger=None): + def notify(self, msg, qid, mailfrom, recipients, logger, + template_vars=defaultdict(str), synchronous=False): "Notify recipients via email." - if logger is None: - logger = self.logger - # extract body from email soup = self.get_email_body_soup(msg, logger) @@ -262,7 +251,8 @@ class EMailNotification(BaseNotification): # sending email notifications for recipient in recipients: - logger.debug(f"generating notification email for '{recipient}'") + logger.debug( + f"generating notification email for '{recipient}'") logger.debug("parsing email template") # generate dict containing all template variables @@ -313,15 +303,40 @@ class EMailNotification(BaseNotification): self.mailfrom, recipient, newmsg.as_string(), "notification email") - def execute(self, milter, pretend=False, - logger=None): - super().execute(milter, pretend, logger) - - if logger is None: - logger = self.logger + def execute(self, milter, logger): + super().execute(milter, logger) self.notify(msg=milter.msg, qid=milter.qid, mailfrom=milter.msginfo["mailfrom"], recipients=milter.msginfo["rcpts"], template_vars=milter.msginfo["vars"], logger=logger) + + +class Notify: + NOTIFICATION_TYPES = { + "email": EMailNotification} + + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) + + nodification_type = cfg["args"]["type"] + del cfg["args"]["type"] + cfg["args"]["pretend"] = cfg["pretend"] + self._notification = self.NOTIFICATION_TYPES[nodification_type]( + **cfg["args"]) + self._headersonly = self._notification._headersonly + + def __str__(self): + cfg = [] + for key, value in self.cfg["args"].items(): + cfg.append(f"{key}={value}") + class_name = type(self._notification).__name__ + return f"{class_name}(" + ", ".join(cfg) + ")" + + def execute(self, milter): + logger = CustomLogger( + self.logger, {"name": self.cfg["name"], "qid": milter.qid}) + self._notification.execute(milter, logger) diff --git a/pymodmilter/rule.py b/pymodmilter/rule.py index 5052d2e..5a831c2 100644 --- a/pymodmilter/rule.py +++ b/pymodmilter/rule.py @@ -12,83 +12,53 @@ # along with PyMod-Milter. If not, see . # -__all__ = [ - "RuleConfig", - "Rule"] +__all__ = ["Rule"] -from pymodmilter import BaseConfig -from pymodmilter.action import ActionConfig, Action -from pymodmilter.conditions import ConditionsConfig, Conditions +import logging - -class RuleConfig(BaseConfig): - def __init__(self, cfg, local_addrs, debug=False): - super().__init__(cfg, debug) - - self.conditions = None - self.actions = [] - - self.pretend = False - if "pretend" in cfg: - assert isinstance(cfg["pretend"], bool), \ - f"{self.name}: pretend: invalid value, should be bool" - self.pretend = cfg["pretend"] - - assert "actions" in cfg, \ - f"{self.name}: mandatory parameter 'actions' not found" - actions = cfg["actions"] - assert isinstance(actions, list), \ - f"{self.name}: actions: invalid value, should be list" - - self.logger.debug(f"{self.name}: pretend={self.pretend}, " - f"loglevel={self.loglevel}") - - if "conditions" in cfg: - assert isinstance(cfg["conditions"], dict), \ - f"{self.name}: conditions: invalid value, should be dict" - cfg["conditions"]["name"] = f"{self.name}: condition" - if "loglevel" not in cfg["conditions"]: - cfg["conditions"]["loglevel"] = self.loglevel - self.conditions = ConditionsConfig( - cfg["conditions"], local_addrs, debug) - else: - self.conditions = None - - for idx, action_cfg in enumerate(cfg["actions"]): - if "name" in action_cfg: - assert isinstance(action_cfg["name"], str), \ - f"{self.name}: Action #{idx}: name: invalid value, " \ - f"should be string" - action_cfg["name"] = f"{self.name}: {action_cfg['name']}" - else: - action_cfg["name"] = f"{self.name}: Action #{idx}" - - if "loglevel" not in action_cfg: - action_cfg["loglevel"] = self.loglevel - if "pretend" not in action_cfg: - action_cfg["pretend"] = self.pretend - self.actions.append( - ActionConfig(action_cfg, local_addrs, debug)) +from pymodmilter.action import Action +from pymodmilter.conditions import Conditions class Rule: """ Rule to implement multiple actions on emails. """ + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) - def __init__(self, cfg): - self.logger = cfg.logger - - if cfg.conditions is None: - self.conditions = None - else: - self.conditions = Conditions(cfg.conditions) + self.conditions = cfg["conditions"] if "conditions" in cfg else None + if self.conditions is not None: + self.conditions["name"] = f"{cfg['name']}: condition" + self.conditions["loglevel"] = cfg["loglevel"] + self.conditions = Conditions(self.conditions, local_addrs, debug) self.actions = [] - for action_cfg in cfg.actions: - self.actions.append(Action(action_cfg)) + for idx, action_cfg in enumerate(cfg["actions"]): + if "name" in action_cfg: + action_cfg["name"] = f"{cfg['name']}: {action_cfg['name']}" + else: + action_cfg["name"] = f"action#{idx}" + if "loglevel" not in action_cfg: + action_cfg["loglevel"] = cfg["loglevel"] + if "pretend" not in action_cfg: + action_cfg["pretend"] = cfg["pretend"] + self.actions.append(Action(action_cfg, local_addrs, debug)) - self.pretend = cfg.pretend + def __str__(self): + cfg = [] + for key in ["name", "loglevel", "pretend"]: + value = self.cfg[key] + cfg.append(f"{key}={value}") + if self.conditions is not None: + cfg.append(f"conditions={self.conditions}") + actions = [] + for action in self.actions: + actions.append(str(action)) + cfg.append("actions=[" + ", ".join(actions) + "]") + return "Rule(" + ", ".join(cfg) + ")" def execute(self, milter): """Execute all actions of this rule.""" diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 6bd6304..bc93699 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -16,13 +16,17 @@ __all__ = ["main"] import Milter import argparse +import json import logging import logging.handlers +import os +import re import sys from pymodmilter import mailer -from pymodmilter import ModifyMilterConfig, ModifyMilter +from pymodmilter import ModifyMilter from pymodmilter import __version__ as version +from pymodmilter.config import MilterConfig def main(): @@ -80,36 +84,52 @@ def main(): logger.setLevel(logging.INFO) try: - logger.debug("prepare milter configuration") - cfg = ModifyMilterConfig(args.config, args.debug) + logger.debug("read milter configuration") + + try: + with open(args.config, "r") as fh: + # remove lines with leading # (comments), they + # are not allowed in json + cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read()) + except IOError as e: + raise RuntimeError(f"unable to open/read config file: {e}") + + try: + cfg = json.loads(cfg) + except json.JSONDecodeError as e: + cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] + msg = "\n".join(cfg_text) + raise RuntimeError(f"{e}\n{msg}") + + cfg = MilterConfig(cfg) if not args.debug: - logger.setLevel(cfg.loglevel) + logger.setLevel(cfg.get_loglevel(args.debug)) if args.socket: socket = args.socket - elif cfg.socket: - socket = cfg.socket + elif cfg["socket"]: + socket = cfg["socket"] else: raise RuntimeError( "listening socket is neither specified on the command line " "nor in the configuration file") - if not cfg.rules: + if not cfg["rules"]: raise RuntimeError("no rules configured") - for rule_cfg in cfg.rules: - if not rule_cfg.actions: + for rule in cfg["rules"]: + if not rule["actions"]: raise RuntimeError( - f"{rule_cfg.name}: no actions configured") + f"{rule['name']}: no actions configured") except (RuntimeError, AssertionError) as e: - logger.error(e) + logger.error(f"error in config file: {e}") sys.exit(255) try: - ModifyMilter.set_config(cfg) - except (RuntimeError, ValueError) as e: + ModifyMilter.set_config(cfg, args.debug) + except RuntimeError as e: logger.error(e) sys.exit(254) @@ -147,6 +167,7 @@ def main(): mailer.queue.put(None) logger.info("pymodmilter stopped") + sys.exit(rc) diff --git a/pymodmilter/storage.py b/pymodmilter/storage.py index de4a2a9..f64d1a0 100644 --- a/pymodmilter/storage.py +++ b/pymodmilter/storage.py @@ -28,24 +28,26 @@ from time import gmtime from pymodmilter.base import CustomLogger from pymodmilter.conditions import Conditions +from pymodmilter.config import ActionConfig +from pymodmilter.notify import Notify class BaseMailStorage: "Mail storage base class" _headersonly = True - def __init__(self, original=False, metadata=False, metavar=None): + def __init__(self, original=False, metadata=False, metavar=None, + pretend=False): self.original = original self.metadata = metadata self.metavar = metavar - return + self.pretend = False def add(self, data, qid, mailfrom="", recipients=[]): "Add email to storage." return ("", "") - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + def execute(self, milter, logger): return def find(self, mailfrom=None, recipients=None, older_than=None): @@ -69,9 +71,9 @@ class FileMailStorage(BaseMailStorage): "Storage class to store mails on filesystem." _headersonly = False - def __init__(self, directory, original=False, metadata=False, - metavar=None): - super().__init__(original, metadata, metavar) + def __init__(self, directory, original=False, metadata=False, metavar=None, + pretend=False): + super().__init__(original, metadata, metavar, pretend) # check if directory exists and is writable if not os.path.isdir(directory) or \ not os.access(directory, os.W_OK): @@ -81,6 +83,15 @@ class FileMailStorage(BaseMailStorage): self.directory = directory self._metadata_suffix = ".metadata" + def __str__(self): + cfg = [] + cfg.append(f"metadata={self.metadata}") + cfg.append(f"metavar={self.metavar}") + cfg.append(f"pretend={self.pretend}") + cfg.append(f"directory={self.directory}") + cfg.append(f"original={self.original}") + return "FileMailStorage(" + ", ".join(cfg) + ")" + def get_storageid(self, qid): timestamp = datetime.now().strftime("%Y%m%d%H%M%S") return f"{timestamp}_{qid}" @@ -144,8 +155,7 @@ class FileMailStorage(BaseMailStorage): return storage_id, metafile, datafile - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + def execute(self, milter, logger): if self.original: milter.fp.seek(0) data = milter.fp.read @@ -158,7 +168,7 @@ class FileMailStorage(BaseMailStorage): recipients = list(milter.msginfo["rcpts"]) subject = milter.msg["subject"] or "" - if not pretend: + if not self.pretend: storage_id, metafile, datafile = self.add( data(), milter.qid, mailfrom, recipients, subject) logger.info(f"stored message in file {datafile}") @@ -174,7 +184,7 @@ class FileMailStorage(BaseMailStorage): def get_metadata(self, storage_id): "Return metadata of email in storage." - super(FileMailStorage, self).get_metadata(storage_id) + super().get_metadata(storage_id) if not self.metadata: return None @@ -197,7 +207,7 @@ class FileMailStorage(BaseMailStorage): def find(self, mailfrom=None, recipients=None, older_than=None): "Find emails in storage." - super(FileMailStorage, self).find(mailfrom, recipients, older_than) + super().find(mailfrom, recipients, older_than) if isinstance(mailfrom, str): mailfrom = [mailfrom] if isinstance(recipients, str): @@ -238,7 +248,7 @@ class FileMailStorage(BaseMailStorage): def delete(self, storage_id, recipients=None): "Delete email from storage." - super(FileMailStorage, self).delete(storage_id, recipients) + super().delete(storage_id, recipients) if not recipients or not self.metadata: self._remove(storage_id) @@ -264,7 +274,7 @@ class FileMailStorage(BaseMailStorage): self._save_metafile(metafile, metadata) def get_mail(self, storage_id): - super(FileMailStorage, self).get_mail(storage_id) + super().get_mail(storage_id) metadata = self.get_metadata(storage_id) _, datafile = self._get_file_paths(storage_id) @@ -275,31 +285,101 @@ class FileMailStorage(BaseMailStorage): return (metadata, data) +class Store: + STORAGE_TYPES = { + "file": FileMailStorage} + + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) + + storage_type = cfg["args"]["type"] + del cfg["args"]["type"] + cfg["args"]["pretend"] = cfg["pretend"] + self._storage = self.STORAGE_TYPES[storage_type]( + **cfg["args"]) + self._headersonly = self._storage._headersonly + + def __str__(self): + cfg = [] + for key, value in self.cfg["args"].items(): + cfg.append(f"{key}={value}") + class_name = type(self._storage).__name__ + return f"{class_name}(" + ", ".join(cfg) + ")" + + def execute(self, milter): + logger = CustomLogger( + self.logger, {"name": self.cfg["name"], "qid": milter.qid}) + self._storage.execute(milter, logger) + + class Quarantine: "Quarantine class." _headersonly = False - def __init__(self, storage, notification=None, whitelist=None, - milter_action=None, reject_reason="Message rejected"): - self.storage = storage.action(**storage.args, metadata=True) - self.storage_name = storage.name - self.storage_logger = storage.logger + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) - self.notification = notification - if self.notification is not None: - self.notification = notification.action(**notification.args) - self.notification_name = notification.name - self.notification_logger = notification.logger - self.whitelist = Conditions(whitelist) - self.milter_action = milter_action - self.reject_reason = reject_reason + store_cfg = ActionConfig({ + "name": cfg["name"], + "loglevel": cfg["loglevel"], + "pretend": cfg["pretend"], + "type": "store", + "args": cfg["args"]["store"].get_config()}) + self.store = Store(store_cfg, local_addrs, debug) - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + self.notify = None + if "notify" in cfg["args"]: + notify_cfg = ActionConfig({ + "name": cfg["name"], + "loglevel": cfg["loglevel"], + "pretend": cfg["pretend"], + "type": "notify", + "args": cfg["args"]["notify"].get_config()}) + self.notify = Notify(notify_cfg, local_addrs, debug) + + self.whitelist = None + if "whitelist" in cfg["args"]: + whitelist_cfg = cfg["args"]["whitelist"] + whitelist_cfg["name"] = cfg["name"] + whitelist_cfg["loglevel"] = cfg["loglevel"] + self.whitelist = Conditions( + whitelist_cfg, + local_addrs=[], + debug=debug) + + self.milter_action = None + if "milter_action" in cfg["args"]: + self.milter_action = cfg["args"]["milter_action"] + self.reject_reason = None + if "reject_reason" in cfg["args"]: + self.reject_reason = cfg["args"]["reject_reason"] + + def __str__(self): + cfg = [] + cfg.append(f"store={str(self.store)}") + if self.notify is not None: + cfg.append(f"notify={str(self.notify)}") + if self.whitelist is not None: + cfg.append(f"whitelist={str(self.whitelist)}") + for key in ["milter_action", "reject_reason"]: + if key not in self.cfg["args"]: + continue + value = self.cfg["args"][key] + cfg.append(f"{key}={value}") + class_name = type(self).__name__ + return f"{class_name}(" + ", ".join(cfg) + ")" + + def execute(self, milter): + logger = CustomLogger( + self.logger, {"name": self.cfg["name"], "qid": milter.qid}) wl_rcpts = [] if self.whitelist: wl_rcpts = self.whitelist.get_wl_rcpts( - milter.msginfo["mailfrom"], milter.msginfo["rcpts"]) + milter.msginfo["mailfrom"], milter.msginfo["rcpts"], logger) logger.info(f"whitelisted recipients: {wl_rcpts}") rcpts = [ @@ -312,14 +392,10 @@ class Quarantine: logger.info(f"add to quarantine for recipients: {rcpts}") milter.msginfo["rcpts"] = rcpts - custom_logger = CustomLogger( - self.storage_logger, {"name": self.storage_name}) - self.storage.execute(milter, pretend, custom_logger) + self.store.execute(milter) - if self.notification is not None: - custom_logger = CustomLogger( - self.notification_logger, {"name": self.notification_name}) - self.notification.execute(milter, pretend, custom_logger) + if self.notify is not None: + self.notify.execute(milter) milter.msginfo["rcpts"].extend(wl_rcpts) milter.delrcpt(rcpts) diff --git a/pymodmilter/whitelist.py b/pymodmilter/whitelist.py index 4a5ac8a..ca34851 100644 --- a/pymodmilter/whitelist.py +++ b/pymodmilter/whitelist.py @@ -26,17 +26,21 @@ from playhouse.db_url import connect class WhitelistBase: "Whitelist base class" - def __init__(self, cfg): - self.name = cfg["name"] - self.logger = logging.getLogger(__name__) + def __init__(self, cfg, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) + + peewee_logger = logging.getLogger("peewee") + peewee_logger.setLevel(cfg.get_loglevel(debug)) + self.valid_entry_regex = re.compile( r"^[a-zA-Z0-9_.=+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$") self.batv_regex = re.compile( r"^prvs=[0-9]{4}[0-9A-Fa-f]{6}=(?P.+?)@") def remove_batv(self, addr): - return self.batv_regex.sub("\g", addr, count=1) - + return self.batv_regex.sub(r"\g", addr, count=1) def check(self, mailfrom, recipient): "Check if mailfrom/recipient combination is whitelisted." @@ -82,8 +86,8 @@ class DatabaseWhitelist(WhitelistBase): _db_connections = {} _db_tables = {} - def __init__(self, cfg): - super().__init__(cfg) + def __init__(self, cfg, debug): + super().__init__(cfg, debug) tablename = cfg["table"] connection_string = cfg["connection"] @@ -110,9 +114,10 @@ class DatabaseWhitelist(WhitelistBase): self.meta = Meta self.meta.database = db self.meta.table_name = tablename - self.model = type(f"WhitelistModel_{self.name}", (WhitelistModel,), { - "Meta": self.meta - }) + self.model = type( + f"WhitelistModel_{self.cfg['name']}", + (WhitelistModel,), + {"Meta": self.meta}) if connection_string not in DatabaseWhitelist._db_tables.keys(): DatabaseWhitelist._db_tables[connection_string] = [] @@ -125,6 +130,13 @@ class DatabaseWhitelist(WhitelistBase): raise RuntimeError( f"unable to initialize table '{tablename}': {e}") + def __str__(self): + cfg = [] + for arg in ("connection", "table"): + if arg in self.cfg: + cfg.append(f"{arg}={self.cfg[arg]}") + return "DatabaseWhitelist(" + ", ".join(cfg) + ")" + def _entry_to_dict(self, entry): result = {} result[entry.id] = { @@ -147,14 +159,14 @@ class DatabaseWhitelist(WhitelistBase): value += 1 return value - def check(self, mailfrom, recipient): + def check(self, mailfrom, recipient, logger): # check if mailfrom/recipient combination is whitelisted super().check(mailfrom, recipient) mailfrom = self.remove_batv(mailfrom) recipient = self.remove_batv(recipient) # generate list of possible mailfroms - self.logger.debug( + logger.debug( f"query database for whitelist entries from <{mailfrom}> " f"to <{recipient}>") mailfroms = [""] diff --git a/setup.py b/setup.py index 5fae1b5..f9d6689 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,6 @@ setup(name = "pymodmilter", ] ) ], - install_requires = ["pymilter", "netaddr", "beautifulsoup4[lxml]", "peewee"], + install_requires = ["pymilter", "jsonschema", "netaddr", "beautifulsoup4[lxml]", "peewee"], python_requires = ">=3.8" ) From f18d4e57f9c4cd7f15f28049d19ad2cefd3c98f0 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 30 Sep 2021 01:17:17 +0200 Subject: [PATCH 140/145] cleanup --- pymodmilter/action.py | 3 --- pymodmilter/config.py | 1 + pymodmilter/run.py | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pymodmilter/action.py b/pymodmilter/action.py index a2bc31a..e82a307 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -17,7 +17,6 @@ __all__ = ["Action"] import logging from pymodmilter import modify, notify, storage -from pymodmilter.base import CustomLogger from pymodmilter.conditions import Conditions @@ -64,8 +63,6 @@ class Action: def execute(self, milter): """Execute configured action.""" - logger = CustomLogger( - self.logger, {"qid": milter.qid, "name": self.cfg["name"]}) if self.conditions is None or \ self.conditions.match(milter): return self.action.execute(milter) diff --git a/pymodmilter/config.py b/pymodmilter/config.py index 48020f5..7aa1a47 100644 --- a/pymodmilter/config.py +++ b/pymodmilter/config.py @@ -85,6 +85,7 @@ class BaseConfig: def get_config(self): return self._config + class WhitelistConfig(BaseConfig): JSON_SCHEMA = { "type": "object", diff --git a/pymodmilter/run.py b/pymodmilter/run.py index bc93699..3675809 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -19,7 +19,6 @@ import argparse import json import logging import logging.handlers -import os import re import sys From e11d78ae4f67f53d5322a6bf1f6ab20901e53c40 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 30 Sep 2021 01:23:59 +0200 Subject: [PATCH 141/145] more cleanup --- pymodmilter/action.py | 2 -- pymodmilter/rule.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/pymodmilter/action.py b/pymodmilter/action.py index e82a307..62b7d4e 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -34,8 +34,6 @@ class Action: def __init__(self, cfg, local_addrs, debug): self.cfg = cfg - self.logger = logging.getLogger(cfg["name"]) - self.logger.setLevel(cfg.get_loglevel(debug)) self.conditions = cfg["conditions"] if "conditions" in cfg else None if self.conditions is not None: diff --git a/pymodmilter/rule.py b/pymodmilter/rule.py index 5a831c2..eea2425 100644 --- a/pymodmilter/rule.py +++ b/pymodmilter/rule.py @@ -26,8 +26,6 @@ class Rule: """ def __init__(self, cfg, local_addrs, debug): self.cfg = cfg - self.logger = logging.getLogger(cfg["name"]) - self.logger.setLevel(cfg.get_loglevel(debug)) self.conditions = cfg["conditions"] if "conditions" in cfg else None if self.conditions is not None: From 7de9cc1bb869199b2793d5c86754fcccbea224a3 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 30 Sep 2021 02:06:16 +0200 Subject: [PATCH 142/145] cleanup again --- pymodmilter/action.py | 2 -- pymodmilter/modify.py | 3 ++- pymodmilter/notify.py | 4 ++-- pymodmilter/rule.py | 2 -- pymodmilter/storage.py | 1 + 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pymodmilter/action.py b/pymodmilter/action.py index 62b7d4e..ce0deda 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -14,8 +14,6 @@ __all__ = ["Action"] -import logging - from pymodmilter import modify, notify, storage from pymodmilter.conditions import Conditions diff --git a/pymodmilter/modify.py b/pymodmilter/modify.py index 8222a81..6cdfac0 100644 --- a/pymodmilter/modify.py +++ b/pymodmilter/modify.py @@ -17,7 +17,8 @@ __all__ = [ "ModHeader", "DelHeader", "AddDisclaimer", - "RewriteLinks"] + "RewriteLinks", + "Modify"] import logging import re diff --git a/pymodmilter/notify.py b/pymodmilter/notify.py index 32d3270..f1ef95e 100644 --- a/pymodmilter/notify.py +++ b/pymodmilter/notify.py @@ -14,7 +14,8 @@ __all__ = [ "BaseNotification", - "EMailNotification"] + "EMailNotification", + "Notify"] import email import logging @@ -256,7 +257,6 @@ class EMailNotification(BaseNotification): logger.debug("parsing email template") # generate dict containing all template variables - variables = defaultdict(str, template_vars) variables.update({ "HTML_TEXT": sanitized_text, diff --git a/pymodmilter/rule.py b/pymodmilter/rule.py index eea2425..a596e34 100644 --- a/pymodmilter/rule.py +++ b/pymodmilter/rule.py @@ -14,8 +14,6 @@ __all__ = ["Rule"] -import logging - from pymodmilter.action import Action from pymodmilter.conditions import Conditions diff --git a/pymodmilter/storage.py b/pymodmilter/storage.py index f64d1a0..423ce2d 100644 --- a/pymodmilter/storage.py +++ b/pymodmilter/storage.py @@ -15,6 +15,7 @@ __all__ = [ "BaseMailStorage", "FileMailStorage", + "Store", "Quarantine"] import json From 60e3f49fe1fc4eba74c70bed6e8ddd14189a1ea1 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 30 Sep 2021 15:12:31 +0200 Subject: [PATCH 143/145] add option for file creation mode --- pymodmilter/config.py | 1 + pymodmilter/storage.py | 36 +++++++++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/pymodmilter/config.py b/pymodmilter/config.py index 7aa1a47..98b831d 100644 --- a/pymodmilter/config.py +++ b/pymodmilter/config.py @@ -192,6 +192,7 @@ class StoreConfig(BaseConfig): "properties": { "type": {"type": "string"}, "directory": {"type": "string"}, + "mode": {"type": "string"}, "metavar": {"type": "string"}, "original": {"type": "boolean", "default": True}}}} diff --git a/pymodmilter/storage.py b/pymodmilter/storage.py index 423ce2d..9dd54af 100644 --- a/pymodmilter/storage.py +++ b/pymodmilter/storage.py @@ -73,7 +73,7 @@ class FileMailStorage(BaseMailStorage): _headersonly = False def __init__(self, directory, original=False, metadata=False, metavar=None, - pretend=False): + mode=None, pretend=False): super().__init__(original, metadata, metavar, pretend) # check if directory exists and is writable if not os.path.isdir(directory) or \ @@ -82,6 +82,13 @@ class FileMailStorage(BaseMailStorage): f"directory '{directory}' does not exist or is " f"not writable") self.directory = directory + try: + self.mode = int(mode, 8) if mode is not None else None + if self.mode is not None and self.mode > 511: + raise ValueError + except ValueError: + raise RuntimeError(f"invalid mode '{mode}'") + self._metadata_suffix = ".metadata" def __str__(self): @@ -104,15 +111,33 @@ class FileMailStorage(BaseMailStorage): def _save_datafile(self, datafile, data): try: - with open(datafile, "wb") as f: - f.write(data) + if self.mode is None: + with open(datafile, "wb") as f: + f.write(data) + else: + umask = os.umask(0) + with open( + os.open(datafile, os.O_CREAT | os.O_WRONLY, self.mode), + "wb") as f: + f.write(data) + os.umask(umask) + except IOError as e: raise RuntimeError(f"unable save data file: {e}") def _save_metafile(self, metafile, metadata): try: - with open(metafile, "w") as f: - json.dump(metadata, f, indent=2) + if self.mode is None: + with open(metafile, "w") as f: + json.dump(metadata, f, indent=2) + else: + umask = os.umask(0) + with open( + os.open(metafile, os.O_CREAT | os.O_WRONLY, self.mode), + "w") as f: + json.dump(metadata, f, indent=2) + os.umask(umask) + except IOError as e: raise RuntimeError(f"unable to save metadata file: {e}") @@ -330,6 +355,7 @@ class Quarantine: "pretend": cfg["pretend"], "type": "store", "args": cfg["args"]["store"].get_config()}) + store_cfg["args"]["metadata"] = True self.store = Store(store_cfg, local_addrs, debug) self.notify = None From cd7e0688dc0b66844c87b4a2e244ca1b63b62133 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 30 Sep 2021 16:54:03 +0200 Subject: [PATCH 144/145] prepare for cli --- pymodmilter/__init__.py | 3 +- pymodmilter/cli.py | 628 ++++++++++++++++++++++++++++++++++++++++ pymodmilter/config.py | 24 +- pymodmilter/run.py | 30 +- 4 files changed, 658 insertions(+), 27 deletions(-) create mode 100644 pymodmilter/cli.py diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index aac9594..34c5d61 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -15,7 +15,9 @@ __all__ = [ "action", "base", + "cli", "conditions", + "config", "mailer", "modify", "notify", @@ -23,7 +25,6 @@ __all__ = [ "run", "storage", "whitelist", - "ModifyMilterConfig", "ModifyMilter"] __version__ = "1.2.0" diff --git a/pymodmilter/cli.py b/pymodmilter/cli.py new file mode 100644 index 0000000..2f7b8c8 --- /dev/null +++ b/pymodmilter/cli.py @@ -0,0 +1,628 @@ +#!/usr/bin/env python +# +# PyQuarantine-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyQuarantine-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyQuarantineMilter. If not, see . +# + +import argparse +import logging +import logging.handlers +import sys +import time + +from pymodmilter.config import get_milter_config +from pymodmilter import __version__ as version + + +def _get_quarantine(quarantines, name): + try: + quarantine = next((q for q in quarantines if q.name == name)) + except StopIteration: + raise RuntimeError("invalid quarantine 'name'") + return quarantine + + +def _get_storage(quarantines, name): + quarantine = _get_quarantine(quarantines, name) + storage = quarantine.get_storage() + if not storage: + raise RuntimeError( + "storage type is set to NONE") + return storage + + +def _get_notification(quarantines, name): + quarantine = _get_quarantine(quarantines, name) + notification = quarantine.get_notification() + if not notification: + raise RuntimeError( + "notification type is set to NONE") + return notification + + +def _get_whitelist(quarantines, name): + quarantine = _get_quarantine(quarantines, name) + whitelist = quarantine.get_whitelist() + if not whitelist: + raise RuntimeError( + "whitelist type is set to NONE") + return whitelist + + +def print_table(columns, rows): + if not rows: + return + + column_lengths = [] + column_formats = [] + + # iterate columns to display + for header, key in columns: + # get the length of the header string + lengths = [len(header)] + # get the length of the longest value + lengths.append( + len(str(max(rows, key=lambda x: len(str(x[key])))[key]))) + # use the longer one + length = max(lengths) + column_lengths.append(length) + column_formats.append(f"{{:<{length}}}") + + # define row format + row_format = " | ".join(column_formats) + + # define header/body separator + separators = [] + for length in column_lengths: + separators.append("-" * length) + separator = "-+-".join(separators) + + # print header and separator + print(row_format.format(*[column[0] for column in columns])) + print(separator) + + keys = [entry[1] for entry in columns] + # print rows + for entry in rows: + row = [] + for key in keys: + row.append(entry[key]) + print(row_format.format(*row)) + + +def list_quarantines(quarantines, args): + if args.batch: + print("\n".join([q.name for q in quarantines])) + else: + qlist = [] + for q in quarantines: + storage = q.get_storage() + if storage: + storage_type = q.get_storage().storage_type + else: + storage_type = "NONE" + + notification = q.get_notification() + if notification: + notification_type = q.get_notification().notification_type + else: + notification_type = "NONE" + + whitelist = q.get_whitelist() + if whitelist: + whitelist_type = q.get_whitelist().whitelist_type + else: + whitelist_type = "NONE" + + qlist.append({ + "name": q.name, + "storage": storage_type, + "notification": notification_type, + "whitelist": whitelist_type, + "action": q.action}) + print_table( + [("Name", "name"), + ("Storage", "storage"), + ("Notification", "notification"), + ("Whitelist", "whitelist"), + ("Action", "action")], + qlist + ) + + +def list_quarantine_emails(quarantines, args): + logger = logging.getLogger(__name__) + storage = _get_storage(quarantines, args.quarantine) + # find emails and transform some metadata values to strings + rows = [] + emails = storage.find( + args.mailfrom, args.recipients, args.older_than) + for storage_id, metadata in emails.items(): + row = emails[storage_id] + row["storage_id"] = storage_id + row["date"] = time.strftime( + '%Y-%m-%d %H:%M:%S', + time.localtime( + metadata["date"])) + row["mailfrom"] = metadata["mailfrom"] + row["recipient"] = metadata["recipients"].pop(0) + if "subject" not in emails[storage_id]["headers"].keys(): + emails[storage_id]["headers"]["subject"] = "" + row["subject"] = emails[storage_id]["headers"]["subject"][:60].strip() + rows.append(row) + + if metadata["recipients"]: + row = { + "storage_id": "", + "date": "", + "mailfrom": "", + "recipient": metadata["recipients"].pop(0), + "subject": "" + } + rows.append(row) + + if args.batch: + # batch mode, print quarantine IDs, each on a new line + print("\n".join(emails.keys())) + return + + if not emails: + logger.info(f"quarantine '{args.quarantine}' is empty") + print_table( + [("Quarantine-ID", "storage_id"), ("Date", "date"), + ("From", "mailfrom"), ("Recipient(s)", "recipient"), + ("Subject", "subject")], + rows + ) + + +def list_whitelist(quarantines, args): + logger = logging.getLogger(__name__) + whitelist = _get_whitelist(quarantines, args.quarantine) + + # find whitelist entries + entries = whitelist.find( + mailfrom=args.mailfrom, + recipients=args.recipients, + older_than=args.older_than) + if not entries: + logger.info( + f"whitelist of quarantine '{args.quarantine}' is empty") + return + + # transform some values to strings + for entry_id, entry in entries.items(): + entries[entry_id]["permanent_str"] = str(entry["permanent"]) + entries[entry_id]["created_str"] = entry["created"].strftime( + '%Y-%m-%d %H:%M:%S') + entries[entry_id]["last_used_str"] = entry["last_used"].strftime( + '%Y-%m-%d %H:%M:%S') + + print_table( + [ + ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"), + ("Created", "created_str"), ("Last used", "last_used_str"), + ("Comment", "comment"), ("Permanent", "permanent_str") + ], + entries.values() + ) + + +def add_whitelist_entry(quarantines, args): + logger = logging.getLogger(__name__) + whitelist = _get_whitelist(quarantines, args.quarantine) + + # check existing entries + entries = whitelist.check(args.mailfrom, args.recipient) + if entries: + # check if the exact entry exists already + for entry in entries.values(): + if entry["mailfrom"] == args.mailfrom and \ + entry["recipient"] == args.recipient: + raise RuntimeError( + "an entry with this from/to combination already exists") + + if not args.force: + # the entry is already covered by others + for entry_id, entry in entries.items(): + entries[entry_id]["permanent_str"] = str(entry["permanent"]) + entries[entry_id]["created_str"] = entry["created"].strftime( + '%Y-%m-%d %H:%M:%S') + entries[entry_id]["last_used_str"] = entry["last_used"].strftime( + '%Y-%m-%d %H:%M:%S') + print_table( + [ + ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"), + ("Created", "created_str"), ("Last used", "last_used_str"), + ("Comment", "comment"), ("Permanent", "permanent_str") + ], + entries.values() + ) + print("") + raise RuntimeError( + "from/to combination is already covered by the entries above, " + "use --force to override.") + + # add entry to whitelist + whitelist.add(args.mailfrom, args.recipient, args.comment, args.permanent) + logger.info("whitelist entry added successfully") + + +def delete_whitelist_entry(quarantines, args): + logger = logging.getLogger(__name__) + whitelist = _get_whitelist(quarantines, args.quarantine) + whitelist.delete(args.whitelist_id) + logger.info("whitelist entry deleted successfully") + + +def notify(quarantines, args): + logger = logging.getLogger(__name__) + quarantine = _get_quarantine(quarantines, args.quarantine) + quarantine.notify(args.quarantine_id, args.recipient) + logger.info("notification sent successfully") + + +def release(quarantines, args): + logger = logging.getLogger(__name__) + quarantine = _get_quarantine(quarantines, args.quarantine) + quarantine.release(args.quarantine_id, args.recipient) + logger.info("quarantined email released successfully") + + +def delete(quarantines, args): + logger = logging.getLogger(__name__) + storage = _get_storage(quarantines, args.quarantine) + storage.delete(args.quarantine_id, args.recipient) + logger.info("quarantined email deleted successfully") + + +def get(quarantines, args): + storage = _get_storage(quarantines, args.quarantine) + fp, _ = storage.get_mail(args.quarantine_id) + print(fp.read().decode()) + fp.close() + + +class StdErrFilter(logging.Filter): + def filter(self, rec): + return rec.levelno in (logging.ERROR, logging.WARNING) + + +class StdOutFilter(logging.Filter): + def filter(self, rec): + return rec.levelno in (logging.DEBUG, logging.INFO) + + +def main(): + python_version = ".".join([str(v) for v in sys.version_info[0:3]]) + python_version = f"{python_version}-{sys.version_info[3]}" + + "PyQuarantine command-line interface." + # parse command line + def formatter_class(prog): return argparse.HelpFormatter( + prog, max_help_position=50, width=140) + parser = argparse.ArgumentParser( + description="PyQuarantine CLI", + formatter_class=formatter_class) + parser.add_argument( + "-c", "--config", help="Config file to read.", + default="/etc/pymodmilter/pymodmilter.conf") + parser.add_argument( + "-d", "--debug", + help="Log debugging messages.", + action="store_true") + parser.add_argument( + "-v", "--version", + help="Print version.", + action="version", + version=f"%(prog)s {version} (python {python_version})") + parser.set_defaults(syslog=False) + subparsers = parser.add_subparsers( + dest="command", + title="Commands") + subparsers.required = True + + # list command + list_parser = subparsers.add_parser( + "list", + help="List available quarantines.", + formatter_class=formatter_class) + list_parser.add_argument( + "-b", "--batch", + help="Print results using only quarantine names, each on a new line.", + action="store_true") + list_parser.set_defaults(func=list_quarantines) + + # quarantine command group + quarantine_parser = subparsers.add_parser( + "quarantine", + description="Manage quarantines.", + help="Manage quarantines.", + formatter_class=formatter_class) + quarantine_parser.add_argument( + "quarantine", + metavar="QUARANTINE", + help="Quarantine name.") + quarantine_subparsers = quarantine_parser.add_subparsers( + dest="command", + title="Quarantine commands") + quarantine_subparsers.required = True + # quarantine list command + quarantine_list_parser = quarantine_subparsers.add_parser( + "list", + description="List emails in quarantines.", + help="List emails in quarantine.", + formatter_class=formatter_class) + quarantine_list_parser.add_argument( + "-f", "--from", + dest="mailfrom", + help="Filter emails by from address.", + default=None, + nargs="+") + quarantine_list_parser.add_argument( + "-t", "--to", + dest="recipients", + help="Filter emails by recipient address.", + default=None, + nargs="+") + quarantine_list_parser.add_argument( + "-o", "--older-than", + dest="older_than", + help="Filter emails by age (days).", + default=None, + type=float) + quarantine_list_parser.add_argument( + "-b", "--batch", + help="Print results using only email quarantine IDs, each on a new line.", + action="store_true") + quarantine_list_parser.set_defaults(func=list_quarantine_emails) + # quarantine notify command + quarantine_notify_parser = quarantine_subparsers.add_parser( + "notify", + description="Notify recipient about email in quarantine.", + help="Notify recipient about email in quarantine.", + formatter_class=formatter_class) + quarantine_notify_parser.add_argument( + "quarantine_id", + metavar="ID", + help="Quarantine ID.") + quarantine_notify_parser_group = quarantine_notify_parser.add_mutually_exclusive_group( + required=True) + quarantine_notify_parser_group.add_argument( + "-t", "--to", + dest="recipient", + help="Release email for one recipient address.") + quarantine_notify_parser_group.add_argument( + "-a", "--all", + help="Release email for all recipients.", + action="store_true") + quarantine_notify_parser.set_defaults(func=notify) + # quarantine release command + quarantine_release_parser = quarantine_subparsers.add_parser( + "release", + description="Release email from quarantine.", + help="Release email from quarantine.", + formatter_class=formatter_class) + quarantine_release_parser.add_argument( + "quarantine_id", + metavar="ID", + help="Quarantine ID.") + quarantine_release_parser.add_argument( + "-n", + "--disable-syslog", + dest="syslog", + help="Disable syslog messages.", + action="store_false") + quarantine_release_parser_group = quarantine_release_parser.add_mutually_exclusive_group( + required=True) + quarantine_release_parser_group.add_argument( + "-t", "--to", + dest="recipient", + help="Release email for one recipient address.") + quarantine_release_parser_group.add_argument( + "-a", "--all", + help="Release email for all recipients.", + action="store_true") + quarantine_release_parser.set_defaults(func=release) + # quarantine delete command + quarantine_delete_parser = quarantine_subparsers.add_parser( + "delete", + description="Delete email from quarantine.", + help="Delete email from quarantine.", + formatter_class=formatter_class) + quarantine_delete_parser.add_argument( + "quarantine_id", + metavar="ID", + help="Quarantine ID.") + quarantine_delete_parser.add_argument( + "-n", "--disable-syslog", + dest="syslog", + help="Disable syslog messages.", + action="store_false") + quarantine_delete_parser_group = quarantine_delete_parser.add_mutually_exclusive_group( + required=True) + quarantine_delete_parser_group.add_argument( + "-t", "--to", + dest="recipient", + help="Delete email for one recipient address.") + quarantine_delete_parser_group.add_argument( + "-a", "--all", + help="Delete email for all recipients.", + action="store_true") + quarantine_delete_parser.set_defaults(func=delete) + # quarantine get command + quarantine_get_parser = quarantine_subparsers.add_parser( + "get", + description="Get email from quarantine.", + help="Get email from quarantine", + formatter_class=formatter_class) + quarantine_get_parser.add_argument( + "quarantine_id", + metavar="ID", + help="Quarantine ID.") + quarantine_get_parser.set_defaults(func=get) + + # whitelist command group + whitelist_parser = subparsers.add_parser( + "whitelist", + description="Manage whitelists.", + help="Manage whitelists.", + formatter_class=formatter_class) + whitelist_parser.add_argument( + "quarantine", + metavar="QUARANTINE", + help="Quarantine name.") + whitelist_subparsers = whitelist_parser.add_subparsers( + dest="command", + title="Whitelist commands") + whitelist_subparsers.required = True + # whitelist list command + whitelist_list_parser = whitelist_subparsers.add_parser( + "list", + description="List whitelist entries.", + help="List whitelist entries.", + formatter_class=formatter_class) + whitelist_list_parser.add_argument( + "-f", "--from", + dest="mailfrom", + help="Filter entries by from address.", + default=None, + nargs="+") + whitelist_list_parser.add_argument( + "-t", "--to", + dest="recipients", + help="Filter entries by recipient address.", + default=None, + nargs="+") + whitelist_list_parser.add_argument( + "-o", "--older-than", + dest="older_than", + help="Filter emails by last used date (days).", + default=None, + type=float) + whitelist_list_parser.set_defaults(func=list_whitelist) + # whitelist add command + whitelist_add_parser = whitelist_subparsers.add_parser( + "add", + description="Add whitelist entry.", + help="Add whitelist entry.", + formatter_class=formatter_class) + whitelist_add_parser.add_argument( + "-f", "--from", + dest="mailfrom", + help="From address.", + required=True) + whitelist_add_parser.add_argument( + "-t", "--to", + dest="recipient", + help="Recipient address.", + required=True) + whitelist_add_parser.add_argument( + "-c", "--comment", + help="Comment.", + default="added by CLI") + whitelist_add_parser.add_argument( + "-p", "--permanent", + help="Add a permanent entry.", + action="store_true") + whitelist_add_parser.add_argument( + "--force", + help="Force adding an entry, even if already covered by another entry.", + action="store_true") + whitelist_add_parser.set_defaults(func=add_whitelist_entry) + # whitelist delete command + whitelist_delete_parser = whitelist_subparsers.add_parser( + "delete", + description="Delete whitelist entry.", + help="Delete whitelist entry.", + formatter_class=formatter_class) + whitelist_delete_parser.add_argument( + "whitelist_id", + metavar="ID", + help="Whitelist ID.") + whitelist_delete_parser.set_defaults(func=delete_whitelist_entry) + + args = parser.parse_args() + + # setup logging + loglevel = logging.INFO + root_logger = logging.getLogger() + root_logger.setLevel(loglevel) + + # setup console log + if args.debug: + formatter = logging.Formatter( + "%(levelname)s: [%(name)s] - %(message)s") + else: + formatter = logging.Formatter("%(levelname)s: %(message)s") + # stdout + stdouthandler = logging.StreamHandler(sys.stdout) + stdouthandler.setLevel(logging.DEBUG) + stdouthandler.setFormatter(formatter) + stdouthandler.addFilter(StdOutFilter()) + root_logger.addHandler(stdouthandler) + # stderr + stderrhandler = logging.StreamHandler(sys.stderr) + stderrhandler.setLevel(logging.WARNING) + stderrhandler.setFormatter(formatter) + stderrhandler.addFilter(StdErrFilter()) + root_logger.addHandler(stderrhandler) + logger = logging.getLogger(__name__) + + try: + logger.debug("read milter configuration") + cfg = get_milter_config(args.config) + if not cfg["rules"]: + raise RuntimeError("no rules configured") + + for rule in cfg["rules"]: + if not rule["actions"]: + raise RuntimeError( + f"{rule['name']}: no actions configured") + except (RuntimeError, AssertionError) as e: + logger.error(f"config error: {e}") + sys.exit(255) + + quarantines = [] + for rule in cfg["rules"]: + for action in rule["actions"]: + if action["type"] == "quarantine": + quarantines.append(action) + + print(quarantines) + sys.exit(0) + + if args.syslog: + # setup syslog + sysloghandler = logging.handlers.SysLogHandler( + address="/dev/log", + facility=logging.handlers.SysLogHandler.LOG_MAIL) + sysloghandler.setLevel(loglevel) + if args.debug: + formatter = logging.Formatter( + "pyquarantine: [%(name)s] [%(levelname)s] %(message)s") + else: + formatter = logging.Formatter("pyquarantine: %(message)s") + sysloghandler.setFormatter(formatter) + root_logger.addHandler(sysloghandler) + + # call the commands function + try: + args.func(cfg, args) + except RuntimeError as e: + logger.error(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/pymodmilter/config.py b/pymodmilter/config.py index 98b831d..6b6f336 100644 --- a/pymodmilter/config.py +++ b/pymodmilter/config.py @@ -26,10 +26,13 @@ __all__ = [ "QuarantineConfig", "ActionConfig", "RuleConfig", - "MilterConfig"] + "MilterConfig", + "get_milter_config"] +import json import jsonschema import logging +import re class BaseConfig: @@ -330,3 +333,22 @@ class MilterConfig(BaseConfig): for idx, rule in enumerate(self["rules"]): rules.append(RuleConfig(rule, rec)) self["rules"] = rules + + +def get_milter_config(cfgfile): + try: + with open(cfgfile, "r") as fh: + # remove lines with leading # (comments), they + # are not allowed in json + cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read()) + except IOError as e: + raise RuntimeError(f"unable to open/read config file: {e}") + + try: + cfg = json.loads(cfg) + except json.JSONDecodeError as e: + cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] + msg = "\n".join(cfg_text) + raise RuntimeError(f"{e}\n{msg}") + + return MilterConfig(cfg) diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 3675809..22548b7 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -16,16 +16,14 @@ __all__ = ["main"] import Milter import argparse -import json import logging import logging.handlers -import re import sys from pymodmilter import mailer from pymodmilter import ModifyMilter from pymodmilter import __version__ as version -from pymodmilter.config import MilterConfig +from pymodmilter.config import get_milter_config def main(): @@ -84,30 +82,12 @@ def main(): try: logger.debug("read milter configuration") - - try: - with open(args.config, "r") as fh: - # remove lines with leading # (comments), they - # are not allowed in json - cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read()) - except IOError as e: - raise RuntimeError(f"unable to open/read config file: {e}") - - try: - cfg = json.loads(cfg) - except json.JSONDecodeError as e: - cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] - msg = "\n".join(cfg_text) - raise RuntimeError(f"{e}\n{msg}") - - cfg = MilterConfig(cfg) - - if not args.debug: - logger.setLevel(cfg.get_loglevel(args.debug)) + cfg = get_milter_config(args.config) + logger.setLevel(cfg.get_loglevel(args.debug)) if args.socket: socket = args.socket - elif cfg["socket"]: + elif "socket" in cfg: socket = cfg["socket"] else: raise RuntimeError( @@ -123,7 +103,7 @@ def main(): f"{rule['name']}: no actions configured") except (RuntimeError, AssertionError) as e: - logger.error(f"error in config file: {e}") + logger.error(f"config error: {e}") sys.exit(255) try: From 0bd88f7cf4293c37527732ace3926ffd29005766 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 30 Sep 2021 18:04:36 +0200 Subject: [PATCH 145/145] prepare to merge with pyquarantine project --- MANIFEST.in | 4 +-- README.md | 20 ++++++------ {pymodmilter => pyquarantine}/__init__.py | 16 +++++----- .../_runtime_patches.py | 6 ++-- {pymodmilter => pyquarantine}/action.py | 10 +++--- {pymodmilter => pyquarantine}/base.py | 6 ++-- {pymodmilter => pyquarantine}/cli.py | 6 ++-- {pymodmilter => pyquarantine}/conditions.py | 10 +++--- {pymodmilter => pyquarantine}/config.py | 6 ++-- .../docs/pyquarantine.conf.example | 6 ++-- .../docs/templates/disclaimer_html.template | 0 .../docs/templates/disclaimer_text.template | 0 .../docs/templates/notification.template | 29 ++++++++++++++++++ pyquarantine/docs/templates/removed.png | Bin 0 -> 2136 bytes {pymodmilter => pyquarantine}/mailer.py | 6 ++-- .../misc/openrc/pyquarantine-milter.confd | 4 +-- .../misc/openrc/pyquarantine-milter.initd | 2 +- {pymodmilter => pyquarantine}/modify.py | 10 +++--- {pymodmilter => pyquarantine}/notify.py | 10 +++--- {pymodmilter => pyquarantine}/rule.py | 10 +++--- {pymodmilter => pyquarantine}/run.py | 28 ++++++++--------- {pymodmilter => pyquarantine}/storage.py | 14 ++++----- {pymodmilter => pyquarantine}/whitelist.py | 6 ++-- setup.cfg | 2 +- setup.py | 21 +++++++------ test-pymodmilter | 9 ------ 26 files changed, 133 insertions(+), 108 deletions(-) rename {pymodmilter => pyquarantine}/__init__.py (96%) rename {pymodmilter => pyquarantine}/_runtime_patches.py (97%) rename {pymodmilter => pyquarantine}/action.py (87%) rename {pymodmilter => pyquarantine}/base.py (91%) rename {pymodmilter => pyquarantine}/cli.py (99%) rename {pymodmilter => pyquarantine}/conditions.py (95%) rename {pymodmilter => pyquarantine}/config.py (98%) rename pymodmilter/docs/pymodmilter.conf.example => pyquarantine/docs/pyquarantine.conf.example (96%) rename {pymodmilter => pyquarantine}/docs/templates/disclaimer_html.template (100%) rename {pymodmilter => pyquarantine}/docs/templates/disclaimer_text.template (100%) create mode 100644 pyquarantine/docs/templates/notification.template create mode 100644 pyquarantine/docs/templates/removed.png rename {pymodmilter => pyquarantine}/mailer.py (92%) rename pymodmilter/misc/openrc/pymodmilter.confd => pyquarantine/misc/openrc/pyquarantine-milter.confd (56%) rename pymodmilter/misc/openrc/pymodmilter.initd => pyquarantine/misc/openrc/pyquarantine-milter.initd (95%) rename {pymodmilter => pyquarantine}/modify.py (97%) rename {pymodmilter => pyquarantine}/notify.py (97%) rename {pymodmilter => pyquarantine}/rule.py (89%) rename {pymodmilter => pyquarantine}/run.py (83%) rename {pymodmilter => pyquarantine}/storage.py (97%) rename {pymodmilter => pyquarantine}/whitelist.py (97%) delete mode 100755 test-pymodmilter diff --git a/MANIFEST.in b/MANIFEST.in index 89bd49f..f29efd2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include LICENSE README.md -recursive-include pymodmilter/docs * -recursive-include pymodmilter/misc * +recursive-include pyquarantine/docs * +recursive-include pyquarantine/misc * diff --git a/README.md b/README.md index a349d98..2c5e99f 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,27 @@ -# pymodmilter +# pyquarantine A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers. The project is currently in beta status, but it is already used in a productive enterprise environment that processes about a million e-mails per month. The basic idea is to define rules with conditions and actions which are processed when all conditions are true. ## Dependencies -Pymodmilter is depending on these python packages, but they are installed automatically if you are working with pip. -* [pymilter](https://pythonhosted.org/pymilter/) -* [netaddr](https://github.com/drkjam/netaddr/) +pyquarantine is depending on these python packages, but they are installed automatically if you are working with pip. +* [jsonschema](https://github.com/Julian/jsonschema) +* [pymilter](https://github.com/sdgathman/pymilter) +* [netaddr](https://github.com/drkjam/netaddr) +* [peewee](https://github.com/coleifer/peewee) * [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) ## Installation -* Install pymodmilter with pip and copy the example config file. +* Install pyquarantine with pip and copy the example config file. ```sh -pip install pymodmilter -cp /etc/pymodmilter/pymodmilter.conf.example /etc/pymodmilter/pymodmilter.conf +pip install pyquarantine +cp /etc/pyquarantine/pyquarantine.conf.example /etc/pyquarantine/pyquarantine.conf ``` -* Modify /etc/pymodmilter/pymodmilter.conf according to your needs. +* Modify /etc/pyquarantine/pyquarantine.conf according to your needs. ## Configuration options -Pymodmilter uses a config file in JSON format. The config file has to be JSON valid with the exception of allowed comment lines starting with **#**. The options are described below. +pyquarantine uses a config file in JSON format. The config file has to be JSON valid with the exception of allowed comment lines starting with **#**. The options are described below. Rules and actions are processed in the given order. ### Global diff --git a/pymodmilter/__init__.py b/pyquarantine/__init__.py similarity index 96% rename from pymodmilter/__init__.py rename to pyquarantine/__init__.py index 34c5d61..c46681b 100644 --- a/pymodmilter/__init__.py +++ b/pyquarantine/__init__.py @@ -1,15 +1,15 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # __all__ = [ @@ -27,9 +27,9 @@ __all__ = [ "whitelist", "ModifyMilter"] -__version__ = "1.2.0" +__version__ = "2.0.0" -from pymodmilter import _runtime_patches +from pyquarantine import _runtime_patches import Milter import logging @@ -44,9 +44,9 @@ from email.policy import SMTPUTF8 from io import BytesIO from netaddr import IPNetwork, AddrFormatError -from pymodmilter.base import CustomLogger, MilterMessage -from pymodmilter.base import replace_illegal_chars -from pymodmilter.rule import Rule +from pyquarantine.base import CustomLogger, MilterMessage +from pyquarantine.base import replace_illegal_chars +from pyquarantine.rule import Rule class ModifyMilter(Milter.Base): diff --git a/pymodmilter/_runtime_patches.py b/pyquarantine/_runtime_patches.py similarity index 97% rename from pymodmilter/_runtime_patches.py rename to pyquarantine/_runtime_patches.py index 4d868ea..aef4656 100644 --- a/pymodmilter/_runtime_patches.py +++ b/pyquarantine/_runtime_patches.py @@ -1,15 +1,15 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # import encodings diff --git a/pymodmilter/action.py b/pyquarantine/action.py similarity index 87% rename from pymodmilter/action.py rename to pyquarantine/action.py index ce0deda..cb30822 100644 --- a/pymodmilter/action.py +++ b/pyquarantine/action.py @@ -1,21 +1,21 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # __all__ = ["Action"] -from pymodmilter import modify, notify, storage -from pymodmilter.conditions import Conditions +from pyquarantine import modify, notify, storage +from pyquarantine.conditions import Conditions class Action: diff --git a/pymodmilter/base.py b/pyquarantine/base.py similarity index 91% rename from pymodmilter/base.py rename to pyquarantine/base.py index a88671a..70be625 100644 --- a/pymodmilter/base.py +++ b/pyquarantine/base.py @@ -1,15 +1,15 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # __all__ = [ diff --git a/pymodmilter/cli.py b/pyquarantine/cli.py similarity index 99% rename from pymodmilter/cli.py rename to pyquarantine/cli.py index 2f7b8c8..d7d733e 100644 --- a/pymodmilter/cli.py +++ b/pyquarantine/cli.py @@ -20,8 +20,8 @@ import logging.handlers import sys import time -from pymodmilter.config import get_milter_config -from pymodmilter import __version__ as version +from pyquarantine.config import get_milter_config +from pyquarantine import __version__ as version def _get_quarantine(quarantines, name): @@ -316,7 +316,7 @@ def main(): formatter_class=formatter_class) parser.add_argument( "-c", "--config", help="Config file to read.", - default="/etc/pymodmilter/pymodmilter.conf") + default="/etc/pyquarantine/pyquarantine.conf") parser.add_argument( "-d", "--debug", help="Log debugging messages.", diff --git a/pymodmilter/conditions.py b/pyquarantine/conditions.py similarity index 95% rename from pymodmilter/conditions.py rename to pyquarantine/conditions.py index 9761c2f..3ec4b88 100644 --- a/pymodmilter/conditions.py +++ b/pyquarantine/conditions.py @@ -1,15 +1,15 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # __all__ = ["Conditions"] @@ -18,8 +18,8 @@ import logging import re from netaddr import IPAddress, IPNetwork, AddrFormatError -from pymodmilter import CustomLogger -from pymodmilter.whitelist import DatabaseWhitelist +from pyquarantine import CustomLogger +from pyquarantine.whitelist import DatabaseWhitelist class Conditions: diff --git a/pymodmilter/config.py b/pyquarantine/config.py similarity index 98% rename from pymodmilter/config.py rename to pyquarantine/config.py index 6b6f336..c729ad3 100644 --- a/pymodmilter/config.py +++ b/pyquarantine/config.py @@ -1,15 +1,15 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # __all__ = [ diff --git a/pymodmilter/docs/pymodmilter.conf.example b/pyquarantine/docs/pyquarantine.conf.example similarity index 96% rename from pymodmilter/docs/pymodmilter.conf.example rename to pyquarantine/docs/pyquarantine.conf.example index e7e5d02..c062f75 100644 --- a/pymodmilter/docs/pymodmilter.conf.example +++ b/pyquarantine/docs/pyquarantine.conf.example @@ -1,4 +1,4 @@ -# This is an example /etc/pymodmilter.conf file. +# This is an example /etc/pyquarantine/pyquarantine.conf file. # Copy it into place before use. # # The file is in JSON format. @@ -176,14 +176,14 @@ # Notes: Path to a file which contains the html representation of the disclaimer. # Value: [ FILE_PATH ] # - "html_template": "/etc/pymodmilter/templates/disclaimer_html.template", + "html_template": "/etc/pyquarantine/templates/disclaimer_html.template", # Option: text_template # Type: String # Notes: Path to a file which contains the text representation of the disclaimer. # Value: [ FILE_PATH ] # - "text_template": "/etc/pymodmilter/templates/disclaimer_text.template", + "text_template": "/etc/pyquarantine/templates/disclaimer_text.template", # Option: error_policy # Type: String diff --git a/pymodmilter/docs/templates/disclaimer_html.template b/pyquarantine/docs/templates/disclaimer_html.template similarity index 100% rename from pymodmilter/docs/templates/disclaimer_html.template rename to pyquarantine/docs/templates/disclaimer_html.template diff --git a/pymodmilter/docs/templates/disclaimer_text.template b/pyquarantine/docs/templates/disclaimer_text.template similarity index 100% rename from pymodmilter/docs/templates/disclaimer_text.template rename to pyquarantine/docs/templates/disclaimer_text.template diff --git a/pyquarantine/docs/templates/notification.template b/pyquarantine/docs/templates/notification.template new file mode 100644 index 0000000..f41a4ad --- /dev/null +++ b/pyquarantine/docs/templates/notification.template @@ -0,0 +1,29 @@ + + +

Quarantine notification

+ + + + + + + + + + + + + + + + + + + + + +
Envelope-From:{ENVELOPE_FROM}
From:{FROM}
Envelope-To:{ENVELOPE_TO}
To:{TO}
Subject:{SUBJECT}

+

Preview of the original e-mail

+ {HTML_TEXT} + + diff --git a/pyquarantine/docs/templates/removed.png b/pyquarantine/docs/templates/removed.png new file mode 100644 index 0000000000000000000000000000000000000000..a5081b7f7c5d1a977fecaaca8076afd98b71283a GIT binary patch literal 2136 zcmZvec|6p67sr3Y2xZWyT!~y{xw13Dlr>8X8s(OK8%)N?jJ+@nYS1*6+_BSG>K2|X z7g-`}V;QncBfG{L5^gn?`*82`yk5^A&mZUeI_G@P`F{U-pU>-%>}<>-$Il)I003fX zVTuF*5cJS@=i@p&U3=~vU4ZaNb7P=l;N0?I0QNOP7y-bm$0znYcn)L!yB4?c03g_Y zWT5WAXWjt7yJl%>bR*n#E!QejwoEXwu?hS*%Jmc?0EU^WoQ>UY8~3?@rLQ|XJ65`2K!8{Ul7i5W5wsM94)}dkOfeDHgmLeQR5Am~2Dz&d>6gBw=!ET@yTP`o@1%n5R3s|H2M) zqrA#{Ic*drEL2oFm{Ws7%@>vC_;|?#0mhgtJ%p;O!4Rjvi?u_=4J9Gwq)5bwk`74` zBPgp=jG3Iq>AzhY9mRTBzh0?ar%V3<^d~+V)VMyX{#Nl4(~YO%YlD5*CRU6;NN$cDIf~FfYwFER72- zz0M!7dek)+1lMPE>Q+%}yM|Sja#9IZ6a<#N2gIgIr25u*70MUed&_+8Q6fwPDJ^={ zL@uX#S104W)%z8dbt2Qu(A2C6kmFH@J#}DpKy(#cp(-BI|2-1UPP-^{^Zl>o0tE?+ z+vCX>)^8}aL$1k{3!I-h6DGdJ6X=olf|yR!Fi+ zr$D>FcfKkeQb7T^^PaWzVh^>(1_p1Bo)RZx|v_~>n7X^Alsz(_5luF8A@mW z^SdJo%Wjem*&px-vzW?1;eh5RM(}v#-hI`y-TbVI6NC+}(@pD2pnCL5-ZA%TCRyux z_#fY7KQ!;=A2UzuR~g_>j)-E7486FDVQ%B(q+e>^zO1gnzC^SMbAB1=$+Q?lI~eqI zP1mVEfJf!pp5so(nR`EI;4#e4Qpsv-JB*)Y* z@CDQh?A$Yv_eyNb(4pt0-G;1Unc|Y;-sQrEtr1ow(pK`;9lfLoqcQqW$__0p(XvD~fHD zr>l;-H^XTz)k%&F`iIIvJ?U{p;^v6WM~de$7e4*%kiGj-RFe?jfCBA1*OY}J#OOE~qsjF$)-0|0Xn&--b-l$h@c57`` zZ?{%=a#ErL=fxy8ulvU>yVy@TlaxDy&1wsk@>`RARS+VK1kZQS(TmozxL{U|?!De_ zi?RlVHYl%&@mL53Z#PZniuxQx!N9exxTItz=s)%|EpGV-WHaukIc%um>zc(JDe27k zKlSQx(GpN%gGrZ+bp}K=>?8dP_jQtmQhPzdxCrrGl69@dp2L+j?0mEP#B9kr&hO-S zh~Wimih{W7cH!wsyxnG9`O=jWs`}M9)6ca}?+HRR7nN?8;ePZnVDXNp2H#`%5xrM^ z8lG$|rtf0x$xgXnO0H}cKT{-oJ;PDW)4&RrAGDH}`ZW#m z{LIVK)4cQMx<3q0mKFQ(jL9V;D-v2yrfZ9@@-wa@gm3w-rHMpu z%3o@=B5k;XzKfg#`Uq8^KyQvi`;i^djRb?58i1Eqoj|7g$ncW%${r zlgL=A9VdfI)wJ1P=peQc>O6jX)S$W2-_0AZ(|bzfNN{hlGal~y{oS;$FD72C#OTA7L(`QLS&A^dDJIw3Ww! zzIA+`N)(EB?`fZKg-3Nb>brDIE|*JDacVz(u`w$sE_wD%Y)uVa;!g;@));0x@6blb zq5REBBIl5q^CM2mtvNOVj<#?!z@&G|)k9!ax2u!!^pVzvRL>B&Jdbkw;faO&i K(+XqvxPJpND*fjG literal 0 HcmV?d00001 diff --git a/pymodmilter/mailer.py b/pyquarantine/mailer.py similarity index 92% rename from pymodmilter/mailer.py rename to pyquarantine/mailer.py index 51a913c..b5bea6c 100644 --- a/pymodmilter/mailer.py +++ b/pyquarantine/mailer.py @@ -1,15 +1,15 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # import logging diff --git a/pymodmilter/misc/openrc/pymodmilter.confd b/pyquarantine/misc/openrc/pyquarantine-milter.confd similarity index 56% rename from pymodmilter/misc/openrc/pymodmilter.confd rename to pyquarantine/misc/openrc/pyquarantine-milter.confd index 6c5e71d..277e442 100644 --- a/pymodmilter/misc/openrc/pymodmilter.confd +++ b/pyquarantine/misc/openrc/pyquarantine-milter.confd @@ -1,8 +1,8 @@ -# /etc/conf.d/pymodmilter: config file for /etc/init.d/pymodmilter +# /etc/conf.d/pyquarantine: config file for /etc/init.d/pyquarantine # Start the daemon as the user. You can optionally append a group name here also. # USER="daemon" # USER="daemon:nobody" -# Optional parameters for pymodmilter +# Optional parameters for pyquarantine # MILTER_OPTS="" diff --git a/pymodmilter/misc/openrc/pymodmilter.initd b/pyquarantine/misc/openrc/pyquarantine-milter.initd similarity index 95% rename from pymodmilter/misc/openrc/pymodmilter.initd rename to pyquarantine/misc/openrc/pyquarantine-milter.initd index 70160aa..8bf697f 100755 --- a/pymodmilter/misc/openrc/pymodmilter.initd +++ b/pyquarantine/misc/openrc/pyquarantine-milter.initd @@ -4,7 +4,7 @@ user=${USER:-daemon} milter_opts="${MILTER_OPTS:-}" pidfile="/run/${RC_SVCNAME}.pid" -command="/usr/bin/pymodmilter" +command="/usr/bin/pyquarantine-milter" command_args="${milter_opts}" command_background=true command_user="${user}" diff --git a/pymodmilter/modify.py b/pyquarantine/modify.py similarity index 97% rename from pymodmilter/modify.py rename to pyquarantine/modify.py index 6cdfac0..f57e460 100644 --- a/pymodmilter/modify.py +++ b/pyquarantine/modify.py @@ -1,15 +1,15 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # __all__ = [ @@ -30,8 +30,8 @@ from copy import copy from email.message import MIMEPart from email.policy import SMTPUTF8 -from pymodmilter import replace_illegal_chars -from pymodmilter.base import CustomLogger +from pyquarantine import replace_illegal_chars +from pyquarantine.base import CustomLogger class AddHeader: diff --git a/pymodmilter/notify.py b/pyquarantine/notify.py similarity index 97% rename from pymodmilter/notify.py rename to pyquarantine/notify.py index f1ef95e..56ca4ba 100644 --- a/pymodmilter/notify.py +++ b/pyquarantine/notify.py @@ -1,15 +1,15 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # __all__ = [ @@ -30,8 +30,8 @@ from html import escape from os.path import basename from urllib.parse import quote -from pymodmilter.base import CustomLogger -from pymodmilter import mailer +from pyquarantine.base import CustomLogger +from pyquarantine import mailer class BaseNotification: diff --git a/pymodmilter/rule.py b/pyquarantine/rule.py similarity index 89% rename from pymodmilter/rule.py rename to pyquarantine/rule.py index a596e34..e0e003b 100644 --- a/pymodmilter/rule.py +++ b/pyquarantine/rule.py @@ -1,21 +1,21 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # __all__ = ["Rule"] -from pymodmilter.action import Action -from pymodmilter.conditions import Conditions +from pyquarantine.action import Action +from pyquarantine.conditions import Conditions class Rule: diff --git a/pymodmilter/run.py b/pyquarantine/run.py similarity index 83% rename from pymodmilter/run.py rename to pyquarantine/run.py index 22548b7..4cbce22 100644 --- a/pymodmilter/run.py +++ b/pyquarantine/run.py @@ -1,15 +1,15 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # __all__ = ["main"] @@ -20,25 +20,25 @@ import logging import logging.handlers import sys -from pymodmilter import mailer -from pymodmilter import ModifyMilter -from pymodmilter import __version__ as version -from pymodmilter.config import get_milter_config +from pyquarantine import mailer +from pyquarantine import ModifyMilter +from pyquarantine import __version__ as version +from pyquarantine.config import get_milter_config def main(): python_version = ".".join([str(v) for v in sys.version_info[0:3]]) python_version = f"{python_version}-{sys.version_info[3]}" - "Run PyMod-Milter." + "Run pyquarantine." parser = argparse.ArgumentParser( - description="PyMod milter daemon", + description="pyquarantine-milter daemon", formatter_class=lambda prog: argparse.HelpFormatter( prog, max_help_position=45, width=140)) parser.add_argument( "-c", "--config", help="Config file to read.", - default="/etc/pymodmilter/pymodmilter.conf") + default="/etc/pyquarantine/pyquarantine.conf") parser.add_argument( "-s", @@ -125,10 +125,10 @@ def main(): sysloghandler = logging.handlers.SysLogHandler( address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) sysloghandler.setFormatter( - logging.Formatter("pymodmilter: %(message)s")) + logging.Formatter("pyquarantine: %(message)s")) root_logger.addHandler(sysloghandler) - logger.info("pymodmilter starting") + logger.info("pyquarantine-milter starting") # register milter factory class Milter.factory = ModifyMilter @@ -139,13 +139,13 @@ def main(): rc = 0 try: - Milter.runmilter("pymodmilter", socketname=socket, timeout=600) + Milter.runmilter("pyquarantine", socketname=socket, timeout=600) except Milter.milter.error as e: logger.error(e) rc = 255 mailer.queue.put(None) - logger.info("pymodmilter stopped") + logger.info("pyquarantine-milter stopped") sys.exit(rc) diff --git a/pymodmilter/storage.py b/pyquarantine/storage.py similarity index 97% rename from pymodmilter/storage.py rename to pyquarantine/storage.py index 9dd54af..597e2dd 100644 --- a/pymodmilter/storage.py +++ b/pyquarantine/storage.py @@ -1,15 +1,15 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # __all__ = [ @@ -27,10 +27,10 @@ from datetime import datetime from glob import glob from time import gmtime -from pymodmilter.base import CustomLogger -from pymodmilter.conditions import Conditions -from pymodmilter.config import ActionConfig -from pymodmilter.notify import Notify +from pyquarantine.base import CustomLogger +from pyquarantine.conditions import Conditions +from pyquarantine.config import ActionConfig +from pyquarantine.notify import Notify class BaseMailStorage: diff --git a/pymodmilter/whitelist.py b/pyquarantine/whitelist.py similarity index 97% rename from pymodmilter/whitelist.py rename to pyquarantine/whitelist.py index ca34851..34a923a 100644 --- a/pymodmilter/whitelist.py +++ b/pyquarantine/whitelist.py @@ -1,15 +1,15 @@ -# PyMod-Milter is free software: you can redistribute it and/or modify +# pyquarantine is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# PyMod-Milter is distributed in the hope that it will be useful, +# pyquarantine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with PyMod-Milter. If not, see . +# along with pyquarantine. If not, see . # __all__ = [ diff --git a/setup.cfg b/setup.cfg index f3d2f8a..90d059a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [metadata] -version = attr: pymodmilter.__version__ +version = attr: pyquarantine.__version__ diff --git a/setup.py b/setup.py index f9d6689..a27fe09 100644 --- a/setup.py +++ b/setup.py @@ -4,14 +4,14 @@ def read_file(fname): with open(fname, 'r') as f: return f.read() -setup(name = "pymodmilter", +setup(name = "pyquarantine", author = "Thomas Oettli", author_email = "spacefreak@noop.ch", description = "A pymilter based sendmail/postfix pre-queue filter.", license = "GPL 3", keywords = "header quarantine milter", - url = "https://github.com/spacefreak86/pymodmilter", - packages = ["pymodmilter"], + url = "https://github.com/spacefreak86/pyquarantine", + packages = ["pyquarantine"], long_description = read_file("README.md"), long_description_content_type = "text/markdown", classifiers = [ @@ -28,20 +28,23 @@ setup(name = "pymodmilter", include_package_data = True, entry_points = { "console_scripts": [ - "pymodmilter=pymodmilter.run:main" + "pyquarantine-milter=pyquarantine.run:main", + "pyquarantine=pyquarantine.run:main", ] }, data_files = [ ( - "/etc/pymodmilter", + "/etc/pyquarantine", [ - "pymodmilter/docs/pymodmilter.conf.example" + "pyquarantine/docs/pyquarantine.conf.example" ] ), ( - "/etc/pymodmilter/templates", + "/etc/pyquarantine/templates", [ - "pymodmilter/docs/templates/disclaimer_html.template", - "pymodmilter/docs/templates/disclaimer_text.template" + "pyquarantine/docs/templates/disclaimer_html.template", + "pyquarantine/docs/templates/disclaimer_text.template", + "pyquarantine/docs/templates/notification.template", + "pyquarantine/docs/templates/removed.png" ] ) ], diff --git a/test-pymodmilter b/test-pymodmilter deleted file mode 100755 index fbe25a8..0000000 --- a/test-pymodmilter +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python - -import sys -import pymodmilter.run - -if __name__ == '__main__': - sys.exit( - pymodmilter.run.main() - )