From d3f91889afc6203b837f199f24948dbcf75787f7 Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Tue, 14 May 2024 15:19:10 -0700 Subject: [PATCH] Adding mailman Signed-off-by: James Ketrenos --- docker-compose.yml | 84 ++- mailman/.circleci/config.yml | 81 +++ mailman/.github/ISSUE_TEMPLATE/bug_report.md | 11 + mailman/.github/dependabot.yml | 34 + mailman/.github/workflows/main.workflow | 9 + mailman/.github/workflows/main.yml | 28 + mailman/.github/workflows/publish_docs.yml | 24 + mailman/.github/workflows/stale.yml | 28 + mailman/.gitignore | 7 + mailman/LICENSE | 21 + mailman/NEWS.md | 132 ++++ mailman/README.md | 585 ++++++++++++++++++ mailman/build.sh | 35 ++ mailman/core/Dockerfile | 37 ++ mailman/core/Dockerfile.dev | 39 ++ mailman/core/README.md | 154 +++++ mailman/core/VERSION | 1 + mailman/core/assets/exim/25_mm3_macros | 17 + mailman/core/assets/exim/455_mm3_router | 15 + mailman/core/assets/exim/55_mm3_transport | 11 + mailman/core/assets/mailman-hyperkitty.cfg | 9 + mailman/core/assets/mailman.cfg | 35 ++ mailman/core/docker-entrypoint.sh | 237 +++++++ mailman/core/requirements.txt | 5 + mailman/deploy.py | 148 +++++ mailman/docker-compose-mysql.yaml | 70 +++ mailman/docker-compose-postorius.yaml | 68 ++ mailman/docker-compose.yaml | 67 ++ mailman/docs/core.md | 1 + mailman/docs/index.md | 1 + mailman/docs/news.md | 1 + mailman/docs/web.md | 1 + mailman/mkdocs.yml | 19 + mailman/postorius/Dockerfile | 46 ++ mailman/postorius/Dockerfile.dev | 54 ++ mailman/postorius/docker-entrypoint.sh | 147 +++++ mailman/postorius/mailman-web/__init__.py | 0 mailman/postorius/mailman-web/manage.py | 10 + mailman/postorius/mailman-web/settings.py | 357 +++++++++++ mailman/postorius/mailman-web/urls.py | 33 + mailman/postorius/mailman-web/uwsgi.ini | 25 + mailman/postorius/mailman-web/wsgi.py | 38 ++ mailman/tests/docker-test.yaml | 11 + mailman/tests/test.sh | 36 ++ mailman/web/Dockerfile | 54 ++ mailman/web/Dockerfile.dev | 61 ++ mailman/web/README.md | 161 +++++ mailman/web/VERSION | 1 + mailman/web/docker-entrypoint.sh | 154 +++++ mailman/web/mailman-web/__init__.py | 0 mailman/web/mailman-web/manage.py | 10 + mailman/web/mailman-web/settings.py | 424 +++++++++++++ mailman/web/mailman-web/urls.py | 34 + mailman/web/mailman-web/uwsgi.ini | 53 ++ mailman/web/mailman-web/wsgi.py | 38 ++ mailman/web/requirements.txt | 4 + .../apache2/sites-available/ketrenos.com-ssl | 2 +- web/etc/nginx/sites-available/default | 5 + 58 files changed, 3764 insertions(+), 9 deletions(-) create mode 100644 mailman/.circleci/config.yml create mode 100644 mailman/.github/ISSUE_TEMPLATE/bug_report.md create mode 100644 mailman/.github/dependabot.yml create mode 100644 mailman/.github/workflows/main.workflow create mode 100644 mailman/.github/workflows/main.yml create mode 100644 mailman/.github/workflows/publish_docs.yml create mode 100644 mailman/.github/workflows/stale.yml create mode 100644 mailman/.gitignore create mode 100644 mailman/LICENSE create mode 100644 mailman/NEWS.md create mode 100644 mailman/README.md create mode 100755 mailman/build.sh create mode 100644 mailman/core/Dockerfile create mode 100644 mailman/core/Dockerfile.dev create mode 100644 mailman/core/README.md create mode 100644 mailman/core/VERSION create mode 100644 mailman/core/assets/exim/25_mm3_macros create mode 100644 mailman/core/assets/exim/455_mm3_router create mode 100644 mailman/core/assets/exim/55_mm3_transport create mode 100644 mailman/core/assets/mailman-hyperkitty.cfg create mode 100644 mailman/core/assets/mailman.cfg create mode 100755 mailman/core/docker-entrypoint.sh create mode 100644 mailman/core/requirements.txt create mode 100644 mailman/deploy.py create mode 100644 mailman/docker-compose-mysql.yaml create mode 100644 mailman/docker-compose-postorius.yaml create mode 100644 mailman/docker-compose.yaml create mode 120000 mailman/docs/core.md create mode 120000 mailman/docs/index.md create mode 120000 mailman/docs/news.md create mode 120000 mailman/docs/web.md create mode 100644 mailman/mkdocs.yml create mode 100644 mailman/postorius/Dockerfile create mode 100644 mailman/postorius/Dockerfile.dev create mode 100755 mailman/postorius/docker-entrypoint.sh create mode 100644 mailman/postorius/mailman-web/__init__.py create mode 100755 mailman/postorius/mailman-web/manage.py create mode 100644 mailman/postorius/mailman-web/settings.py create mode 100644 mailman/postorius/mailman-web/urls.py create mode 100644 mailman/postorius/mailman-web/uwsgi.ini create mode 100755 mailman/postorius/mailman-web/wsgi.py create mode 100644 mailman/tests/docker-test.yaml create mode 100755 mailman/tests/test.sh create mode 100644 mailman/web/Dockerfile create mode 100644 mailman/web/Dockerfile.dev create mode 100644 mailman/web/README.md create mode 100644 mailman/web/VERSION create mode 100755 mailman/web/docker-entrypoint.sh create mode 100644 mailman/web/mailman-web/__init__.py create mode 100755 mailman/web/mailman-web/manage.py create mode 100644 mailman/web/mailman-web/settings.py create mode 100644 mailman/web/mailman-web/urls.py create mode 100644 mailman/web/mailman-web/uwsgi.ini create mode 100755 mailman/web/mailman-web/wsgi.py create mode 100644 mailman/web/requirements.txt diff --git a/docker-compose.yml b/docker-compose.yml index 1431974..1fbb827 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,8 @@ services: - /home/jketreno/docker/webserver/web/etc/nginx/sites-enabled:/etc/nginx/sites-enabled:ro - /home/jketreno/docker/webserver/web/etc/nginx/sites-available:/etc/nginx/sites-available:ro - /home/jketreno/docker/webserver/web/etc/apache2:/etc/apache2:ro + - /home/jketreno/docker/webserver/keys/cron/etc/letsencrypt/options-ssl-apache.conf:/etc/letsencrypt/options-ssl-apache.conf:ro + - /home/jketreno/docker/webserver/keys/cron/etc/letsencrypt/options-ssl-nginx.conf:/etc/letsencrypt/options-ssl-nginx.conf:ro - /home/jketreno/docker/webserver/keys/cron/etc/letsencrypt/live:/etc/letsencrypt/live:ro - /home/jketreno/docker/webserver/keys/cron/etc/letsencrypt/archive:/etc/letsencrypt/archive:ro - /home/jketreno/docker/webserver/web/entrypoint.sh:/entrypoint.sh:ro @@ -30,10 +32,10 @@ services: dockerfile: Dockerfile.mail restart: always ports: - - 143:143 # dovecot imap - - 993:993 # dovecot imaps - - 465:465 # postfix smtps - - 587:587 # postfix submission + - 143:143 # dovecot imap + - 993:993 # dovecot imaps + - 465:465 # postfix smtps + - 587:587 # postfix submission volumes: - /home/jketreno/docker/webserver/keys/cron/etc/letsencrypt/live:/etc/letsencrypt/live:ro - /home/jketreno/docker/webserver/keys/cron/etc/letsencrypt/archive:/etc/letsencrypt/archive:ro @@ -63,7 +65,6 @@ services: - /home/jketreno/docker/webserver/keys/mail/etc/spamassassin/sa-update-keys/:/etc/spamassassin/sa-update-keys:rw - /home/jketreno/docker/webserver/keys/mail/etc/dovecot/private:/etc/dovecot-private:ro - /home/jketreno/docker/webserver/keys/mail/etc/opendkim:/etc/opendkim-private:rw - # Authentication of dovecot users via pam # #- /etc/passwd:/etc/systempasswd:ro @@ -81,9 +82,9 @@ services: dockerfile: Dockerfile.roundcube restart: always environment: - - ROUNDCUBEMAIL_DEFAULT_HOST=tls://ketrenos.com - - ROUNDCUBEMAIL_SMTP_SERVER=tls://ketrenos.com - - ROUNDCUBEMAIL_SMTP_PORT=587 + - ROUNDCUBEMAIL_DEFAULT_HOST=tls://ketrenos.com + - ROUNDCUBEMAIL_SMTP_SERVER=tls://ketrenos.com + - ROUNDCUBEMAIL_SMTP_PORT=587 ports: - 8124:80 volumes: @@ -129,3 +130,70 @@ services: - /home/jketreno/docker/webserver/dns/entrypoint.sh:/entrypoint.sh:ro - /home/jketreno/docker/webserver/data/log:/var/log:rw - /home/jketreno/docker/webserver/data/dns/var/lib/:/var/lib:rw + + ketrenet-mailman-core: + image: maxking/mailman-core:0.4 # Use a specific version tag (tag latest is not published) + container_name: ketrenet-mailman-core + hostname: mailman-core + restart: unless-stopped + volumes: + - /home/jketreno/docker/webserver/data/mailman/core:/opt/mailman/ + stop_grace_period: 30s + links: + - database:database + depends_on: + - database + environment: + - DATABASE_URL=postgresql://mailman:1mailm3np2ss@database/mailmandb + - DATABASE_TYPE=postgres + - DATABASE_CLASS=mailman.database.postgresql.PostgreSQLDatabase + - HYPERKITTY_API_KEY=s0meap1k3y + ports: + - "127.0.0.1:8001:8001" # API + - "127.0.0.1:8024:8024" # LMTP - incoming emails + networks: + ketrenet-mailman: + + + ketrenet-mailman-web: + image: maxking/mailman-web:0.4 # Use a specific version tag (tag latest is not published) + container_name: ketrenet-mailman-web + hostname: mailman-web + restart: unless-stopped + depends_on: + - database + links: + - mailman-core:mailman-core + - database:database + volumes: + - /home/jketreno/docker/webserver/data/mailman/web:/opt/mailman-web-data + environment: + - DATABASE_TYPE=postgres + - DATABASE_URL=postgresql://mailman:1mailm3np2ss@database/mailmandb + - HYPERKITTY_API_KEY=s0meap1k3y + ports: + - "127.0.0.1:8000:8000" # HTTP + - "127.0.0.1:8080:8080" # uwsgi + networks: + ketrenet-mailman: + + + ketrenet-database: + environment: + - POSTGRES_DB=mailmandb + - POSTGRES_USER=mailman + - POSTGRES_PASSWORD=1mailm@np@ss + image: postgres:12-alpine + volumes: + - /home/jketreno/docker/webserver/data/mailman/database:/var/lib/postgresql/data + networks: + ketrenet-mailman: + + +networks: + ketrenet-mailman: + driver: bridge + ipam: + driver: default + config: + - subnet: 172.19.199.0/24 diff --git a/mailman/.circleci/config.yml b/mailman/.circleci/config.yml new file mode 100644 index 0000000..1cc8dcb --- /dev/null +++ b/mailman/.circleci/config.yml @@ -0,0 +1,81 @@ +version: '2.1' + +jobs: + build: + parameters: + rolling: + type: string + default: "no" + machine: + image: ubuntu-2004:202010-01 + environment: + DOCKER_BUILDKIT: 1 + BUILDKIT_PROGRESS: plain + BUILD_ROLLING: << parameters.rolling >> + steps: + - checkout + - run: + name: Install Python dependencies + command: python3 -m pip install packaging + - restore_cache: + keys: + - python-deps-cache-v1 + - run: + name: Building Container Images + command: ./build.sh << parameters.rolling >> + - save_cache: + key: python-deps-cache-v1 + paths: + - /root/.cache + - run: + environment: + DB: postgres + name: Postgres Tests + command: bash tests/test.sh + - run: + environment: + DB: mysql + name: MySQL Test + command: bash tests/test.sh + - deploy: + name: Deploy + command: | + python3 --version + python3 deploy.py + - store_artifacts: + path: /opt/mailman/web/logs/ + + - store_artifacts: + path: /opt/mailman/core/var/logs + +workflows: + version: 2 + test-stable: + jobs: + - build: + rolling: "no" + filters: + tags: + only: /^v\d+\.\d+\.\d+$/ + + cron-builds: + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: main + jobs: + - build: + rolling: "yes" + context: org-global + + test-rolling: + jobs: + - build: + rolling: "yes" + context: org-global + filters: + branches: + # Forked pull requests have CIRCLE_BRANCH set to pull/XXX + ignore: /pull\/[0-9]+/ diff --git a/mailman/.github/ISSUE_TEMPLATE/bug_report.md b/mailman/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..b409cec --- /dev/null +++ b/mailman/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,11 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: bug +assignees: '' + +--- + +**Tag/version of Container Images** +Choose from: rolling, 0.3, 0.3.*, 0.4.* diff --git a/mailman/.github/dependabot.yml b/mailman/.github/dependabot.yml new file mode 100644 index 0000000..50043f6 --- /dev/null +++ b/mailman/.github/dependabot.yml @@ -0,0 +1,34 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "docker" # See documentation for possible values + directory: "/core" # Location of package manifests + schedule: + interval: "weekly" + - package-ecosystem: "docker" # See documentation for possible values + directory: "/web" # Location of package manifests + schedule: + interval: "weekly" + - package-ecosystem: "docker" # See documentation for possible values + directory: "/postorius" # Location of package manifests + schedule: + interval: "weekly" + # Enable version updates for Actions + - package-ecosystem: "github-actions" + # Look for `.github/workflows` in the `root` directory + directory: "/" + # Check for updates once a week + schedule: + interval: "weekly" + - package-ecosystem: "pip" + directory: "/core" + schedule: + interval: "daily" + - package-ecosystem: "pip" + directory: "/web" + schedule: + interval: "daily" \ No newline at end of file diff --git a/mailman/.github/workflows/main.workflow b/mailman/.github/workflows/main.workflow new file mode 100644 index 0000000..6ebf824 --- /dev/null +++ b/mailman/.github/workflows/main.workflow @@ -0,0 +1,9 @@ +workflow "Add PR to release notes" { + on = "pull_request" + resolves = ["Chronicler"] +} + +action "Chronicler" { + uses = "crosscompile/chronicler-action@v1.0.0" + secrets = ["GITHUB_TOKEN"] +} diff --git a/mailman/.github/workflows/main.yml b/mailman/.github/workflows/main.yml new file mode 100644 index 0000000..d123643 --- /dev/null +++ b/mailman/.github/workflows/main.yml @@ -0,0 +1,28 @@ +# This is a basic workflow to help you get started with Actions + +name: CI + +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master ] + pull_request: + branches: [ master ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: Chronicler Action + # You may pin to the exact commit or the version. + # uses: crosscompile/chronicler-action@5c25dbce26b0789724a92902c69217b46f023b51 + uses: crosscompile/chronicler-action@v1.0.1 diff --git a/mailman/.github/workflows/publish_docs.yml b/mailman/.github/workflows/publish_docs.yml new file mode 100644 index 0000000..856a7d0 --- /dev/null +++ b/mailman/.github/workflows/publish_docs.yml @@ -0,0 +1,24 @@ + + +name: Publish docs via GitHub Pages +on: + push: + branches: + - main + - master + +jobs: + build: + name: Deploy docs + runs-on: ubuntu-latest + steps: + - name: Checkout main + uses: actions/checkout@v4 + + - name: Deploy docs + uses: mhausenblas/mkdocs-deploy-gh-pages@master + # Or use mhausenblas/mkdocs-deploy-gh-pages@nomaterial to build without the mkdocs-material theme + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CUSTOM_DOMAIN: asynchronous.in + CONFIG_FILE: mkdocs.yml diff --git a/mailman/.github/workflows/stale.yml b/mailman/.github/workflows/stale.yml new file mode 100644 index 0000000..bc81445 --- /dev/null +++ b/mailman/.github/workflows/stale.yml @@ -0,0 +1,28 @@ +# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. +# +# You can adjust the behavior by modifying this file. +# For more information, see: +# https://github.com/actions/stale +name: Mark stale issues and pull requests + +on: + schedule: + - cron: '19 22 * * *' + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v9 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'This issue has not been updated for more than 1year' + stale-pr-message: 'This Pull Request has not been updated for more than 1year' + stale-issue-label: 'no-issue-activity' + stale-pr-label: 'no-pr-activity' + days-before-stale: 365 diff --git a/mailman/.gitignore b/mailman/.gitignore new file mode 100644 index 0000000..feb8f08 --- /dev/null +++ b/mailman/.gitignore @@ -0,0 +1,7 @@ +*/.sass-cache/* +*.log +*.log +*.sass-cache/ +/web/mailman-web/settings_local.py +pythonenv3.8/* +.venv/* diff --git a/mailman/LICENSE b/mailman/LICENSE new file mode 100644 index 0000000..095f16e --- /dev/null +++ b/mailman/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Abhilash Raj + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/mailman/NEWS.md b/mailman/NEWS.md new file mode 100644 index 0000000..c00791f --- /dev/null +++ b/mailman/NEWS.md @@ -0,0 +1,132 @@ +# NEWS + +## Upgrading to 0.4.0 Release + +Some configurations in the release are backwards incompatible with +what was working before. + +### Web server configuration + +With 0.4.0 version, we added Port mapping from host's Port 8000/8080 +to mailman-web container's port 8000/8080. Make sure you have this +in your docker-compose.yaml + +```yaml + mailman-web: + ports: + - "127.0.0.1:8000:8000" # HTTP + - "127.0.0.1:8080:8080" # uwsgi +``` + +You should update your web server to proxy 127.0.0.1:8000. + +#### Nginx + +Update the Nginx configuration to look like this, notice the +actual **URL for `proxy_pass` is the only thing that has changed** +along with some options like `uwsgi_read_timeout` and `include uwsgi_params` +that have been removed from the previous version. + +``` + location / { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } +``` + +For other web servers like Apache2, update the URL accordingly. + +**Note** that if you are using `uwsgi_pass` instead of `proxy_pass` +then you should update the URL accordingly to `https://127.0.0.1:8080`. + +### MTA configuration + +MTA configuration needs updating to ensure that all IPs from the +`172.19.199.0/24` subnet is added to `mynetworks` in Postfix configs. + +Please verify that the network configuration generated by the containers +look like this: + +```bash +$ docker exec mailman-core cat /etc/mailman.cfg +# This file is autogenerated at container startup. +[database] +class: mailman.database.postgresql.PostgreSQLDatabase +url: postgres://mailman:mailmanpass@database/mailmandb +[runner.retry] +sleep_time: 10s + +[webservice] +hostname: 172.19.199.3 +port: 8001 +admin_user: restadmin +admin_pass: restpass +configuration: /etc/gunicorn.cfg + +[mta] +incoming: mailman.mta.postfix.LMTP +outgoing: mailman.mta.deliver.deliver +lmtp_host: 172.19.199.3 +lmtp_port: 8024 +smtp_host: 172.19.199.1 +smtp_port: 25 +configuration: /etc/postfix-mailman.cfg + +[archiver.hyperkitty] +class: mailman_hyperkitty.Archiver +enable: yes +configuration: /etc/mailman-hyperkitty.cfg +``` + +**Note that lmtp_host and webserver hostname can be different than +before since new containers don't have static IP addresses. They +are automatically parsed from the output of "ip route" command +from inside mailman-core container.** + +You can verify that the IP address of the containers by running the +following commands, note that the **output can be different** and it is +fine if that is the case. + +```bash +$ docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mailman-core +172.19.199.3 +$ docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mailman-web +172.19.199.4 +``` + +---- +## Mailman Core + +### v1.1.1 (released Aug 9 2017) + +- The MM_HOSTNAME now defaults to output of `hostname -i` instead of `mailman-core`. This + is the hostname Core binds to for Webservice. +- Added pymysql to the image to use MySQL as database. +- The default settings for using SQLITE are now more sane. +- Postfix's transport maps are generated at the container startup now even when + there is no lists exist. + + +## Mailman Web + +### v1.1.1 (released Aug 9 2017) + +- The default search_index for whoosh now exists on persistent storage at + `/opt/mailman-web-data` +- Move to using Alpine instead of Debian for this image, python2.7:alpine-3.6 + image is now the base image +- Django compressor is now using `sassc` from alpine repo. +- Default value of SECRET_KEY is now removed. It is MUST to set SECRET_KEY + environment variable to run this image now. +- If a SERVE_FROM_DOMAIN environment variable is defined, the default Django's + example.com site is renamed to this domain. The SITE_ID remains same so there + is no change required to serve this domain. +- If MAILMAN_ADMIN_USER and MAILMAN_ADMIN_EMAIL environment variables are + defined a Django Superuser is created by default. The password for this user + would have to be reset on the first login. +- Fix cron configuration which would run them in wrong order. +- Removed facebook as default social auth provider in the settings.py +- Uwsgi now listens on port 8080 for uwsgi protocol and 8000 for http protocol. +- Threads are enabled by default in the uwsgi configuration now. +- Hyperkitty updated to v1.1.1 diff --git a/mailman/README.md b/mailman/README.md new file mode 100644 index 0000000..c5988de --- /dev/null +++ b/mailman/README.md @@ -0,0 +1,585 @@ +--- +permalink: / +--- + + +# GNU Mailman 3 Deployment with Docker + + + +[![CircleCI](https://circleci.com/gh/maxking/docker-mailman/tree/main.svg?style=svg)](https://circleci.com/gh/maxking/docker-mailman/tree/main) + +This repository hosts code for two docker images `maxking/mailman-core` and +`maxking/mailman-web` both of which are meant to deploy [GNU Mailman 3][1] in +a production environment. + +[Docker][2] is a container ecosystem which can run containers on several +platforms. It consists of a tool called [docker-compose][3] which can be used to +run multi-container applications. This repository consists of a +[`docker-compose.yaml`][19] file which is a set of +configurations that can be used to deploy the [Mailman 3 Suite][4]. + +Please see [release page](https://github.com/maxking/docker-mailman/releases) +for the releases and change log. + +## Release + + +The tags for the images are assumed to be release versions for images. This is +going to be a somewhat common philosophy of distributing Container images where +the images with same tags are usually updated with the new functionality. + +Releases will follow the following rules: + +* Images tagged like A.B.C will never change. If you want to pin down versions + of Images, use these tags. + +* Images tagged with A.B will correspond to the latest A.B.C version + released. Releases in A.B series are supposed to be backwards compatible, + i.e., any existing installation should not break when upgrading between + subversions of A.B.C. So, if you want the latest updates and want to + frequently update your installation without having to change the version + numbers, you can use this. + +* Any changes in the minor version of Mailman components of the images will + cause a bump in the minor version, e.g., A.(B+1) can have one or more + updated Mailman components from A.B. Also, significant change in functionality, + that might change how Images work or how people interact with the containers + can also cause a bump in the minor version. + +* Major versions will change either when there are backwards incompatible + changes or when the releases reach a certain set milestone or when there are + bugfix releases for the internal components or both. + + +## Container Registries + +The container images are available from multiple container registries. Do specify an [explicit version tag](https://hub.docker.com/r/maxking/mailman-web/tags?page=1&ordering=last_updated&name=0.) (e.g. `0.4.5` , MAJOR.MINOR like `0.4` also works as floating tag pointing to latest patch version) as tag `latest` is **not** updated anymore. + +### Mailman Core + +- `ghcr.io/maxking/mailman-core` +- `docker.io/maxking/mailman-core` + +### Mailman Web + +- `ghcr.io/maxking/mailman-web` +- `docker.io/maxking/mailman-web` + +### Postorius + +- `ghcr.io/maxking/postorius` +- `docker.io/maxking/postorius` + +## Rolling Releases + +Rolling releases are made up of Mailman Components installed from [git +source](https://gitlab.com/mailman). **Note that these releases are made up of +un-released software and should be assumed to be beta quality.** + +Every commit is tested with Mailman's CI infrastructure and is included in +rolling releases only if they have passed the complete test suite. + +```bash +$ docker pull docker.io/maxking/mailman-web:rolling +$ docker pull docker.io/maxking/mailman-core:rolling +``` + +Rolling releases are built with every commit and also re-generated nightly. You +can inspect the images to get which commit it was built using: + +```bash +$ docker inspect --format '{{json .Config.Labels }}' mailman-core | python -m json.tool +{ + "version.git_commit": "45a4d7805b2b3d0e7c51679f59682d64ba02f05f", +} + +$ docker inspect --format '{{json .Config.Labels }}' mailman-web | python -m json.tool +{ + "version.git_commit": "45a4d7805b2b3d0e7c51679f59682d64ba02f05f", +} + +``` + +- `version.git_commit` : This is the commit hash of the Dockerfile in the + [Github repo](https://github.com/maxking/docker-mailman) + +## Dependencies + +- Docker +- Docker-compose + +To install these on Ubuntu/Debian: + +``` +$ sudo apt install docker.io docker-compose +``` + +For other systems, you can read the official Docker documentation to install +[Docker from here][5] and [docker-compose from here][6]. + + +## Configuration + +Most of the common configuration is handled through environment variables in the +`docker-compose.yaml`. However, there is need for some extra configuration that +interacts directly with the application. There are two configuration files on +the host that interact directly with Mailman's settings. These files exist on +the host running the containers and are imported at runtime in the containers. + +* `/opt/mailman/core/mailman-extra.cfg` : This is the configuration for Mailman + Core and anything that you add here will be added to Core's configuration. You + need to restart your mailman-core container for the changes in this file to + take effect. + +* `/opt/mailman/web/settings_local.py` : This is the Django configuration that + is imported by the [existing configuration][2] + provided by the mailman-web container. **This file is referred to as + `settings.py` in most of the Postorius and Django documentation.** To change + or override any settings in Django/Postorius, you need to create/edit this file. + A useful configuration for troubleshooting is `DEBUG = True`. + +[2]: https://github.com/maxking/docker-mailman/blob/master/web/mailman-web/settings.py + +Also, note that if you need any other files to be accessible from the host to +inside the container, you can place them at certain directories which are +mounted inside the containers. + + +* `/opt/mailman/core` in host maps to `/opt/mailman/` in mailman-core container. +* `/opt/mailman/web` in host maps to `/opt/mailman-web-data` in mailman-web + container. + +### Mailman-web +These are the settings that you MUST change in your docker-compose.yaml before deploying: + +- `SERVE_FROM_DOMAIN`: The domain name from which Django will be served. To be + added to `ALLOWED_HOSTS` in django settings. Default value is not set. This + also replaces Django's default `example.com` SITE and becomes the default SITE + (with SITE_ID=1). + +- `HYPERKITTY_API_KEY`: Hyperkitty's API Key, should be set to the same value + as set for the mailman-core. (Not needed in case of Postorius-only version.) + +- `MAILMAN_ADMIN_USER`: The username for the admin user to be created by default. + +- `MAILMAN_ADMIN_EMAIL`: The email for the admin user to be created by default. + +- `SECRET_KEY`: Django's secret key, mainly used for signing cookies and others. + +Please note here that if you choose to create the admin user using the +environment variables mentioned above (`MAILMAN_ADMIN_USER` & +`MAILMAN_ADMIN_EMAIL`), no password is set for your admin account. To set a +password, plese follow the "Forgot Password" link on the "Sign In" page. + +Mailman web is already configured to send emails through `$SMTP_HOST` as the +MTA's address. If you want to modify it, you can set the value in under +docker-compose.yaml for mailman-web container. By default, `SMTP_HOST` points +to the gateway of the web container, which is the host itself. + +You can also use the environment variables `SMTP_HOST` (defaults to +the container's gateway), `SMTP_PORT` (defaults to `25`), `SMTP_HOST_USER` (defaults to +an empty string), `SMTP_HOST_PASSWORD` (defaults to an empty string), +`SMTP_USE_TLS` (defaults to `False`) and `SMTP_USE_SSL` (defaults to `False`). + +This is required in addition to the [Setup your MTA](#setting-up-your-mta) +section below, which covers email setup for Mailman Core. + +For more details on how to configure this image, please look at +[Mailman-web's Readme](web/) + +### Mailman-Core + +These are the variables that you MUST change in your docker-compose.yaml before deploying: + +- `HYPERKITTY_API_KEY`: Hyperkitty's API Key, should be set to the same value as + set for the mailman-web. Skip the variable in case of non-Hyperkitty deployment. + +- `DATABASE_URL`: URL of the type + `driver://user:password@hostname:port/databasename` for the django to use. If + not set, the default is set to + `sqlite:///opt/mailman-web-data/mailmanweb.db`. The standard + docker-compose.yaml comes with it set to a postgres database. There is no need + to change this if you are happy with PostgreSQL. + +- `DATABASE_TYPE`: Its value can be one of `sqlite`, `postgres` or `mysql` as + these are the only three database types that Mailman 3 supports. Its default + value is set to `sqlite` along with the default database class and default + database url above. + +- `DATABASE_CLASS`: Default value is + `mailman.database.sqlite.SQLiteDatabase`. The values for this can be found in + the mailman's documentation [here][11]. +- `SMTP_HOST` : outgoing host for SMTP connections +- `SMTP_PORT` : use this port. 25, 587, whatever your host asks for. +- `SMTP_HOST_USER`: authenticate this user +- `SMTP_HOST_PASSWORD`: and use this password + +For more details on how to configure this image, please look [Mailman-core's +Readme](core/) + + +While the above configuration will allow you to run the images and possibly view +the Web Frontend, it won't be functional until it is fully configured to to send +emails. + +To configure the mailman-core container to send emails, see the [Setting your MTA +section below](#setting-up-your-mta). + +## Running + +To run the containers, simply run: + +```bash +$ mkdir -p /opt/mailman/core +$ mkdir -p /opt/mailman/web +$ git clone https://github.com/maxking/docker-mailman +$ cd docker-mailman +# Change some configuration variables as mentioned above. +$ docker-compose up -d +``` + +Note that the web frontend in the mailman-web container is, by default, only +configured to serve dynamic content. Anything static like stylesheets, etc., is +expected to be served directly by the web server. The static content exists at +`/opt/mailman/web/static` and should be _aliased_ to `/static/` in the web +server configuration. + +See [the nginx configuration][17] as an example. + +This command will do several things, most importantly: + +- Run a wsgi server using [`uwsgi`][7] for the Mailman's Django-based web + frontend listening on port 8000. It will run 2 worker + processes with 4 threads each. You may want to change the setting + `ALLOWED_HOSTS` in the settings before deploying the application in + production. + +- Run a PostgreSQL server with a default database, username, and password as + mentioned in the `docker-compose.yaml`. You will have to change configuration + files too if you change any of these. + +- Run mailman-core listening on port 8001 for REST API and port 8024 (LMTP + server) for messages from your MTA. You will have to configure your MTA to + send messages at this address. + +Some more details about what the above system achieves is mentioned below. If you +are only going to deploy a simple configuration, you don't need to read +this. However, these are very easy to understand if you know how docker works. + +- First create a bridge network called `mailman` in the + `docker-compose.yaml`. It will probably be named something else in your + machine. All the containers + mentioned (mailman-core, mailman-web, database) will join this network and are + assigned static IPs. The host operating system is the default gateway + from within these containers. + +- Spin off a mailman-core container attached to the mailman bridge network created above. It has + GNU Mailman 3 core running inside it. Mailman core's REST API is available at + port 8001 and LMTP server listens at port 8024. + +- Spin off a mailman-web container which has a Django application running with + both Mailman's web frontend Postorius and Mailman's web-based Archiver + running. [Uwsgi][7] server is used to run a web server with the configuration + provided in this repository [here][2]. You may want to + change the setting `ALLOWED_HOSTS` in the settings before deploying the + application in production. You can do that by adding a + `/opt/mailman/web/settings_local.py` which is imported by the Django when + running. + +- Spin off a PostgreSQL database container which is used by both mailman-core + and mailman-web as their primary database. + +- mailman-core mounts `/opt/mailman/core` from host OS at `/opt/mailman` in the + container. Mailman's var directory is stored there so that it is accessible + from the host operating system. Configuration for Mailman core is generated on + every run from the environment variables provided. Extra configuration can + also be provided at `/opt/mailman/core/mailman-extra.cfg` (on host), and will + be added to generated configuration file. Mailman also needs another + configuration file called + [mailman-hyperkitty.cfg][3] and is also + expected to be at `/opt/mailman/core/` on the host OS. + +[3]: https://github.com/maxking/docker-mailman/blob/master/core/assets/mailman-hyperkitty.cfg + +- mailman-web mounts `/opt/mailman/web` from the host OS to + `/opt/mailman-web-data` in the container. It consists of the logs and + settings_local.py file for Django. + +- database mounts `/opt/mailman/database` at `/var/lib/postgresql/data` so that + PostgreSQL can persist its data even if the database containers are + updated/changed/removed. + +## Setting up your MTA + +The provided docker containers do not have an MTA in-built. You can either run +your own MTA inside a container and have them relay emails to the mailman-core +container or just install an MTA on the host and have them relay emails. + +### Exim4 + +To use [Exim4][8], it should be setup to relay emails from mailman-core and +mailman-web. The mailman specific configuration is provided in the +repository at `core/assets/exim`. There are three files + +- [25_mm3_macros](core/assets/exim/25_mm3_macros) to be placed at + `/etc/exim4/conf.d/main/25_mm3_macros` in a typical Debian install of + exim4. Please change MY_DOMAIN_NAME to the domain name that will be used to + serve mailman. Multi-domain setups will be added later. + +- [455_mm3_router](core/assets/exim/455_mm3_router) to be placed at + `/etc/exim4/conf.d/router/455_mm3_router` in a typical Debian install of exim4. + +- [55_mm3_transport](core/assets/exim/55_mm3_transport) to be placed at + `/etc/exim4/conf.d/transport/55_mm3_transport` in a typical Debian install of exim4. + + +Also, the default configuration inside the mailman-core image has the MTA set +to Exim, but just for reference, it looks like this: + +``` +# mailman.cfg +[mta] +incoming: mailman.mta.exim4.LMTP +outgoing: mailman.mta.deliver.deliver +lmtp_host: $MM_HOSTNAME +lmtp_port: 8024 +smtp_host: $SMTP_HOST +smtp_port: $SMTP_PORT +configuration: python:mailman.config.exim4 +``` + +### Postfix + +To use [Postfix][12], edit the `main.cf` configuration file, which is typically +at `/etc/postfix/main.cf` on Debian-based operating systems. Add +mailman-core and mailman-web to `mynetworks` so it will relay emails from +the containers and add the following configuration lines: + +``` +# main.cf + +# Support the default VERP delimiter. +recipient_delimiter = + +unknown_local_recipient_reject_code = 550 +owner_request_special = no + +transport_maps = + regexp:/opt/mailman/core/var/data/postfix_lmtp +local_recipient_maps = + regexp:/opt/mailman/core/var/data/postfix_lmtp +relay_domains = + regexp:/opt/mailman/core/var/data/postfix_domains +``` + +To configure Mailman to use Postfix, add `MTA=postfix` under mailman-core's +environment section in the `docker-compose.yaml`: + +``` + mailman-core: + + environment: + - MTA=postfix +``` + +This will auto-generate the configuration to talk to Postfix assuming that +Postfix is available at the gateway address for the container's bridge network +at port 25. The final configuration can be found by executing: + +``` +$ docker exec mailman-core cat /etc/mailman.cfg +``` + +The postfix configuration that is generated looks like this: +``` +[mta] +incoming: mailman.mta.postfix.LMTP +outgoing: mailman.mta.deliver.deliver +lmtp_host: $MM_HOSTNAME +lmtp_port: 8024 +smtp_host: $SMTP_HOST +smtp_port: $SMTP_PORT +configuration: /etc/postfix-mailman.cfg +``` + +So, if you need to update the values, you can set `SMTP_HOST`, `SMTP_PORT`, +`MM_HOSTNAME` environment variables in `mailman-core` container. + +Please verify the output for `[mta]` section to ensure that it points to +the right `smtp_host` (address to reach postfix from mailman-core container) +and `lmtp_host` (address to reach mailman-core container from postfix). + +The configuration file `/etc/postfix-mailman.cfg` is also generated automatically +inside the `mailman-core` container and contains the configuration specific +for Postfix. + +## Site Owner + +Setup site owner address. By default, mailman is setup with the site_owner set to 'changeme@example.com'. This should be pointing to a valid mailbox. Add the following to the '/opt/mailman/core/mailman-extra.cfg'. + +``` +[mailman] +# This address is the "site owner" address. Certain messages which must be +# delivered to a human, but which can't be delivered to a list owner (e.g. a +# bounce from a list owner), will be sent to this address. It should point to +# a human. +site_owner: changeme@example.com +``` + +## Setting up search indexing + +Hyperkitty in mailman-web image support full-text indexing. The current default +indexing engine is [Whoosh](https://whoosh.readthedocs.io/en/latest/intro.html) +for historical reasons. It is highly recommended that you instead use Xapian for +production use cases. The default will change when the next major version bump +happens. + +To configure your Mailman-web container to use Xapian, add the following to your +`settings_local.py`: + +```python +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'xapian_backend.XapianEngine', + 'PATH': "/opt/mailman-web-data/fulltext_index", + }, +} +``` + +If you have been using the default search indexing engine, you might have to +re-index emails using the following command: + +```bash +$ docker-compose exec mailman-web ./manage.py rebuild_index +``` + +This command can take some time if you a lot of emails, so please be patient! + +## Setting up your web server + +It is advisable to run your Django (interfaced through WSGI server) through an +_actual_ webserver in production for better performance. + +If you are using v0.1.0, the uwsgi server is configured to listen to requests at +port `8000` using the `HTTP` protocol. Make sure that you preserve the `HOST` +header when you proxy the requests from your Web Server. In Nginx you can do +that by adding the following to your configuration: + +``` + # Nginx configuration. + location /static { + alias /opt/mailman/web/static; + autoindex off; + } + + location / { + proxy_pass http://127.0.0.1:8000; + include uwsgi_params; + uwsgi_read_timeout 300; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + +``` + +Make sure you are using `proxy_pass` for the `HTTP` protocol. + +### uwsgi + + +Starting from v0.1.1, the uwsgi server is configured to listen to requests at +port `8000` with the http protocol and port `8080` for the uwsgi +protocol. + +**Please make sure that you are using port 8080 for uwsgi protocol.** + +It is advised to use the uwsgi protocol as it has better performance. Both +Apache and Nginx have native support for the uwsgi protocol through plugins which +are generally included in the distro packages. + +To move to uwsgi protocol in the above nginx configuration use this + +``` + # Nginx configuration. + location /static { + alias /opt/mailman/web/static; + autoindex off; + } + + location / { + uwsgi_pass localhost:8080; + include uwsgi_params; + uwsgi_read_timeout 300; + } +``` + +Please make sure that you are using v0.1.1 or greater if you use this configuration. + + +### Serving static files + +UWSGI by default doesn't serve static files so, when running +`mailman-web` using the provided `docker-compose.yaml` file, you won't see any +CSS or JS files being served. + +To enable serving of static files using UWSGI, add the following environment +variable to your `docker-compose.yaml` file under `mailman-web`: + +``` +UWSGI_STATIC_MAP=/static=/opt/mailman-web-data/static +``` + +It is recommended to use web-server to serve static files instead of UWSGI for +better performance. You will have to add an alias rule in your web server to +serve the static files. See [here][18] for instructions on how to configure your +web server. The STATIC_ROOT for you would be `/opt/mailman/web/static`. + +### SSL certificates + +SSL Certificates from Lets Encrypt need to be renewed every 90 days. You can +setup a cron job to do the job. I have this small shell script (certbot-renew.sh) +that you can put up in `/etc/cron.monthly` to get the job done. + +``` +#! /bin/bash + +cd /opt/letsencrypt/ +./certbot-auto --config /etc/letsencrypt/renewal/MY_DOMAIN_NAME.conf certonly + +if [ $? -ne 0 ] + then + ERRORLOG=`tail /var/log/letsencrypt/letsencrypt.log` + echo -e "The Let's Encrypt cert has not been renewed! \n \n" \ + $ERRORLOG + else + nginx -s reload +fi + +exit 0 +``` + +**Please do not forget to make the script executable (`chmod +x certbot-renew.sh`).** + +## LICENSE + +This repository is licensed under the MIT License. Please see the LICENSE file for +more details. + +[1]: http://list.org +[2]: https://www.docker.com/ +[3]: https://docs.docker.com/compose/ +[4]: http://docs.mailman3.org/en/latest/ +[5]: https://docs.docker.com/engine/installation/ +[6]: https://docs.docker.com/compose/install/ +[7]: https://uwsgi-docs.readthedocs.io/en/latest/ +[8]: http://exim.org/ +[9]: https://letsencrypt.org/ +[10]: https://certbot.eff.org/ +[11]: https://mailman.readthedocs.io/en/latest/src/mailman/docs/database.html +[12]: http://www.postfix.org/ +[13]: http://semver.org/ +[14]: https://docs.docker.com/engine/security/trust/content_trust/ +[15]: http://docs.mailman3.org/en/latest/config-web.html#setting-up-email +[17]: https://docs.mailman3.org/en/latest/install/virtualenv.html#nginx-configuration +[18]: http://docs.list.org/en/latest/pre-installation-guide.html#django-static-files +[19]: https://github.com/maxking/docker-mailman/blob/master/docker-compose.yaml diff --git a/mailman/build.sh b/mailman/build.sh new file mode 100755 index 0000000..9d8fe7a --- /dev/null +++ b/mailman/build.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -ex + +# Since all the dockerfiles now require buildkit features. +export DOCKER_BUILDKIT=1 + +# Set the default value of BUILD_ROLLING to no. +export BUILD_ROLLING="${1:-no}" + +DOCKER=docker + +if [ "$BUILD_ROLLING" = "yes" ]; then + echo "Building rolling releases..." + # Build the mailman-core image. + $DOCKER build -f core/Dockerfile.dev \ + --label version.git_commit="$COMMIT_ID" \ + -t maxking/mailman-core:rolling core/ + + # Build the mailman-web image. + $DOCKER build -f web/Dockerfile.dev \ + --label version.git_commit="$COMMIT_ID" \ + -t maxking/mailman-web:rolling web/ + + # build the postorius image. + $DOCKER build -f postorius/Dockerfile.dev\ + --label version.git_commit="$COMMIT_ID"\ + -t maxking/postorius:rolling postorius/ +else + echo "Building stable releases..." + # Build the stable releases. + $DOCKER build -t maxking/mailman-core:rolling core/ + $DOCKER build -t maxking/mailman-web:rolling web/ + $DOCKER build -t maxking/postorius:rolling postorius/ +fi diff --git a/mailman/core/Dockerfile b/mailman/core/Dockerfile new file mode 100644 index 0000000..65a13e0 --- /dev/null +++ b/mailman/core/Dockerfile @@ -0,0 +1,37 @@ +# syntax = docker/dockerfile:1.3 +# Use 3.15 for Core since it has Python 3.9 +FROM alpine:3.19 + +# Add requirements file. +COPY requirements.txt /tmp/ + +#Install all required packages, add user for executing mailman and set execution rights for startup script +RUN --mount=type=cache,target=/root/.cache \ + apk update \ + && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev \ + libffi-dev \ + # Mailman html to plaintext conversion uses lynx. + # psutil needs linux-headers to compile on musl c library. + && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c lynx tzdata \ + && python3 -m pip install --break-system-packages -U pip setuptools wheel \ + && python3 -m pip install --break-system-packages psycopg2 \ + gunicorn==19.9.0 \ + pymysql \ + -r /tmp/requirements.txt \ + 'importlib-resources<6.0.0' \ + && apk del build-deps \ + && adduser -S mailman + +#Add startup script to container +COPY docker-entrypoint.sh /usr/local/bin/ + +# Change the working directory. +WORKDIR /opt/mailman + +#Expose the ports for the api (8001) and lmtp (8024) +EXPOSE 8001 8024 + +ENV MAILMAN_CONFIG_FILE /etc/mailman.cfg + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["master", "--force"] diff --git a/mailman/core/Dockerfile.dev b/mailman/core/Dockerfile.dev new file mode 100644 index 0000000..7064e34 --- /dev/null +++ b/mailman/core/Dockerfile.dev @@ -0,0 +1,39 @@ +# syntax = docker/dockerfile:1.3 +# Use 3.15 for Core since it has Python 3.9 +FROM alpine:3.19 + +# Set the commits that we are building. +ARG CORE_REF +ARG MM3_HK_REF + +#Install all required packages, add user for executing mailman and set execution +#rights for startup script +RUN --mount=type=cache,target=/root/.cache \ + apk update \ + && apk add --no-cache --virtual build-deps gcc python3-dev musl-dev \ + postgresql-dev git libffi-dev g++ \ + && apk add --no-cache bash su-exec postgresql-client mysql-client \ + curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c tzdata \ + && python3 -m pip install -U --break-system-packages psycopg2 pymysql setuptools wheel \ + && python3 -m pip install --break-system-packages \ + git+https://gitlab.com/mailman/mailman \ + git+https://gitlab.com/mailman/mailman-hyperkitty \ + gunicorn==19.9.0 \ + && apk del build-deps \ + && adduser -S mailman + +#Add startup script to container +COPY docker-entrypoint.sh /usr/local/bin/ + +# Change the working directory. +WORKDIR /opt/mailman + +#Expose the ports for the api (8001) and lmtp (8024) +EXPOSE 8001 8024 + +# Set the default configuration file. +ENV MAILMAN_CONFIG_FILE /etc/mailman.cfg + +ENTRYPOINT ["docker-entrypoint.sh"] + +CMD ["master"] diff --git a/mailman/core/README.md b/mailman/core/README.md new file mode 100644 index 0000000..ec55bbc --- /dev/null +++ b/mailman/core/README.md @@ -0,0 +1,154 @@ +Mailman3 Core Docker Image +========================== + +When you spawn off this container, you must mount `/opt/mailman` to the +container. Mailman's `var` directory will also be stored here so that it can +persist across different sessions and containers. Any configuration at +`/opt/mailman/core/mailman-extra.cfg` (on the host) will be added to the mailman's default +generated confifguration (see below). + +It is not advised to run multiple mailman processes on the same host sharing the +same `/opt/mailman` (`/opt/mailman/core` on the host) directory as this will +almost certainly be dangerous. + + +Configuration +============= + +These are the variables that you MUST change before deploying: + +- `HYPERKITTY_API_KEY`: Hyperkitty's API Key, should be set to the same value as + set for the mailman-core. + +- `DATABASE_URL`: URL of the type + `driver://user:password@hostname:port/databasename` for the django to use. If + not set, the default is set to + `sqlite:///opt/mailman-web-data/mailmanweb.db`. The standard + docker-compose.yaml comes with it set to a postgres database. It is not must + to change this if you are happy with PostgreSQL. + +- `DATABASE_TYPE`: Its value can be one of `sqlite`, `postgres` or `mysql` as + these are the only three database types that Mailman 3 supports. Its default + value is set to `sqlite` along with the default database class and default + database url above. + +- `DATABASE_CLASS`: Default value is + `mailman.database.sqlite.SQLiteDatabase`. The values for this can be found in + the mailman's documentation [here][11]. + + +These are the variables that you don't need to change if you are using a +standard version of docker-compose.yaml from this repository. + +- `MM_HOSTNAME`: Which hostname or IP should Core bind to for REST API and + LMTP. If not defined output from the `hostname -i` command is used. + +- `MAILMAN_REST_PORT`: Which port should Core use for the REST API. If not defined + the default is `8001`. + +- `MAILMAN_REST_USER`: Which username should Core use for the REST API. If not + defined the default is `restadmin`. + +- `MAILMAN_REST_PASSWORD`: Which password should Core use for the REST API. If + not defined the default is `restpass`. + +- `MTA`: Mail Transfer Agent to use. Either `exim` or `postfix`. Default value is `exim`. + +- `SMTP_HOST`: IP Address/hostname from which you will be sending + emails. Default value is the container's gateway retrieved from: + /sbin/ip route | awk '/default/ { print $3 }' + +- `SMTP_PORT`: Port used for SMTP. Default is `25`. + +- `HYPERKITTY_URL`: Default value is `http://mailman-web:8000/hyperkitty` + +In case of a need for fine tuning of REST API web-server that uses [Gunicorn](https://docs.gunicorn.org/en/stable/settings.html) (e.g. for raising of timeouts) `/opt/mailman/core/gunicorn-extra.cfg` file could be provided holding necessary configuration options. + +Configuration file, [shipped with Mailman Core](https://gitlab.com/mailman/mailman/-/blob/master/src/mailman/config/gunicorn.cfg), is used by default. + +For example, to increase the default 30 sec timeout, which won't work for some API calls to highly populated lists, provide the following `gunicorn-extra.cfg` file: + +``` +[gunicorn] +graceful_timeout = 30 +timeout = 300 +``` + +Running Mailman-Core +==================== + +It is highly recomended that you run this image along with the +docker-compose.yaml file provided at the [github repo][1] of this +image. However, it is possible to run this image as a standalone container if +you want just a mailman-core. + +```bash +$ mkdir -p /opt/mailman/core +$ docker run -it -e "HYPERKITTY_API_KEY=changeme" -h mailman-core -v /opt/mailman/core:/opt/mailman mailman-core +``` + +However, if you don't provide the environment `DATABASE_URL`, the database _may_ +not be persisted. All the configuration options are explained in more detail. + +If you need mode advanced configuration for mailman, you can create +`/opt/mailman/mailman.cfg` and it be added to the configuration inside the +container. Note that anything inside this configuration will override the +settings provided via the environment variables and their default values. + +By default, the following settings are generated: + +``` +# mailman.cfg +[mta] +incoming: mailman.mta.exim4.LMTP +outgoing: mailman.mta.deliver.deliver +lmtp_host: $MM_HOSTNAME +lmtp_port: 8024 +smtp_host: $SMTP_HOST +smtp_port: $SMTP_PORT +configuration: python:mailman.config.exim4 + +[runner.retry] +sleep_time: 10s + +[webservice] +hostname: $MM_HOSTNAME +port: $MAILMAN_REST_PORT +admin_user: $MAILMAN_REST_USER +admin_pass: $MAILMAN_REST_PASSWORD +configuration: /etc/gunicorn.cfg + +[archiver.hyperkitty] +class: mailman_hyperkitty.Archiver +enable: yes +configuration: /etc/mailman-hyperkitty.cfg + +[database] +class: $DATABASE_CLASS +url: $DATABASE_URL +``` + +``` +# mailman-hyperkitty.cfg +[general] +base_url: $HYPERKITTY_URL +api_key: $HYPERKITTY_API_KEY +``` + +MTA +=== + +You can use Postfix or [Exim][2] with this image to send emails. Mailman Core +can interact with any modern MTA which can deliver emails over LMTP. The +documentation for Mailman Core has configuration settigs for using them. + +Only Exim and Postfix have been tested with these images and are supported as of +now. There _might_ be some limitations with using other MTAs in a containerized +environments. Contributions are welcome for anything additional needed to +support other MTAs. + +To setup Exim or Posfix, checkout the [documentation][3]. + +[1]: https://github.com/maxking/docker-mailman +[2]: http://www.exim.org +[3]: https://asynchronous.in/docker-mailman#setting-up-your-mta diff --git a/mailman/core/VERSION b/mailman/core/VERSION new file mode 100644 index 0000000..17e51c3 --- /dev/null +++ b/mailman/core/VERSION @@ -0,0 +1 @@ +0.1.1 diff --git a/mailman/core/assets/exim/25_mm3_macros b/mailman/core/assets/exim/25_mm3_macros new file mode 100644 index 0000000..b823fa2 --- /dev/null +++ b/mailman/core/assets/exim/25_mm3_macros @@ -0,0 +1,17 @@ +# Place this file at +# /etc/exim4/conf.d/main/25_mm3_macros + +domainlist mm3_domains=MY_DOMAIN_NAME +# Depending on your network configuration +#MM3_LMTP_HOST=mailman-core +MM3_LMTP_HOST=localhost +MM3_LMTP_PORT=8024 +MM3_HOME=/opt/mailman/core/var + +################################################################ +# The configuration below is boilerplate: +# you should not need to change it. + +# The path to the list receipt (used as the required file when +# matching list addresses) +MM3_LISTCHK=MM3_HOME/lists/${local_part}.${domain} diff --git a/mailman/core/assets/exim/455_mm3_router b/mailman/core/assets/exim/455_mm3_router new file mode 100644 index 0000000..d0220a6 --- /dev/null +++ b/mailman/core/assets/exim/455_mm3_router @@ -0,0 +1,15 @@ +# Place this file at +# /etc/exim4/conf.d/router/455_mm3_router + +mailman3_router: + driver = accept + domains = +mm3_domains + require_files = MM3_LISTCHK + local_part_suffix_optional + local_part_suffix = -admin : \ + -bounces : -bounces+* : \ + -confirm : -confirm+* : \ + -join : -leave : \ + -owner : -request : \ + -subscribe : -unsubscribe + transport = mailman3_transport diff --git a/mailman/core/assets/exim/55_mm3_transport b/mailman/core/assets/exim/55_mm3_transport new file mode 100644 index 0000000..e5cd356 --- /dev/null +++ b/mailman/core/assets/exim/55_mm3_transport @@ -0,0 +1,11 @@ +# Place this file at +# /etc/exim4/conf.d/transport/55_mm3_transport + +mailman3_transport: + debug_print = "Email for mailman" + driver = smtp + protocol = lmtp + allow_localhost + hosts = MM3_LMTP_HOST + port = MM3_LMTP_PORT + rcpt_include_affixes = true diff --git a/mailman/core/assets/mailman-hyperkitty.cfg b/mailman/core/assets/mailman-hyperkitty.cfg new file mode 100644 index 0000000..ce9f0de --- /dev/null +++ b/mailman/core/assets/mailman-hyperkitty.cfg @@ -0,0 +1,9 @@ +[general] +# This is your HyperKitty installation, preferably on the localhost. This +# address will be used by Mailman to forward incoming emails to HyperKitty +# for archiving. It does not need to be publicly available, in fact it's +# better if it is not. +base_url: http://mailman-web:8000/hyperkitty/ +# Shared API key, must be the identical to the value in HyperKitty's +# settings. +api_key: ASmallAPIKey diff --git a/mailman/core/assets/mailman.cfg b/mailman/core/assets/mailman.cfg new file mode 100644 index 0000000..8115aec --- /dev/null +++ b/mailman/core/assets/mailman.cfg @@ -0,0 +1,35 @@ +[mta] +incoming: mailman.mta.exim4.LMTP +outgoing: mailman.mta.deliver.deliver +lmtp_host: mailman-core +lmtp_port: 8024 +smtp_host: 172.19.199.1 +smtp_port: 25 +configuration: python:mailman.config.exim4 + +# [archiver.mhonarc] +# enable: yes + +# [archiver.mail_archive] +# enable: yes + +# [archiver.prototype] +# enable: yes + +[runner.retry] +sleep_time: 10s + +[shell] +use_ipython: yes + +[webservice] +hostname: mailman-core + +[archiver.hyperkitty] +class: mailman_hyperkitty.Archiver +enable: yes +configuration: /opt/mailman/mailman-hyperkitty.cfg + +[database] +class: mailman.database.postgresql.PostgreSQLDatabase +url: postgres://mailman:mailmanpass@database/mailmandb diff --git a/mailman/core/docker-entrypoint.sh b/mailman/core/docker-entrypoint.sh new file mode 100755 index 0000000..d0bd089 --- /dev/null +++ b/mailman/core/docker-entrypoint.sh @@ -0,0 +1,237 @@ +#! /bin/bash +set -e + +function wait_for_postgres () { + # Check if the postgres database is up and accepting connections before + # moving forward. + # TODO: Use python3's psycopg2 module to do this in python3 instead of + # installing postgres-client in the image. + until psql -P pager=off $DATABASE_URL -c '\l'; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 + done + >&2 echo "Postgres is up - continuing" +} + +function wait_for_mysql () { + # Check if MySQL is up and accepting connections. + readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));") + until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do + >&2 echo "MySQL is unavailable - sleeping" + sleep 1 + done + >&2 echo "MySQL is up - continuing" +} + +# Empty the config file. +echo "# This file is autogenerated at container startup." > /etc/mailman.cfg + +# Check if $MM_HOSTNAME is set, if not, set it to the value returned by +# `hostname -i` command to set it to whatever IP address is assigned to the +# container. +if [[ ! -v MM_HOSTNAME ]]; then + export MM_HOSTNAME=`hostname -i` +fi + +# SMTP_HOST defaults to the gateway +if [[ ! -v SMTP_HOST ]]; then + export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }') + echo "SMTP_HOST not specified, using the gateway ($SMTP_HOST) as default" +fi + +if [[ ! -v SMTP_PORT ]]; then + export SMTP_PORT=25 +fi + +# Check if REST port, username, and password are set, if not, set them +# to default values. +if [[ ! -v MAILMAN_REST_PORT ]]; then + export MAILMAN_REST_PORT='8001' +fi + +if [[ ! -v MAILMAN_REST_USER ]]; then + export MAILMAN_REST_USER='restadmin' +fi + +if [[ ! -v MAILMAN_REST_PASSWORD ]]; then + export MAILMAN_REST_PASSWORD='restpass' +fi + +function setup_database () { + if [[ ! -v DATABASE_URL ]] + then + echo "Environment variable DATABASE_URL should be defined..." + exit 1 + fi + + # Translate mysql:// urls to mysql+mysql:// backend: + if [[ "$DATABASE_URL" == mysql://* ]]; then + DATABASE_URL="mysql+pymysql://${DATABASE_URL:8}" + echo "Database URL prefix was automatically rewritten to: mysql+pymysql://" + fi + + # If DATABASE_CLASS is not set, guess it for common databases: + if [ -z "$DATABASE_CLASS" ]; then + if [[ ("$DATABASE_URL" == mysql:*) || + ("$DATABASE_URL" == mysql+*) ]]; then + DATABASE_CLASS=mailman.database.mysql.MySQLDatabase + fi + if [[ ("$DATABASE_URL" == postgres:*) || + ("$DATABASE_URL" == postgres+*) ]]; then + DATABASE_CLASS=mailman.database.postgresql.PostgreSQLDatabase + fi + fi + + cat >> /etc/mailman.cfg <> /etc/mailman.cfg << EOF +[runner.retry] +sleep_time: 10s + +[webservice] +hostname: $MM_HOSTNAME +port: $MAILMAN_REST_PORT +admin_user: $MAILMAN_REST_USER +admin_pass: $MAILMAN_REST_PASSWORD +configuration: /etc/gunicorn.cfg + +EOF + +# Generate a basic gunicorn.cfg. +SITE_DIR=$(python3 -c 'import site; print(site.getsitepackages()[0])') +cp "${SITE_DIR}/mailman/config/gunicorn.cfg" /etc/gunicorn.cfg + +# Generate a basic configuration to use exim +cat > /tmp/exim-mailman.cfg < /etc/postfix-mailman.cfg << EOF +[postfix] +transport_file_type: regex +# While in regex mode, postmap_command is never used, a placeholder +# is added here so that it doesn't break anything. +postmap_command: true +EOF + +# Generate a basic configuration to use postfix. +cat > /tmp/postfix-mailman.cfg <> /etc/mailman.cfg +elif [ "$MTA" == "postfix" ] +then + echo "Using Postfix configuration" + cat /tmp/postfix-mailman.cfg >> /etc/mailman.cfg +else + echo "No MTA environment variable found, defaulting to Exim" + cat /tmp/exim-mailman.cfg >> /etc/mailman.cfg +fi + +rm -f /tmp/{postfix,exim}-mailman.cfg + +if [[ -e /opt/mailman/mailman-extra.cfg ]] +then + echo "Found configuration file at /opt/mailman/mailman-extra.cfg" + cat /opt/mailman/mailman-extra.cfg >> /etc/mailman.cfg +fi + +if [[ -e /opt/mailman/gunicorn-extra.cfg ]] +then + echo "Found [webserver] configuration file at /opt/mailman/gunicorn-extra.cfg" + cat /opt/mailman/gunicorn-extra.cfg > /etc/gunicorn.cfg +fi + +if [[ -v HYPERKITTY_API_KEY ]]; then + +echo "HYPERKITTY_API_KEY found, setting up HyperKitty archiver..." + +cat >> /etc/mailman.cfg << EOF +[archiver.hyperkitty] +class: mailman_hyperkitty.Archiver +enable: yes +configuration: /etc/mailman-hyperkitty.cfg + +EOF + +if [[ ! -v HYPERKITTY_URL ]]; then + echo "HYPERKITTY_URL not set, using the default value of http://mailman-web:8000/hyperkitty" + export HYPERKITTY_URL="http://mailman-web:8000/hyperkitty/" +fi + +# Generate a basic mailman-hyperkitty.cfg. +cat > /etc/mailman-hyperkitty.cfg <&2 echo "Postgres is unavailable - sleeping" + sleep 1 + done + >&2 echo "Postgres is up - continuing" +} + +function wait_for_mysql () { + # Check if MySQL is up and accepting connections. + readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));") + until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do + >&2 echo "MySQL is unavailable - sleeping" + sleep 1 + done + >&2 echo "MySQL is up - continuing" +} + +function check_or_create () { + # Check if the path exists, if not, create the directory. + if [[ ! -e dir ]]; then + echo "$1 does not exist, creating ..." + mkdir "$1" + fi +} + +# function postgres_ready(){ +# python << END +# import sys +# import psycopg2 +# try: +# conn = psycopg2.connect(dbname="$POSTGRES_DB", user="$POSTGRES_USER", password="$POSTGRES_PASSWORD", host="postgres") +# except psycopg2.OperationalError: +# sys.exit(-1) +# sys.exit(0) +# END +# } + +# SMTP_HOST defaults to the gateway +if [[ ! -v SMTP_HOST ]]; then + export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }') +fi + +# Check if $SECRET_KEY is defined, if not, bail out. +if [[ ! -v SECRET_KEY ]]; then + echo "SECRET_KEY is not defined. Aborting." + exit 1 +fi + +# Check if $DATABASE_URL is defined, if not, use a standard sqlite database. +# +# If the $DATABASE_URL is defined and is postgres, check if it is available +# yet. Do not start the container before the postgresql boots up. +# +# If the $DATABASE_URL is defined and is mysql, check if the database is +# available before the container boots up. +# +# TODO: Check the database type and detect if it is up based on that. For now, +# assume that postgres is being used if DATABASE_URL is defined. + +if [[ ! -v DATABASE_URL ]]; then + echo "DATABASE_URL is not defined. Using sqlite database..." + export DATABASE_URL=sqlite://mailmanweb.db + export DATABASE_TYPE='sqlite' +fi + +if [[ "$DATABASE_TYPE" = 'postgres' ]] +then + wait_for_postgres +elif [[ "$DATABASE_TYPE" = 'mysql' ]] +then + wait_for_mysql +fi + +# Check if we are in the correct directory before running commands. +if [[ ! $(pwd) == '/opt/mailman-web' ]]; then + echo "Running in the wrong directory...switching to /opt/mailman-web" + cd /opt/mailman-web +fi + +# Check if the logs directory is setup. +if [[ ! -e /opt/mailman-web-data/logs/mailmanweb.log ]]; then + echo "Creating log file for mailman web" + mkdir -p /opt/mailman-web-data/logs/ + touch /opt/mailman-web-data/logs/mailmanweb.log +fi + +if [[ ! -e /opt/mailman-web-data/logs/uwsgi.log ]]; then + echo "Creating log file for uwsgi.." + touch /opt/mailman-web-data/logs/uwsgi.log +fi + +# Check if the settings_local.py file exists, if yes, copy it too. +if [[ -e /opt/mailman-web-data/settings_local.py ]]; then + echo "Copying settings_local.py ..." + cp /opt/mailman-web-data/settings_local.py /opt/mailman-web/settings_local.py + chown mailman:mailman /opt/mailman-web/settings_local.py +else + echo "settings_local.py not found, it is highly recommended that you provide one" + echo "Using default configuration to run." +fi + +# Collect static for the django installation. +python3 manage.py collectstatic --noinput --clear --verbosity 0 + +# Compile all the installed po files to mo. +SITE_DIR=$(python3 -c 'import site; print(site.getsitepackages()[0])') +echo "Compiling locale files in $SITE_DIR" +cd $SITE_DIR && python3 /opt/mailman-web/manage.py compilemessages && cd - + +# Migrate all the data to the database if this is a new installation, otherwise +# this command will upgrade the database. +python3 manage.py migrate + +# If MAILMAN_ADMIN_USER and MAILMAN_ADMIN_EMAIL is defined create a new +# superuser for Django. There is no password setup so it can't login yet unless +# the password is reset. +if [[ -v MAILMAN_ADMIN_USER ]] && [[ -v MAILMAN_ADMIN_EMAIL ]]; +then + echo "Creating admin user $MAILMAN_ADMIN_USER ..." + python3 manage.py createsuperuser --noinput --username "$MAILMAN_ADMIN_USER"\ + --email "$MAILMAN_ADMIN_EMAIL" 2> /dev/null || \ + echo "Superuser $MAILMAN_ADMIN_USER already exists" +fi + +# If SERVE_FROM_DOMAIN is defined then rename the default `example.com` +# domain to the defined domain. +if [[ -v SERVE_FROM_DOMAIN ]]; +then + echo "Setting $SERVE_FROM_DOMAIN as the default domain ..." + python3 manage.py shell -c \ + "from django.contrib.sites.models import Site; Site.objects.filter(domain='example.com').update(domain='$SERVE_FROM_DOMAIN', name='$SERVE_FROM_DOMAIN')" +fi + +# Create a mailman user with the specific UID and GID and do not create home +# directory for it. Also chown the logs directory to write the files. +chown mailman:mailman /opt/mailman-web-data -R + +exec $@ diff --git a/mailman/postorius/mailman-web/__init__.py b/mailman/postorius/mailman-web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mailman/postorius/mailman-web/manage.py b/mailman/postorius/mailman-web/manage.py new file mode 100755 index 0000000..f9726f9 --- /dev/null +++ b/mailman/postorius/mailman-web/manage.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/mailman/postorius/mailman-web/settings.py b/mailman/postorius/mailman-web/settings.py new file mode 100644 index 0000000..ab10f2b --- /dev/null +++ b/mailman/postorius/mailman-web/settings.py @@ -0,0 +1,357 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. +# +# This file is part of Mailman Suite. +# +# Mailman Suite is free sofware: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# Mailman Suite is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + +# You should have received a copy of the GNU General Public License along +# with Mailman Suite. If not, see . +""" +Django Settings for Mailman Suite (hyperkitty + postorius) + +For more information on this file, see +https://docs.djangoproject.com/en/1.8/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.8/ref/settings/ +""" + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +import os +import dj_database_url +import sys +from socket import gethostbyname + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.environ.get('SECRET_KEY') + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False + +ADMINS = ( + ('Mailman Suite Admin', 'root@localhost'), +) + +SITE_ID = 1 + +# Hosts/domain names that are valid for this site; required if DEBUG is False +# See https://docs.djangoproject.com/en/3.1/ref/settings/#allowed-hosts +ALLOWED_HOSTS = [ + "localhost", # Archiving API from Mailman, keep it. + "mailman-web", + gethostbyname("mailman-web"), + os.environ.get('SERVE_FROM_DOMAIN'), +] +ALLOWED_HOSTS.extend(os.getenv("DJANGO_ALLOWED_HOSTS", "").split(",")) + +# Mailman API credentials +MAILMAN_REST_API_URL = os.environ.get('MAILMAN_REST_URL', 'http://mailman-core:8001') +MAILMAN_REST_API_USER = os.environ.get('MAILMAN_REST_USER', 'restadmin') +MAILMAN_REST_API_PASS = os.environ.get('MAILMAN_REST_PASSWORD', 'restpass') + +# Application definition + +INSTALLED_APPS = [] +DEFAULT_APPS = [ + 'postorius', + 'django_mailman3', + # Uncomment the next line to enable the admin: + 'django.contrib.admin', + # Uncomment the next line to enable admin documentation: + # 'django.contrib.admindocs', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.humanize', + 'django_gravatar', + 'allauth', + 'allauth.account', + 'allauth.socialaccount', +] +MAILMAN_WEB_SOCIAL_AUTH = [ + 'django_mailman3.lib.auth.fedora', + 'allauth.socialaccount.providers.openid', + 'allauth.socialaccount.providers.github', + 'allauth.socialaccount.providers.gitlab', + 'allauth.socialaccount.providers.google', +] + +MIDDLEWARE = ( + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.middleware.locale.LocaleMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'django_mailman3.middleware.TimezoneMiddleware', + 'allauth.account.middleware.AccountMiddleware', + 'postorius.middleware.PostoriusMiddleware', +) + +ROOT_URLCONF = 'urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.template.context_processors.tz', + 'django.template.context_processors.csrf', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'django_mailman3.context_processors.common', + 'postorius.context_processors.postorius', + ], + }, + }, +] + +WSGI_APPLICATION = 'wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases + + +# This uses $DATABASE_URL from the environment variable to create a +# django-style-config-dict. +# https://github.com/kennethreitz/dj-database-url +DATABASES = { + 'default': dj_database_url.config(conn_max_age=600) +} + +# Avoid Django 3.2+ warning +# https://github.com/maxking/docker-mailman/issues/595 +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + + +# If you're behind a proxy, use the X-Forwarded-Host header +# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host +USE_X_FORWARDED_HOST = True + + +# Password validation +# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': +'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': +'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': +'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': +'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +# Internationalization +# https://docs.djangoproject.com/en/1.8/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +STATIC_ROOT = '/opt/mailman-web-data/static' + +STATIC_URL = '/static/' + +# Additional locations of static files + + +# List of finder classes that know how to find static files in +# various locations. +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', +) + + +SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer' + +LOGIN_URL = 'account_login' +LOGIN_REDIRECT_URL = 'list_index' +LOGOUT_URL = 'account_logout' + + +# Use SERVE_FROM_DOMAIN as the default domain in the email. +hostname = os.environ.get('SERVE_FROM_DOMAIN', 'localhost.local') +DEFAULT_FROM_EMAIL = 'postorius@{}'.format(hostname) +SERVER_EMAIL = 'root@{}'.format(hostname) + +# Change this when you have a real email backend +EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST = os.environ.get('SMTP_HOST', '') +EMAIL_PORT = os.environ.get('SMTP_PORT', 25) +EMAIL_HOST_USER = os.environ.get('SMTP_HOST_USER', '') +EMAIL_HOST_PASSWORD = os.environ.get('SMTP_HOST_PASSWORD', '') +EMAIL_USE_TLS = os.environ.get('SMTP_USE_TLS', False) +EMAIL_USE_SSL = os.environ.get('SMTP_USE_SSL', False) + +# Compatibility with Bootstrap 3 +from django.contrib.messages import constants as messages # flake8: noqa +MESSAGE_TAGS = { + messages.ERROR: 'danger' +} + + +# +# Social auth +# +AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'allauth.account.auth_backends.AuthenticationBackend', +) + +# Django Allauth +ACCOUNT_AUTHENTICATION_METHOD = "username_email" +ACCOUNT_EMAIL_REQUIRED = True +ACCOUNT_EMAIL_VERIFICATION = "mandatory" +# You probably want https in production, but this is a dev setup file +ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https" +ACCOUNT_UNIQUE_EMAIL = True + +SOCIALACCOUNT_PROVIDERS = { + 'openid': { + 'SERVERS': [ + dict(id='yahoo', + name='Yahoo', + openid_url='http://me.yahoo.com'), + ], + }, + 'google': { + 'SCOPE': ['profile', 'email'], + 'AUTH_PARAMS': {'access_type': 'online'}, + }, + 'facebook': { + 'METHOD': 'oauth2', + 'SCOPE': ['email'], + 'FIELDS': [ + 'email', + 'name', + 'first_name', + 'last_name', + 'locale', + 'timezone', + ], + 'VERSION': 'v2.4', + }, +} + + +import sys +# A sample logging configuration. The only tangible logging +# performed by this configuration is to send an email to +# the site admins on every HTTP 500 error when DEBUG=False. +# See http://docs.djangoproject.com/en/dev/topics/logging for +# more details on how to customize your logging configuration. +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler' + }, + 'file':{ + 'level': 'INFO', + 'class': 'logging.handlers.RotatingFileHandler', + #'class': 'logging.handlers.WatchedFileHandler', + 'filename': os.environ.get('DJANGO_LOG_URL','/opt/mailman-web-data/logs/mailmanweb.log'), + 'formatter': 'verbose', + }, + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'simple', + 'level': 'INFO', + 'stream': sys.stdout, + }, + # TODO: use an environment variable $DJ_LOG_URL to configure the logging + # using an environment variable. + }, + 'loggers': { + 'django.request': { + 'handlers': ['mail_admins', 'file'], + 'level': 'INFO', + 'propagate': True, + }, + 'django': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True, + }, + 'postorius': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True + }, + }, + 'formatters': { + 'verbose': { + 'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s' + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + }, + #'root': { + # 'handlers': ['file'], + # 'level': 'INFO', + #}, +} + + +if os.environ.get('LOG_TO_CONSOLE') == 'yes': + LOGGING['loggers']['django']['handlers'].append('console') + LOGGING['loggers']['django.request']['handlers'].append('console') +POSTORIUS_TEMPLATE_BASE_URL = os.environ.get('POSTORIUS_TEMPLATE_BASE_URL', 'http://mailman-web:8000') + +try: + from settings_local import * +except ImportError: + pass + +# Compatibility for older installs that override INSTALLED_APPS +if not INSTALLED_APPS: + INSTALLED_APPS = DEFAULT_APPS + MAILMAN_WEB_SOCIAL_AUTH diff --git a/mailman/postorius/mailman-web/urls.py b/mailman/postorius/mailman-web/urls.py new file mode 100644 index 0000000..3e3c312 --- /dev/null +++ b/mailman/postorius/mailman-web/urls.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. +# +# This file is part of Postorius. +# +# Postorius is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# Postorius is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Postorius. If not, see . + +from django.conf.urls import include +from django.contrib import admin +from django.urls import re_path, reverse_lazy +from django.views.generic import RedirectView + +urlpatterns = [ + re_path(r'^$', RedirectView.as_view( + url=reverse_lazy('list_index'), + permanent=True)), + re_path(r'postorius/', include('postorius.urls')), + re_path(r'', include('django_mailman3.urls')), + re_path(r'accounts/', include('allauth.urls')), + # Django admin + re_path(r'^admin/', admin.site.urls), +] diff --git a/mailman/postorius/mailman-web/uwsgi.ini b/mailman/postorius/mailman-web/uwsgi.ini new file mode 100644 index 0000000..e5d00a6 --- /dev/null +++ b/mailman/postorius/mailman-web/uwsgi.ini @@ -0,0 +1,25 @@ +[uwsgi] +# Port on which uwsgi will be listening. +uwsgi-socket = 0.0.0.0:8080 +http-socket = 0.0.0.0:8000 + +# Move to the directory where the django files are. +chdir = /opt/mailman-web + +# Use the wsgi file provided with the django project. +wsgi-file = wsgi.py + +# Setup default number of processes and threads per process. +master = true +processes = 2 +threads = 2 + +# Drop privileges and don't run as root. +uid = mailman +gid = mailman + +# Setup the request log. +req-logger = file:/opt/mailman-web-data/logs/uwsgi.log + +# Last log and it logs the rest of the stuff. +logger = file:/opt/mailman-web-data/logs/uwsgi-error.log diff --git a/mailman/postorius/mailman-web/wsgi.py b/mailman/postorius/mailman-web/wsgi.py new file mode 100755 index 0000000..8e59cd8 --- /dev/null +++ b/mailman/postorius/mailman-web/wsgi.py @@ -0,0 +1,38 @@ +""" +WSGI config for HyperKitty project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/wsgi/ +""" + +import os + +# import sys +# import site + +# For some unknown reason, sometimes mod_wsgi fails to set the python paths to +# the virtualenv, with the 'python-path' option. You can do it here too. +# +# # Remember original sys.path. +# prev_sys_path = list(sys.path) +# # Add here, for the settings module +# site.addsitedir(os.path.abspath(os.path.dirname(__file__))) +# # Add the virtualenv +# venv = os.path.join(os.path.abspath(os.path.dirname(__file__)), +# '..', 'lib', 'python2.6', 'site-packages') +# site.addsitedir(venv) +# # Reorder sys.path so new directories at the front. +# new_sys_path = [] +# for item in list(sys.path): +# if item not in prev_sys_path: +# new_sys_path.append(item) +# sys.path.remove(item) +# sys.path[:0] = new_sys_path + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") + +application = get_wsgi_application() diff --git a/mailman/tests/docker-test.yaml b/mailman/tests/docker-test.yaml new file mode 100644 index 0000000..2911cfe --- /dev/null +++ b/mailman/tests/docker-test.yaml @@ -0,0 +1,11 @@ +version: '2' + +services: + mailman-core: + image: maxking/mailman-core:rolling + + mailman-web: + image: maxking/mailman-web:rolling + environment: + - SECRET_KEY=abcdefghijklmnopqrstuv + - SERVE_FROM_DOMAIN=araj.me diff --git a/mailman/tests/test.sh b/mailman/tests/test.sh new file mode 100755 index 0000000..58f6a69 --- /dev/null +++ b/mailman/tests/test.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -e + +# If the DB environment variable is not set, use postgres.x +if [ "$DB" = "postgres" ] || [ -z $DB ] +then + docker-compose -f docker-compose.yaml -f tests/docker-test.yaml up -d +elif [ "$DB" = "mysql" ] +then + docker-compose -f docker-compose-mysql.yaml -f tests/docker-test.yaml up -d +fi + +# Print the IP Addresses of the Containers. +docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mailman-core +docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mailman-web + +# Make sure all the containers are running. +docker-compose ps + +# Sleep for a while and check again if the containers are up. +sleep 60 +docker ps + +# Check if there is anything interesting in the logs. +docker logs mailman-web +docker logs mailman-core + + +# Check to see if the core is working as expected. +docker exec mailman-core curl -u restadmin:restpass http://mailman-core:8001/3.1/system | grep "GNU Mailman" + +# Check to see if postorius is working. +docker exec mailman-web curl -L http://mailman-web:8000/postorius/lists | grep "Mailing List" + +# Check to see if hyperkitty is working. +docker exec mailman-web curl -L http://mailman-web:8000/hyperkitty/ | grep "Available lists" diff --git a/mailman/web/Dockerfile b/mailman/web/Dockerfile new file mode 100644 index 0000000..b01125d --- /dev/null +++ b/mailman/web/Dockerfile @@ -0,0 +1,54 @@ +# syntax = docker/dockerfile:1.3 +FROM alpine:3.19.1 + +# Add requirements file. +COPY requirements.txt /tmp/ + +# Install packages and dependencies for postorius and hyperkitty Add user for +# executing apps, change ownership for uwsgi+django files and set execution +# rights for management script +RUN --mount=type=cache,target=/root/.cache \ + set -ex \ + && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers libldap \ + postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \ + && apk add --no-cache --virtual .mailman-rundeps bash sassc tzdata \ + postgresql-client mysql-client py3-mysqlclient curl mailcap gettext \ + python3 py3-pip xapian-core xapian-bindings-python3 libffi pcre-dev py-cryptography \ + && python3 -m pip install --break-system-packages -U 'Django<4.2' pip setuptools wheel \ + && pip install --break-system-packages -r /tmp/requirements.txt \ + whoosh \ + uwsgi \ + psycopg2 \ + dj-database-url \ + mysqlclient \ + typing \ + xapian-haystack \ + django-auth-ldap \ + pymemcache \ + diskcache \ + django-utils-six \ + tzdata \ + && apk del .build-deps \ + && addgroup -S mailman \ + && adduser -S -G mailman mailman + +# Add needed files for uwsgi server + settings for django +COPY mailman-web /opt/mailman-web +# Add startup script to container +COPY docker-entrypoint.sh /usr/local/bin/ + +RUN chown -R mailman /opt/mailman-web/ \ + && chmod u+x /opt/mailman-web/manage.py + +WORKDIR /opt/mailman-web + +# Expose port 8000 for http and port 8080 for uwsgi +# (see web/mailman-web/uwsgi.ini#L2-L4) +EXPOSE 8000 8080 + +# Use stop signal for uwsgi server +STOPSIGNAL SIGINT + +ENTRYPOINT ["docker-entrypoint.sh"] + +CMD ["uwsgi", "--ini", "/opt/mailman-web/uwsgi.ini"] diff --git a/mailman/web/Dockerfile.dev b/mailman/web/Dockerfile.dev new file mode 100644 index 0000000..ece0282 --- /dev/null +++ b/mailman/web/Dockerfile.dev @@ -0,0 +1,61 @@ +# syntax = docker/dockerfile:1.3 +FROM alpine:3.19.1 + +ARG POSTORIUS_REF +ARG HYPERKITTY_REF +ARG DJ_MM3_REF +ARG CLIENT_REF + +# Install packages and dependencies for postorius and hyperkitty Add user for +# executing apps, change ownership for uwsgi+django files and set execution +# rights for management script +RUN --mount=type=cache,target=/root/.cache \ + set -ex \ + && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers git libldap \ + postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \ + && apk add --no-cache --virtual .mailman-rundeps bash sassc pcre-dev tzdata \ + python3 py3-pip postgresql-client mysql-client py3-mysqlclient \ + curl mailcap xapian-core xapian-bindings-python3 libffi gettext py-cryptography \ + && python3 -m pip install --break-system-packages -U pip setuptools wheel \ + && python3 -m pip install --break-system-packages -U \ + git+https://gitlab.com/mailman/mailmanclient \ + git+https://gitlab.com/mailman/postorius \ + git+https://gitlab.com/mailman/hyperkitty \ + whoosh \ + uwsgi \ + psycopg2 \ + dj-database-url \ + mysqlclient \ + xapian-haystack \ + django-auth-ldap \ + pymemcache \ + tzdata \ + diskcache \ + django-utils-six \ + && python3 -m pip install --break-system-packages -U 'Django<4.2' \ + && python3 -m pip install --break-system-packages -U \ + git+https://gitlab.com/mailman/django-mailman3 \ + && apk del .build-deps \ + && addgroup -S mailman \ + && adduser -S -G mailman mailman + +# Add needed files for uwsgi server + settings for django +COPY mailman-web /opt/mailman-web +# Add startup script to container +COPY docker-entrypoint.sh /usr/local/bin/ + +RUN chown -R mailman /opt/mailman-web/ \ + && chmod u+x /opt/mailman-web/manage.py + +WORKDIR /opt/mailman-web + +# Expose port 8000 for http and port 8080 for uwsgi +# (see web/mailman-web/uwsgi.ini#L2-L4) +EXPOSE 8000 8080 + +# Use stop signal for uwsgi server +STOPSIGNAL SIGINT + +ENTRYPOINT ["docker-entrypoint.sh"] + +CMD ["uwsgi", "--ini", "/opt/mailman-web/uwsgi.ini"] diff --git a/mailman/web/README.md b/mailman/web/README.md new file mode 100644 index 0000000..c1116cf --- /dev/null +++ b/mailman/web/README.md @@ -0,0 +1,161 @@ +# Mailman 3 Web UI + +This image consists of Mailman3's Web UI(Postorius) and Archiver +(Hyperkitty). This image is built from latest sources on [gitlab][1]. In future, +latest and stable releases will be seperate. I am looking forward to the release +of Mailman Suite 3.1 before that. + +## Configuration + + +These are the settings that you MUST change before deploying: + +- `SERVE_FROM_DOMAIN`: The domain name from which Django will be served. To be + added to `ALLOWED_HOSTS` in django settings. Default value is not set. This + also replaces Django's default `example.com` SITE and becomes the default SITE + (with SITE_ID=1). + +- `HYPERKITTY_API_KEY`: Hyperkitty's API Key, should be set to the same value as + set for the mailman-core. + +- `MAILMAN_ADMIN_USER`: The username for the admin user to be created by default. + +- `MAILMAN_ADMIN_EMAIL`: The email for the admin user to be created by default. + +- `SECRET_KEY`: Django's secret key, mainly used for signing cookies and others. + +These are the settings that are set to sane default and you do not need to +change them unless you know what you want. + +- `DATABASE_URL`: URL of the type + `driver://user:password@hostname:port/databasename` for the django to use. If + not set, the default is set to + `sqlite:///opt/mailman-web-data/mailmanweb.db`. The standard + docker-compose.yaml comes with it set to a postgres database. It is not must + to change this if you are happy with PostgreSQL. + +- `MAILMAN_REST_URL`: The URL to the Mailman core's REST API server. Defaut + value is `http://mailman-core:8001`. + +- `MAILMAN_REST_USER`: Mailman's REST API username. Default value is `restadmin` + +- `MAILMAN_REST_PASSWORD`: Mailman's REST API user's password. Default value is + `restpass` + +- `MAILMAN_HOSTNAME`: IP of the Container from which Mailman will send emails to + hyperkitty (django). Set to `mailman-core` by default. + +- `SMTP_HOST`: IP Address/hostname from which you will be sending + emails. Default value is the container's gateway retrieved from: + /sbin/ip route | awk '/default/ { print $3 }' + +- `SMTP_PORT`: Port used for SMTP. Default is `25`. + +- `SMTP_HOST_USER`: Used for SMTP authentication. Default is an empty string. + +- `SMTP_HOST_PASSWORD`: Default is an empty string. + +- `SMTP_USE_TLS`: Specifies wheather the SMTP connection is encrypted + via TLS. Default is `False`. (`EMAIL_USE_TLS`/`EMAIL_USE_SSL` are mutually exclusive, so only set one of those settings.) + +- `SMTP_USE_SSL`: Specifies wheather the SMTP connection is encrypted + via SSL. Default is `False`. (EMAIL_USE_TLS/EMAIL_USE_SSL are mutually exclusive, so only set one of those settings.) + +- `DJANGO_LOG_URL`: Path to the django's log file. Defaults to + `/opt/mailman-web-data/logs/mailmanweb.log`. + +- `DJANGO_ALLOWED_HOSTS`: Entry to add to ALLOWED_HOSTS in Django + configuration. Format as comma-separated list (no whitespace). This is a separate configuration from`SERVE_FROM_DOMAIN` as + latter is used for other purposes too. + +- `POSTORIUS_TEMPLATE_BASE_URL`: The base url at which the `mailman-web` + container can be reached from `mailman-core` container. This is set to + `http://mailman-web:8000` by default so that Core can fetch templates from + Web. + +- `DISKCACHE_PATH` and `DISKCACHE_SIZE`: Django Diskcache location path and + size respectively. Defaults are `/opt/mailman-web-data/diskcache` and 1G. + +[1]: https://github.com/maxking/docker-mailman/blob/master/web/mailman-web/settings.py + +## Social Auth + +In order to separate `INSTALLED_APPS` from the social authentication plugins a new settings `MAILMAN_WEB_SOCIAL_AUTH` is created. This includes all the enabled social auth plugins. + +### Disable social auth + +In order to disable social auth, you can add the following to your +settings_local.py + +```python +MAILMAN_WEB_SOCIAL_AUTH = [] +``` + +In older versions of continer images (0.3.*), you had to override +`INSTALLED_APPS` in order to disable social auth, but addition of +this new setting will make it easier to disable social auth making +sure that you get any updates to the django apps that are added in +future. + +The default behavior will remain the same as 0.3 release if you +have not overriden `INSTALLED_APPS` though. + +## Running + +It is highly recommended that you run this using the [docker-compose.yaml][2] +provided in the [github repo][3] of this project. You will need to proxy the +requests the container that you create with this image using an actual web +server like Nginx. The [github repo][3] provides the setup instructions for +Nginx. + +Since the setup has `USE_SSL` set to `True` in django's `settings.py`, you may +also want to get a SSL certificate if you don't already have one. [Lets +Encrypt][4] provides free SSL certiticates for everyone and there are _some_ +instructions about that also. + +After the first run, you can create a superuser for django using the following +command: + +```bash +$ docker exec -it mailman-web python3 manage.py createsuperuser +``` + +## Django management commands + +In order to run Django management commands in the `mailman-web` container, you +can run following: + +```bash +$ docker exec -it mailman-web python3 manage.py +``` + +And replace `` with the appropriate management command. + + +## Importing Archives from Mailman 2 + +In order to import archvies from Mailman 2, you need to get the `listname.mbox` +file in a location that is readable inside `mailman-web` container. + +Please place `listname.mbox` file at `/opt/mailman/web` **on the host**. Verify +that the file is present inside the `mailman-web` contianer by running: + +```bash +$ docker exec -it mailman-web ls /opt/mailman-web-data +``` +And verify that you can see `listname.mbox` in the `ls` output above. After you +have verified that, you can then run the `hyperkitty_import` command to do the +actual import: + +```bash +$ docker exec -it mailman-web python3 manage.py hyperkitty_import -l listname@domain /opt/mailman-web-data/listname.mbox +``` + +This should take some time to import depending on how many emails are in the +archives. + + +[1]: https://gitlab.com/mailman +[3]: https://github.com/maxking/docker-mailman/ +[2]: https://github.com/maxking/docker-mailman/blob/master/docker-compose.yaml +[4]: https://letsencrypt.org diff --git a/mailman/web/VERSION b/mailman/web/VERSION new file mode 100644 index 0000000..17e51c3 --- /dev/null +++ b/mailman/web/VERSION @@ -0,0 +1 @@ +0.1.1 diff --git a/mailman/web/docker-entrypoint.sh b/mailman/web/docker-entrypoint.sh new file mode 100755 index 0000000..3324d05 --- /dev/null +++ b/mailman/web/docker-entrypoint.sh @@ -0,0 +1,154 @@ +#! /bin/bash +set -e + + +function wait_for_postgres () { + # Check if the postgres database is up and accepting connections before + # moving forward. + # TODO: Use python's psycopg2 module to do this in python instead of + # installing postgres-client in the image. + until psql -P pager=off $DATABASE_URL -c '\l'; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 + done + >&2 echo "Postgres is up - continuing" +} + +function wait_for_mysql () { + # Check if MySQL is up and accepting connections. + readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));") + until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do + >&2 echo "MySQL is unavailable - sleeping" + sleep 1 + done + >&2 echo "MySQL is up - continuing" +} + +function check_or_create () { + # Check if the path exists, if not, create the directory. + if [[ ! -e dir ]]; then + echo "$1 does not exist, creating ..." + mkdir "$1" + fi +} + +# function postgres_ready(){ +# python << END +# import sys +# import psycopg2 +# try: +# conn = psycopg2.connect(dbname="$POSTGRES_DB", user="$POSTGRES_USER", password="$POSTGRES_PASSWORD", host="postgres") +# except psycopg2.OperationalError: +# sys.exit(-1) +# sys.exit(0) +# END +# } + +# SMTP_HOST defaults to the gateway +if [[ ! -v SMTP_HOST ]]; then + export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }') +fi + +# Check if $SECRET_KEY is defined, if not, bail out. +if [[ ! -v SECRET_KEY ]]; then + echo "SECRET_KEY is not defined. Aborting." + exit 1 +fi + +# Check if $DATABASE_URL is defined, if not, use a standard sqlite database. +# +# If the $DATABASE_URL is defined and is postgres, check if it is available +# yet. Do not start the container before the postgresql boots up. +# +# If the $DATABASE_URL is defined and is mysql, check if the database is +# available before the container boots up. +# +# TODO: Check the database type and detect if it is up based on that. For now, +# assume that postgres is being used if DATABASE_URL is defined. + +if [[ ! -v DATABASE_URL ]]; then + echo "DATABASE_URL is not defined. Using sqlite database..." + export DATABASE_URL=sqlite://mailmanweb.db + export DATABASE_TYPE='sqlite' +fi + +if [[ "$DATABASE_TYPE" = 'postgres' ]] +then + wait_for_postgres +elif [[ "$DATABASE_TYPE" = 'mysql' ]] +then + wait_for_mysql +fi + +# Check if we are in the correct directory before running commands. +if [[ ! $(pwd) == '/opt/mailman-web' ]]; then + echo "Running in the wrong directory...switching to /opt/mailman-web" + cd /opt/mailman-web +fi + +# Check if the logs directory is setup. +if [[ ! -e /opt/mailman-web-data/logs/mailmanweb.log ]]; then + echo "Creating log file for mailman web" + mkdir -p /opt/mailman-web-data/logs/ + touch /opt/mailman-web-data/logs/mailmanweb.log +fi + +if [[ ! -e /opt/mailman-web-data/logs/uwsgi.log ]]; then + echo "Creating log file for uwsgi.." + touch /opt/mailman-web-data/logs/uwsgi.log +fi + +# Check if the settings_local.py file exists, if yes, copy it too. +if [[ -e /opt/mailman-web-data/settings_local.py ]]; then + echo "Copying settings_local.py ..." + cp /opt/mailman-web-data/settings_local.py /opt/mailman-web/settings_local.py + chown mailman:mailman /opt/mailman-web/settings_local.py +else + echo "settings_local.py not found, it is highly recommended that you provide one" + echo "Using default configuration to run." +fi + +# Collect static for the django installation. +python3 manage.py collectstatic --noinput --clear --verbosity 0 + + +# Compile all the installed po files to mo. +SITE_DIR=$(python3 -c 'import site; print(site.getsitepackages()[0])') +echo "Compiling locale files in $SITE_DIR" +cd $SITE_DIR && /opt/mailman-web/manage.py compilemessages && cd - + +# Compress static files. +python3 /opt/mailman-web/manage.py compress --force + + +# Migrate all the data to the database if this is a new installation, otherwise +# this command will upgrade the database. +python3 /opt/mailman-web/manage.py migrate + +# If MAILMAN_ADMIN_USER and MAILMAN_ADMIN_EMAIL is defined create a new +# superuser for Django. There is no password setup so it can't login yet unless +# the password is reset. +if [[ -v MAILMAN_ADMIN_USER ]] && [[ -v MAILMAN_ADMIN_EMAIL ]]; +then + echo "Creating admin user $MAILMAN_ADMIN_USER ..." + python3 /opt/mailman-web/manage.py createsuperuser --noinput --username "$MAILMAN_ADMIN_USER"\ + --email "$MAILMAN_ADMIN_EMAIL" 2> /dev/null || \ + echo "Superuser $MAILMAN_ADMIN_USER already exists" +fi + +# If SERVE_FROM_DOMAIN is defined then rename the default `example.com` +# domain to the defined domain. +if [[ -v SERVE_FROM_DOMAIN ]]; +then + echo "Setting $SERVE_FROM_DOMAIN as the default domain ..." + python3 /opt/mailman-web/manage.py shell -c \ + "from django.contrib.sites.models import Site; Site.objects.filter(domain='example.com').update(domain='$SERVE_FROM_DOMAIN', name='$SERVE_FROM_DOMAIN')" +fi + +# Create a mailman user with the specific UID and GID and do not create home +# directory for it. Also chown the logs directory to write the files. +chown mailman:mailman /opt/mailman-web-data -R + +[[ -v DISKCACHE_PATH ]] && chown mailman:mailman "${DISKCACHE_PATH}" -R + +exec $@ diff --git a/mailman/web/mailman-web/__init__.py b/mailman/web/mailman-web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mailman/web/mailman-web/manage.py b/mailman/web/mailman-web/manage.py new file mode 100755 index 0000000..4eb6f34 --- /dev/null +++ b/mailman/web/mailman-web/manage.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/mailman/web/mailman-web/settings.py b/mailman/web/mailman-web/settings.py new file mode 100644 index 0000000..c19a827 --- /dev/null +++ b/mailman/web/mailman-web/settings.py @@ -0,0 +1,424 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. +# +# This file is part of Mailman Suite. +# +# Mailman Suite is free sofware: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# Mailman Suite is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + +# You should have received a copy of the GNU General Public License along +# with Mailman Suite. If not, see . +""" +Django Settings for Mailman Suite (hyperkitty + postorius) + +For more information on this file, see +https://docs.djangoproject.com/en/1.8/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.8/ref/settings/ +""" + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +import os +import dj_database_url +import sys +from socket import gethostbyname, gaierror + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.environ.get('SECRET_KEY') + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False + +ADMINS = ( + ('Mailman Suite Admin', 'root@localhost'), +) + +SITE_ID = 1 + +# Hosts/domain names that are valid for this site; required if DEBUG is False +# See https://docs.djangoproject.com/en/3.1/ref/settings/#allowed-hosts +ALLOWED_HOSTS = [ + "localhost", # Archiving API from Mailman, keep it. + "mailman-web", + os.environ.get('SERVE_FROM_DOMAIN'), +] + +try: + ALLOWED_HOSTS.append(gethostbyname("mailman-web")) # only add if this resolves +except gaierror: + pass + +ALLOWED_HOSTS.extend(os.getenv("DJANGO_ALLOWED_HOSTS", "").split(",")) + +# Mailman API credentials +MAILMAN_REST_API_URL = os.environ.get('MAILMAN_REST_URL', 'http://mailman-core:8001') +MAILMAN_REST_API_USER = os.environ.get('MAILMAN_REST_USER', 'restadmin') +MAILMAN_REST_API_PASS = os.environ.get('MAILMAN_REST_PASSWORD', 'restpass') +MAILMAN_ARCHIVER_KEY = os.environ.get('HYPERKITTY_API_KEY') +MAILMAN_ARCHIVER_FROM = (os.environ.get('MAILMAN_HOST_IP', gethostbyname(os.environ.get('MAILMAN_HOSTNAME', 'mailman-core'))),) + +# Application definition + +INSTALLED_APPS = [] +DEFAULT_APPS = [ + 'hyperkitty', + 'postorius', + 'django_mailman3', + # Uncomment the next line to enable the admin: + 'django.contrib.admin', + # Uncomment the next line to enable admin documentation: + # 'django.contrib.admindocs', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.humanize', + 'rest_framework', + 'django_gravatar', + 'compressor', + 'haystack', + 'django_extensions', + 'django_q', + 'allauth', + 'allauth.account', + 'allauth.socialaccount', +] + +MAILMAN_WEB_SOCIAL_AUTH = [ + 'django_mailman3.lib.auth.fedora', + 'allauth.socialaccount.providers.openid', + 'allauth.socialaccount.providers.github', + 'allauth.socialaccount.providers.gitlab', + 'allauth.socialaccount.providers.google', +] + +MIDDLEWARE = ( + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.middleware.locale.LocaleMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'allauth.account.middleware.AccountMiddleware', + 'django_mailman3.middleware.TimezoneMiddleware', + 'postorius.middleware.PostoriusMiddleware', +) + +ROOT_URLCONF = 'urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.template.context_processors.tz', + 'django.template.context_processors.csrf', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'django_mailman3.context_processors.common', + 'hyperkitty.context_processors.common', + 'postorius.context_processors.postorius', + ], + }, + }, +] + +WSGI_APPLICATION = 'wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases +# dj_database_url uses $DATABASE_URL environment variable to create a +# django-style-config-dict. +# https://github.com/kennethreitz/dj-database-url +DATABASES = { + 'default': dj_database_url.config(conn_max_age=600) +} + +# Avoid Django 3.2+ warning +# https://github.com/maxking/docker-mailman/issues/595 +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + + +# If you're behind a proxy, use the X-Forwarded-Host header +# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host +USE_X_FORWARDED_HOST = True + + +# Password validation +# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +# Internationalization +# https://docs.djangoproject.com/en/1.8/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = os.environ.get('TZ', 'UTC') + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +STATIC_ROOT = '/opt/mailman-web-data/static' + +STATIC_URL = '/static/' + +# Additional locations of static files + + +# List of finder classes that know how to find static files in +# various locations. +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + 'compressor.finders.CompressorFinder', +) + + +SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer' + +LOGIN_URL = 'account_login' +LOGIN_REDIRECT_URL = 'list_index' +LOGOUT_URL = 'account_logout' + + +# Use SERVE_FROM_DOMAIN as the default domain in the email. +hostname = os.environ.get('SERVE_FROM_DOMAIN', 'localhost.local') +DEFAULT_FROM_EMAIL = 'postorius@{}'.format(hostname) +SERVER_EMAIL = 'root@{}'.format(hostname) + +# Change this when you have a real email backend +EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST = os.environ.get('SMTP_HOST', '') +EMAIL_PORT = os.environ.get('SMTP_PORT', 25) +EMAIL_HOST_USER = os.environ.get('SMTP_HOST_USER', '') +EMAIL_HOST_PASSWORD = os.environ.get('SMTP_HOST_PASSWORD', '') +EMAIL_USE_TLS = os.environ.get('SMTP_USE_TLS', False) +EMAIL_USE_SSL = os.environ.get('SMTP_USE_SSL', False) + +# Compatibility with Bootstrap 3 +from django.contrib.messages import constants as messages # flake8: noqa +MESSAGE_TAGS = { + messages.ERROR: 'danger' +} + + +# +# Social auth +# +AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'allauth.account.auth_backends.AuthenticationBackend', +) + +# Django Allauth +ACCOUNT_AUTHENTICATION_METHOD = "username_email" +ACCOUNT_EMAIL_REQUIRED = True +ACCOUNT_EMAIL_VERIFICATION = "mandatory" +# You probably want https in production, but this is a dev setup file +ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https" +ACCOUNT_UNIQUE_EMAIL = True + +SOCIALACCOUNT_PROVIDERS = { + 'openid': { + 'SERVERS': [ + dict(id='yahoo', + name='Yahoo', + openid_url='http://me.yahoo.com'), + ], + }, + 'google': { + 'SCOPE': ['profile', 'email'], + 'AUTH_PARAMS': {'access_type': 'online'}, + }, + 'facebook': { + 'METHOD': 'oauth2', + 'SCOPE': ['email'], + 'FIELDS': [ + 'email', + 'name', + 'first_name', + 'last_name', + 'locale', + 'timezone', + ], + 'VERSION': 'v2.4', + }, +} + + +# django-compressor +# https://pypi.python.org/pypi/django_compressor +# +COMPRESS_PRECOMPILERS = ( + ('text/less', 'lessc {infile} {outfile}'), + ('text/x-scss', 'sassc -t compressed {infile} {outfile}'), + ('text/x-sass', 'sassc -t compressed {infile} {outfile}'), +) + +# On a production setup, setting COMPRESS_OFFLINE to True will bring a +# significant performance improvement, as CSS files will not need to be +# recompiled on each requests. It means running an additional "compress" +# management command after each code upgrade. +# http://django-compressor.readthedocs.io/en/latest/usage/#offline-compression +# COMPRESS_OFFLINE = True + +# +# Full-text search engine +# +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', + 'PATH': "/opt/mailman-web-data/fulltext_index", + # You can also use the Xapian engine, it's faster and more accurate, + # but requires another library. + # http://django-haystack.readthedocs.io/en/v2.4.1/installing_search_engines.html#xapian + # Example configuration for Xapian: + #'ENGINE': 'xapian_backend.XapianEngine' + }, +} + +import sys +# A sample logging configuration. The only tangible logging +# performed by this configuration is to send an email to +# the site admins on every HTTP 500 error when DEBUG=False. +# See http://docs.djangoproject.com/en/dev/topics/logging for +# more details on how to customize your logging configuration. +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler' + }, + 'file':{ + 'level': 'INFO', + 'class': 'logging.handlers.RotatingFileHandler', + #'class': 'logging.handlers.WatchedFileHandler', + 'filename': os.environ.get('DJANGO_LOG_URL','/opt/mailman-web-data/logs/mailmanweb.log'), + 'formatter': 'verbose', + }, + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'simple', + 'level': 'INFO', + 'stream': sys.stdout, + }, + }, + 'loggers': { + 'django.request': { + 'handlers': ['mail_admins', 'file'], + 'level': 'INFO', + 'propagate': True, + }, + 'django': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True, + }, + 'hyperkitty': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True, + }, + 'postorius': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True + }, + }, + 'formatters': { + 'verbose': { + 'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s' + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + }, +} + + +if os.environ.get('LOG_TO_CONSOLE') == 'yes': + LOGGING['loggers']['django']['handlers'].append('console') + LOGGING['loggers']['django.request']['handlers'].append('console') + +# HyperKitty-specific +# +# Only display mailing-lists from the same virtual host as the webserver +FILTER_VHOST = False + + +Q_CLUSTER = { + 'timeout': 300, + 'retry': 300, + 'save_limit': 100, + 'orm': 'default', +} + +POSTORIUS_TEMPLATE_BASE_URL = os.environ.get('POSTORIUS_TEMPLATE_BASE_URL', 'http://mailman-web:8000') + +DISKCACHE_PATH = os.environ.get('DISKCACHE_PATH', '/opt/mailman-web-data/diskcache') +DISKCACHE_SIZE = os.environ.get('DISKCACHE_SIZE', 2 ** 30) # 1 gigabyte + +CACHES = { + 'default': { + 'BACKEND': 'diskcache.DjangoCache', + 'LOCATION': DISKCACHE_PATH, + 'OPTIONS': { + 'size_limit': DISKCACHE_SIZE, + }, + }, +} + +try: + from settings_local import * +except ImportError: + pass + +# Compatibility for older installs that override INSTALLED_APPS +if not INSTALLED_APPS: + INSTALLED_APPS = DEFAULT_APPS + MAILMAN_WEB_SOCIAL_AUTH diff --git a/mailman/web/mailman-web/urls.py b/mailman/web/mailman-web/urls.py new file mode 100644 index 0000000..19d49f9 --- /dev/null +++ b/mailman/web/mailman-web/urls.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. +# +# This file is part of Postorius. +# +# Postorius is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# Postorius is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Postorius. If not, see . + +from django.conf.urls import include +from django.contrib import admin +from django.urls import path, reverse_lazy +from django.views.generic import RedirectView + +urlpatterns = [ + path(r'', RedirectView.as_view( + url=reverse_lazy('list_index'), + permanent=True)), + path(r'postorius/', include('postorius.urls')), + path(r'hyperkitty/', include('hyperkitty.urls')), + path(r'', include('django_mailman3.urls')), + path(r'accounts/', include('allauth.urls')), + # Django admin + path(r'admin/', admin.site.urls), +] diff --git a/mailman/web/mailman-web/uwsgi.ini b/mailman/web/mailman-web/uwsgi.ini new file mode 100644 index 0000000..897ebad --- /dev/null +++ b/mailman/web/mailman-web/uwsgi.ini @@ -0,0 +1,53 @@ +[uwsgi] +# Port on which uwsgi will be listening. +uwsgi-socket = 0.0.0.0:8080 +http-socket = 0.0.0.0:8000 + +# Enable threading for python +enable-threads = true + +# Setting uwsgi buffer size to what Apache2 supports. +buffer-size = 8190 + +# Move to the directory where the django files are. +chdir = /opt/mailman-web + +# Use the wsgi file provided with the django project. +wsgi-file = wsgi.py + +# Setup default number of processes and threads per process. +master = true +processes = 2 +threads = 2 + +# Drop privileges and don't run as root. +uid = mailman +gid = mailman + +# Setup the django_q related worker processes. +attach-daemon = ./manage.py qcluster + +# Setup hyperkitty's cron jobs. +# 'minutely' jobs are run hourly for perf reasons. +# See https://github.com/maxking/docker-mailman/issues/327 +unique-cron = 0 -1 -1 -1 -1 ./manage.py runjobs minutely +unique-cron = -15 -1 -1 -1 -1 ./manage.py runjobs quarter_hourly +unique-cron = 0 -1 -1 -1 -1 ./manage.py runjobs hourly +unique-cron = 0 0 -1 -1 -1 ./manage.py runjobs daily +unique-cron = 0 0 1 -1 -1 ./manage.py runjobs monthly +unique-cron = 0 0 -1 -1 0 ./manage.py runjobs weekly +unique-cron = 0 0 1 1 -1 ./manage.py runjobs yearly + +# Setup the request log. +req-logger = file:/opt/mailman-web-data/logs/uwsgi.log + +# Log cron seperately. +logger = cron file:/opt/mailman-web-data/logs/uwsgi-cron.log +log-route = cron uwsgi-cron + +# Log qcluster commands seperately. +logger = qcluster file:/opt/mailman-web-data/logs/uwsgi-qcluster.log +log-route = qcluster uwsgi-daemons + +# Last log and it logs the rest of the stuff. +logger = file:/opt/mailman-web-data/logs/uwsgi-error.log diff --git a/mailman/web/mailman-web/wsgi.py b/mailman/web/mailman-web/wsgi.py new file mode 100755 index 0000000..8e59cd8 --- /dev/null +++ b/mailman/web/mailman-web/wsgi.py @@ -0,0 +1,38 @@ +""" +WSGI config for HyperKitty project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/wsgi/ +""" + +import os + +# import sys +# import site + +# For some unknown reason, sometimes mod_wsgi fails to set the python paths to +# the virtualenv, with the 'python-path' option. You can do it here too. +# +# # Remember original sys.path. +# prev_sys_path = list(sys.path) +# # Add here, for the settings module +# site.addsitedir(os.path.abspath(os.path.dirname(__file__))) +# # Add the virtualenv +# venv = os.path.join(os.path.abspath(os.path.dirname(__file__)), +# '..', 'lib', 'python2.6', 'site-packages') +# site.addsitedir(venv) +# # Reorder sys.path so new directories at the front. +# new_sys_path = [] +# for item in list(sys.path): +# if item not in prev_sys_path: +# new_sys_path.append(item) +# sys.path.remove(item) +# sys.path[:0] = new_sys_path + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") + +application = get_wsgi_application() diff --git a/mailman/web/requirements.txt b/mailman/web/requirements.txt new file mode 100644 index 0000000..e6dcc61 --- /dev/null +++ b/mailman/web/requirements.txt @@ -0,0 +1,4 @@ +mailmanclient==3.3.5 +postorius==1.3.10 +hyperkitty==1.3.9 +django-mailman3==1.3.11 \ No newline at end of file diff --git a/web/etc/apache2/sites-available/ketrenos.com-ssl b/web/etc/apache2/sites-available/ketrenos.com-ssl index 672b785..497d71c 100644 --- a/web/etc/apache2/sites-available/ketrenos.com-ssl +++ b/web/etc/apache2/sites-available/ketrenos.com-ssl @@ -1,5 +1,5 @@ - SSLPassPhraseDialog exec:/etc/apache2/ssl/passphrase +# SSLPassPhraseDialog exec:/etc/apache2/ssl/passphrase ServerAdmin james_webmaster@ketrenos.com ServerName ketrenos.com diff --git a/web/etc/nginx/sites-available/default b/web/etc/nginx/sites-available/default index fc3d49c..3e57817 100644 --- a/web/etc/nginx/sites-available/default +++ b/web/etc/nginx/sites-available/default @@ -31,6 +31,11 @@ server { autoindex on; } + location /3d-sbs { + alias /var/www/ketrenos.com/3d-sbs; + autoindex on; + } + location /funeral { alias /var/www/ketrenos.com/funeral; autoindex on;