From 2a3df086d314fcf5c249cc48a83029662fc50ed0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 20 Jun 2021 21:58:30 -0700 Subject: [PATCH] Docker: add [c]git, nginx, fastapi, php-fpm, ca Now, we have a full collection of services used to run aurweb over HTTPS using a self-signed CA. New Docker services: - `ca` - Certificate authority services - When the `ca` service is run, it will (if needed) generate a CA certificate and leaf certificate for localhost AUR access. This ca is then shared with things like nginx to use the leaf certificate. Users can import `./cache/ca.root.pem` into their browser or ca-certificates as a root CA who issued aurweb's certificate. - `git` - Start sshd and set it up for aur git access - `cgit` - Serve cgit with uwsgi on port 3000 - `fastapi` - Serve our FastAPI app with `hypercorn` on port 8000 - `php-fpm` - Serve our PHP-wise aurweb - `nginx` - Serve FastAPI, PHP and CGit with an HTTPS certificate. - PHP: https://localhost:8443 - PHP CGit: https://localhost:8443/cgit - FastAPI: https://localhost:8444 - FastAPI CGit: https://localhost:8444/cgit Short of it: Run the following in a shell to run PHP and FastAPI servers on port **8443** and **8444**, respectively. $ docker-compose up nginx This will host the PHP, FastAPI, CGit and Git ecosystems. Git SSH can be knocked at `aur@localhost:2222` as long as you have a valid public key in the aurweb database. Signed-off-by: Kevin Morris --- .gitignore | 1 + Dockerfile | 2 +- docker-compose.yml | 158 ++++++++++++++++++++++++++++++++++ docker/ca-entrypoint.sh | 36 ++++++++ docker/ca.ext | 7 ++ docker/cgit-entrypoint.sh | 13 +++ docker/fastapi-entrypoint.sh | 10 +++ docker/git-entrypoint.sh | 45 ++++++++++ docker/health/cgit.sh | 2 + docker/health/fastapi.sh | 2 + docker/health/nginx.sh | 2 + docker/health/php.sh | 2 + docker/health/sshd.sh | 2 + docker/logging.conf | 27 ++++++ docker/nginx-entrypoint.sh | 123 ++++++++++++++++++++++++++ docker/php-entrypoint.sh | 18 ++++ docker/scripts/run-fastapi.sh | 11 +++ docker/scripts/run-nginx.sh | 16 ++++ docker/scripts/run-php.sh | 7 ++ docker/scripts/run-sshd.sh | 2 + logs/.gitkeep | 0 21 files changed, 485 insertions(+), 1 deletion(-) create mode 100755 docker/ca-entrypoint.sh create mode 100644 docker/ca.ext create mode 100755 docker/cgit-entrypoint.sh create mode 100755 docker/fastapi-entrypoint.sh create mode 100755 docker/git-entrypoint.sh create mode 100755 docker/health/cgit.sh create mode 100755 docker/health/fastapi.sh create mode 100755 docker/health/nginx.sh create mode 100755 docker/health/php.sh create mode 100755 docker/health/sshd.sh create mode 100644 docker/logging.conf create mode 100755 docker/nginx-entrypoint.sh create mode 100755 docker/php-entrypoint.sh create mode 100755 docker/scripts/run-fastapi.sh create mode 100755 docker/scripts/run-nginx.sh create mode 100755 docker/scripts/run-php.sh create mode 100755 docker/scripts/run-sshd.sh create mode 100644 logs/.gitkeep diff --git a/.gitignore b/.gitignore index 2006e838..5d1a4de7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ fastapi_aw/ .coverage .idea /cache/* +/logs/* diff --git a/Dockerfile b/Dockerfile index 6858bedb..002f135a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN pacman -Syu --noconfirm --noprogressbar \ python-requests python-aiofiles python-python-multipart \ python-pytest-asyncio python-coverage hypercorn python-bcrypt \ python-email-validator openssh python-lxml mariadb mariadb-libs \ - python-isort flake8 + python-isort flake8 cgit uwsgi uwsgi-plugin-cgi php php-fpm RUN useradd -U -d /aurweb -c 'AUR User' aur diff --git a/docker-compose.yml b/docker-compose.yml index 7fbd20fd..e21b91bf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,12 +7,28 @@ # - `pytest-sqlite` - Run pytest suites with SQLite # - `test` - Run sharness, pytest-mysql and pytest-sqlite # - `mariadb` - `port 13306` - MariaDB server for docker +# - `ca` - Certificate Authority generation +# - `git` - `port 2222` - Git over SSH server +# - `fastapi` - hypercorn service for aurweb's FastAPI app +# - `php-fpm` - Execution server for PHP aurweb +# - `nginx` - `ports 8444 (FastAPI), 8443 (PHP)` - Everything +# - You can reach `nginx` via FastAPI at `https://localhost:8444/` +# or via PHP at `https://localhost:8443/`. CGit can be reached +# via the `/cgit/` request uri on either server. # # Copyright (C) 2021 aurweb Development # All Rights Reserved. version: "3.8" services: + ca: + image: aurweb:latest + init: true + entrypoint: /docker/ca-entrypoint.sh + command: exit 0 + volumes: + - ./cache:/cache + mariadb: image: aurweb:latest init: true @@ -30,6 +46,132 @@ services: interval: 2s timeout: 60s + git: + image: aurweb:latest + init: true + environment: + - AUR_CONFIG=conf/config + entrypoint: /docker/git-entrypoint.sh + command: /docker/scripts/run-sshd.sh + ports: + - "2222:22" + healthcheck: + test: "bash /docker/health/sshd.sh" + interval: 2s + timeout: 30s + volumes: + - mariadb_run:/var/run/mysqld + - mariadb_data:/var/lib/mysql + - git_data:/aurweb/aur.git + - ./cache:/cache + + cgit: + image: aurweb:latest + init: true + environment: + - AUR_CONFIG=/aurweb/conf/config + entrypoint: /docker/cgit-entrypoint.sh + command: >- + uwsgi --socket 0.0.0.0:3000 + --plugins cgi + --cgi /usr/share/webapps/cgit/cgit.cgi + healthcheck: + test: "bash /docker/health/cgit.sh" + interval: 2s + timeout: 30s + depends_on: + git: + condition: service_healthy + links: + - git + volumes: + - git_data:/aurweb/aur.git + + php-fpm: + image: aurweb:latest + init: true + environment: + - AUR_CONFIG=/aurweb/conf/config + entrypoint: /docker/php-entrypoint.sh + command: /docker/scripts/run-php.sh + healthcheck: + test: "bash /docker/health/php.sh" + interval: 2s + timeout: 30s + depends_on: + ca: + condition: service_started + git: + condition: service_healthy + mariadb: + condition: service_healthy + links: + - ca + - git + - mariadb + volumes: + - mariadb_run:/var/run/mysqld # Bind socket in this volume. + - mariadb_data:/var/lib/mysql + - ./cache:/cache + + + fastapi: + image: aurweb:latest + init: true + environment: + - AUR_CONFIG=conf/config + entrypoint: /docker/fastapi-entrypoint.sh + command: /docker/scripts/run-fastapi.sh + healthcheck: + test: "bash /docker/health/fastapi.sh" + interval: 2s + timeout: 30s + depends_on: + ca: + condition: service_started + git: + condition: service_healthy + mariadb: + condition: service_healthy + links: + - ca + - git + - mariadb + volumes: + - mariadb_run:/var/run/mysqld # Bind socket in this volume. + - mariadb_data:/var/lib/mysql + - ./cache:/cache + + nginx: + image: aurweb:latest + init: true + environment: + - AUR_CONFIG=conf/config + entrypoint: /docker/nginx-entrypoint.sh + command: /docker/scripts/run-nginx.sh + ports: + - "8443:8443" # PHP + - "8444:8444" # FastAPI + healthcheck: + test: "bash /docker/health/nginx.sh" + interval: 2s + timeout: 30s + depends_on: + cgit: + condition: service_healthy + fastapi: + condition: service_healthy + php-fpm: + condition: service_healthy + links: + - cgit + - fastapi + - php-fpm + volumes: + - git_data:/aurweb/aur.git + - ./cache:/cache + - ./logs:/var/log/nginx + sharness: image: aurweb:latest init: true @@ -37,7 +179,13 @@ services: - AUR_CONFIG=conf/config.sqlite entrypoint: /docker/test-sqlite-entrypoint.sh command: /docker/scripts/run-sharness.sh + depends_on: + git: + condition: service_healthy + links: + - git volumes: + - git_data:/aurweb/aur.git - ./cache:/cache pytest-mysql: @@ -50,10 +198,14 @@ services: depends_on: mariadb: condition: service_healthy + git: + condition: service_healthy links: - mariadb + - git volumes: - mariadb_run:/var/run/mysqld + - git_data:/aurweb/aur.git - ./cache:/cache pytest-sqlite: @@ -65,7 +217,11 @@ services: command: /docker/scripts/run-pytests.sh clean volumes: - mariadb_run:/var/run/mysqld + - git_data:/aurweb/aur.git - ./cache:/cache + depends_on: + git: + condition: service_healthy test: image: aurweb:latest @@ -81,8 +237,10 @@ services: - mariadb volumes: - mariadb_run:/var/run/mysqld + - git_data:/aurweb/aur.git - ./cache:/cache volumes: mariadb_run: {} # Share /var/run/mysqld/mysqld.sock mariadb_data: {} # Share /var/lib/mysql + git_data: {} # Share aurweb/aur.git diff --git a/docker/ca-entrypoint.sh b/docker/ca-entrypoint.sh new file mode 100755 index 00000000..b7514187 --- /dev/null +++ b/docker/ca-entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -eou pipefail + +if [ -f /cache/localhost.cert.pem ] && \ + [ -f /cache/localhost.key.pem ] && \ + [ -f /cache/ca.root.pem ]; then + echo "Already have certs, skipping." + exec "$@" +fi + +openssl genrsa -des3 -out ca.key \ + -passout pass:devca 2048 + +openssl req -x509 -new -nodes \ + -key ca.key -sha256 -days 1825 \ + -out /cache/ca.root.pem \ + -subj "/C=US/ST=California/L=Nowhere/O=aurweb/CN=localhost" \ + --passin pass:devca + +# Generate keys for aurweb. +openssl req -nodes -newkey rsa:2048 -keyout /cache/localhost.key.pem \ + -out localhost.csr \ + -subj "/C=US/ST=California/L=Nowhere/O=aurweb/CN=localhost" + +echo "$(hexdump -n 16 -e '4/4 "%08X" 1 "\n"' /dev/random)" \ + > /cache/ca.root.srl +openssl x509 -req -in localhost.csr -CA /cache/ca.root.pem \ + -CAkey ca.key -CAserial /cache/ca.root.srl \ + -out /cache/localhost.cert.pem \ + -days 825 -sha256 -extfile /docker/ca.ext \ + --passin pass:devca + +chmod 666 /cache/localhost.{key,cert}.pem +chmod 666 /cache/ca.root.pem + +exec "$@" diff --git a/docker/ca.ext b/docker/ca.ext new file mode 100644 index 00000000..ab9de5fb --- /dev/null +++ b/docker/ca.ext @@ -0,0 +1,7 @@ +authorityKeyIdentifier=keyid,issuer +basicConstraints=CA:FALSE +keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment +subjectAltName = @alt_names + +[alt_names] +DNS.1 = localhost diff --git a/docker/cgit-entrypoint.sh b/docker/cgit-entrypoint.sh new file mode 100755 index 00000000..e05e1b7a --- /dev/null +++ b/docker/cgit-entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -eou pipefail + +cp -vf conf/cgitrc.proto /etc/cgitrc + +sed -ri 's|clone-prefix=.*|clone-prefix=https://localhost:8443|' /etc/cgitrc +sed -ri 's|header=.*|header=/aurweb/web/template/cgit/header.html|' /etc/cgitrc +sed -ri 's|footer=.*|footer=/aurweb/web/template/cgit/footer.html|' /etc/cgitrc +sed -ri 's|repo\.path=.*|repo.path=/aurweb/aur.git|' /etc/cgitrc + +mkdir -p /var/cache/cgit + +exec "$@" diff --git a/docker/fastapi-entrypoint.sh b/docker/fastapi-entrypoint.sh new file mode 100755 index 00000000..2f04c29f --- /dev/null +++ b/docker/fastapi-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -eou pipefail +dir="$(dirname $0)" + +bash $dir/test-mysql-entrypoint.sh + +sed -ri "s;^(aur_location) = .+;\1 = https://localhost:8444;" conf/config +sed -ri 's/^(name) = .+/\1 = aurweb/' conf/config + +exec "$@" diff --git a/docker/git-entrypoint.sh b/docker/git-entrypoint.sh new file mode 100755 index 00000000..d17ceeaf --- /dev/null +++ b/docker/git-entrypoint.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -eou pipefail + +SSHD_CONFIG=/etc/ssh/sshd_config + +GIT_REPO=aur.git +GIT_KEY=/cache/git.key + +# Setup SSH Keys. +ssh-keygen -A + +# Add AUR SSH config. +cat >> $SSHD_CONFIG << EOF +Match User aur + PasswordAuthentication no + AuthorizedKeysCommand /usr/local/bin/aurweb-git-auth "%t" "%k" + AuthorizedKeysCommandUser aur + AcceptEnv AUR_OVERWRITE + SetEnv AUR_CONFIG=/aurweb/config/config +EOF + +# Taken from INSTALL. +mkdir -pv $GIT_REPO + +# Initialize git repository. +if [ ! -f $GIT_REPO/config ]; then + cd $GIT_REPO + git init --bare + git config --local transfer.hideRefs '^refs/' + git config --local --add transfer.hideRefs '!refs/' + git config --local --add transfer.hideRefs '!HEAD' + ln -sf /usr/local/bin/aurweb-git-update hooks/update + chown -R aur . + cd .. +fi + +if [ ! -f $GIT_KEY ]; then + # Create a DSA ssh private/pubkey at /cache/git.key{.pub,}. + ssh-keygen -f $GIT_KEY -t dsa -N '' -C 'AUR Git Key' +fi + +# Users should modify these permissions on their local machines. +chmod 666 ${GIT_KEY}{.pub,} + +exec "$@" diff --git a/docker/health/cgit.sh b/docker/health/cgit.sh new file mode 100755 index 00000000..add33031 --- /dev/null +++ b/docker/health/cgit.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec printf "" >>/dev/tcp/127.0.0.1/3000 diff --git a/docker/health/fastapi.sh b/docker/health/fastapi.sh new file mode 100755 index 00000000..19a1c733 --- /dev/null +++ b/docker/health/fastapi.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec pgrep hypercorn diff --git a/docker/health/nginx.sh b/docker/health/nginx.sh new file mode 100755 index 00000000..c530103d --- /dev/null +++ b/docker/health/nginx.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec curl --no-verify -q https://localhost:8444 diff --git a/docker/health/php.sh b/docker/health/php.sh new file mode 100755 index 00000000..7325946b --- /dev/null +++ b/docker/health/php.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec printf "" >>/dev/tcp/127.0.0.1/9000 diff --git a/docker/health/sshd.sh b/docker/health/sshd.sh new file mode 100755 index 00000000..6befdfb5 --- /dev/null +++ b/docker/health/sshd.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec printf "" >>/dev/tcp/127.0.0.1/22 diff --git a/docker/logging.conf b/docker/logging.conf new file mode 100644 index 00000000..ab754c9f --- /dev/null +++ b/docker/logging.conf @@ -0,0 +1,27 @@ +[loggers] +keys=root,sampleLogger + +[handlers] +keys=consoleHandler + +[formatters] +keys=sampleFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler + +[logger_sampleLogger] +level=DEBUG +handlers=consoleHandler +qualname=sampleLogger +propagate=0 + +[handler_consoleHandler] +class=StreamHandler +level=DEBUG +formatter=sampleFormatter +args=(sys.stdout,) + +[formatter_sampleFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s diff --git a/docker/nginx-entrypoint.sh b/docker/nginx-entrypoint.sh new file mode 100755 index 00000000..1e442ef7 --- /dev/null +++ b/docker/nginx-entrypoint.sh @@ -0,0 +1,123 @@ +#!/bin/bash +set -eou pipefail + +# Setup a config for our mysql db. +cp -vf conf/config.dev conf/config +sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config +sed -ri 's/^(host) = .+/\1 = mariadb/' conf/config +sed -ri 's/^(user) = .+/\1 = aur/' conf/config +sed -ri 's/^;?(password) = .+/\1 = aur/' conf/config + +# Setup http(s) stuff. +sed -ri "s|^(aur_location) = .+|\1 = https://localhost:8444|" conf/config +sed -ri 's/^(disable_http_login) = .+/\1 = 1/' conf/config + +cp -vf /cache/localhost.cert.pem /etc/ssl/certs/localhost.cert.pem +cp -vf /cache/localhost.key.pem /etc/ssl/private/localhost.key.pem + +cat > /etc/nginx/nginx.conf << EOF +daemon off; +user root; +worker_processes auto; +pid /var/run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; + +events { + worker_connections 256; +} + +http { + sendfile on; + tcp_nopush on; + types_hash_max_size 4096; + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers on; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + gzip on; + + upstream hypercorn { + server fastapi:8000; + } + + upstream cgit { + server cgit:3000; + } + + server { + listen 8443 ssl http2; + server_name localhost default_server; + + ssl_certificate /etc/ssl/certs/localhost.cert.pem; + ssl_certificate_key /etc/ssl/private/localhost.key.pem; + + root /aurweb/web/html; + index index.php; + + location ~ ^/cgit { + include uwsgi_params; + rewrite ^/cgit/([^?/]+/[^?]*)?(?:\?(.*))?$ /cgit.cgi?url=\$1&\$2 last; + uwsgi_modifier1 9; + uwsgi_param CGIT_CONFIG /etc/cgitrc; + uwsgi_pass uwsgi://cgit; + } + + location ~ ^/[^/]+\.php($|/) { + fastcgi_pass php-fpm:9000; + fastcgi_index index.php; + fastcgi_split_path_info ^(/[^/]+\.php)(/.*)\$; + fastcgi_param SCRIPT_FILENAME \$document_root\$fastcgi_script_name; + fastcgi_param PATH_INFO \$fastcgi_path_info; + include fastcgi_params; + } + + location ~ .* { + rewrite ^/(.*)$ /index.php/\$1 last; + } + + } + + server { + listen 8444 ssl http2; + server_name localhost default_server; + + ssl_certificate /etc/ssl/certs/localhost.cert.pem; + ssl_certificate_key /etc/ssl/private/localhost.key.pem; + + root /aurweb/web/html; + + location / { + try_files \$uri @proxy_to_app; + } + + location ~ ^/cgit { + include uwsgi_params; + rewrite ^/cgit/([^?/]+/[^?]*)?(?:\?(.*))?$ /cgit.cgi?url=\$1&\$2 last; + uwsgi_modifier1 9; + uwsgi_param CGIT_CONFIG /etc/cgitrc; + uwsgi_pass uwsgi://cgit; + } + + location @proxy_to_app { + proxy_set_header Host \$http_host; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto \$scheme; + proxy_redirect off; + proxy_buffering off; + proxy_pass https://hypercorn; + } + } + + map \$http_upgrade \$connection_upgrade { + default upgrade; + '' close; + } +} +EOF + +exec "$@" diff --git a/docker/php-entrypoint.sh b/docker/php-entrypoint.sh new file mode 100755 index 00000000..19c6d059 --- /dev/null +++ b/docker/php-entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -eou pipefail +dir="$(dirname $0)" + +bash $dir/test-mysql-entrypoint.sh + +sed -ri "s;^(aur_location) = .+;\1 = https://localhost:8443;" conf/config +sed -ri 's/^(name) = .+/\1 = aurweb/' conf/config + +sed -ri 's/^(listen).*/\1 = 0.0.0.0:9000/' /etc/php/php-fpm.d/www.conf +sed -ri 's/^;?(clear_env).*/\1 = no/' /etc/php/php-fpm.d/www.conf + +sed -ri 's/^;?(extension=pdo_mysql)/\1/' /etc/php/php.ini +sed -ri 's/^;?(open_basedir).*$/\1 = \//' /etc/php/php.ini + +python -m aurweb.initdb 2>/dev/null || /bin/true + +exec "$@" diff --git a/docker/scripts/run-fastapi.sh b/docker/scripts/run-fastapi.sh new file mode 100755 index 00000000..6c420497 --- /dev/null +++ b/docker/scripts/run-fastapi.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Initialize the new database; ignore errors. +python -m aurweb.initdb 2>/dev/null || /bin/true + +exec hypercorn --reload \ + --certfile /cache/localhost.cert.pem \ + --keyfile /cache/localhost.key.pem \ + --error-logfile - \ + --log-config docker/logging.conf \ + -b "0.0.0.0:8000" aurweb.asgi:app diff --git a/docker/scripts/run-nginx.sh b/docker/scripts/run-nginx.sh new file mode 100755 index 00000000..7780dae8 --- /dev/null +++ b/docker/scripts/run-nginx.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +echo "=== Running nginx server! ===" +echo +echo " Services:" +echo " - FastAPI : https://localhost:8444/" +echo " (cgit) : https://localhost:8444/cgit/" +echo " - PHP : https://localhost:8443/" +echo " (cgit) : https://localhost:8443/cgit/" +echo +echo " Note: Copy root CA (./cache/ca.root.pem) to ca-certificates or browser." +echo +echo " Thanks for using aurweb!" +echo + +exec nginx -c /etc/nginx/nginx.conf diff --git a/docker/scripts/run-php.sh b/docker/scripts/run-php.sh new file mode 100755 index 00000000..22346b47 --- /dev/null +++ b/docker/scripts/run-php.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -eou pipefail + +# Initialize the new database; ignore errors. +python -m aurweb.initdb 2>/dev/null || /bin/true + +exec php-fpm --fpm-config /etc/php/php-fpm.conf --nodaemonize diff --git a/docker/scripts/run-sshd.sh b/docker/scripts/run-sshd.sh new file mode 100755 index 00000000..a69af7e2 --- /dev/null +++ b/docker/scripts/run-sshd.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec /usr/sbin/sshd -D diff --git a/logs/.gitkeep b/logs/.gitkeep new file mode 100644 index 00000000..e69de29b