Docker: add [c]git, nginx, fastapi, php-fpm, ca

Now, we have a full collection of services used to run
aurweb over HTTPS using a self-signed CA.

New Docker services:

- `ca` - Certificate authority services
    - When the `ca` service is run, it will (if needed) generate
      a CA certificate and leaf certificate for localhost AUR
      access. This ca is then shared with things like nginx to
      use the leaf certificate. Users can import
      `./cache/ca.root.pem` into their browser or ca-certificates
      as a root CA who issued aurweb's certificate.
- `git` - Start sshd and set it up for aur git access
- `cgit` - Serve cgit with uwsgi on port 3000
- `fastapi` - Serve our FastAPI app with `hypercorn` on port 8000
- `php-fpm` - Serve our PHP-wise aurweb
- `nginx` - Serve FastAPI, PHP and CGit with an HTTPS certificate.
    - PHP: https://localhost:8443
    - PHP CGit: https://localhost:8443/cgit
    - FastAPI: https://localhost:8444
    - FastAPI CGit: https://localhost:8444/cgit

Short of it: Run the following in a shell to run PHP and FastAPI
servers on port **8443** and **8444**, respectively.

    $ docker-compose up nginx

This will host the PHP, FastAPI, CGit and Git ecosystems.

Git SSH can be knocked at `aur@localhost:2222` as long as you have a
valid public key in the aurweb database.

Signed-off-by: Kevin Morris <kevr@0cost.org>

.gitignore

@@ -19,3 +19,4 @@ fastapi_aw/
 .coverage
 .idea
 /cache/*
+/logs/*

Dockerfile

@@ -17,7 +17,7 @@ RUN pacman -Syu --noconfirm --noprogressbar \
     python-requests python-aiofiles python-python-multipart \
     python-pytest-asyncio python-coverage hypercorn python-bcrypt \
     python-email-validator openssh python-lxml mariadb mariadb-libs \
-    python-isort flake8
+    python-isort flake8 cgit uwsgi uwsgi-plugin-cgi php php-fpm
 
 RUN useradd -U -d /aurweb -c 'AUR User' aur
 

docker-compose.yml

@@ -7,12 +7,28 @@
 #   - `pytest-sqlite` - Run pytest suites with SQLite
 #   - `test` - Run sharness, pytest-mysql and pytest-sqlite
 #   - `mariadb` - `port 13306` - MariaDB server for docker
+#   - `ca` - Certificate Authority generation
+#   - `git` - `port 2222` - Git over SSH server
+#   - `fastapi` - hypercorn service for aurweb's FastAPI app
+#   - `php-fpm` - Execution server for PHP aurweb
+#   - `nginx` - `ports 8444 (FastAPI), 8443 (PHP)` - Everything
+#     - You can reach `nginx` via FastAPI at `https://localhost:8444/`
+#       or via PHP at `https://localhost:8443/`. CGit can be reached
+#       via the `/cgit/` request uri on either server.
 #
 # Copyright (C) 2021 aurweb Development
 # All Rights Reserved.
 version: "3.8"
 
 services:
+  ca:
+    image: aurweb:latest
+    init: true
+    entrypoint: /docker/ca-entrypoint.sh
+    command: exit 0
+    volumes:
+      - ./cache:/cache
+
   mariadb:
     image: aurweb:latest
     init: true
@@ -30,6 +46,132 @@ services:
       interval: 2s
       timeout: 60s
 
+  git:
+    image: aurweb:latest
+    init: true
+    environment:
+      - AUR_CONFIG=conf/config
+    entrypoint: /docker/git-entrypoint.sh
+    command: /docker/scripts/run-sshd.sh
+    ports:
+      - "2222:22"
+    healthcheck:
+      test: "bash /docker/health/sshd.sh"
+      interval: 2s
+      timeout: 30s
+    volumes:
+      - mariadb_run:/var/run/mysqld
+      - mariadb_data:/var/lib/mysql
+      - git_data:/aurweb/aur.git
+      - ./cache:/cache
+
+  cgit:
+    image: aurweb:latest
+    init: true
+    environment:
+      - AUR_CONFIG=/aurweb/conf/config
+    entrypoint: /docker/cgit-entrypoint.sh
+    command: >-
+      uwsgi --socket 0.0.0.0:3000
+            --plugins cgi
+            --cgi /usr/share/webapps/cgit/cgit.cgi
+    healthcheck:
+      test: "bash /docker/health/cgit.sh"
+      interval: 2s
+      timeout: 30s
+    depends_on:
+      git:
+        condition: service_healthy
+    links:
+      - git
+    volumes:
+      - git_data:/aurweb/aur.git
+
+  php-fpm:
+    image: aurweb:latest
+    init: true
+    environment:
+      - AUR_CONFIG=/aurweb/conf/config
+    entrypoint: /docker/php-entrypoint.sh
+    command: /docker/scripts/run-php.sh
+    healthcheck:
+      test: "bash /docker/health/php.sh"
+      interval: 2s
+      timeout: 30s
+    depends_on:
+      ca:
+        condition: service_started
+      git:
+        condition: service_healthy
+      mariadb:
+        condition: service_healthy
+    links:
+      - ca
+      - git
+      - mariadb
+    volumes:
+      - mariadb_run:/var/run/mysqld # Bind socket in this volume.
+      - mariadb_data:/var/lib/mysql
+      - ./cache:/cache
+
+
+  fastapi:
+    image: aurweb:latest
+    init: true
+    environment:
+      - AUR_CONFIG=conf/config
+    entrypoint: /docker/fastapi-entrypoint.sh
+    command: /docker/scripts/run-fastapi.sh
+    healthcheck:
+      test: "bash /docker/health/fastapi.sh"
+      interval: 2s
+      timeout: 30s
+    depends_on:
+      ca:
+        condition: service_started
+      git:
+        condition: service_healthy
+      mariadb:
+        condition: service_healthy
+    links:
+      - ca
+      - git
+      - mariadb
+    volumes:
+      - mariadb_run:/var/run/mysqld # Bind socket in this volume.
+      - mariadb_data:/var/lib/mysql
+      - ./cache:/cache
+
+  nginx:
+    image: aurweb:latest
+    init: true
+    environment:
+      - AUR_CONFIG=conf/config
+    entrypoint: /docker/nginx-entrypoint.sh
+    command: /docker/scripts/run-nginx.sh
+    ports:
+      - "8443:8443" # PHP
+      - "8444:8444" # FastAPI
+    healthcheck:
+      test: "bash /docker/health/nginx.sh"
+      interval: 2s
+      timeout: 30s
+    depends_on:
+      cgit:
+        condition: service_healthy
+      fastapi:
+        condition: service_healthy
+      php-fpm:
+        condition: service_healthy
+    links:
+      - cgit
+      - fastapi
+      - php-fpm
+    volumes:
+      - git_data:/aurweb/aur.git
+      - ./cache:/cache
+      - ./logs:/var/log/nginx
+
   sharness:
     image: aurweb:latest
     init: true
@@ -37,7 +179,13 @@ services:
       - AUR_CONFIG=conf/config.sqlite
     entrypoint: /docker/test-sqlite-entrypoint.sh
     command: /docker/scripts/run-sharness.sh
+    depends_on:
+      git:
+        condition: service_healthy
+    links:
+      - git
     volumes:
+      - git_data:/aurweb/aur.git
       - ./cache:/cache
 
   pytest-mysql:
@@ -50,10 +198,14 @@ services:
     depends_on:
       mariadb:
         condition: service_healthy
+      git:
+        condition: service_healthy
     links:
       - mariadb
+      - git
     volumes:
       - mariadb_run:/var/run/mysqld
+      - git_data:/aurweb/aur.git
       - ./cache:/cache
 
   pytest-sqlite:
@@ -65,7 +217,11 @@ services:
     command: /docker/scripts/run-pytests.sh clean
     volumes:
       - mariadb_run:/var/run/mysqld
+      - git_data:/aurweb/aur.git
       - ./cache:/cache
+    depends_on:
+      git:
+        condition: service_healthy
 
   test:
     image: aurweb:latest
@@ -81,8 +237,10 @@ services:
       - mariadb
     volumes:
       - mariadb_run:/var/run/mysqld
+      - git_data:/aurweb/aur.git
       - ./cache:/cache
 
 volumes:
   mariadb_run: {} # Share /var/run/mysqld/mysqld.sock
   mariadb_data: {} # Share /var/lib/mysql
+  git_data: {} # Share aurweb/aur.git

docker/ca-entrypoint.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -eou pipefail
+
+if [ -f /cache/localhost.cert.pem ] && \
+    [ -f /cache/localhost.key.pem ] && \
+    [ -f /cache/ca.root.pem ]; then
+    echo "Already have certs, skipping."
+    exec "$@"
+fi
+
+openssl genrsa -des3 -out ca.key \
+    -passout pass:devca 2048
+
+openssl req -x509 -new -nodes \
+    -key ca.key -sha256 -days 1825 \
+    -out /cache/ca.root.pem \
+    -subj "/C=US/ST=California/L=Nowhere/O=aurweb/CN=localhost" \
+    --passin pass:devca
+
+# Generate keys for aurweb.
+openssl req -nodes -newkey rsa:2048 -keyout /cache/localhost.key.pem \
+    -out localhost.csr \
+    -subj "/C=US/ST=California/L=Nowhere/O=aurweb/CN=localhost"
+
+echo "$(hexdump -n 16 -e '4/4 "%08X" 1 "\n"' /dev/random)" \
+    > /cache/ca.root.srl
+openssl x509 -req -in localhost.csr -CA /cache/ca.root.pem \
+    -CAkey ca.key -CAserial /cache/ca.root.srl \
+    -out /cache/localhost.cert.pem \
+    -days 825 -sha256 -extfile /docker/ca.ext \
+    --passin pass:devca
+
+chmod 666 /cache/localhost.{key,cert}.pem
+chmod 666 /cache/ca.root.pem
+
+exec "$@"

docker/ca.ext

@@ -0,0 +1,7 @@
+authorityKeyIdentifier=keyid,issuer
+basicConstraints=CA:FALSE
+keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = localhost

docker/cgit-entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -eou pipefail
+
+cp -vf conf/cgitrc.proto /etc/cgitrc
+
+sed -ri 's|clone-prefix=.*|clone-prefix=https://localhost:8443|' /etc/cgitrc
+sed -ri 's|header=.*|header=/aurweb/web/template/cgit/header.html|' /etc/cgitrc
+sed -ri 's|footer=.*|footer=/aurweb/web/template/cgit/footer.html|' /etc/cgitrc
+sed -ri 's|repo\.path=.*|repo.path=/aurweb/aur.git|' /etc/cgitrc
+
+mkdir -p /var/cache/cgit
+
+exec "$@"

docker/fastapi-entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eou pipefail
+dir="$(dirname $0)"
+
+bash $dir/test-mysql-entrypoint.sh
+
+sed -ri "s;^(aur_location) = .+;\1 = https://localhost:8444;" conf/config
+sed -ri 's/^(name) = .+/\1 = aurweb/' conf/config
+
+exec "$@"

docker/git-entrypoint.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -eou pipefail
+
+SSHD_CONFIG=/etc/ssh/sshd_config
+
+GIT_REPO=aur.git
+GIT_KEY=/cache/git.key
+
+# Setup SSH Keys.
+ssh-keygen -A
+
+# Add AUR SSH config.
+cat >> $SSHD_CONFIG << EOF
+Match User aur
+    PasswordAuthentication no
+    AuthorizedKeysCommand /usr/local/bin/aurweb-git-auth "%t" "%k"
+    AuthorizedKeysCommandUser aur
+    AcceptEnv AUR_OVERWRITE
+    SetEnv AUR_CONFIG=/aurweb/config/config
+EOF
+
+# Taken from INSTALL.
+mkdir -pv $GIT_REPO
+
+# Initialize git repository.
+if [ ! -f $GIT_REPO/config ]; then
+    cd $GIT_REPO
+    git init --bare
+    git config --local transfer.hideRefs '^refs/'
+    git config --local --add transfer.hideRefs '!refs/'
+    git config --local --add transfer.hideRefs '!HEAD'
+    ln -sf /usr/local/bin/aurweb-git-update hooks/update
+    chown -R aur .
+    cd ..
+fi
+
+if [ ! -f $GIT_KEY ]; then
+    # Create a DSA ssh private/pubkey at /cache/git.key{.pub,}.
+    ssh-keygen -f $GIT_KEY -t dsa -N '' -C 'AUR Git Key'
+fi
+
+# Users should modify these permissions on their local machines.
+chmod 666 ${GIT_KEY}{.pub,}
+
+exec "$@"

docker/health/cgit.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec printf "" >>/dev/tcp/127.0.0.1/3000

docker/health/fastapi.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec pgrep hypercorn

docker/health/nginx.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec curl --no-verify -q https://localhost:8444

docker/health/php.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec printf "" >>/dev/tcp/127.0.0.1/9000

docker/health/sshd.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec printf "" >>/dev/tcp/127.0.0.1/22

docker/logging.conf

@@ -0,0 +1,27 @@
+[loggers]
+keys=root,sampleLogger
+
+[handlers]
+keys=consoleHandler
+
+[formatters]
+keys=sampleFormatter
+
+[logger_root]
+level=DEBUG
+handlers=consoleHandler
+
+[logger_sampleLogger]
+level=DEBUG
+handlers=consoleHandler
+qualname=sampleLogger
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+formatter=sampleFormatter
+args=(sys.stdout,)
+
+[formatter_sampleFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s

docker/nginx-entrypoint.sh

@@ -0,0 +1,123 @@
+#!/bin/bash
+set -eou pipefail
+
+# Setup a config for our mysql db.
+cp -vf conf/config.dev conf/config
+sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config
+sed -ri 's/^(host) = .+/\1 = mariadb/' conf/config
+sed -ri 's/^(user) = .+/\1 = aur/' conf/config
+sed -ri 's/^;?(password) = .+/\1 = aur/' conf/config
+
+# Setup http(s) stuff.
+sed -ri "s|^(aur_location) = .+|\1 = https://localhost:8444|" conf/config
+sed -ri 's/^(disable_http_login) = .+/\1 = 1/' conf/config
+
+cp -vf /cache/localhost.cert.pem /etc/ssl/certs/localhost.cert.pem
+cp -vf /cache/localhost.key.pem /etc/ssl/private/localhost.key.pem
+
+cat > /etc/nginx/nginx.conf << EOF
+daemon off;
+user root;
+worker_processes auto;
+pid /var/run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
+
+events {
+    worker_connections 256;
+}
+
+http {
+    sendfile on;
+    tcp_nopush on;
+    types_hash_max_size 4096;
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
+    ssl_prefer_server_ciphers on;
+
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log;
+
+    gzip on;
+
+    upstream hypercorn {
+        server fastapi:8000;
+    }
+
+    upstream cgit {
+        server cgit:3000;
+    }
+
+    server {
+        listen 8443 ssl http2;
+        server_name localhost default_server;
+
+        ssl_certificate /etc/ssl/certs/localhost.cert.pem;
+        ssl_certificate_key /etc/ssl/private/localhost.key.pem;
+
+        root /aurweb/web/html;
+        index index.php;
+
+        location ~ ^/cgit {
+            include uwsgi_params;
+            rewrite ^/cgit/([^?/]+/[^?]*)?(?:\?(.*))?$ /cgit.cgi?url=\$1&\$2 last;
+            uwsgi_modifier1 9;
+            uwsgi_param CGIT_CONFIG /etc/cgitrc;
+            uwsgi_pass uwsgi://cgit;
+        }
+
+        location ~ ^/[^/]+\.php($|/) {
+            fastcgi_pass   php-fpm:9000;
+            fastcgi_index  index.php;
+            fastcgi_split_path_info ^(/[^/]+\.php)(/.*)\$;
+            fastcgi_param  SCRIPT_FILENAME  \$document_root\$fastcgi_script_name;
+            fastcgi_param  PATH_INFO        \$fastcgi_path_info;
+            include        fastcgi_params;
+        }
+
+        location ~ .* {
+            rewrite ^/(.*)$ /index.php/\$1 last;
+        }
+
+    }
+
+    server {
+        listen 8444 ssl http2;
+        server_name localhost default_server;
+
+        ssl_certificate /etc/ssl/certs/localhost.cert.pem;
+        ssl_certificate_key /etc/ssl/private/localhost.key.pem;
+
+        root /aurweb/web/html;
+
+        location / {
+            try_files \$uri @proxy_to_app;
+        }
+
+        location ~ ^/cgit {
+            include uwsgi_params;
+            rewrite ^/cgit/([^?/]+/[^?]*)?(?:\?(.*))?$ /cgit.cgi?url=\$1&\$2 last;
+            uwsgi_modifier1 9;
+            uwsgi_param CGIT_CONFIG /etc/cgitrc;
+            uwsgi_pass uwsgi://cgit;
+        }
+
+        location @proxy_to_app {
+            proxy_set_header Host \$http_host;
+            proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto \$scheme;
+            proxy_redirect off;
+            proxy_buffering off;
+            proxy_pass https://hypercorn;
+        }
+    }
+
+    map \$http_upgrade \$connection_upgrade {
+        default upgrade;
+        '' close;
+    }
+}
+EOF
+
+exec "$@"

docker/php-entrypoint.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -eou pipefail
+dir="$(dirname $0)"
+
+bash $dir/test-mysql-entrypoint.sh
+
+sed -ri "s;^(aur_location) = .+;\1 = https://localhost:8443;" conf/config
+sed -ri 's/^(name) = .+/\1 = aurweb/' conf/config
+
+sed -ri 's/^(listen).*/\1 = 0.0.0.0:9000/' /etc/php/php-fpm.d/www.conf
+sed -ri 's/^;?(clear_env).*/\1 = no/' /etc/php/php-fpm.d/www.conf
+
+sed -ri 's/^;?(extension=pdo_mysql)/\1/' /etc/php/php.ini
+sed -ri 's/^;?(open_basedir).*$/\1 = \//' /etc/php/php.ini
+
+python -m aurweb.initdb 2>/dev/null || /bin/true
+
+exec "$@"

docker/scripts/run-fastapi.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Initialize the new database; ignore errors.
+python -m aurweb.initdb 2>/dev/null || /bin/true
+
+exec hypercorn --reload \
+    --certfile /cache/localhost.cert.pem \
+    --keyfile /cache/localhost.key.pem \
+    --error-logfile - \
+    --log-config docker/logging.conf \
+    -b "0.0.0.0:8000" aurweb.asgi:app

docker/scripts/run-nginx.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+echo "=== Running nginx server! ==="
+echo
+echo " Services:"
+echo "  - FastAPI : https://localhost:8444/"
+echo "     (cgit) : https://localhost:8444/cgit/"
+echo "  - PHP     : https://localhost:8443/"
+echo "     (cgit) : https://localhost:8443/cgit/"
+echo
+echo " Note: Copy root CA (./cache/ca.root.pem) to ca-certificates or browser."
+echo
+echo " Thanks for using aurweb!"
+echo
+
+exec nginx -c /etc/nginx/nginx.conf

docker/scripts/run-php.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -eou pipefail
+
+# Initialize the new database; ignore errors.
+python -m aurweb.initdb 2>/dev/null || /bin/true
+
+exec php-fpm --fpm-config /etc/php/php-fpm.conf --nodaemonize

docker/scripts/run-sshd.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec /usr/sbin/sshd -D