Improve Docker ecosystem

Instead of using Dockerfile for everything, we've introduced
a docker-compose.yml file and kept the Dockerfile to producing
a pure base image for the services defined.

docker-compose services:

- `mariadb` - Setup mariadb
- `sharness` - Run sharness suites
- `pytest-mysql` - Run pytest suites with MariaDB
- `pytest-sqlite` - Run pytest suites with SQLite
- `test` - Run all tests and produce a collective coverage report
    - This target mounts a cache volume and copies any successful
      coverage report back to `./cache/.coverage`. Users can run
      `./util/fix-coverage ./cache/.coverage` to rewrite source
      code paths and move coverage into place to view reports
      on your local system.

== Get Started ==

Build `aurweb:latest`.

    $ docker build -t aurweb:latest .

Run all tests via `docker-compose`.

    $ docker-compose up test

You can also purely run `pytest` in SQLite or MariaDB modes.

    $ docker-compose up pytest-sqlite
    $ docker-compose up pytest-mysql

Or `sharness` alone, which only uses SQLite internally.

    $ docker-compose up sharness

After running tests, coverage reports are stored in `./cache/.coverage`.
This database was most likely created in a different path, and so it
needs to be sanitized with `./util/fix-coverage`.

    $ ./util/fix-coverage cache/.coverage
    Copied coverage db to /path/to/aurweb/.coverage.
    $ coverage report
    ...
    $ coverage html
    $ coverage xml
    ...

Defined components:

**Entrypoints**

- mariadb-entrypoint.sh - setup mariadb and run its daemon
- test-mysql-entrypoint.sh - setup mysql configurations
- test-sqlite-entrypoint.sh - setup sqlite configurations
- tests-entrypoint.sh - setup mysql and sqlite configurations

**Scripts**

- run-mariadb.sh - setup databases
- run-pytests.sh - run pytest suites
- run-sharness.sh - run sharness suites
- run-tests.sh - run both pytests and sharness

**Health**

- mariadb.sh - A healthcheck script for the mariadb service
- pytest.sh - A healthcheck script for the pytest-* services
- sharness.sh - A healthcheck script for the sharness service

This Docker configuration is setup for tests, but should be
extendable for web and git servers.

**Changes to Makefile**

- Remove `.coverage` in the `clean` target
- Add a `coverage` target which prints a report and outputs xml

Signed-off-by: Kevin Morris <kevr@0cost.org>

.gitignore

@@ -18,3 +18,4 @@ fastapi_aw/
 .pylintrc
 .coverage
 .idea
+/cache/*

Dockerfile

@@ -1,31 +1,33 @@
-FROM archlinux
+FROM archlinux:base-devel
-COPY . /aurweb
+
-WORKDIR /aurweb
+# Setup some default system stuff.
+RUN bash -c 'echo "127.0.0.1 localhost" >> /etc/hosts'
+RUN bash -c 'echo "::1 localhost" >> /etc/hosts'
+RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime
+
+RUN mkdir -p .pkg-cache
 
 # Install dependencies.
-RUN pacman -Syu --noconfirm base-devel git gpgme protobuf pyalpm \
+RUN pacman -Syu --noconfirm --noprogressbar \
-    python-mysql-connector python-pygit2 python-srcinfo python-bleach \
+    --cachedir .pkg-cache git gpgme protobuf pyalpm \
+    python-mysqlclient python-pygit2 python-srcinfo python-bleach \
     python-markdown python-sqlalchemy python-alembic python-pytest \
     python-werkzeug python-pytest-tap python-fastapi nginx python-authlib \
     python-itsdangerous python-httpx python-jinja python-pytest-cov \
     python-requests python-aiofiles python-python-multipart \
     python-pytest-asyncio python-coverage hypercorn python-bcrypt \
-    python-email-validator openssh python-lxml
+    python-email-validator openssh python-lxml mariadb mariadb-libs \
+    python-isort flake8
 
-# Remove aurweb.sqlite3 if it was copied over via COPY.
+RUN useradd -U -d /aurweb -c 'AUR User' aur
-RUN rm -fv aurweb.sqlite3
 
-# Setup our test config.
+COPY docker /docker
-RUN sed -r "s;YOUR_AUR_ROOT;/aurweb;g" conf/config.dev > conf/config
 
-# Install translations.
+WORKDIR /aurweb
-RUN AUR_CONFIG=conf/config make -C po all install
+COPY . .
 
-# Initialize the database.
+ENV PYTHONPATH=/aurweb
-RUN AUR_CONFIG=conf/config python -m aurweb.initdb
+ENV AUR_CONFIG=conf/config
 
-# Test everything!
+RUN make -C po all install
-RUN make -C test
+RUN python setup.py install
-
-# Produce a coverage report.
-RUN coverage report --include='aurweb/*'

docker-compose.yml

@@ -0,0 +1,88 @@
+#
+# Docker service definitions for the aurweb project.
+#
+# Notable services:
+#   - `sharness` - Run sharness test suites
+#   - `pytest-mysql` - Run pytest suites with MariaDB
+#   - `pytest-sqlite` - Run pytest suites with SQLite
+#   - `test` - Run sharness, pytest-mysql and pytest-sqlite
+#   - `mariadb` - `port 13306` - MariaDB server for docker
+#
+# Copyright (C) 2021 aurweb Development
+# All Rights Reserved.
+version: "3.8"
+
+services:
+  mariadb:
+    image: aurweb:latest
+    init: true
+    entrypoint: /docker/mariadb-entrypoint.sh
+    command: /docker/scripts/run-mariadb.sh mysqld_safe --datadir=/var/lib/mysql
+    ports:
+      # This will expose mariadbd on 127.0.0.1:13306 in the host.
+      # Ex: `mysql -uaur -paur -h 127.0.0.1 -P 13306 aurweb`
+      - "13306:3306"
+    volumes:
+      - mariadb_run:/var/run/mysqld # Bind socket in this volume.
+      - mariadb_data:/var/lib/mysql
+    healthcheck:
+      test: "bash /docker/health/mariadb.sh"
+      interval: 2s
+      timeout: 60s
+
+  sharness:
+    image: aurweb:latest
+    init: true
+    environment:
+      - AUR_CONFIG=conf/config.sqlite
+    entrypoint: /docker/test-sqlite-entrypoint.sh
+    command: /docker/scripts/run-sharness.sh
+    volumes:
+      - ./cache:/cache
+
+  pytest-mysql:
+    image: aurweb:latest
+    init: true
+    environment:
+      - AUR_CONFIG=conf/config
+    entrypoint: /docker/test-mysql-entrypoint.sh
+    command: /docker/scripts/run-pytests.sh clean
+    depends_on:
+      mariadb:
+        condition: service_healthy
+    links:
+      - mariadb
+    volumes:
+      - mariadb_run:/var/run/mysqld
+      - ./cache:/cache
+
+  pytest-sqlite:
+    image: aurweb:latest
+    init: true
+    environment:
+      - AUR_CONFIG=conf/config.sqlite
+    entrypoint: /docker/test-sqlite-entrypoint.sh
+    command: /docker/scripts/run-pytests.sh clean
+    volumes:
+      - mariadb_run:/var/run/mysqld
+      - ./cache:/cache
+
+  test:
+    image: aurweb:latest
+    init: true
+    environment:
+      - AUR_CONFIG=conf/config
+    entrypoint: /docker/tests-entrypoint.sh
+    command: /docker/scripts/run-tests.sh
+    depends_on:
+      mariadb:
+        condition: service_healthy
+    links:
+      - mariadb
+    volumes:
+      - mariadb_run:/var/run/mysqld
+      - ./cache:/cache
+
+volumes:
+  mariadb_run: {} # Share /var/run/mysqld/mysqld.sock
+  mariadb_data: {} # Share /var/lib/mysql

docker/health/mariadb.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec mysqladmin ping --silent

docker/mariadb-entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -eou pipefail
+
+mariadb-install-db --user=mysql --basedir=/usr --datadir=/var/lib/mysql
+
+exec "$@"

docker/scripts/run-mariadb.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+mysqld_safe --datadir=/var/lib/mysql --skip-networking &
+until mysqladmin ping --silent; do
+    sleep 1s
+done
+
+# Create test database.
+mysql -u root -e "CREATE USER 'aur'@'%' IDENTIFIED BY 'aur'" \
+    2>/dev/null || /bin/true
+mysql -u root -e "DROP DATABASE aurweb_test" 2>/dev/null || /bin/true
+mysql -u root -e "CREATE DATABASE aurweb_test"
+mysql -u root -e "GRANT ALL PRIVILEGES ON aurweb_test.* TO 'aur'@'%'"
+mysql -u root -e "FLUSH PRIVILEGES"
+
+# Shutdown mariadb.
+mysqladmin -uroot shutdown
+
+exec "$@"

docker/scripts/run-pytests.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+set -eou pipefail
+
+COVERAGE=1
+PARAMS=()
+
+while [ $# -ne 0 ]; do
+    key="$1"
+    case "$key" in
+        --no-coverage)
+            COVERAGE=0
+            shift
+            ;;
+        -*)
+            echo "usage: $0 [--no-coverage] targets ..."
+            exit 1
+            ;;
+        *)
+            PARAMS+=("$key")
+            shift
+            ;;
+    esac
+done
+
+# Initialize the new database; ignore errors.
+python -m aurweb.initdb 2>/dev/null || /bin/true
+
+# Run pytest with optional targets in front of it.
+make -C test "${PARAMS[@]}" pytest
+
+# By default, report coverage and move it into cache.
+if [ $COVERAGE -eq 1 ]; then
+    make -C test coverage
+
+    # /cache is mounted as a volume. Copy coverage into it.
+    # Users can then sanitize the coverage locally in their
+    # aurweb root directory: ./util/fix-coverage ./cache/.coverage
+    rm -f /cache/.coverage
+    cp -v .coverage /cache/.coverage
+    chmod 666 /cache/.coverage
+fi

docker/scripts/run-sharness.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -eou pipefail
+
+# Initialize the new database; ignore errors.
+python -m aurweb.initdb 2>/dev/null || /bin/true
+
+make -C test sh

docker/scripts/run-tests.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -eou pipefail
+dir=$(dirname $0)
+
+# Clean up coverage and stuff.
+make -C test clean
+
+# Run sharness tests.
+bash $dir/run-sharness.sh
+
+# Run Python tests with MariaDB database.
+# Pass --silence to avoid reporting coverage. We will do that below.
+bash $dir/run-pytests.sh --no-coverage
+
+# Export SQLite aurweb configuration.
+export AUR_CONFIG=conf/config.sqlite
+
+# Run Python tests.
+bash $dir/run-pytests.sh --no-coverage
+
+make -C test coverage
+
+# /cache is mounted as a volume. Copy coverage into it.
+# Users can then sanitize the coverage locally in their
+# aurweb root directory: ./util/fix-coverage ./cache/.coverage
+rm -f /cache/.coverage
+cp -v .coverage /cache/.coverage
+chmod 666 /cache/.coverage
+
+# Run flake8 and isort checks.
+for dir in aurweb test migrations; do
+    flake8 --count $dir
+    isort --check-only $dir
+done

docker/test-mysql-entrypoint.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -eou pipefail
+
+DB_NAME="aurweb_test"
+DB_HOST="mariadb"
+DB_USER="aur"
+DB_PASS="aur"
+
+# Setup a config for our mysql db.
+cp -vf conf/config.dev conf/config
+sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config
+sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" conf/config
+sed -ri "s/^(host) = .+/\1 = ${DB_HOST}/" conf/config
+sed -ri "s/^(user) = .+/\1 = ${DB_USER}/" conf/config
+sed -ri "s/^;?(password) = .+/\1 = ${DB_PASS}/" conf/config
+
+# The port can be excluded from use if properly using
+# volumes to share the mysql socket from the mariadb service.
+# Example port sed:
+# sed -i "s/^;?(port = .+)$/\1/" conf/config
+
+# Continue onto the main command.
+exec "$@"

docker/test-sqlite-entrypoint.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -eou pipefail
+
+DB_BACKEND="sqlite"
+DB_NAME="aurweb.sqlite3"
+
+# Create an SQLite config from the default dev config.
+cp -vf conf/config.dev conf/config.sqlite
+cp -vf conf/config.defaults conf/config.sqlite.defaults
+
+# Modify it for SQLite.
+sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config.sqlite
+sed -ri "s/^(backend) = .+/\1 = ${DB_BACKEND}/" conf/config.sqlite
+sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" conf/config.sqlite
+
+exec "$@"

docker/tests-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -eou pipefail
+dir="$(dirname $0)"
+
+bash $dir/test-mysql-entrypoint.sh
+bash $dir/test-sqlite-entrypoint.sh
+
+exec "$@"

test/Makefile

@@ -17,10 +17,15 @@ else
 sh: $(T)
 endif
 
+coverage:
+	cd .. && coverage report --include='aurweb/*'
+	cd .. && coverage xml --include='aurweb/*'
+
 clean:
 	$(RM) -r test-results/
+	rm -f ../.coverage
 
 $(T):
 	@echo "*** $@ ***"; $(SHELL) $@
 
-.PHONY: check $(FOREIGN_TARGETS) clean $(T)
+.PHONY: check coverage $(FOREIGN_TARGETS) clean $(T)

util/fix-coverage

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+""" A simple script which updates paths for .coverage executed
+in another directory. This is particularly useful for fixing
+.coverage files received via ./cache/.coverage after Docker runs.
+
+Copyright (C) 2021 aurweb Development
+All Rights Reserved.
+"""
+import os
+import re
+import sqlite3
+import sys
+import traceback
+
+import aurweb.config
+
+
+def eprint(*args, **kwargs):
+    print(*args, **kwargs, file=sys.stderru)
+
+
+def main():
+    if len(sys.argv) == 1:
+        print(f"usage: {sys.argv[0]} .coverage")
+        return 1
+
+    coverage_db = sys.argv[1]
+    if not os.path.exists(coverage_db):
+        eprint("error: coverage file not found")
+        return 2
+
+    aurwebdir = aurweb.config.get("options", "aurwebdir")
+    new_coverage_db = os.path.join(aurwebdir, ".coverage")
+    with open(coverage_db, "rb") as i:
+        with open(new_coverage_db, "wb") as f:
+            f.write(i.read())
+        print(f"Copied coverage db to {new_coverage_db}.")
+    coverage_db = new_coverage_db
+
+    sqlite3.paramstyle = "format"
+    db = sqlite3.connect(coverage_db)
+
+    cursor = db.cursor()
+    results = cursor.execute("SELECT * FROM file")
+
+    files = dict()
+    for i, path in results:
+        files[i] = path
+
+    for _, i in enumerate(files.keys()):
+        new_path = re.sub(r'^/aurweb', aurwebdir, files[i])
+        cursor.execute("UPDATE file SET path = ? WHERE id = ?", (
+            new_path, i))
+
+    db.commit()
+    db.close()
+
+    return 0
+
+
+if __name__ == "__main__":
+    e = 1
+    try:
+        e = main()
+    except Exception:
+        traceback.print_exc()
+        e = 1
+    sys.exit(e)