From 5bd46d18a3cd645ae2164ee3e151ac10524355a2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 20 Jun 2021 12:19:33 -0700 Subject: [PATCH] Improve Docker ecosystem Instead of using Dockerfile for everything, we've introduced a docker-compose.yml file and kept the Dockerfile to producing a pure base image for the services defined. docker-compose services: - `mariadb` - Setup mariadb - `sharness` - Run sharness suites - `pytest-mysql` - Run pytest suites with MariaDB - `pytest-sqlite` - Run pytest suites with SQLite - `test` - Run all tests and produce a collective coverage report - This target mounts a cache volume and copies any successful coverage report back to `./cache/.coverage`. Users can run `./util/fix-coverage ./cache/.coverage` to rewrite source code paths and move coverage into place to view reports on your local system. == Get Started == Build `aurweb:latest`. $ docker build -t aurweb:latest . Run all tests via `docker-compose`. $ docker-compose up test You can also purely run `pytest` in SQLite or MariaDB modes. $ docker-compose up pytest-sqlite $ docker-compose up pytest-mysql Or `sharness` alone, which only uses SQLite internally. $ docker-compose up sharness After running tests, coverage reports are stored in `./cache/.coverage`. This database was most likely created in a different path, and so it needs to be sanitized with `./util/fix-coverage`. $ ./util/fix-coverage cache/.coverage Copied coverage db to /path/to/aurweb/.coverage. $ coverage report ... $ coverage html $ coverage xml ... Defined components: **Entrypoints** - mariadb-entrypoint.sh - setup mariadb and run its daemon - test-mysql-entrypoint.sh - setup mysql configurations - test-sqlite-entrypoint.sh - setup sqlite configurations - tests-entrypoint.sh - setup mysql and sqlite configurations **Scripts** - run-mariadb.sh - setup databases - run-pytests.sh - run pytest suites - run-sharness.sh - run sharness suites - run-tests.sh - run both pytests and sharness **Health** - mariadb.sh - A healthcheck script for the mariadb service - pytest.sh - A healthcheck script for the pytest-* services - sharness.sh - A healthcheck script for the sharness service This Docker configuration is setup for tests, but should be extendable for web and git servers. **Changes to Makefile** - Remove `.coverage` in the `clean` target - Add a `coverage` target which prints a report and outputs xml Signed-off-by: Kevin Morris --- .gitignore | 1 + Dockerfile | 40 ++++++++------- cache/.gitkeep | 0 docker-compose.yml | 88 ++++++++++++++++++++++++++++++++ docker/health/mariadb.sh | 2 + docker/mariadb-entrypoint.sh | 6 +++ docker/scripts/run-mariadb.sh | 19 +++++++ docker/scripts/run-pytests.sh | 41 +++++++++++++++ docker/scripts/run-sharness.sh | 7 +++ docker/scripts/run-tests.sh | 34 ++++++++++++ docker/test-mysql-entrypoint.sh | 23 +++++++++ docker/test-sqlite-entrypoint.sh | 16 ++++++ docker/tests-entrypoint.sh | 8 +++ test/Makefile | 7 ++- util/fix-coverage | 68 ++++++++++++++++++++++++ 15 files changed, 340 insertions(+), 20 deletions(-) create mode 100644 cache/.gitkeep create mode 100644 docker-compose.yml create mode 100755 docker/health/mariadb.sh create mode 100755 docker/mariadb-entrypoint.sh create mode 100755 docker/scripts/run-mariadb.sh create mode 100755 docker/scripts/run-pytests.sh create mode 100755 docker/scripts/run-sharness.sh create mode 100755 docker/scripts/run-tests.sh create mode 100755 docker/test-mysql-entrypoint.sh create mode 100755 docker/test-sqlite-entrypoint.sh create mode 100755 docker/tests-entrypoint.sh create mode 100755 util/fix-coverage diff --git a/.gitignore b/.gitignore index 35b571d7..2006e838 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ fastapi_aw/ .pylintrc .coverage .idea +/cache/* diff --git a/Dockerfile b/Dockerfile index c432f73f..6858bedb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,33 @@ -FROM archlinux -COPY . /aurweb -WORKDIR /aurweb +FROM archlinux:base-devel + +# Setup some default system stuff. +RUN bash -c 'echo "127.0.0.1 localhost" >> /etc/hosts' +RUN bash -c 'echo "::1 localhost" >> /etc/hosts' +RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime + +RUN mkdir -p .pkg-cache # Install dependencies. -RUN pacman -Syu --noconfirm base-devel git gpgme protobuf pyalpm \ - python-mysql-connector python-pygit2 python-srcinfo python-bleach \ +RUN pacman -Syu --noconfirm --noprogressbar \ + --cachedir .pkg-cache git gpgme protobuf pyalpm \ + python-mysqlclient python-pygit2 python-srcinfo python-bleach \ python-markdown python-sqlalchemy python-alembic python-pytest \ python-werkzeug python-pytest-tap python-fastapi nginx python-authlib \ python-itsdangerous python-httpx python-jinja python-pytest-cov \ python-requests python-aiofiles python-python-multipart \ python-pytest-asyncio python-coverage hypercorn python-bcrypt \ - python-email-validator openssh python-lxml + python-email-validator openssh python-lxml mariadb mariadb-libs \ + python-isort flake8 -# Remove aurweb.sqlite3 if it was copied over via COPY. -RUN rm -fv aurweb.sqlite3 +RUN useradd -U -d /aurweb -c 'AUR User' aur -# Setup our test config. -RUN sed -r "s;YOUR_AUR_ROOT;/aurweb;g" conf/config.dev > conf/config +COPY docker /docker -# Install translations. -RUN AUR_CONFIG=conf/config make -C po all install +WORKDIR /aurweb +COPY . . -# Initialize the database. -RUN AUR_CONFIG=conf/config python -m aurweb.initdb +ENV PYTHONPATH=/aurweb +ENV AUR_CONFIG=conf/config -# Test everything! -RUN make -C test - -# Produce a coverage report. -RUN coverage report --include='aurweb/*' +RUN make -C po all install +RUN python setup.py install diff --git a/cache/.gitkeep b/cache/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..7fbd20fd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,88 @@ +# +# Docker service definitions for the aurweb project. +# +# Notable services: +# - `sharness` - Run sharness test suites +# - `pytest-mysql` - Run pytest suites with MariaDB +# - `pytest-sqlite` - Run pytest suites with SQLite +# - `test` - Run sharness, pytest-mysql and pytest-sqlite +# - `mariadb` - `port 13306` - MariaDB server for docker +# +# Copyright (C) 2021 aurweb Development +# All Rights Reserved. +version: "3.8" + +services: + mariadb: + image: aurweb:latest + init: true + entrypoint: /docker/mariadb-entrypoint.sh + command: /docker/scripts/run-mariadb.sh mysqld_safe --datadir=/var/lib/mysql + ports: + # This will expose mariadbd on 127.0.0.1:13306 in the host. + # Ex: `mysql -uaur -paur -h 127.0.0.1 -P 13306 aurweb` + - "13306:3306" + volumes: + - mariadb_run:/var/run/mysqld # Bind socket in this volume. + - mariadb_data:/var/lib/mysql + healthcheck: + test: "bash /docker/health/mariadb.sh" + interval: 2s + timeout: 60s + + sharness: + image: aurweb:latest + init: true + environment: + - AUR_CONFIG=conf/config.sqlite + entrypoint: /docker/test-sqlite-entrypoint.sh + command: /docker/scripts/run-sharness.sh + volumes: + - ./cache:/cache + + pytest-mysql: + image: aurweb:latest + init: true + environment: + - AUR_CONFIG=conf/config + entrypoint: /docker/test-mysql-entrypoint.sh + command: /docker/scripts/run-pytests.sh clean + depends_on: + mariadb: + condition: service_healthy + links: + - mariadb + volumes: + - mariadb_run:/var/run/mysqld + - ./cache:/cache + + pytest-sqlite: + image: aurweb:latest + init: true + environment: + - AUR_CONFIG=conf/config.sqlite + entrypoint: /docker/test-sqlite-entrypoint.sh + command: /docker/scripts/run-pytests.sh clean + volumes: + - mariadb_run:/var/run/mysqld + - ./cache:/cache + + test: + image: aurweb:latest + init: true + environment: + - AUR_CONFIG=conf/config + entrypoint: /docker/tests-entrypoint.sh + command: /docker/scripts/run-tests.sh + depends_on: + mariadb: + condition: service_healthy + links: + - mariadb + volumes: + - mariadb_run:/var/run/mysqld + - ./cache:/cache + +volumes: + mariadb_run: {} # Share /var/run/mysqld/mysqld.sock + mariadb_data: {} # Share /var/lib/mysql diff --git a/docker/health/mariadb.sh b/docker/health/mariadb.sh new file mode 100755 index 00000000..cbae37bd --- /dev/null +++ b/docker/health/mariadb.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec mysqladmin ping --silent diff --git a/docker/mariadb-entrypoint.sh b/docker/mariadb-entrypoint.sh new file mode 100755 index 00000000..e33c61c7 --- /dev/null +++ b/docker/mariadb-entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -eou pipefail + +mariadb-install-db --user=mysql --basedir=/usr --datadir=/var/lib/mysql + +exec "$@" diff --git a/docker/scripts/run-mariadb.sh b/docker/scripts/run-mariadb.sh new file mode 100755 index 00000000..7e908129 --- /dev/null +++ b/docker/scripts/run-mariadb.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +mysqld_safe --datadir=/var/lib/mysql --skip-networking & +until mysqladmin ping --silent; do + sleep 1s +done + +# Create test database. +mysql -u root -e "CREATE USER 'aur'@'%' IDENTIFIED BY 'aur'" \ + 2>/dev/null || /bin/true +mysql -u root -e "DROP DATABASE aurweb_test" 2>/dev/null || /bin/true +mysql -u root -e "CREATE DATABASE aurweb_test" +mysql -u root -e "GRANT ALL PRIVILEGES ON aurweb_test.* TO 'aur'@'%'" +mysql -u root -e "FLUSH PRIVILEGES" + +# Shutdown mariadb. +mysqladmin -uroot shutdown + +exec "$@" diff --git a/docker/scripts/run-pytests.sh b/docker/scripts/run-pytests.sh new file mode 100755 index 00000000..021603b1 --- /dev/null +++ b/docker/scripts/run-pytests.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -eou pipefail + +COVERAGE=1 +PARAMS=() + +while [ $# -ne 0 ]; do + key="$1" + case "$key" in + --no-coverage) + COVERAGE=0 + shift + ;; + -*) + echo "usage: $0 [--no-coverage] targets ..." + exit 1 + ;; + *) + PARAMS+=("$key") + shift + ;; + esac +done + +# Initialize the new database; ignore errors. +python -m aurweb.initdb 2>/dev/null || /bin/true + +# Run pytest with optional targets in front of it. +make -C test "${PARAMS[@]}" pytest + +# By default, report coverage and move it into cache. +if [ $COVERAGE -eq 1 ]; then + make -C test coverage + + # /cache is mounted as a volume. Copy coverage into it. + # Users can then sanitize the coverage locally in their + # aurweb root directory: ./util/fix-coverage ./cache/.coverage + rm -f /cache/.coverage + cp -v .coverage /cache/.coverage + chmod 666 /cache/.coverage +fi diff --git a/docker/scripts/run-sharness.sh b/docker/scripts/run-sharness.sh new file mode 100755 index 00000000..8e928b3f --- /dev/null +++ b/docker/scripts/run-sharness.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -eou pipefail + +# Initialize the new database; ignore errors. +python -m aurweb.initdb 2>/dev/null || /bin/true + +make -C test sh diff --git a/docker/scripts/run-tests.sh b/docker/scripts/run-tests.sh new file mode 100755 index 00000000..3181a623 --- /dev/null +++ b/docker/scripts/run-tests.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -eou pipefail +dir=$(dirname $0) + +# Clean up coverage and stuff. +make -C test clean + +# Run sharness tests. +bash $dir/run-sharness.sh + +# Run Python tests with MariaDB database. +# Pass --silence to avoid reporting coverage. We will do that below. +bash $dir/run-pytests.sh --no-coverage + +# Export SQLite aurweb configuration. +export AUR_CONFIG=conf/config.sqlite + +# Run Python tests. +bash $dir/run-pytests.sh --no-coverage + +make -C test coverage + +# /cache is mounted as a volume. Copy coverage into it. +# Users can then sanitize the coverage locally in their +# aurweb root directory: ./util/fix-coverage ./cache/.coverage +rm -f /cache/.coverage +cp -v .coverage /cache/.coverage +chmod 666 /cache/.coverage + +# Run flake8 and isort checks. +for dir in aurweb test migrations; do + flake8 --count $dir + isort --check-only $dir +done diff --git a/docker/test-mysql-entrypoint.sh b/docker/test-mysql-entrypoint.sh new file mode 100755 index 00000000..ea4df868 --- /dev/null +++ b/docker/test-mysql-entrypoint.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -eou pipefail + +DB_NAME="aurweb_test" +DB_HOST="mariadb" +DB_USER="aur" +DB_PASS="aur" + +# Setup a config for our mysql db. +cp -vf conf/config.dev conf/config +sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config +sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" conf/config +sed -ri "s/^(host) = .+/\1 = ${DB_HOST}/" conf/config +sed -ri "s/^(user) = .+/\1 = ${DB_USER}/" conf/config +sed -ri "s/^;?(password) = .+/\1 = ${DB_PASS}/" conf/config + +# The port can be excluded from use if properly using +# volumes to share the mysql socket from the mariadb service. +# Example port sed: +# sed -i "s/^;?(port = .+)$/\1/" conf/config + +# Continue onto the main command. +exec "$@" diff --git a/docker/test-sqlite-entrypoint.sh b/docker/test-sqlite-entrypoint.sh new file mode 100755 index 00000000..c26f6735 --- /dev/null +++ b/docker/test-sqlite-entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -eou pipefail + +DB_BACKEND="sqlite" +DB_NAME="aurweb.sqlite3" + +# Create an SQLite config from the default dev config. +cp -vf conf/config.dev conf/config.sqlite +cp -vf conf/config.defaults conf/config.sqlite.defaults + +# Modify it for SQLite. +sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config.sqlite +sed -ri "s/^(backend) = .+/\1 = ${DB_BACKEND}/" conf/config.sqlite +sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" conf/config.sqlite + +exec "$@" diff --git a/docker/tests-entrypoint.sh b/docker/tests-entrypoint.sh new file mode 100755 index 00000000..ca3d3d9a --- /dev/null +++ b/docker/tests-entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -eou pipefail +dir="$(dirname $0)" + +bash $dir/test-mysql-entrypoint.sh +bash $dir/test-sqlite-entrypoint.sh + +exec "$@" diff --git a/test/Makefile b/test/Makefile index 920c7113..4a8207f8 100644 --- a/test/Makefile +++ b/test/Makefile @@ -17,10 +17,15 @@ else sh: $(T) endif +coverage: + cd .. && coverage report --include='aurweb/*' + cd .. && coverage xml --include='aurweb/*' + clean: $(RM) -r test-results/ + rm -f ../.coverage $(T): @echo "*** $@ ***"; $(SHELL) $@ -.PHONY: check $(FOREIGN_TARGETS) clean $(T) +.PHONY: check coverage $(FOREIGN_TARGETS) clean $(T) diff --git a/util/fix-coverage b/util/fix-coverage new file mode 100755 index 00000000..aa56591d --- /dev/null +++ b/util/fix-coverage @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +""" A simple script which updates paths for .coverage executed +in another directory. This is particularly useful for fixing +.coverage files received via ./cache/.coverage after Docker runs. + +Copyright (C) 2021 aurweb Development +All Rights Reserved. +""" +import os +import re +import sqlite3 +import sys +import traceback + +import aurweb.config + + +def eprint(*args, **kwargs): + print(*args, **kwargs, file=sys.stderru) + + +def main(): + if len(sys.argv) == 1: + print(f"usage: {sys.argv[0]} .coverage") + return 1 + + coverage_db = sys.argv[1] + if not os.path.exists(coverage_db): + eprint("error: coverage file not found") + return 2 + + aurwebdir = aurweb.config.get("options", "aurwebdir") + new_coverage_db = os.path.join(aurwebdir, ".coverage") + with open(coverage_db, "rb") as i: + with open(new_coverage_db, "wb") as f: + f.write(i.read()) + print(f"Copied coverage db to {new_coverage_db}.") + coverage_db = new_coverage_db + + sqlite3.paramstyle = "format" + db = sqlite3.connect(coverage_db) + + cursor = db.cursor() + results = cursor.execute("SELECT * FROM file") + + files = dict() + for i, path in results: + files[i] = path + + for _, i in enumerate(files.keys()): + new_path = re.sub(r'^/aurweb', aurwebdir, files[i]) + cursor.execute("UPDATE file SET path = ? WHERE id = ?", ( + new_path, i)) + + db.commit() + db.close() + + return 0 + + +if __name__ == "__main__": + e = 1 + try: + e = main() + except Exception: + traceback.print_exc() + e = 1 + sys.exit(e)