From db02227cc467ac9b9abcd9ee28bb10b2ef62cf4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Wed, 20 May 2020 23:15:16 +0100 Subject: [PATCH 001/844] ci: add gitlab ci MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns Signed-off-by: Lukas Fleischer --- .gitlab-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..d463b109 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,12 @@ +image: archlinux + +before_script: + - pacman -Syu --noconfirm --noprogressbar --needed + base-devel git gpgme protobuf pyalpm python-mysql-connector + python-pygit2 python-srcinfo python-bleach python-markdown + python-sqlalchemy python-alembic python-pytest python-werkzeug + python-pytest-tap + +test: + script: + - make -C test From 23f6dd16a7c8b6f81c229e2307838edd213d6149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Wed, 20 May 2020 23:27:50 +0100 Subject: [PATCH 002/844] ci: add cache to gitlab ci MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns Signed-off-by: Lukas Fleischer --- .gitlab-ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d463b109..74784fce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,13 @@ image: archlinux +cache: + key: system-v1 + paths: + # For some reason Gitlab CI only supports storing cache/artifacts in a path relative to the build directory + - .pkg-cache + before_script: - - pacman -Syu --noconfirm --noprogressbar --needed + - pacman -Syu --noconfirm --noprogressbar --needed --cachedir .pkg-cache base-devel git gpgme protobuf pyalpm python-mysql-connector python-pygit2 python-srcinfo python-bleach python-markdown python-sqlalchemy python-alembic python-pytest python-werkzeug From 8a13500535942a1c99b97ae4de46e5a1c0297cd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Sun, 19 Apr 2020 20:11:02 +0200 Subject: [PATCH 003/844] Create aurweb.spawn for spawing the test server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This program makes it easier for developers to spawn the PHP server since it fetches automatically what it needs from the configuration file, rather than having the user explicitly pass arguments to the php executable. When the setup gets more complicated as we introduce Python, aurweb.spawn will keep providing the same interface, while under the hood it is planned to support running multiple sub-processes. Its Python interface provides an way for the test suite to spawn the test server when it needs to perform HTTP requests to the test server. The current implementation is somewhat weak as it doesn’t detect when a child process dies, but this is not supposed to happen often, and it is only meant for aurweb developers. In the long term, aurweb.spawn will eventually become obsolete, and replaced by Docker or Flask’s tools. Signed-off-by: Lukas Fleischer --- TESTING | 7 +-- aurweb/spawn.py | 107 +++++++++++++++++++++++++++++++++++++++++++ conf/config.defaults | 3 ++ 3 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 aurweb/spawn.py diff --git a/TESTING b/TESTING index 4a1e6f4c..a5e08cb8 100644 --- a/TESTING +++ b/TESTING @@ -17,7 +17,8 @@ INSTALL. Ensure to enable the pdo_sqlite extension in php.ini. 3) Copy conf/config.defaults to conf/config and adjust the configuration - (pay attention to disable_http_login, enable_maintenance and aur_location). + Pay attention to disable_http_login, enable_maintenance, aur_location and + htmldir. Be sure to change backend to sqlite and name to the file location of your created test database. @@ -31,6 +32,6 @@ INSTALL. $ ./gendummydata.py out.sql $ sqlite3 path/to/aurweb.sqlite3 < out.sql -5) Run the PHP built-in web server: +5) Run the test server: - $ AUR_CONFIG='/path/to/aurweb/conf/config' php -S localhost:8080 -t /path/to/aurweb/web/html + $ AUR_CONFIG='/path/to/aurweb/conf/config' python -m aurweb.spawn diff --git a/aurweb/spawn.py b/aurweb/spawn.py new file mode 100644 index 00000000..5fa646b5 --- /dev/null +++ b/aurweb/spawn.py @@ -0,0 +1,107 @@ +""" +Provide an automatic way of spawing an HTTP test server running aurweb. + +It can be called from the command-line or from another Python module. + +This module uses a global state, since you can’t open two servers with the same +configuration anyway. +""" + + +import atexit +import argparse +import subprocess +import sys +import time +import urllib + +import aurweb.config +import aurweb.schema + + +children = [] +verbosity = 0 + + +class ProcessExceptions(Exception): + """ + Compound exception used by stop() to list all the errors that happened when + terminating child processes. + """ + def __init__(self, message, exceptions): + self.message = message + self.exceptions = exceptions + messages = [message] + [str(e) for e in exceptions] + super().__init__("\n- ".join(messages)) + + +def spawn_child(args): + """Open a subprocess and add it to the global state.""" + if verbosity >= 1: + print(f"Spawning {args}", file=sys.stderr) + children.append(subprocess.Popen(args)) + + +def start(): + """ + Spawn the test server. If it is already running, do nothing. + + The server can be stopped with stop(), or is automatically stopped when the + Python process ends using atexit. + """ + if children: + return + atexit.register(stop) + aur_location = aurweb.config.get("options", "aur_location") + aur_location_parts = urllib.parse.urlsplit(aur_location) + htmldir = aurweb.config.get("options", "htmldir") + spawn_child(["php", "-S", aur_location_parts.netloc, "-t", htmldir]) + + +def stop(): + """ + Stop all the child processes. + + If an exception occurs during the process, the process continues anyway + because we don’t want to leave runaway processes around, and all the + exceptions are finally raised as a single ProcessExceptions. + """ + global children + atexit.unregister(stop) + exceptions = [] + for p in children: + try: + p.terminate() + if verbosity >= 1: + print(f"Sent SIGTERM to {p.args}", file=sys.stderr) + except Exception as e: + exceptions.append(e) + for p in children: + try: + rc = p.wait() + if rc != 0 and rc != -15: + # rc = -15 indicates the process was terminated with SIGTERM, + # which is to be expected since we called terminate on them. + raise Exception(f"Process {p.args} exited with {rc}") + except Exception as e: + exceptions.append(e) + children = [] + if exceptions: + raise ProcessExceptions("Errors terminating the child processes:", + exceptions) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + prog='python -m aurweb.spawn', + description='Start aurweb\'s test server.') + parser.add_argument('-v', '--verbose', action='count', default=0, + help='increase verbosity') + args = parser.parse_args() + verbosity = args.verbose + start() + try: + while True: + time.sleep(60) + except KeyboardInterrupt: + stop() diff --git a/conf/config.defaults b/conf/config.defaults index 447dacac..86fe765c 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -41,6 +41,9 @@ cache = none cache_pkginfo_ttl = 86400 memcache_servers = 127.0.0.1:11211 +; Directory containing aurweb's PHP code, required by aurweb.spawn. +;htmldir = /path/to/web/html + [ratelimit] request_limit = 4000 window_length = 86400 From 48b58b1c2f74df0906231d2affd9f2b352a8e330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 23 May 2020 17:54:07 +0100 Subject: [PATCH 004/844] ci: remove Travis CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are are moving to Gitlab CI. Signed-off-by: Filipe Laíns Signed-off-by: Lukas Fleischer --- .travis.yml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5bbfda1f..00000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -language: python - -python: 3.6 - -addons: - apt: - packages: - - bsdtar - - libarchive-dev - - libgpgme11-dev - - libprotobuf-dev - -install: - - curl https://codeload.github.com/libgit2/libgit2/tar.gz/v0.26.0 | tar -xz - - curl https://sources.archlinux.org/other/pacman/pacman-5.0.2.tar.gz | tar -xz - - curl https://git.archlinux.org/pyalpm.git/snapshot/pyalpm-0.8.1.tar.gz | tar -xz - - ( cd libgit2-0.26.0 && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr && make && sudo make install ) - - ( cd pacman-5.0.2 && ./configure --prefix=/usr && make && sudo make install ) - - ( cd pyalpm-0.8.1 && python setup.py build && python setup.py install ) - - pip install mysql-connector-python-rf pygit2==0.26 srcinfo - - pip install bleach Markdown - -script: make -C test From 8d1be7ea8a8d7c270f692a6c375ef2614c5ac601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 1 Jun 2020 23:35:25 +0100 Subject: [PATCH 005/844] Refactor code to comply with flake8 and isort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns Signed-off-by: Lukas Fleischer --- aurweb/git/auth.py | 3 +- aurweb/git/serve.py | 14 +- aurweb/git/update.py | 6 +- aurweb/initdb.py | 7 +- aurweb/l10n.py | 2 +- aurweb/schema.py | 4 +- aurweb/scripts/aurblup.py | 3 +- aurweb/scripts/rendercomment.py | 6 +- migrations/env.py | 9 +- schema/gendummydata.py | 346 ++++++++++++++++---------------- setup.py | 3 +- 11 files changed, 206 insertions(+), 197 deletions(-) diff --git a/aurweb/git/auth.py b/aurweb/git/auth.py index 3b1e485f..abecd276 100755 --- a/aurweb/git/auth.py +++ b/aurweb/git/auth.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 -import os -import shlex import re +import shlex import sys import aurweb.config diff --git a/aurweb/git/serve.py b/aurweb/git/serve.py index 64d51b9e..b91f1a13 100755 --- a/aurweb/git/serve.py +++ b/aurweb/git/serve.py @@ -175,11 +175,11 @@ def pkgbase_set_comaintainers(pkgbase, userlist, user, privileged): i += 1 for userid in uids_rem: - cur = conn.execute("DELETE FROM PackageComaintainers " + - "WHERE PackageBaseID = ? AND UsersID = ?", - [pkgbase_id, userid]) - subprocess.Popen((notify_cmd, 'comaintainer-remove', - str(userid), str(pkgbase_id))) + cur = conn.execute("DELETE FROM PackageComaintainers " + + "WHERE PackageBaseID = ? AND UsersID = ?", + [pkgbase_id, userid]) + subprocess.Popen((notify_cmd, 'comaintainer-remove', + str(userid), str(pkgbase_id))) conn.commit() conn.close() @@ -268,7 +268,7 @@ def pkgbase_disown(pkgbase, user, privileged): cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) userid = cur.fetchone()[0] if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) + raise aurweb.exceptions.InvalidUserException(user) subprocess.Popen((notify_cmd, 'disown', str(userid), str(pkgbase_id))) @@ -472,7 +472,7 @@ def checkarg(cmdargv, *argdesc): checkarg_atmost(cmdargv, *argdesc) -def serve(action, cmdargv, user, privileged, remote_addr): +def serve(action, cmdargv, user, privileged, remote_addr): # noqa: C901 if enable_maintenance: if remote_addr not in maintenance_exc: raise aurweb.exceptions.MaintenanceException diff --git a/aurweb/git/update.py b/aurweb/git/update.py index 39128f8b..929b254e 100755 --- a/aurweb/git/update.py +++ b/aurweb/git/update.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 import os -import pygit2 import re import subprocess import sys import time +import pygit2 import srcinfo.parse import srcinfo.utils @@ -75,7 +75,7 @@ def create_pkgbase(conn, pkgbase, user): return pkgbase_id -def save_metadata(metadata, conn, user): +def save_metadata(metadata, conn, user): # noqa: C901 # Obtain package base ID and previous maintainer. pkgbase = metadata['pkgbase'] cur = conn.execute("SELECT ID, MaintainerUID FROM PackageBases " @@ -232,7 +232,7 @@ def die_commit(msg, commit): exit(1) -def main(): +def main(): # noqa: C901 repo = pygit2.Repository(repo_path) user = os.environ.get("AUR_USER") diff --git a/aurweb/initdb.py b/aurweb/initdb.py index 91777f7e..c8d0b2ae 100644 --- a/aurweb/initdb.py +++ b/aurweb/initdb.py @@ -1,11 +1,12 @@ -import aurweb.db -import aurweb.schema +import argparse import alembic.command import alembic.config -import argparse import sqlalchemy +import aurweb.db +import aurweb.schema + def feed_initial_data(conn): conn.execute(aurweb.schema.AccountTypes.insert(), [ diff --git a/aurweb/l10n.py b/aurweb/l10n.py index a7c0103e..492200b3 100644 --- a/aurweb/l10n.py +++ b/aurweb/l10n.py @@ -16,4 +16,4 @@ class Translator: self._localedir, languages=[lang]) self._translator[lang].install() - return _(s) + return _(s) # _ is not defined, what is this? # noqa: F821 diff --git a/aurweb/schema.py b/aurweb/schema.py index 6792cf1d..20f3e5ce 100644 --- a/aurweb/schema.py +++ b/aurweb/schema.py @@ -6,7 +6,7 @@ usually be automatically generated. See `migrations/README` for details. """ -from sqlalchemy import CHAR, Column, ForeignKey, Index, MetaData, String, TIMESTAMP, Table, Text, text +from sqlalchemy import CHAR, TIMESTAMP, Column, ForeignKey, Index, MetaData, String, Table, Text, text from sqlalchemy.dialects.mysql import BIGINT, DECIMAL, INTEGER, TINYINT from sqlalchemy.ext.compiler import compiles @@ -24,7 +24,7 @@ def compile_bigint_sqlite(type_, compiler, **kw): to INTEGER. Aside from that, BIGINT is the same as INTEGER for SQLite. See https://docs.sqlalchemy.org/en/13/dialects/sqlite.html#allowing-autoincrement-behavior-sqlalchemy-types-other-than-integer-integer - """ + """ # noqa: E501 return 'INTEGER' diff --git a/aurweb/scripts/aurblup.py b/aurweb/scripts/aurblup.py index a7d43f12..e32937ce 100755 --- a/aurweb/scripts/aurblup.py +++ b/aurweb/scripts/aurblup.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -import pyalpm import re +import pyalpm + import aurweb.config import aurweb.db diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index 76865d27..422dd33b 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -import re -import pygit2 import sys + import bleach import markdown +import pygit2 import aurweb.config import aurweb.db @@ -47,7 +47,7 @@ class FlysprayLinksInlineProcessor(markdown.inlinepatterns.InlineProcessor): class FlysprayLinksExtension(markdown.extensions.Extension): def extendMarkdown(self, md, md_globals): - processor = FlysprayLinksInlineProcessor(r'\bFS#(\d+)\b',md) + processor = FlysprayLinksInlineProcessor(r'\bFS#(\d+)\b', md) md.inlinePatterns.register(processor, 'flyspray-links', 118) diff --git a/migrations/env.py b/migrations/env.py index 1627e693..c2ff58c1 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -1,10 +1,11 @@ -import aurweb.db -import aurweb.schema - -from alembic import context import logging.config + import sqlalchemy +from alembic import context + +import aurweb.db +import aurweb.schema # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/schema/gendummydata.py b/schema/gendummydata.py index 1f3d0476..b3a73ef2 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -10,33 +10,32 @@ usage: gendummydata.py outputfilename.sql # insert these users/packages into the AUR database. # import hashlib -import random -import time -import os -import sys -import io import logging +import os +import random +import sys +import time -LOG_LEVEL = logging.DEBUG # logging level. set to logging.INFO to reduce output +LOG_LEVEL = logging.DEBUG # logging level. set to logging.INFO to reduce output SEED_FILE = "/usr/share/dict/words" -DB_HOST = os.getenv("DB_HOST", "localhost") -DB_NAME = os.getenv("DB_NAME", "AUR") -DB_USER = os.getenv("DB_USER", "aur") -DB_PASS = os.getenv("DB_PASS", "aur") -USER_ID = 5 # Users.ID of first bogus user -PKG_ID = 1 # Packages.ID of first package +DB_HOST = os.getenv("DB_HOST", "localhost") +DB_NAME = os.getenv("DB_NAME", "AUR") +DB_USER = os.getenv("DB_USER", "aur") +DB_PASS = os.getenv("DB_PASS", "aur") +USER_ID = 5 # Users.ID of first bogus user +PKG_ID = 1 # Packages.ID of first package MAX_USERS = 300 # how many users to 'register' -MAX_DEVS = .1 # what percentage of MAX_USERS are Developers -MAX_TUS = .2 # what percentage of MAX_USERS are Trusted Users -MAX_PKGS = 900 # how many packages to load -PKG_DEPS = (1, 15) # min/max depends a package has -PKG_RELS = (1, 5) # min/max relations a package has -PKG_SRC = (1, 3) # min/max sources a package has +MAX_DEVS = .1 # what percentage of MAX_USERS are Developers +MAX_TUS = .2 # what percentage of MAX_USERS are Trusted Users +MAX_PKGS = 900 # how many packages to load +PKG_DEPS = (1, 15) # min/max depends a package has +PKG_RELS = (1, 5) # min/max relations a package has +PKG_SRC = (1, 3) # min/max sources a package has PKG_CMNTS = (1, 5) # min/max number of comments a package has CATEGORIES_COUNT = 17 # the number of categories from aur-schema -VOTING = (0, .30) # percentage range for package voting -OPEN_PROPOSALS = 5 # number of open trusted user proposals -CLOSE_PROPOSALS = 15 # number of closed trusted user proposals +VOTING = (0, .30) # percentage range for package voting +OPEN_PROPOSALS = 5 # number of open trusted user proposals +CLOSE_PROPOSALS = 15 # number of closed trusted user proposals RANDOM_TLDS = ("edu", "com", "org", "net", "tw", "ru", "pl", "de", "es") RANDOM_URL = ("http://www.", "ftp://ftp.", "http://", "ftp://") RANDOM_LOCS = ("pub", "release", "files", "downloads", "src") @@ -48,20 +47,20 @@ logging.basicConfig(format=logformat, level=LOG_LEVEL) log = logging.getLogger() if len(sys.argv) != 2: - log.error("Missing output filename argument") - raise SystemExit + log.error("Missing output filename argument") + raise SystemExit # make sure the seed file exists # if not os.path.exists(SEED_FILE): - log.error("Please install the 'words' Arch package") - raise SystemExit + log.error("Please install the 'words' Arch package") + raise SystemExit # make sure comments can be created # if not os.path.exists(FORTUNE_FILE): - log.error("Please install the 'fortune-mod' Arch package") - raise SystemExit + log.error("Please install the 'fortune-mod' Arch package") + raise SystemExit # track what users/package names have been used # @@ -69,21 +68,28 @@ seen_users = {} seen_pkgs = {} user_keys = [] + # some functions to generate random data # def genVersion(): - ver = [] - ver.append("%d" % random.randrange(0,10)) - ver.append("%d" % random.randrange(0,20)) - if random.randrange(0,2) == 0: - ver.append("%d" % random.randrange(0,100)) - return ".".join(ver) + "-%d" % random.randrange(1,11) + ver = [] + ver.append("%d" % random.randrange(0, 10)) + ver.append("%d" % random.randrange(0, 20)) + if random.randrange(0, 2) == 0: + ver.append("%d" % random.randrange(0, 100)) + return ".".join(ver) + "-%d" % random.randrange(1, 11) + + def genCategory(): - return random.randrange(1,CATEGORIES_COUNT) + return random.randrange(1, CATEGORIES_COUNT) + + def genUID(): - return seen_users[user_keys[random.randrange(0,len(user_keys))]] + return seen_users[user_keys[random.randrange(0, len(user_keys))]] + + def genFortune(): - return fortunes[random.randrange(0,len(fortunes))].replace("'", "") + return fortunes[random.randrange(0, len(fortunes))].replace("'", "") # load the words, and make sure there are enough words for users/pkgs @@ -93,25 +99,25 @@ fp = open(SEED_FILE, "r", encoding="utf-8") contents = fp.readlines() fp.close() if MAX_USERS > len(contents): - MAX_USERS = len(contents) + MAX_USERS = len(contents) if MAX_PKGS > len(contents): - MAX_PKGS = len(contents) + MAX_PKGS = len(contents) if len(contents) - MAX_USERS > MAX_PKGS: - need_dupes = 0 + need_dupes = 0 else: - need_dupes = 1 + need_dupes = 1 # select random usernames # log.debug("Generating random user names...") user_id = USER_ID while len(seen_users) < MAX_USERS: - user = random.randrange(0, len(contents)) - word = contents[user].replace("'", "").replace(".","").replace(" ", "_") - word = word.strip().lower() - if word not in seen_users: - seen_users[word] = user_id - user_id += 1 + user = random.randrange(0, len(contents)) + word = contents[user].replace("'", "").replace(".", "").replace(" ", "_") + word = word.strip().lower() + if word not in seen_users: + seen_users[word] = user_id + user_id += 1 user_keys = list(seen_users.keys()) # select random package names @@ -119,17 +125,17 @@ user_keys = list(seen_users.keys()) log.debug("Generating random package names...") num_pkgs = PKG_ID while len(seen_pkgs) < MAX_PKGS: - pkg = random.randrange(0, len(contents)) - word = contents[pkg].replace("'", "").replace(".","").replace(" ", "_") - word = word.strip().lower() - if not need_dupes: - if word not in seen_pkgs and word not in seen_users: - seen_pkgs[word] = num_pkgs - num_pkgs += 1 - else: - if word not in seen_pkgs: - seen_pkgs[word] = num_pkgs - num_pkgs += 1 + pkg = random.randrange(0, len(contents)) + word = contents[pkg].replace("'", "").replace(".", "").replace(" ", "_") + word = word.strip().lower() + if not need_dupes: + if word not in seen_pkgs and word not in seen_users: + seen_pkgs[word] = num_pkgs + num_pkgs += 1 + else: + if word not in seen_pkgs: + seen_pkgs[word] = num_pkgs + num_pkgs += 1 # free up contents memory # @@ -151,32 +157,32 @@ out.write("BEGIN;\n") # log.debug("Creating SQL statements for users.") for u in user_keys: - account_type = 1 # default to normal user - if not has_devs or not has_tus: - account_type = random.randrange(1, 4) - if account_type == 3 and not has_devs: - # this will be a dev account - # - developers.append(seen_users[u]) - if len(developers) >= MAX_DEVS * MAX_USERS: - has_devs = 1 - elif account_type == 2 and not has_tus: - # this will be a trusted user account - # - trustedusers.append(seen_users[u]) - if len(trustedusers) >= MAX_TUS * MAX_USERS: - has_tus = 1 - else: - # a normal user account - # - pass + account_type = 1 # default to normal user + if not has_devs or not has_tus: + account_type = random.randrange(1, 4) + if account_type == 3 and not has_devs: + # this will be a dev account + # + developers.append(seen_users[u]) + if len(developers) >= MAX_DEVS * MAX_USERS: + has_devs = 1 + elif account_type == 2 and not has_tus: + # this will be a trusted user account + # + trustedusers.append(seen_users[u]) + if len(trustedusers) >= MAX_TUS * MAX_USERS: + has_tus = 1 + else: + # a normal user account + # + pass - h = hashlib.new('md5') - h.update(u.encode()); - s = ("INSERT INTO Users (ID, AccountTypeID, Username, Email, Passwd)" - " VALUES (%d, %d, '%s', '%s@example.com', '%s');\n") - s = s % (seen_users[u], account_type, u, u, h.hexdigest()) - out.write(s) + h = hashlib.new('md5') + h.update(u.encode()) + s = ("INSERT INTO Users (ID, AccountTypeID, Username, Email, Passwd)" + " VALUES (%d, %d, '%s', '%s@example.com', '%s');\n") + s = s % (seen_users[u], account_type, u, u, h.hexdigest()) + out.write(s) log.debug("Number of developers: %d" % len(developers)) log.debug("Number of trusted users: %d" % len(trustedusers)) @@ -193,123 +199,123 @@ fp.close() log.debug("Creating SQL statements for packages.") count = 0 for p in list(seen_pkgs.keys()): - NOW = int(time.time()) - if count % 2 == 0: - muid = developers[random.randrange(0,len(developers))] - puid = developers[random.randrange(0,len(developers))] - else: - muid = trustedusers[random.randrange(0,len(trustedusers))] - puid = trustedusers[random.randrange(0,len(trustedusers))] - if count % 20 == 0: # every so often, there are orphans... - muid = "NULL" + NOW = int(time.time()) + if count % 2 == 0: + muid = developers[random.randrange(0, len(developers))] + puid = developers[random.randrange(0, len(developers))] + else: + muid = trustedusers[random.randrange(0, len(trustedusers))] + puid = trustedusers[random.randrange(0, len(trustedusers))] + if count % 20 == 0: # every so often, there are orphans... + muid = "NULL" - uuid = genUID() # the submitter/user + uuid = genUID() # the submitter/user - s = ("INSERT INTO PackageBases (ID, Name, FlaggerComment, SubmittedTS, ModifiedTS, " + s = ("INSERT INTO PackageBases (ID, Name, FlaggerComment, SubmittedTS, ModifiedTS, " "SubmitterUID, MaintainerUID, PackagerUID) VALUES (%d, '%s', '', %d, %d, %d, %s, %s);\n") - s = s % (seen_pkgs[p], p, NOW, NOW, uuid, muid, puid) - out.write(s) + s = s % (seen_pkgs[p], p, NOW, NOW, uuid, muid, puid) + out.write(s) - s = ("INSERT INTO Packages (ID, PackageBaseID, Name, Version) VALUES " + s = ("INSERT INTO Packages (ID, PackageBaseID, Name, Version) VALUES " "(%d, %d, '%s', '%s');\n") - s = s % (seen_pkgs[p], seen_pkgs[p], p, genVersion()) - out.write(s) + s = s % (seen_pkgs[p], seen_pkgs[p], p, genVersion()) + out.write(s) - count += 1 + count += 1 - # create random comments for this package - # - num_comments = random.randrange(PKG_CMNTS[0], PKG_CMNTS[1]) - for i in range(0, num_comments): - now = NOW + random.randrange(400, 86400*3) - s = ("INSERT INTO PackageComments (PackageBaseID, UsersID," - " Comments, RenderedComment, CommentTS) VALUES (%d, %d, '%s', '', %d);\n") - s = s % (seen_pkgs[p], genUID(), genFortune(), now) - out.write(s) + # create random comments for this package + # + num_comments = random.randrange(PKG_CMNTS[0], PKG_CMNTS[1]) + for i in range(0, num_comments): + now = NOW + random.randrange(400, 86400*3) + s = ("INSERT INTO PackageComments (PackageBaseID, UsersID," + " Comments, RenderedComment, CommentTS) VALUES (%d, %d, '%s', '', %d);\n") + s = s % (seen_pkgs[p], genUID(), genFortune(), now) + out.write(s) # Cast votes # track_votes = {} log.debug("Casting votes for packages.") for u in user_keys: - num_votes = random.randrange(int(len(seen_pkgs)*VOTING[0]), - int(len(seen_pkgs)*VOTING[1])) - pkgvote = {} - for v in range(num_votes): - pkg = random.randrange(1, len(seen_pkgs) + 1) - if pkg not in pkgvote: - s = ("INSERT INTO PackageVotes (UsersID, PackageBaseID)" - " VALUES (%d, %d);\n") - s = s % (seen_users[u], pkg) - pkgvote[pkg] = 1 - if pkg not in track_votes: - track_votes[pkg] = 0 - track_votes[pkg] += 1 - out.write(s) + num_votes = random.randrange(int(len(seen_pkgs)*VOTING[0]), + int(len(seen_pkgs)*VOTING[1])) + pkgvote = {} + for v in range(num_votes): + pkg = random.randrange(1, len(seen_pkgs) + 1) + if pkg not in pkgvote: + s = ("INSERT INTO PackageVotes (UsersID, PackageBaseID)" + " VALUES (%d, %d);\n") + s = s % (seen_users[u], pkg) + pkgvote[pkg] = 1 + if pkg not in track_votes: + track_votes[pkg] = 0 + track_votes[pkg] += 1 + out.write(s) # Update statements for package votes # for p in list(track_votes.keys()): - s = "UPDATE PackageBases SET NumVotes = %d WHERE ID = %d;\n" - s = s % (track_votes[p], p) - out.write(s) + s = "UPDATE PackageBases SET NumVotes = %d WHERE ID = %d;\n" + s = s % (track_votes[p], p) + out.write(s) # Create package dependencies and sources # log.debug("Creating statements for package depends/sources.") for p in list(seen_pkgs.keys()): - num_deps = random.randrange(PKG_DEPS[0], PKG_DEPS[1]) - for i in range(0, num_deps): - dep = random.choice([k for k in seen_pkgs]) - deptype = random.randrange(1, 5) - if deptype == 4: - dep += ": for " + random.choice([k for k in seen_pkgs]) - s = "INSERT INTO PackageDepends(PackageID, DepTypeID, DepName) VALUES (%d, %d, '%s');\n" - s = s % (seen_pkgs[p], deptype, dep) - out.write(s) + num_deps = random.randrange(PKG_DEPS[0], PKG_DEPS[1]) + for i in range(0, num_deps): + dep = random.choice([k for k in seen_pkgs]) + deptype = random.randrange(1, 5) + if deptype == 4: + dep += ": for " + random.choice([k for k in seen_pkgs]) + s = "INSERT INTO PackageDepends(PackageID, DepTypeID, DepName) VALUES (%d, %d, '%s');\n" + s = s % (seen_pkgs[p], deptype, dep) + out.write(s) - num_rels = random.randrange(PKG_RELS[0], PKG_RELS[1]) - for i in range(0, num_deps): - rel = random.choice([k for k in seen_pkgs]) - reltype = random.randrange(1, 4) - s = "INSERT INTO PackageRelations(PackageID, RelTypeID, RelName) VALUES (%d, %d, '%s');\n" - s = s % (seen_pkgs[p], reltype, rel) - out.write(s) + num_rels = random.randrange(PKG_RELS[0], PKG_RELS[1]) + for i in range(0, num_deps): + rel = random.choice([k for k in seen_pkgs]) + reltype = random.randrange(1, 4) + s = "INSERT INTO PackageRelations(PackageID, RelTypeID, RelName) VALUES (%d, %d, '%s');\n" + s = s % (seen_pkgs[p], reltype, rel) + out.write(s) - num_sources = random.randrange(PKG_SRC[0], PKG_SRC[1]) - for i in range(num_sources): - src_file = user_keys[random.randrange(0, len(user_keys))] - src = "%s%s.%s/%s/%s-%s.tar.gz" % ( - RANDOM_URL[random.randrange(0,len(RANDOM_URL))], - p, RANDOM_TLDS[random.randrange(0,len(RANDOM_TLDS))], - RANDOM_LOCS[random.randrange(0,len(RANDOM_LOCS))], - src_file, genVersion()) - s = "INSERT INTO PackageSources(PackageID, Source) VALUES (%d, '%s');\n" - s = s % (seen_pkgs[p], src) - out.write(s) + num_sources = random.randrange(PKG_SRC[0], PKG_SRC[1]) + for i in range(num_sources): + src_file = user_keys[random.randrange(0, len(user_keys))] + src = "%s%s.%s/%s/%s-%s.tar.gz" % ( + RANDOM_URL[random.randrange(0, len(RANDOM_URL))], + p, RANDOM_TLDS[random.randrange(0, len(RANDOM_TLDS))], + RANDOM_LOCS[random.randrange(0, len(RANDOM_LOCS))], + src_file, genVersion()) + s = "INSERT INTO PackageSources(PackageID, Source) VALUES (%d, '%s');\n" + s = s % (seen_pkgs[p], src) + out.write(s) # Create trusted user proposals # log.debug("Creating SQL statements for trusted user proposals.") -count=0 +count = 0 for t in range(0, OPEN_PROPOSALS+CLOSE_PROPOSALS): - now = int(time.time()) - if count < CLOSE_PROPOSALS: - start = now - random.randrange(3600*24*7, 3600*24*21) - end = now - random.randrange(0, 3600*24*7) - else: - start = now - end = now + random.randrange(3600*24, 3600*24*7) - if count % 5 == 0: # Don't make the vote about anyone once in a while - user = "" - else: - user = user_keys[random.randrange(0,len(user_keys))] - suid = trustedusers[random.randrange(0,len(trustedusers))] - s = ("INSERT INTO TU_VoteInfo (Agenda, User, Submitted, End," - " Quorum, SubmitterID) VALUES ('%s', '%s', %d, %d, 0.0, %d);\n") - s = s % (genFortune(), user, start, end, suid) - out.write(s) - count += 1 + now = int(time.time()) + if count < CLOSE_PROPOSALS: + start = now - random.randrange(3600*24*7, 3600*24*21) + end = now - random.randrange(0, 3600*24*7) + else: + start = now + end = now + random.randrange(3600*24, 3600*24*7) + if count % 5 == 0: # Don't make the vote about anyone once in a while + user = "" + else: + user = user_keys[random.randrange(0, len(user_keys))] + suid = trustedusers[random.randrange(0, len(trustedusers))] + s = ("INSERT INTO TU_VoteInfo (Agenda, User, Submitted, End," + " Quorum, SubmitterID) VALUES ('%s', '%s', %d, %d, 0.0, %d);\n") + s = s % (genFortune(), user, start, end, suid) + out.write(s) + count += 1 # close output file # diff --git a/setup.py b/setup.py index ca26f0d8..cf88488c 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ import re -from setuptools import setup, find_packages import sys +from setuptools import find_packages, setup + version = None with open('web/lib/version.inc.php', 'r') as f: for line in f.readlines(): From 4cf94816ae022be88540a25356a3abbe10a452eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 1 Jun 2020 23:35:26 +0100 Subject: [PATCH 006/844] flake8: add initial config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns Signed-off-by: Lukas Fleischer --- setup.cfg | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..04f5b8ba --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 127 +max-complexity = 10 + From 8f47b8d731e0d650ea671e169cddaafa64c44055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 1 Jun 2020 23:35:27 +0100 Subject: [PATCH 007/844] isort: add initial config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns Signed-off-by: Lukas Fleischer --- setup.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.cfg b/setup.cfg index 04f5b8ba..b868c096 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,3 +2,7 @@ max-line-length = 127 max-complexity = 10 +[isort] +line_length = 127 +lines_between_types = 1 + From 41a84934114f5d77f51d7064c4a64b2e279c1ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 1 Jun 2020 23:35:28 +0100 Subject: [PATCH 008/844] pre-commit: add initial config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns Signed-off-by: Lukas Fleischer --- .pre-commit-config.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..525c7eb8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +hooks: + - &base + language: python + types: [python] + require_serial: true + exclude: ^migrations/versions + - &flake8 + id: flake8 + name: flake8 + entry: flake8 + <<: *base + - &isort + id: isort + name: isort + entry: isort + <<: *base + +repos: + - repo: local + hooks: + - <<: *flake8 + - <<: *isort + args: ['--check-only', '--diff'] + From d4abe0b72d1215906806ee84474115961e79f7c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 1 Jun 2020 23:35:29 +0100 Subject: [PATCH 009/844] Add CONTRIBUTING.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns Signed-off-by: Lukas Fleischer --- CONTRIBUTING.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..a37d980a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,10 @@ +# Contributing + +Patches should be sent to the [aur-dev@archlinux.org][1] mailing list. + +Before sending patched you are recomended to run the `flake8` and `isort`. + +You can add git hook to do this by installing `python-pre-install` and running +`pre-install install`. + +[1] https://lists.archlinux.org/listinfo/aur-dev From 5be07a8a9e9777d54cf7122be52aa0a7b1b51e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Mon, 1 Jun 2020 18:49:37 +0200 Subject: [PATCH 010/844] aurweb.spawn: Integrate FastAPI and nginx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aurweb.spawn used to launch only PHP’s built-in server. Now it spawns a dummy FastAPI application too. Since both stacks spawn their own HTTP server, aurweb.spawn also spawns nginx as a reverse proxy to mount them under the same base URL, defined by aur_location in the configuration. Signed-off-by: Lukas Fleischer --- .gitlab-ci.yml | 2 +- TESTING | 3 +- aurweb/asgi.py | 8 +++++ aurweb/spawn.py | 80 +++++++++++++++++++++++++++++++++++++------- conf/config.defaults | 7 ++++ 5 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 aurweb/asgi.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 74784fce..f6260ebb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ before_script: base-devel git gpgme protobuf pyalpm python-mysql-connector python-pygit2 python-srcinfo python-bleach python-markdown python-sqlalchemy python-alembic python-pytest python-werkzeug - python-pytest-tap + python-pytest-tap python-fastapi uvicorn nginx test: script: diff --git a/TESTING b/TESTING index a5e08cb8..31e3bcbd 100644 --- a/TESTING +++ b/TESTING @@ -12,7 +12,8 @@ INSTALL. 2) Install the necessary packages: # pacman -S --needed php php-sqlite sqlite words fortune-mod \ - python python-sqlalchemy python-alembic + python python-sqlalchemy python-alembic \ + python-fastapi uvicorn nginx Ensure to enable the pdo_sqlite extension in php.ini. diff --git a/aurweb/asgi.py b/aurweb/asgi.py new file mode 100644 index 00000000..5f30471a --- /dev/null +++ b/aurweb/asgi.py @@ -0,0 +1,8 @@ +from fastapi import FastAPI + +app = FastAPI() + + +@app.get("/hello/") +async def hello(): + return {"message": "Hello from FastAPI!"} diff --git a/aurweb/spawn.py b/aurweb/spawn.py index 5fa646b5..0506afa4 100644 --- a/aurweb/spawn.py +++ b/aurweb/spawn.py @@ -10,8 +10,10 @@ configuration anyway. import atexit import argparse +import os import subprocess import sys +import tempfile import time import urllib @@ -20,6 +22,7 @@ import aurweb.schema children = [] +temporary_dir = None verbosity = 0 @@ -35,10 +38,42 @@ class ProcessExceptions(Exception): super().__init__("\n- ".join(messages)) +def generate_nginx_config(): + """ + Generate an nginx configuration based on aurweb's configuration. + The file is generated under `temporary_dir`. + Returns the path to the created configuration file. + """ + aur_location = aurweb.config.get("options", "aur_location") + aur_location_parts = urllib.parse.urlsplit(aur_location) + config_path = os.path.join(temporary_dir, "nginx.conf") + config = open(config_path, "w") + # We double nginx's braces because they conflict with Python's f-strings. + config.write(f""" + events {{}} + daemon off; + error_log /dev/stderr info; + pid {os.path.join(temporary_dir, "nginx.pid")}; + http {{ + access_log /dev/stdout; + server {{ + listen {aur_location_parts.netloc}; + location / {{ + proxy_pass http://{aurweb.config.get("php", "bind_address")}; + }} + location /hello {{ + proxy_pass http://{aurweb.config.get("fastapi", "bind_address")}; + }} + }} + }} + """) + return config_path + + def spawn_child(args): """Open a subprocess and add it to the global state.""" if verbosity >= 1: - print(f"Spawning {args}", file=sys.stderr) + print(f":: Spawning {args}", file=sys.stderr) children.append(subprocess.Popen(args)) @@ -52,10 +87,29 @@ def start(): if children: return atexit.register(stop) - aur_location = aurweb.config.get("options", "aur_location") - aur_location_parts = urllib.parse.urlsplit(aur_location) - htmldir = aurweb.config.get("options", "htmldir") - spawn_child(["php", "-S", aur_location_parts.netloc, "-t", htmldir]) + + print("{ruler}\n" + "Spawing PHP and FastAPI, then nginx as a reverse proxy.\n" + "Check out {aur_location}\n" + "Hit ^C to terminate everything.\n" + "{ruler}" + .format(ruler=("-" * os.get_terminal_size().columns), + aur_location=aurweb.config.get('options', 'aur_location'))) + + # PHP + php_address = aurweb.config.get("php", "bind_address") + htmldir = aurweb.config.get("php", "htmldir") + spawn_child(["php", "-S", php_address, "-t", htmldir]) + + # FastAPI + host, port = aurweb.config.get("fastapi", "bind_address").rsplit(":", 1) + spawn_child(["python", "-m", "uvicorn", + "--host", host, + "--port", port, + "aurweb.asgi:app"]) + + # nginx + spawn_child(["nginx", "-p", temporary_dir, "-c", generate_nginx_config()]) def stop(): @@ -73,7 +127,7 @@ def stop(): try: p.terminate() if verbosity >= 1: - print(f"Sent SIGTERM to {p.args}", file=sys.stderr) + print(f":: Sent SIGTERM to {p.args}", file=sys.stderr) except Exception as e: exceptions.append(e) for p in children: @@ -99,9 +153,11 @@ if __name__ == '__main__': help='increase verbosity') args = parser.parse_args() verbosity = args.verbose - start() - try: - while True: - time.sleep(60) - except KeyboardInterrupt: - stop() + with tempfile.TemporaryDirectory(prefix="aurweb-") as tmpdirname: + temporary_dir = tmpdirname + start() + try: + while True: + time.sleep(60) + except KeyboardInterrupt: + stop() diff --git a/conf/config.defaults b/conf/config.defaults index 86fe765c..ed495168 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -41,9 +41,16 @@ cache = none cache_pkginfo_ttl = 86400 memcache_servers = 127.0.0.1:11211 +[php] +; Address PHP should bind when spawned in development mode by aurweb.spawn. +bind_address = 127.0.0.1:8081 ; Directory containing aurweb's PHP code, required by aurweb.spawn. ;htmldir = /path/to/web/html +[fastapi] +; Address uvicorn should bind when spawned in development mode by aurweb.spawn. +bind_address = 127.0.0.1:8082 + [ratelimit] request_limit = 4000 window_length = 86400 From 8c868e088c8becc7640327db2e5e2a1cb10bab41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Wed, 3 Jun 2020 02:04:02 +0200 Subject: [PATCH 011/844] Introduce conf/config.dev for development MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit conf/config.dev’s purpose is to provide a lighter configuration template for developers, and split development-specific options off the default configuration file. Signed-off-by: Lukas Fleischer --- TESTING | 11 ++++++----- conf/config.defaults | 10 ---------- conf/config.dev | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 15 deletions(-) create mode 100644 conf/config.dev diff --git a/TESTING b/TESTING index 31e3bcbd..7261df92 100644 --- a/TESTING +++ b/TESTING @@ -17,12 +17,13 @@ INSTALL. Ensure to enable the pdo_sqlite extension in php.ini. -3) Copy conf/config.defaults to conf/config and adjust the configuration - Pay attention to disable_http_login, enable_maintenance, aur_location and - htmldir. +3) Copy conf/config.dev to conf/config and replace YOUR_AUR_ROOT by the absolute + path to the root of your aurweb clone. sed can do both tasks for you: - Be sure to change backend to sqlite and name to the file location of your - created test database. + $ sed -e "s;YOUR_AUR_ROOT;$PWD;g" conf/config.dev > conf/config + + Note that when the upstream config.dev is updated, you should compare it to + your conf/config, or regenerate your configuration with the command above. 4) Prepare the testing database: diff --git a/conf/config.defaults b/conf/config.defaults index ed495168..447dacac 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -41,16 +41,6 @@ cache = none cache_pkginfo_ttl = 86400 memcache_servers = 127.0.0.1:11211 -[php] -; Address PHP should bind when spawned in development mode by aurweb.spawn. -bind_address = 127.0.0.1:8081 -; Directory containing aurweb's PHP code, required by aurweb.spawn. -;htmldir = /path/to/web/html - -[fastapi] -; Address uvicorn should bind when spawned in development mode by aurweb.spawn. -bind_address = 127.0.0.1:8082 - [ratelimit] request_limit = 4000 window_length = 86400 diff --git a/conf/config.dev b/conf/config.dev new file mode 100644 index 00000000..d752f61f --- /dev/null +++ b/conf/config.dev @@ -0,0 +1,32 @@ +; Configuration file for aurweb development. +; +; Options are implicitly inherited from conf/config.defaults, which lists all +; available options for productions, and their default values. This current file +; overrides only options useful for development, and introduces +; development-specific options too. + +[database] +backend = sqlite +name = YOUR_AUR_ROOT/aurweb.sqlite3 + +; Alternative MySQL configuration +;backend = mysql +;name = aurweb +;user = aur +;password = aur + +[options] +aur_location = http://127.0.0.1:8080 +disable_http_login = 0 +enable-maintenance = 0 + +[php] +; Address PHP should bind when spawned in development mode by aurweb.spawn. +bind_address = 127.0.0.1:8081 + +; Directory containing aurweb's PHP code, required by aurweb.spawn. +htmldir = YOUR_AUR_ROOT/web/html + +[fastapi] +; Address uvicorn should bind when spawned in development mode by aurweb.spawn. +bind_address = 127.0.0.1:8082 From 0e3bd8b5969f3c3d6ba9273b15ae1e093fc934e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Thu, 4 Jun 2020 21:59:34 +0200 Subject: [PATCH 012/844] Remove the FastAPI /hello test route Signed-off-by: Lukas Fleischer --- aurweb/asgi.py | 5 ----- aurweb/spawn.py | 3 --- 2 files changed, 8 deletions(-) diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 5f30471a..9bb71ecc 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -1,8 +1,3 @@ from fastapi import FastAPI app = FastAPI() - - -@app.get("/hello/") -async def hello(): - return {"message": "Hello from FastAPI!"} diff --git a/aurweb/spawn.py b/aurweb/spawn.py index 0506afa4..7fe59e65 100644 --- a/aurweb/spawn.py +++ b/aurweb/spawn.py @@ -61,9 +61,6 @@ def generate_nginx_config(): location / {{ proxy_pass http://{aurweb.config.get("php", "bind_address")}; }} - location /hello {{ - proxy_pass http://{aurweb.config.get("fastapi", "bind_address")}; - }} }} }} """) From b1300117ac6fc0f5e9cf1048576db8fb97470bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Thu, 4 Jun 2020 21:59:48 +0200 Subject: [PATCH 013/844] aurweb.spawn: Fix isort errors Signed-off-by: Lukas Fleischer --- aurweb/spawn.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aurweb/spawn.py b/aurweb/spawn.py index 7fe59e65..e86f29fe 100644 --- a/aurweb/spawn.py +++ b/aurweb/spawn.py @@ -8,8 +8,8 @@ configuration anyway. """ -import atexit import argparse +import atexit import os import subprocess import sys @@ -20,7 +20,6 @@ import urllib import aurweb.config import aurweb.schema - children = [] temporary_dir = None verbosity = 0 From 3b347d3989592293661a47a5bac7645afb8d61d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Thu, 4 Jun 2020 22:00:20 +0200 Subject: [PATCH 014/844] Crude OpenID Connect client using Authlib Developers can go to /sso/login to get redirected to the SSO. On successful login, the ID token is displayed. Signed-off-by: Lukas Fleischer --- .gitlab-ci.yml | 3 ++- TESTING | 3 ++- aurweb/asgi.py | 13 +++++++++++++ aurweb/routers/__init__.py | 5 +++++ aurweb/routers/sso.py | 30 ++++++++++++++++++++++++++++++ aurweb/spawn.py | 3 +++ conf/config.defaults | 8 ++++++++ conf/config.dev | 9 +++++++++ 8 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 aurweb/routers/__init__.py create mode 100644 aurweb/routers/sso.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f6260ebb..9dc951aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,8 @@ before_script: base-devel git gpgme protobuf pyalpm python-mysql-connector python-pygit2 python-srcinfo python-bleach python-markdown python-sqlalchemy python-alembic python-pytest python-werkzeug - python-pytest-tap python-fastapi uvicorn nginx + python-pytest-tap python-fastapi uvicorn nginx python-authlib + python-itsdangerous python-httpx test: script: diff --git a/TESTING b/TESTING index 7261df92..d7df3672 100644 --- a/TESTING +++ b/TESTING @@ -13,7 +13,8 @@ INSTALL. # pacman -S --needed php php-sqlite sqlite words fortune-mod \ python python-sqlalchemy python-alembic \ - python-fastapi uvicorn nginx + python-fastapi uvicorn nginx \ + python-authlib python-itsdangerous python-httpx Ensure to enable the pdo_sqlite extension in php.ini. diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 9bb71ecc..60c7ade7 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -1,3 +1,16 @@ from fastapi import FastAPI +from starlette.middleware.sessions import SessionMiddleware + +import aurweb.config + +from aurweb.routers import sso app = FastAPI() + +session_secret = aurweb.config.get("fastapi", "session_secret") +if not session_secret: + raise Exception("[fastapi] session_secret must not be empty") + +app.add_middleware(SessionMiddleware, secret_key=session_secret) + +app.include_router(sso.router) diff --git a/aurweb/routers/__init__.py b/aurweb/routers/__init__.py new file mode 100644 index 00000000..35d43c03 --- /dev/null +++ b/aurweb/routers/__init__.py @@ -0,0 +1,5 @@ +""" +API routers for FastAPI. + +See https://fastapi.tiangolo.com/tutorial/bigger-applications/ +""" diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py new file mode 100644 index 00000000..b16edffb --- /dev/null +++ b/aurweb/routers/sso.py @@ -0,0 +1,30 @@ +import fastapi + +from authlib.integrations.starlette_client import OAuth +from starlette.requests import Request + +import aurweb.config + +router = fastapi.APIRouter() + +oauth = OAuth() +oauth.register( + name="sso", + server_metadata_url=aurweb.config.get("sso", "openid_configuration"), + client_kwargs={"scope": "openid"}, + client_id=aurweb.config.get("sso", "client_id"), + client_secret=aurweb.config.get("sso", "client_secret"), +) + + +@router.get("/sso/login") +async def login(request: Request): + redirect_uri = aurweb.config.get("options", "aur_location") + "/sso/authenticate" + return await oauth.sso.authorize_redirect(request, redirect_uri, prompt="login") + + +@router.get("/sso/authenticate") +async def authenticate(request: Request): + token = await oauth.sso.authorize_access_token(request) + user = await oauth.sso.parse_id_token(request, token) + return dict(user) diff --git a/aurweb/spawn.py b/aurweb/spawn.py index e86f29fe..5da8587e 100644 --- a/aurweb/spawn.py +++ b/aurweb/spawn.py @@ -60,6 +60,9 @@ def generate_nginx_config(): location / {{ proxy_pass http://{aurweb.config.get("php", "bind_address")}; }} + location /sso {{ + proxy_pass http://{aurweb.config.get("fastapi", "bind_address")}; + }} }} }} """) diff --git a/conf/config.defaults b/conf/config.defaults index 447dacac..49259754 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -68,6 +68,14 @@ username-regex = [a-zA-Z0-9]+[.\-_]?[a-zA-Z0-9]+$ git-serve-cmd = /usr/local/bin/aurweb-git-serve ssh-options = restrict +[sso] +openid_configuration = +client_id = +client_secret = + +[fastapi] +session_secret = + [serve] repo-path = /srv/http/aurweb/aur.git/ repo-regex = [a-z0-9][a-z0-9.+_-]*$ diff --git a/conf/config.dev b/conf/config.dev index d752f61f..893e8fd6 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -20,6 +20,12 @@ aur_location = http://127.0.0.1:8080 disable_http_login = 0 enable-maintenance = 0 +; Single sign-on +[sso] +openid_configuration = http://127.0.0.1:8083/auth/realms/aurweb/.well-known/openid-configuration +client_id = aurweb +client_secret = + [php] ; Address PHP should bind when spawned in development mode by aurweb.spawn. bind_address = 127.0.0.1:8081 @@ -30,3 +36,6 @@ htmldir = YOUR_AUR_ROOT/web/html [fastapi] ; Address uvicorn should bind when spawned in development mode by aurweb.spawn. bind_address = 127.0.0.1:8082 + +; Passphrase FastAPI uses to sign client-side sessions. +session_secret = secret From 2b439b819908a7f6e9cbd9029d82de617230312f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Thu, 4 Jun 2020 22:00:34 +0200 Subject: [PATCH 015/844] Guide to setting up Keycloak for the SSO Signed-off-by: Lukas Fleischer --- conf/config.dev | 2 +- doc/sso.txt | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 doc/sso.txt diff --git a/conf/config.dev b/conf/config.dev index 893e8fd6..37f38c45 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -20,7 +20,7 @@ aur_location = http://127.0.0.1:8080 disable_http_login = 0 enable-maintenance = 0 -; Single sign-on +; Single sign-on; see doc/sso.txt. [sso] openid_configuration = http://127.0.0.1:8083/auth/realms/aurweb/.well-known/openid-configuration client_id = aurweb diff --git a/doc/sso.txt b/doc/sso.txt new file mode 100644 index 00000000..1b0b1f7d --- /dev/null +++ b/doc/sso.txt @@ -0,0 +1,38 @@ +Single Sign-On (SSO) +==================== + +This guide will walk you through setting up Keycloak for use with aurweb. For +extensive documentation, see . + +Installing Keycloak +------------------- + +Keycloak is in the official Arch repositories: + + # pacman -S keycloak + +The default port is 8080, which conflicts with aurweb’s default port. You need +to edit `/etc/keycloak/standalone.xml`, looking for this line: + + + +The default developer configuration assumes it is set to 8083. Alternatively, +you may customize [options] aur_location and [sso] openid_configuration in +`conf/config`. + +You may then start `keycloak.service` through systemd. + +See also ArchWiki . + +Configuring a realm +------------------- + +Go to and log in as administrator. Then, hover the +text right below the Keycloak logo at the top left, by default *Master*. Click +*Add realm* and name it *aurweb*. + +Open the *Clients* tab, and create a new *openid-connect* client. Call it +*aurweb*, and set the root URL to (your aur_location). + +Create a user from the *Users* tab and try logging in from +. From 3f31d149a6dd736007c6583a6162aeda1bcc37b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Tue, 9 Jun 2020 20:25:22 +0200 Subject: [PATCH 016/844] aurweb.l10n: Translate without side effects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The install method in Python’s gettext API aliases the translator’s gettext method to an application-global _(). We don’t use that anywhere, and it’s clear from aurweb’s Translator interface that we want to translate a piece of text without affecting any global namespace. Signed-off-by: Lukas Fleischer --- aurweb/l10n.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aurweb/l10n.py b/aurweb/l10n.py index 492200b3..51b56abb 100644 --- a/aurweb/l10n.py +++ b/aurweb/l10n.py @@ -15,5 +15,4 @@ class Translator: self._translator[lang] = gettext.translation("aurweb", self._localedir, languages=[lang]) - self._translator[lang].install() - return _(s) # _ is not defined, what is this? # noqa: F821 + return self._translator[lang].gettext(s) From a5554c19a9712ede5fe5a996bd1bec11cfc9f66a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Mon, 8 Jun 2020 20:16:27 +0200 Subject: [PATCH 017/844] Add SSO account ID in table Users This column holds a user ID issed by the single sign-on provider. For Keycloak, it is an UUID. For more flexibility, we will be using a standardly-sized VARCHAR field. Signed-off-by: Lukas Fleischer --- aurweb/schema.py | 1 + ...6e1cd_add_sso_account_id_in_table_users.py | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 migrations/versions/ef39fcd6e1cd_add_sso_account_id_in_table_users.py diff --git a/aurweb/schema.py b/aurweb/schema.py index 20f3e5ce..a1d56281 100644 --- a/aurweb/schema.py +++ b/aurweb/schema.py @@ -67,6 +67,7 @@ Users = Table( Column('CommentNotify', TINYINT(1), nullable=False, server_default=text("1")), Column('UpdateNotify', TINYINT(1), nullable=False, server_default=text("0")), Column('OwnershipNotify', TINYINT(1), nullable=False, server_default=text("1")), + Column('SSOAccountID', String(255), nullable=True, unique=True), Index('UsersAccountTypeID', 'AccountTypeID'), mysql_engine='InnoDB', ) diff --git a/migrations/versions/ef39fcd6e1cd_add_sso_account_id_in_table_users.py b/migrations/versions/ef39fcd6e1cd_add_sso_account_id_in_table_users.py new file mode 100644 index 00000000..9e125165 --- /dev/null +++ b/migrations/versions/ef39fcd6e1cd_add_sso_account_id_in_table_users.py @@ -0,0 +1,30 @@ +"""Add SSO account ID in table Users + +Revision ID: ef39fcd6e1cd +Revises: f47cad5d6d03 +Create Date: 2020-06-08 10:04:13.898617 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'ef39fcd6e1cd' +down_revision = 'f47cad5d6d03' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('Users', sa.Column('SSOAccountID', sa.String(length=255), nullable=True)) + op.create_unique_constraint(None, 'Users', ['SSOAccountID']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'Users', type_='unique') + op.drop_column('Users', 'SSOAccountID') + # ### end Alembic commands ### From c77e9d1de0d14253ab3c3b958f459b04b233aeb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Mon, 8 Jun 2020 20:16:36 +0200 Subject: [PATCH 018/844] Integrate SQLAlchemy into FastAPI Signed-off-by: Lukas Fleischer --- aurweb/db.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/aurweb/db.py b/aurweb/db.py index 1ccd9a07..02aeba38 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -10,6 +10,8 @@ except ImportError: import aurweb.config +engine = None # See get_engine + def get_sqlalchemy_url(): """ @@ -38,6 +40,34 @@ def get_sqlalchemy_url(): raise ValueError('unsupported database backend') +def get_engine(): + """ + Return the global SQLAlchemy engine. + + The engine is created on the first call to get_engine and then stored in the + `engine` global variable for the next calls. + """ + from sqlalchemy import create_engine + global engine + if engine is None: + engine = create_engine(get_sqlalchemy_url(), + # check_same_thread is for a SQLite technicality + # https://fastapi.tiangolo.com/tutorial/sql-databases/#note + connect_args={"check_same_thread": False}) + return engine + + +def connect(): + """ + Return an SQLAlchemy connection. Connections are usually pooled. See + . + + Since SQLAlchemy connections are context managers too, you should use it + with Python’s `with` operator, or with FastAPI’s dependency injection. + """ + return get_engine().connect() + + class Connection: _conn = None _paramstyle = None From 42f8f160b6c556a8c2006788a25b6fafe7be1c08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Mon, 8 Jun 2020 20:16:49 +0200 Subject: [PATCH 019/844] Open AUR sessions from SSO Only the core functionality is implemented here. See the TODOs. Signed-off-by: Lukas Fleischer --- aurweb/routers/sso.py | 51 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index b16edffb..d0802c34 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -1,9 +1,18 @@ +import time +import uuid + import fastapi from authlib.integrations.starlette_client import OAuth +from fastapi import Depends, HTTPException +from fastapi.responses import RedirectResponse +from sqlalchemy.sql import select from starlette.requests import Request import aurweb.config +import aurweb.db + +from aurweb.schema import Sessions, Users router = fastapi.APIRouter() @@ -23,8 +32,46 @@ async def login(request: Request): return await oauth.sso.authorize_redirect(request, redirect_uri, prompt="login") +def open_session(conn, user_id): + """ + Create a new user session into the database. Return its SID. + """ + # TODO check for account suspension + # TODO apply [options] max_sessions_per_user + sid = uuid.uuid4().hex + conn.execute(Sessions.insert().values( + UsersID=user_id, + SessionID=sid, + LastUpdateTS=time.time(), + )) + # TODO update Users.LastLogin and Users.LastLoginIPAddress + return sid + + @router.get("/sso/authenticate") -async def authenticate(request: Request): +async def authenticate(request: Request, conn=Depends(aurweb.db.connect)): + """ + Receive an OpenID Connect ID token, validate it, then process it to create + an new AUR session. + """ + # TODO check for banned IPs token = await oauth.sso.authorize_access_token(request) user = await oauth.sso.parse_id_token(request, token) - return dict(user) + sub = user.get("sub") # this is the SSO account ID in JWT terminology + if not sub: + raise HTTPException(status_code=400, detail="JWT is missing its `sub` field.") + + aur_accounts = conn.execute(select([Users.c.ID]).where(Users.c.SSOAccountID == sub)) \ + .fetchall() + if not aur_accounts: + return "Sorry, we don’t seem to know you Sir " + sub + elif len(aur_accounts) == 1: + sid = open_session(conn, aur_accounts[0][Users.c.ID]) + response = RedirectResponse("/") + # TODO redirect to the referrer + response.set_cookie(key="AURSID", value=sid, httponly=True, + secure=request.url.scheme == "https") + return response + else: + # We’ve got a severe integrity violation. + raise Exception("Multiple accounts found for SSO account " + sub) From 8d5244d0c0c3b38f29b9b087c5e85082f0bb1f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Mon, 13 Jul 2020 17:05:37 +0200 Subject: [PATCH 020/844] Fix typos in CONTRIBUTING.md Signed-off-by: Lukas Fleischer --- CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a37d980a..7b9ff466 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,9 +2,9 @@ Patches should be sent to the [aur-dev@archlinux.org][1] mailing list. -Before sending patched you are recomended to run the `flake8` and `isort`. +Before sending patches, you are recommended to run `flake8` and `isort`. -You can add git hook to do this by installing `python-pre-install` and running -`pre-install install`. +You can add a git hook to do this by installing `python-pre-commit` and running +`pre-commit install`. [1] https://lists.archlinux.org/listinfo/aur-dev From 4bf8228324e4c3811b48069a4b2ae7fd840c78a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Tue, 14 Jul 2020 15:34:06 +0200 Subject: [PATCH 021/844] SSO: Explain the rationale behind prompt=login We might reconsider it in the future. Signed-off-by: Lukas Fleischer --- aurweb/routers/sso.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index d0802c34..e1ec7efe 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -28,6 +28,13 @@ oauth.register( @router.get("/sso/login") async def login(request: Request): + """ + Redirect the user to the SSO provider’s login page. + + We specify prompt=login to force the user to input their credentials even + if they’re already logged on the SSO. This is less practical, but given AUR + has the potential to impact many users, better safe than sorry. + """ redirect_uri = aurweb.config.get("options", "aur_location") + "/sso/authenticate" return await oauth.sso.authorize_redirect(request, redirect_uri, prompt="login") From d12ea08fcaa62211cbf4d83bba91124b90f861cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Tue, 14 Jul 2020 15:34:23 +0200 Subject: [PATCH 022/844] SSO: Add an SSO option in the login page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We’ll probably change the whole login page in the future, but this makes development easier. Signed-off-by: Lukas Fleischer --- web/html/login.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/html/login.php b/web/html/login.php index 01454414..3a146f60 100644 --- a/web/html/login.php +++ b/web/html/login.php @@ -40,6 +40,9 @@ html_header('AUR ' . __("Login"));

" /> [] + + [] + From 4d0f2d2279ed9fcdf6bb76015ac0da9c6e938d69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Tue, 14 Jul 2020 15:35:05 +0200 Subject: [PATCH 023/844] Implement SSO logout Signed-off-by: Lukas Fleischer --- aurweb/routers/sso.py | 18 ++++++++++++++++++ web/html/logout.php | 14 +++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index e1ec7efe..a8d4b141 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -1,6 +1,8 @@ import time import uuid +from urllib.parse import urlencode + import fastapi from authlib.integrations.starlette_client import OAuth @@ -82,3 +84,19 @@ async def authenticate(request: Request, conn=Depends(aurweb.db.connect)): else: # We’ve got a severe integrity violation. raise Exception("Multiple accounts found for SSO account " + sub) + + +@router.get("/sso/logout") +async def logout(): + """ + Disconnect the user from the SSO provider, potentially affecting every + other Arch service. AUR logout is performed by `/logout`, before it + redirects to `/sso/logout`. + + Based on the OpenID Connect Session Management specification: + https://openid.net/specs/openid-connect-session-1_0.html#RPLogout + """ + metadata = await oauth.sso.load_server_metadata() + # TODO Supply id_token_hint to the end session endpoint. + query = urlencode({'post_logout_redirect_uri': aurweb.config.get('options', 'aur_location')}) + return RedirectResponse(metadata["end_session_endpoint"] + '?' + query) diff --git a/web/html/logout.php b/web/html/logout.php index 14022001..9fd63943 100644 --- a/web/html/logout.php +++ b/web/html/logout.php @@ -5,16 +5,28 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); # access AUR common functions include_once("acctfuncs.inc.php"); # access AUR common functions +$redirect_uri = '/'; + # if they've got a cookie, log them out - need to do this before # sending any HTML output. # if (isset($_COOKIE["AURSID"])) { + $uid = uid_from_sid($_COOKIE['AURSID']); delete_session_id($_COOKIE["AURSID"]); # setting expiration to 1 means '1 second after midnight January 1, 1970' setcookie("AURSID", "", 1, "/", null, !empty($_SERVER['HTTPS']), true); unset($_COOKIE['AURSID']); clear_expired_sessions(); + + # If the account is linked to an SSO account, disconnect the user from the SSO too. + if (isset($uid)) { + $dbh = DB::connect(); + $sso_account_id = $dbh->query("SELECT SSOAccountID FROM Users WHERE ID = " . $dbh->quote($uid)) + ->fetchColumn(); + if ($sso_account_id) + $redirect_uri = '/sso/logout'; + } } -header('Location: /'); +header("Location: $redirect_uri"); From 357dba87b3ee784a4201a7bb56befb105b81bbf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Tue, 14 Jul 2020 15:35:24 +0200 Subject: [PATCH 024/844] Save id_token for the SSO logout As far as I can see, Keycloak ignores it entirely. I can login in as SSO user A, then disconnect from the SSO directly and reconnect as user B, but when I disconnect user A from AUR, Keycloak disconnects B even though AUR passed it an ID token for A. Signed-off-by: Lukas Fleischer --- aurweb/routers/sso.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index a8d4b141..04ecdca6 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -80,6 +80,11 @@ async def authenticate(request: Request, conn=Depends(aurweb.db.connect)): # TODO redirect to the referrer response.set_cookie(key="AURSID", value=sid, httponly=True, secure=request.url.scheme == "https") + if "id_token" in token: + # We save the id_token for the SSO logout. It’s not too important + # though, so if we can’t find it, we can live without it. + response.set_cookie(key="SSO_ID_TOKEN", value=token["id_token"], path="/sso/", + httponly=True, secure=request.url.scheme == "https") return response else: # We’ve got a severe integrity violation. @@ -87,7 +92,7 @@ async def authenticate(request: Request, conn=Depends(aurweb.db.connect)): @router.get("/sso/logout") -async def logout(): +async def logout(request: Request): """ Disconnect the user from the SSO provider, potentially affecting every other Arch service. AUR logout is performed by `/logout`, before it @@ -96,7 +101,13 @@ async def logout(): Based on the OpenID Connect Session Management specification: https://openid.net/specs/openid-connect-session-1_0.html#RPLogout """ + id_token = request.cookies.get("SSO_ID_TOKEN") + if not id_token: + return RedirectResponse("/") + metadata = await oauth.sso.load_server_metadata() - # TODO Supply id_token_hint to the end session endpoint. - query = urlencode({'post_logout_redirect_uri': aurweb.config.get('options', 'aur_location')}) - return RedirectResponse(metadata["end_session_endpoint"] + '?' + query) + query = urlencode({'post_logout_redirect_uri': aurweb.config.get('options', 'aur_location'), + 'id_token_hint': id_token}) + response = RedirectResponse(metadata["end_session_endpoint"] + '?' + query) + response.delete_cookie("SSO_ID_TOKEN", path="/sso/") + return response From 0e08b151e5c3606e573b1f7113466b5dd6efdcef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Mon, 20 Jul 2020 16:25:11 +0200 Subject: [PATCH 025/844] SSO: Port IP ban checking Signed-off-by: Lukas Fleischer --- aurweb/routers/sso.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index 04ecdca6..efd4462c 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -14,7 +14,7 @@ from starlette.requests import Request import aurweb.config import aurweb.db -from aurweb.schema import Sessions, Users +from aurweb.schema import Bans, Sessions, Users router = fastapi.APIRouter() @@ -57,13 +57,28 @@ def open_session(conn, user_id): return sid +def is_ip_banned(conn, ip): + """ + Check if an IP is banned. `ip` is a string and may be an IPv4 as well as an + IPv6, depending on the server’s configuration. + """ + result = conn.execute(Bans.select().where(Bans.c.IPAddress == ip)) + return result.fetchone() is not None + + @router.get("/sso/authenticate") async def authenticate(request: Request, conn=Depends(aurweb.db.connect)): """ Receive an OpenID Connect ID token, validate it, then process it to create an new AUR session. """ - # TODO check for banned IPs + # TODO Handle translations + if is_ip_banned(conn, request.client.host): + raise HTTPException( + status_code=403, + detail='The login form is currently disabled for your IP address, ' + 'probably due to sustained spam attacks. Sorry for the ' + 'inconvenience.') token = await oauth.sso.authorize_access_token(request) user = await oauth.sso.parse_id_token(request, token) sub = user.get("sub") # this is the SSO account ID in JWT terminology From e323156947a93ba65a99f927ed2d99c738c34f2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Mon, 20 Jul 2020 16:25:22 +0200 Subject: [PATCH 026/844] SSO: Port account suspension Signed-off-by: Lukas Fleischer --- aurweb/routers/sso.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index efd4462c..3e3b743d 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -41,11 +41,20 @@ async def login(request: Request): return await oauth.sso.authorize_redirect(request, redirect_uri, prompt="login") +def is_account_suspended(conn, user_id): + row = conn.execute(select([Users.c.Suspended]).where(Users.c.ID == user_id)).fetchone() + return row is not None and bool(row[0]) + + def open_session(conn, user_id): """ Create a new user session into the database. Return its SID. """ - # TODO check for account suspension + # TODO Handle translations. + if is_account_suspended(conn, user_id): + raise HTTPException(status_code=403, detail='Account suspended') + # TODO This is a terrible message because it could imply the attempt at + # logging in just caused the suspension. # TODO apply [options] max_sessions_per_user sid = uuid.uuid4().hex conn.execute(Sessions.insert().values( From 239988def7479cba6407901ee4671b6794d0b2ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Mon, 20 Jul 2020 16:25:28 +0200 Subject: [PATCH 027/844] Build a translation facility for FastAPI Signed-off-by: Lukas Fleischer --- aurweb/l10n.py | 22 ++++++++++++++++++++++ aurweb/routers/sso.py | 20 +++++++++++--------- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/aurweb/l10n.py b/aurweb/l10n.py index 51b56abb..a476ecd8 100644 --- a/aurweb/l10n.py +++ b/aurweb/l10n.py @@ -16,3 +16,25 @@ class Translator: self._localedir, languages=[lang]) return self._translator[lang].gettext(s) + + +def get_translator_for_request(request): + """ + Determine the preferred language from a FastAPI request object and build a + translator function for it. + + Example: + ```python + _ = get_translator_for_request(request) + print(_("Hello")) + ``` + """ + lang = request.cookies.get("AURLANG") + if lang is None: + lang = aurweb.config.get("options", "default_lang") + translator = Translator() + + def translate(message): + return translator.translate(message, lang) + + return translate diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index 3e3b743d..7b9c67c8 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -14,6 +14,7 @@ from starlette.requests import Request import aurweb.config import aurweb.db +from aurweb.l10n import get_translator_for_request from aurweb.schema import Bans, Sessions, Users router = fastapi.APIRouter() @@ -46,13 +47,13 @@ def is_account_suspended(conn, user_id): return row is not None and bool(row[0]) -def open_session(conn, user_id): +def open_session(request, conn, user_id): """ Create a new user session into the database. Return its SID. """ - # TODO Handle translations. if is_account_suspended(conn, user_id): - raise HTTPException(status_code=403, detail='Account suspended') + _ = get_translator_for_request(request) + raise HTTPException(status_code=403, detail=_('Account suspended')) # TODO This is a terrible message because it could imply the attempt at # logging in just caused the suspension. # TODO apply [options] max_sessions_per_user @@ -81,25 +82,26 @@ async def authenticate(request: Request, conn=Depends(aurweb.db.connect)): Receive an OpenID Connect ID token, validate it, then process it to create an new AUR session. """ - # TODO Handle translations if is_ip_banned(conn, request.client.host): + _ = get_translator_for_request(request) raise HTTPException( status_code=403, - detail='The login form is currently disabled for your IP address, ' - 'probably due to sustained spam attacks. Sorry for the ' - 'inconvenience.') + detail=_('The login form is currently disabled for your IP address, ' + 'probably due to sustained spam attacks. Sorry for the ' + 'inconvenience.')) token = await oauth.sso.authorize_access_token(request) user = await oauth.sso.parse_id_token(request, token) sub = user.get("sub") # this is the SSO account ID in JWT terminology if not sub: - raise HTTPException(status_code=400, detail="JWT is missing its `sub` field.") + _ = get_translator_for_request(request) + raise HTTPException(status_code=400, detail=_("JWT is missing its `sub` field.")) aur_accounts = conn.execute(select([Users.c.ID]).where(Users.c.SSOAccountID == sub)) \ .fetchall() if not aur_accounts: return "Sorry, we don’t seem to know you Sir " + sub elif len(aur_accounts) == 1: - sid = open_session(conn, aur_accounts[0][Users.c.ID]) + sid = open_session(request, conn, aur_accounts[0][Users.c.ID]) response = RedirectResponse("/") # TODO redirect to the referrer response.set_cookie(key="AURSID", value=sid, httponly=True, From efe99dc16f2a94be1d4e5917a07fee2260f3d547 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 5 Jul 2020 18:19:06 -0700 Subject: [PATCH 028/844] Support conjunctive keyword search in RPC interface Newly supported API Version 6 modifies `type=search` for _by_ type `name-desc`: it now behaves the same as `name-desc` search through the https://aur.archlinux.org/packages/ search page. Search for packages containing the literal keyword `blah blah` AND `haha`: https://aur.archlinux.org/rpc/?v=6&type=search&arg="blah blah"%20haha Search for packages containing the literal keyword `abc 123`: https://aur.archlinux.org/rpc/?v=6&type=search&arg="abc 123" The following example searches for packages that contain `blah` AND `abc`: https://aur.archlinux.org/rpc/?v=6&type=search&arg=blah%20abc The legacy method still searches for packages that contain `blah abc`: https://aur.archlinux.org/rpc/?v=5&type=search&arg=blah%20abc https://aur.archlinux.org/rpc/?v=5&type=search&arg=blah%20abc API Version 6 is currently only considered during a `search` of `name-desc`. Note: This change was written as a solution to https://bugs.archlinux.org/task/49133. PS: + Some spacing issues fixed in comments. Signed-off-by: Kevin Morris Signed-off-by: Lukas Fleischer --- doc/rpc.txt | 4 ++++ web/lib/aurjson.class.php | 29 +++++++++++++++++++---------- web/lib/pkgfuncs.inc.php | 34 ++++++++++++++++++++-------------- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/doc/rpc.txt b/doc/rpc.txt index 3148ebea..b0f5c4e1 100644 --- a/doc/rpc.txt +++ b/doc/rpc.txt @@ -39,6 +39,10 @@ Examples `/rpc/?v=5&type=search&by=makedepends&arg=boost` `search` with callback:: `/rpc/?v=5&type=search&arg=foobar&callback=jsonp1192244621103` +`search` with API Version 6 for packages containing `cookie` AND `milk`:: + `/rpc/?v=6&type=search&arg=cookie%20milk` +`search` with API Version 6 for packages containing `cookie milk`:: + `/rpc/?v=6&type=search&arg="cookie milk"` `info`:: `/rpc/?v=5&type=info&arg[]=foobar` `info` with multiple packages:: diff --git a/web/lib/aurjson.class.php b/web/lib/aurjson.class.php index 0ac586fe..86eae22b 100644 --- a/web/lib/aurjson.class.php +++ b/web/lib/aurjson.class.php @@ -1,6 +1,7 @@ version = intval($http_data['v']); } - if ($this->version < 1 || $this->version > 5) { + if ($this->version < 1 || $this->version > 6) { return $this->json_error('Invalid version specified.'); } @@ -140,7 +141,7 @@ class AurJSON { } /* - * Check if an IP needs to be rate limited. + * Check if an IP needs to be rate limited. * * @param $ip IP of the current request * @@ -192,7 +193,7 @@ class AurJSON { $value = get_cache_value('ratelimit-ws:' . $ip, $status); if (!$status || ($status && $value < $deletion_time)) { if (set_cache_value('ratelimit-ws:' . $ip, $time, $window_length) && - set_cache_value('ratelimit:' . $ip, 1, $window_length)) { + set_cache_value('ratelimit:' . $ip, 1, $window_length)) { return; } } else { @@ -370,7 +371,7 @@ class AurJSON { } elseif ($this->version >= 2) { if ($this->version == 2 || $this->version == 3) { $fields = implode(',', self::$fields_v2); - } else if ($this->version == 4 || $this->version == 5) { + } else if ($this->version >= 4 && $this->version <= 6) { $fields = implode(',', self::$fields_v4); } $query = "SELECT {$fields} " . @@ -492,13 +493,21 @@ class AurJSON { if (strlen($keyword_string) < 2) { return $this->json_error('Query arg too small.'); } - $keyword_string = $this->dbh->quote("%" . addcslashes($keyword_string, '%_') . "%"); - if ($search_by === 'name') { - $where_condition = "(Packages.Name LIKE $keyword_string)"; - } else if ($search_by === 'name-desc') { - $where_condition = "(Packages.Name LIKE $keyword_string OR "; - $where_condition .= "Description LIKE $keyword_string)"; + if ($this->version >= 6 && $search_by === 'name-desc') { + $where_condition = construct_keyword_search($this->dbh, + $keyword_string, true, false); + } else { + $keyword_string = $this->dbh->quote( + "%" . addcslashes($keyword_string, '%_') . "%"); + + if ($search_by === 'name') { + $where_condition = "(Packages.Name LIKE $keyword_string)"; + } else if ($search_by === 'name-desc') { + $where_condition = "(Packages.Name LIKE $keyword_string "; + $where_condition .= "OR Description LIKE $keyword_string)"; + } + } } else if ($search_by === 'maintainer') { if (empty($keyword_string)) { diff --git a/web/lib/pkgfuncs.inc.php b/web/lib/pkgfuncs.inc.php index 80758005..ac5c8cfe 100644 --- a/web/lib/pkgfuncs.inc.php +++ b/web/lib/pkgfuncs.inc.php @@ -696,8 +696,10 @@ function pkg_search_page($params, $show_headers=true, $SID="") { $q_where .= "AND (PackageBases.Name LIKE " . $dbh->quote($K) . ") "; } elseif (isset($params["SeB"]) && $params["SeB"] == "k") { - /* Search by keywords. */ - $q_where .= construct_keyword_search($dbh, $params['K'], false); + /* Search by name. */ + $q_where .= "AND ("; + $q_where .= construct_keyword_search($dbh, $params['K'], false, true); + $q_where .= ") "; } elseif (isset($params["SeB"]) && $params["SeB"] == "N") { /* Search by name (exact match). */ @@ -709,7 +711,9 @@ function pkg_search_page($params, $show_headers=true, $SID="") { } else { /* Keyword search (default). */ - $q_where .= construct_keyword_search($dbh, $params['K'], true); + $q_where .= "AND ("; + $q_where .= construct_keyword_search($dbh, $params['K'], true, true); + $q_where .= ") "; } } @@ -832,10 +836,11 @@ function pkg_search_page($params, $show_headers=true, $SID="") { * @param handle $dbh Database handle * @param string $keywords The search term * @param bool $namedesc Search name and description fields + * @param bool $keyword Search packages with a matching PackageBases.Keyword * * @return string WHERE part of the SQL clause */ -function construct_keyword_search($dbh, $keywords, $namedesc) { +function construct_keyword_search($dbh, $keywords, $namedesc, $keyword=false) { $count = 0; $where_part = ""; $q_keywords = ""; @@ -860,13 +865,18 @@ function construct_keyword_search($dbh, $keywords, $namedesc) { $term = "%" . addcslashes($term, '%_') . "%"; $q_keywords .= $op . " ("; + $q_keywords .= "Packages.Name LIKE " . $dbh->quote($term) . " "; if ($namedesc) { - $q_keywords .= "Packages.Name LIKE " . $dbh->quote($term) . " OR "; - $q_keywords .= "Description LIKE " . $dbh->quote($term) . " OR "; + $q_keywords .= "OR Description LIKE " . $dbh->quote($term) . " "; + } + + if ($keyword) { + $q_keywords .= "OR EXISTS (SELECT * FROM PackageKeywords WHERE "; + $q_keywords .= "PackageKeywords.PackageBaseID = Packages.PackageBaseID AND "; + $q_keywords .= "PackageKeywords.Keyword LIKE " . $dbh->quote($term) . ")) "; + } else { + $q_keywords .= ") "; } - $q_keywords .= "EXISTS (SELECT * FROM PackageKeywords WHERE "; - $q_keywords .= "PackageKeywords.PackageBaseID = Packages.PackageBaseID AND "; - $q_keywords .= "PackageKeywords.Keyword LIKE " . $dbh->quote($term) . ")) "; $count++; if ($count >= 20) { @@ -875,11 +885,7 @@ function construct_keyword_search($dbh, $keywords, $namedesc) { $op = "AND "; } - if (!empty($q_keywords)) { - $where_part = "AND (" . $q_keywords . ") "; - } - - return $where_part; + return $q_keywords; } /** From 445a991ef1b8c76fb1d51837c4fb692dbfb080e6 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 15 Jul 2020 11:45:54 -0700 Subject: [PATCH 029/844] Exclude suspended Users from being notified The existing notify.py script was grabbing entries regardless of user suspension. This has been modified to only send notifications to unsuspended users. This change was written as a solution to https://bugs.archlinux.org/task/65554. Signed-off-by: Kevin Morris Signed-off-by: Lukas Fleischer --- aurweb/scripts/notify.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/aurweb/scripts/notify.py b/aurweb/scripts/notify.py index edae76f8..7f8e7168 100755 --- a/aurweb/scripts/notify.py +++ b/aurweb/scripts/notify.py @@ -126,7 +126,7 @@ class ResetKeyNotification(Notification): def __init__(self, conn, uid): cur = conn.execute('SELECT UserName, Email, BackupEmail, ' + 'LangPreference, ResetKey ' + - 'FROM Users WHERE ID = ?', [uid]) + 'FROM Users WHERE ID = ? AND Suspended = 0', [uid]) self._username, self._to, self._backup, self._lang, self._resetkey = cur.fetchone() super().__init__() @@ -173,7 +173,8 @@ class CommentNotification(Notification): 'ON PackageNotifications.UserID = Users.ID WHERE ' + 'Users.CommentNotify = 1 AND ' + 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ?', + 'PackageNotifications.PackageBaseID = ? AND ' + + 'Users.Suspended = 0', [uid, pkgbase_id]) self._recipients = cur.fetchall() cur = conn.execute('SELECT Comments FROM PackageComments WHERE ID = ?', @@ -220,7 +221,8 @@ class UpdateNotification(Notification): 'ON PackageNotifications.UserID = Users.ID WHERE ' + 'Users.UpdateNotify = 1 AND ' + 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ?', + 'PackageNotifications.PackageBaseID = ? AND ' + + 'Users.Suspended = 0', [uid, pkgbase_id]) self._recipients = cur.fetchall() super().__init__() @@ -266,7 +268,8 @@ class FlagNotification(Notification): 'INNER JOIN PackageBases ' + 'ON PackageBases.MaintainerUID = Users.ID OR ' + 'PackageBases.ID = PackageComaintainers.PackageBaseID ' + - 'WHERE PackageBases.ID = ?', [pkgbase_id]) + 'WHERE PackageBases.ID = ? AND ' + + 'Users.Suspended = 0', [pkgbase_id]) self._recipients = cur.fetchall() cur = conn.execute('SELECT FlaggerComment FROM PackageBases WHERE ' + 'ID = ?', [pkgbase_id]) @@ -304,7 +307,8 @@ class OwnershipEventNotification(Notification): 'ON PackageNotifications.UserID = Users.ID WHERE ' + 'Users.OwnershipNotify = 1 AND ' + 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ?', + 'PackageNotifications.PackageBaseID = ? AND ' + + 'Users.Suspended = 0', [uid, pkgbase_id]) self._recipients = cur.fetchall() cur = conn.execute('SELECT FlaggerComment FROM PackageBases WHERE ' + @@ -343,7 +347,7 @@ class ComaintainershipEventNotification(Notification): def __init__(self, conn, uid, pkgbase_id): self._pkgbase = pkgbase_from_id(conn, pkgbase_id) cur = conn.execute('SELECT Email, LangPreference FROM Users ' + - 'WHERE ID = ?', [uid]) + 'WHERE ID = ? AND Suspended = 0', [uid]) self._to, self._lang = cur.fetchone() super().__init__() @@ -386,7 +390,8 @@ class DeleteNotification(Notification): 'INNER JOIN PackageNotifications ' + 'ON PackageNotifications.UserID = Users.ID WHERE ' + 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ?', + 'PackageNotifications.PackageBaseID = ? AND ' + + 'Users.Suspended = 0', [uid, old_pkgbase_id]) self._recipients = cur.fetchall() super().__init__() @@ -433,7 +438,8 @@ class RequestOpenNotification(Notification): 'INNER JOIN Users ' + 'ON Users.ID = PackageRequests.UsersID ' + 'OR Users.ID = PackageBases.MaintainerUID ' + - 'WHERE PackageRequests.ID = ?', [reqid]) + 'WHERE PackageRequests.ID = ? AND ' + + 'Users.Suspended = 0', [reqid]) self._to = aurweb.config.get('options', 'aur_request_ml') self._cc = [row[0] for row in cur.fetchall()] cur = conn.execute('SELECT Comments FROM PackageRequests WHERE ID = ?', @@ -489,7 +495,8 @@ class RequestCloseNotification(Notification): 'INNER JOIN Users ' + 'ON Users.ID = PackageRequests.UsersID ' + 'OR Users.ID = PackageBases.MaintainerUID ' + - 'WHERE PackageRequests.ID = ?', [reqid]) + 'WHERE PackageRequests.ID = ? AND ' + + 'Users.Suspended = 0', [reqid]) self._to = aurweb.config.get('options', 'aur_request_ml') self._cc = [row[0] for row in cur.fetchall()] cur = conn.execute('SELECT PackageRequests.ClosureComment, ' + @@ -547,7 +554,8 @@ class TUVoteReminderNotification(Notification): cur = conn.execute('SELECT Email, LangPreference FROM Users ' + 'WHERE AccountTypeID IN (2, 4) AND ID NOT IN ' + '(SELECT UserID FROM TU_Votes ' + - 'WHERE TU_Votes.VoteID = ?)', [vote_id]) + 'WHERE TU_Votes.VoteID = ?) AND ' + + 'Users.Suspended = 0', [vote_id]) self._recipients = cur.fetchall() super().__init__() From a1a742b518b7ead1ea32b13dd52d5ea5248e8bb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Mon, 27 Jul 2020 14:43:48 +0200 Subject: [PATCH 030/844] aurweb.spawn: Support stdout redirections to non-tty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only ttys have a terminal size. If we can’t obtain it, we’ll just use 80 as a sane default. Signed-off-by: Lukas Fleischer --- aurweb/spawn.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/aurweb/spawn.py b/aurweb/spawn.py index 5da8587e..46d534d9 100644 --- a/aurweb/spawn.py +++ b/aurweb/spawn.py @@ -87,12 +87,16 @@ def start(): return atexit.register(stop) + try: + terminal_width = os.get_terminal_size().columns + except OSError: + terminal_width = 80 print("{ruler}\n" "Spawing PHP and FastAPI, then nginx as a reverse proxy.\n" "Check out {aur_location}\n" "Hit ^C to terminate everything.\n" "{ruler}" - .format(ruler=("-" * os.get_terminal_size().columns), + .format(ruler=("-" * terminal_width), aur_location=aurweb.config.get('options', 'aur_location'))) # PHP From 9290eee1385b4ac9e09fec4a784868789ea5a15d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Mon, 27 Jul 2020 14:44:03 +0200 Subject: [PATCH 031/844] Stop redirecting stderr with proc_open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Error outputs were piped to a temporary buffer that wasn’t read by anyone, making debugging hard because errors were completely silenced. By not explicitly redirecting stderr on proc_open, the subprocess inherits its parent stderr. Signed-off-by: Lukas Fleischer --- web/lib/acctfuncs.inc.php | 2 -- web/lib/pkgbasefuncs.inc.php | 2 -- 2 files changed, 4 deletions(-) diff --git a/web/lib/acctfuncs.inc.php b/web/lib/acctfuncs.inc.php index 752abe97..b3822eaf 100644 --- a/web/lib/acctfuncs.inc.php +++ b/web/lib/acctfuncs.inc.php @@ -1347,7 +1347,6 @@ function notify($params) { $descspec = array( 0 => array('pipe', 'r'), 1 => array('pipe', 'w'), - 2 => array('pipe', 'w') ); $p = proc_open($cmd, $descspec, $pipes); @@ -1358,7 +1357,6 @@ function notify($params) { fclose($pipes[0]); fclose($pipes[1]); - fclose($pipes[2]); return proc_close($p); } diff --git a/web/lib/pkgbasefuncs.inc.php b/web/lib/pkgbasefuncs.inc.php index 4c8abba7..4a49898c 100644 --- a/web/lib/pkgbasefuncs.inc.php +++ b/web/lib/pkgbasefuncs.inc.php @@ -96,7 +96,6 @@ function render_comment($id) { $descspec = array( 0 => array('pipe', 'r'), 1 => array('pipe', 'w'), - 2 => array('pipe', 'w') ); $p = proc_open($cmd, $descspec, $pipes); @@ -107,7 +106,6 @@ function render_comment($id) { fclose($pipes[0]); fclose($pipes[1]); - fclose($pipes[2]); return proc_close($p); } From 202ffd8923bc3a08bc6d4f18ac6d91441b0b0cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Tue, 28 Jul 2020 16:33:12 +0200 Subject: [PATCH 032/844] Update last login information on SSO login Signed-off-by: Lukas Fleischer --- aurweb/routers/sso.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index 7b9c67c8..817adadb 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -63,7 +63,13 @@ def open_session(request, conn, user_id): SessionID=sid, LastUpdateTS=time.time(), )) - # TODO update Users.LastLogin and Users.LastLoginIPAddress + + # Update user’s last login information. + conn.execute(Users.update() + .where(Users.c.ID == user_id) + .values(LastLogin=int(time.time()), + LastLoginIPAddress=request.client.host)) + return sid From 5fb4fc12de1dc374395340724d192271d4aa31f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Tue, 28 Jul 2020 16:33:27 +0200 Subject: [PATCH 033/844] HTML error pages for FastAPI Signed-off-by: Lukas Fleischer --- aurweb/asgi.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 60c7ade7..9293ed77 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -1,4 +1,7 @@ -from fastapi import FastAPI +import http + +from fastapi import FastAPI, HTTPException +from fastapi.responses import HTMLResponse from starlette.middleware.sessions import SessionMiddleware import aurweb.config @@ -14,3 +17,14 @@ if not session_secret: app.add_middleware(SessionMiddleware, secret_key=session_secret) app.include_router(sso.router) + + +@app.exception_handler(HTTPException) +async def http_exception_handler(request, exc): + """ + Dirty HTML error page to replace the default JSON error responses. + In the future this should use a proper Arch-themed HTML template. + """ + phrase = http.HTTPStatus(exc.status_code).phrase + return HTMLResponse(f"

{exc.status_code} {phrase}

{exc.detail}

", + status_code=exc.status_code) From be31675b6589e66c8b10a64b44591b594d2eb735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Tue, 28 Jul 2020 16:33:41 +0200 Subject: [PATCH 034/844] Guard OAuth exceptions to provide better messages Signed-off-by: Lukas Fleischer --- aurweb/routers/sso.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index 817adadb..2e4fbacc 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -5,7 +5,7 @@ from urllib.parse import urlencode import fastapi -from authlib.integrations.starlette_client import OAuth +from authlib.integrations.starlette_client import OAuth, OAuthError from fastapi import Depends, HTTPException from fastapi.responses import RedirectResponse from sqlalchemy.sql import select @@ -95,8 +95,18 @@ async def authenticate(request: Request, conn=Depends(aurweb.db.connect)): detail=_('The login form is currently disabled for your IP address, ' 'probably due to sustained spam attacks. Sorry for the ' 'inconvenience.')) - token = await oauth.sso.authorize_access_token(request) - user = await oauth.sso.parse_id_token(request, token) + + try: + token = await oauth.sso.authorize_access_token(request) + user = await oauth.sso.parse_id_token(request, token) + except OAuthError: + # Here, most OAuth errors should be caused by forged or expired tokens. + # Let’s give attackers as little information as possible. + _ = get_translator_for_request(request) + raise HTTPException( + status_code=400, + detail=_('Bad OAuth token. Please retry logging in from the start.')) + sub = user.get("sub") # this is the SSO account ID in JWT terminology if not sub: _ = get_translator_for_request(request) From 87815d37c078c315ac3254741973cfba2bfccace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Wed, 29 Jul 2020 13:46:10 +0200 Subject: [PATCH 035/844] Remove the per-user session limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This feature was originally introduced by f961ffd9c7f2d3d51d3e3b060990a4fef9e56c1b as a fix for FS#12898 . As of today, it is broken because of the `q.SessionID IS NULL` condition in the WHERE clause, which can’t be true because SessionID is not nullable. As a consequence, the session limit was not applied. The fact the absence of the session limit hasn’t caused any issue so far, and hadn’t even been noticed, suggests the feature is unneeded. Signed-off-by: Lukas Fleischer --- aurweb/routers/sso.py | 2 +- conf/config.defaults | 1 - web/lib/acctfuncs.inc.php | 15 --------------- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index 2e4fbacc..73c884a4 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -56,7 +56,7 @@ def open_session(request, conn, user_id): raise HTTPException(status_code=403, detail=_('Account suspended')) # TODO This is a terrible message because it could imply the attempt at # logging in just caused the suspension. - # TODO apply [options] max_sessions_per_user + sid = uuid.uuid4().hex conn.execute(Sessions.insert().values( UsersID=user_id, diff --git a/conf/config.defaults b/conf/config.defaults index 49259754..98e033b7 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -13,7 +13,6 @@ passwd_min_len = 8 default_lang = en default_timezone = UTC sql_debug = 0 -max_sessions_per_user = 8 login_timeout = 7200 persistent_cookie_timeout = 2592000 max_filesize_uncompressed = 8388608 diff --git a/web/lib/acctfuncs.inc.php b/web/lib/acctfuncs.inc.php index b3822eaf..bc603d3b 100644 --- a/web/lib/acctfuncs.inc.php +++ b/web/lib/acctfuncs.inc.php @@ -596,21 +596,6 @@ function try_login() { /* Generate a session ID and store it. */ while (!$logged_in && $num_tries < 5) { - $session_limit = config_get_int('options', 'max_sessions_per_user'); - if ($session_limit) { - /* - * Delete all user sessions except the - * last ($session_limit - 1). - */ - $q = "DELETE FROM Sessions "; - $q.= "WHERE UsersId = " . $userID . " "; - $q.= "AND SessionID NOT IN (SELECT SessionID FROM Sessions "; - $q.= "WHERE UsersID = " . $userID . " "; - $q.= "ORDER BY LastUpdateTS DESC "; - $q.= "LIMIT " . ($session_limit - 1) . ")"; - $dbh->query($q); - } - $new_sid = new_sid(); $q = "INSERT INTO Sessions (UsersID, SessionID, LastUpdateTS)" ." VALUES (" . $userID . ", '" . $new_sid . "', " . strval(time()) . ")"; From 8c28ba6e7f1c99f4b16c651857224b1d19f93466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Mangano-Tarumi?= Date: Wed, 29 Jul 2020 17:25:44 +0200 Subject: [PATCH 036/844] Redirect to referer after SSO login Introduce a `redirect` query argument to SSO login endpoints so that users are redirected to the page they were originally on when they clicked the Login link. Signed-off-by: Lukas Fleischer --- aurweb/routers/sso.py | 23 +++++++++++++++++------ web/html/login.php | 18 ++++++++++++------ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index 73c884a4..4b12b932 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -30,16 +30,21 @@ oauth.register( @router.get("/sso/login") -async def login(request: Request): +async def login(request: Request, redirect: str = None): """ Redirect the user to the SSO provider’s login page. We specify prompt=login to force the user to input their credentials even if they’re already logged on the SSO. This is less practical, but given AUR has the potential to impact many users, better safe than sorry. + + The `redirect` argument is a query parameter specifying the post-login + redirect URL. """ - redirect_uri = aurweb.config.get("options", "aur_location") + "/sso/authenticate" - return await oauth.sso.authorize_redirect(request, redirect_uri, prompt="login") + authenticate_url = aurweb.config.get("options", "aur_location") + "/sso/authenticate" + if redirect: + authenticate_url = authenticate_url + "?" + urlencode([("redirect", redirect)]) + return await oauth.sso.authorize_redirect(request, authenticate_url, prompt="login") def is_account_suspended(conn, user_id): @@ -82,8 +87,15 @@ def is_ip_banned(conn, ip): return result.fetchone() is not None +def is_aur_url(url): + aur_location = aurweb.config.get("options", "aur_location") + if not aur_location.endswith("/"): + aur_location = aur_location + "/" + return url.startswith(aur_location) + + @router.get("/sso/authenticate") -async def authenticate(request: Request, conn=Depends(aurweb.db.connect)): +async def authenticate(request: Request, redirect: str = None, conn=Depends(aurweb.db.connect)): """ Receive an OpenID Connect ID token, validate it, then process it to create an new AUR session. @@ -118,8 +130,7 @@ async def authenticate(request: Request, conn=Depends(aurweb.db.connect)): return "Sorry, we don’t seem to know you Sir " + sub elif len(aur_accounts) == 1: sid = open_session(request, conn, aur_accounts[0][Users.c.ID]) - response = RedirectResponse("/") - # TODO redirect to the referrer + response = RedirectResponse(redirect if redirect and is_aur_url(redirect) else "/") response.set_cookie(key="AURSID", value=sid, httponly=True, secure=request.url.scheme == "https") if "id_token" in token: diff --git a/web/html/login.php b/web/html/login.php index 3a146f60..3f3d66cc 100644 --- a/web/html/login.php +++ b/web/html/login.php @@ -9,6 +9,10 @@ if (!$disable_http_login || (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'])) { $login_error = $login['error']; } +$referer = in_request('referer'); +if ($referer === '') + $referer = $_SERVER['HTTP_REFERER']; + html_header('AUR ' . __("Login")); ?>
@@ -40,13 +44,15 @@ html_header('AUR ' . __("Login"));

" /> [] - - [] + + [] - - - - + +

From 83d228d9e8c2b067dfd5565d22437363f7590d69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Thu, 13 Aug 2020 15:45:58 +0100 Subject: [PATCH 037/844] spawn: expand AUR_CONFIG to the full path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows using a relative path for the config. PHP didn't play well with it. Signed-off-by: Filipe Laíns Signed-off-by: Lukas Fleischer --- aurweb/spawn.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aurweb/spawn.py b/aurweb/spawn.py index 46d534d9..3c5130d7 100644 --- a/aurweb/spawn.py +++ b/aurweb/spawn.py @@ -11,6 +11,7 @@ configuration anyway. import argparse import atexit import os +import os.path import subprocess import sys import tempfile @@ -87,6 +88,9 @@ def start(): return atexit.register(stop) + if 'AUR_CONFIG' in os.environ: + os.environ['AUR_CONFIG'] = os.path.realpath(os.environ['AUR_CONFIG']) + try: terminal_width = os.get_terminal_size().columns except OSError: From 4e4f5855f17acc2bd353093566ea853aa523f933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Thu, 13 Aug 2020 15:46:00 +0100 Subject: [PATCH 038/844] doc: fix AUR_CONFIG in TESTING MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns Signed-off-by: Lukas Fleischer --- TESTING | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TESTING b/TESTING index d7df3672..17c6fbc7 100644 --- a/TESTING +++ b/TESTING @@ -29,7 +29,7 @@ INSTALL. 4) Prepare the testing database: $ cd /path/to/aurweb/ - $ python -m aurweb.initdb + $ AUR_CONFIG=conf/config python -m aurweb.initdb $ cd /path/to/aurweb/schema $ ./gendummydata.py out.sql @@ -37,4 +37,4 @@ INSTALL. 5) Run the test server: - $ AUR_CONFIG='/path/to/aurweb/conf/config' python -m aurweb.spawn + $ AUR_CONFIG=conf/config python -m aurweb.spawn From e62d472708e722e6123f9bfa01d2c32ec9838755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Thu, 13 Aug 2020 15:46:01 +0100 Subject: [PATCH 039/844] doc: add missing gendummydata.py dependencies in TESTING MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns Signed-off-by: Lukas Fleischer --- TESTING | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/TESTING b/TESTING index 17c6fbc7..d666b3ca 100644 --- a/TESTING +++ b/TESTING @@ -14,7 +14,8 @@ INSTALL. # pacman -S --needed php php-sqlite sqlite words fortune-mod \ python python-sqlalchemy python-alembic \ python-fastapi uvicorn nginx \ - python-authlib python-itsdangerous python-httpx + python-authlib python-itsdangerous python-httpx \ + words fortune-mod Ensure to enable the pdo_sqlite extension in php.ini. From db75a5528e45c13a427813c177bf9657a1cb8350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Thu, 13 Aug 2020 15:46:02 +0100 Subject: [PATCH 040/844] doc: simplify database setup instructions in TESTING MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns Signed-off-by: Lukas Fleischer --- TESTING | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/TESTING b/TESTING index d666b3ca..972bce2c 100644 --- a/TESTING +++ b/TESTING @@ -30,11 +30,11 @@ INSTALL. 4) Prepare the testing database: $ cd /path/to/aurweb/ + $ AUR_CONFIG=conf/config python -m aurweb.initdb - $ cd /path/to/aurweb/schema - $ ./gendummydata.py out.sql - $ sqlite3 path/to/aurweb.sqlite3 < out.sql + $ schema/gendummydata.py data.sql + $ sqlite3 aurweb.sqlite3 < data.sql 5) Run the test server: From 92e315465b19e18b67ba7bff0b14c9658db579f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Fri, 4 Sep 2020 23:06:43 +0200 Subject: [PATCH 041/844] gendummydata.py: remove unused database connection variables Signed-off-by: Lukas Fleischer --- schema/gendummydata.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/schema/gendummydata.py b/schema/gendummydata.py index b3a73ef2..8b15ac69 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -18,10 +18,6 @@ import time LOG_LEVEL = logging.DEBUG # logging level. set to logging.INFO to reduce output SEED_FILE = "/usr/share/dict/words" -DB_HOST = os.getenv("DB_HOST", "localhost") -DB_NAME = os.getenv("DB_NAME", "AUR") -DB_USER = os.getenv("DB_USER", "aur") -DB_PASS = os.getenv("DB_PASS", "aur") USER_ID = 5 # Users.ID of first bogus user PKG_ID = 1 # Packages.ID of first package MAX_USERS = 300 # how many users to 'register' From 879c0622d66a04b38b281f879ebbf06ba97c62c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Fri, 4 Sep 2020 23:13:56 +0200 Subject: [PATCH 042/844] gendummydata.py: set exit code to 1 when there is an error Of course the default exit code is 0... Signed-off-by: Lukas Fleischer --- schema/gendummydata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schema/gendummydata.py b/schema/gendummydata.py index 8b15ac69..224c82e5 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -44,19 +44,19 @@ log = logging.getLogger() if len(sys.argv) != 2: log.error("Missing output filename argument") - raise SystemExit + raise SystemExit(1) # make sure the seed file exists # if not os.path.exists(SEED_FILE): log.error("Please install the 'words' Arch package") - raise SystemExit + raise SystemExit(1) # make sure comments can be created # if not os.path.exists(FORTUNE_FILE): log.error("Please install the 'fortune-mod' Arch package") - raise SystemExit + raise SystemExit(1) # track what users/package names have been used # From 51a353582010a45b2121c25a5ad995111f1842a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Fri, 4 Sep 2020 23:10:20 +0200 Subject: [PATCH 043/844] gendummydata.py: set MAX_USERS and MAX_PKGS to more realistic values Signed-off-by: Lukas Fleischer --- schema/gendummydata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schema/gendummydata.py b/schema/gendummydata.py index 224c82e5..91a580c2 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -20,16 +20,16 @@ LOG_LEVEL = logging.DEBUG # logging level. set to logging.INFO to reduce output SEED_FILE = "/usr/share/dict/words" USER_ID = 5 # Users.ID of first bogus user PKG_ID = 1 # Packages.ID of first package -MAX_USERS = 300 # how many users to 'register' +MAX_USERS = 76000 # how many users to 'register' MAX_DEVS = .1 # what percentage of MAX_USERS are Developers MAX_TUS = .2 # what percentage of MAX_USERS are Trusted Users -MAX_PKGS = 900 # how many packages to load +MAX_PKGS = 64000 # how many packages to load PKG_DEPS = (1, 15) # min/max depends a package has PKG_RELS = (1, 5) # min/max relations a package has PKG_SRC = (1, 3) # min/max sources a package has PKG_CMNTS = (1, 5) # min/max number of comments a package has CATEGORIES_COUNT = 17 # the number of categories from aur-schema -VOTING = (0, .30) # percentage range for package voting +VOTING = (0, .001) # percentage range for package voting OPEN_PROPOSALS = 5 # number of open trusted user proposals CLOSE_PROPOSALS = 15 # number of closed trusted user proposals RANDOM_TLDS = ("edu", "com", "org", "net", "tw", "ru", "pl", "de", "es") From 3062a78a92d32144f423fdb460e96a309732d9d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Fri, 4 Sep 2020 23:53:07 +0200 Subject: [PATCH 044/844] gendummydata.py: optimize iteration for big numbers of pkgs Signed-off-by: Lukas Fleischer --- schema/gendummydata.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/schema/gendummydata.py b/schema/gendummydata.py index 91a580c2..c7b3a06d 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -259,20 +259,23 @@ for p in list(track_votes.keys()): # Create package dependencies and sources # log.debug("Creating statements for package depends/sources.") -for p in list(seen_pkgs.keys()): +# the keys of seen_pkgs are accessed many times by random.choice, +# so the list has to be created outside the loops to keep it efficient +seen_pkgs_keys = list(seen_pkgs.keys()) +for p in seen_pkgs_keys: num_deps = random.randrange(PKG_DEPS[0], PKG_DEPS[1]) for i in range(0, num_deps): - dep = random.choice([k for k in seen_pkgs]) + dep = random.choice(seen_pkgs_keys) deptype = random.randrange(1, 5) if deptype == 4: - dep += ": for " + random.choice([k for k in seen_pkgs]) + dep += ": for " + random.choice(seen_pkgs_keys) s = "INSERT INTO PackageDepends(PackageID, DepTypeID, DepName) VALUES (%d, %d, '%s');\n" s = s % (seen_pkgs[p], deptype, dep) out.write(s) num_rels = random.randrange(PKG_RELS[0], PKG_RELS[1]) for i in range(0, num_deps): - rel = random.choice([k for k in seen_pkgs]) + rel = random.choice(seen_pkgs_keys) reltype = random.randrange(1, 4) s = "INSERT INTO PackageRelations(PackageID, RelTypeID, RelName) VALUES (%d, %d, '%s');\n" s = s % (seen_pkgs[p], reltype, rel) From bc972089a158459005700a7eaa8cee3ba666e2d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Sat, 5 Sep 2020 22:15:22 +0200 Subject: [PATCH 045/844] Fix WHERE clause for keyword search queries with empty keywords When the keyword parameter is empty, the AND clause has to be omitted, otherwise we get an SQL syntax error: ... WHERE PackageBases.PackagerUID IS NOT NULL AND () ... This got broken in commit 9e30013aa4fc6ce3a3c9f6f83a6fe789c1fc2456 Author: Kevin Morris Date: Sun Jul 5 18:19:06 2020 -0700 Support conjunctive keyword search in RPC interface Signed-off-by: Lukas Fleischer --- web/lib/pkgfuncs.inc.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/web/lib/pkgfuncs.inc.php b/web/lib/pkgfuncs.inc.php index ac5c8cfe..eb3afab6 100644 --- a/web/lib/pkgfuncs.inc.php +++ b/web/lib/pkgfuncs.inc.php @@ -697,9 +697,7 @@ function pkg_search_page($params, $show_headers=true, $SID="") { } elseif (isset($params["SeB"]) && $params["SeB"] == "k") { /* Search by name. */ - $q_where .= "AND ("; $q_where .= construct_keyword_search($dbh, $params['K'], false, true); - $q_where .= ") "; } elseif (isset($params["SeB"]) && $params["SeB"] == "N") { /* Search by name (exact match). */ @@ -711,9 +709,7 @@ function pkg_search_page($params, $show_headers=true, $SID="") { } else { /* Keyword search (default). */ - $q_where .= "AND ("; $q_where .= construct_keyword_search($dbh, $params['K'], true, true); - $q_where .= ") "; } } @@ -885,7 +881,11 @@ function construct_keyword_search($dbh, $keywords, $namedesc, $keyword=false) { $op = "AND "; } - return $q_keywords; + if (!empty($q_keywords)) { + $where_part = "AND (" . $q_keywords . ") "; + } + + return $where_part; } /** From 568e0d2fa33d17ea4c45d046d9870d8ba0376789 Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Thu, 19 Nov 2020 23:16:17 +0100 Subject: [PATCH 046/844] RSS: Add atom self link https://validator.w3.org/feed/docs/warning/MissingAtomSelfLink.html Signed-off-by: Lukas Fleischer --- web/lib/feedcreator.class.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/lib/feedcreator.class.php b/web/lib/feedcreator.class.php index 802eebbe..e881f252 100644 --- a/web/lib/feedcreator.class.php +++ b/web/lib/feedcreator.class.php @@ -906,12 +906,13 @@ class RSSCreator091 extends FeedCreator { $feed = "encoding."\"?>\n"; $feed.= $this->_createGeneratorComment(); $feed.= $this->_createStylesheetReferences(); - $feed.= "RSSVersion."\">\n"; + $feed.= "RSSVersion."\" xmlns:atom=\"http://www.w3.org/2005/Atom\">\n"; $feed.= " \n"; $feed.= " ".FeedCreator::iTrunc(htmlspecialchars($this->title),100)."\n"; $this->descriptionTruncSize = 500; $feed.= " ".$this->getDescription()."\n"; $feed.= " ".$this->link."\n"; + $feed.= " syndicationURL."\" rel=\"self\" type=\"application/rss+xml\" />\n"; $now = new FeedDate(); $feed.= " ".htmlspecialchars($now->rfc822())."\n"; $feed.= " ".FEEDCREATOR_VERSION."\n"; From 78dbbd3dfa916e0b054a231ff7cd56049ff7dc2f Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Thu, 19 Nov 2020 23:17:49 +0100 Subject: [PATCH 047/844] RSS: Set proper content type header https://validator.w3.org/feed/docs/warning/UnexpectedContentType.html Signed-off-by: Lukas Fleischer --- web/html/rss.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/html/rss.php b/web/html/rss.php index d6e7825a..245a2171 100644 --- a/web/html/rss.php +++ b/web/html/rss.php @@ -11,6 +11,8 @@ $host = $_SERVER['HTTP_HOST']; $feed_key = 'pkg-feed-' . $protocol; +header("Content-Type: application/rss+xml"); + $bool = false; $ret = get_cache_value($feed_key, $bool); if ($bool) { From 1d0c6ffe24c692d4279ce6ad6cb03aeb38a2303e Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Thu, 19 Nov 2020 23:18:33 +0100 Subject: [PATCH 048/844] RSS: Make sure image title matches channel title https://validator.w3.org/feed/docs/warning/ImageTitleDoesntMatch.html Signed-off-by: Lukas Fleischer --- web/html/rss.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/html/rss.php b/web/html/rss.php index 245a2171..5720d3d1 100644 --- a/web/html/rss.php +++ b/web/html/rss.php @@ -33,7 +33,7 @@ $rss->description = "The latest and greatest packages in the AUR"; $rss->link = "${protocol}://{$host}"; $rss->syndicationURL = "{$protocol}://{$host}" . get_uri('/rss/'); $image = new FeedImage(); -$image->title = "AUR"; +$image->title = "AUR Newest Packages"; $image->url = "{$protocol}://{$host}/css/archnavbar/aurlogo.png"; $image->link = $rss->link; $image->description = "AUR Newest Packages Feed"; From eb11943fed16462e089891b128fd01f9e460b114 Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Thu, 19 Nov 2020 23:22:11 +0100 Subject: [PATCH 049/844] RSS: Always provide a GUID https://validator.w3.org/feed/docs/warning/MissingGuid.html Signed-off-by: Lukas Fleischer --- web/html/rss.php | 1 + 1 file changed, 1 insertion(+) diff --git a/web/html/rss.php b/web/html/rss.php index 5720d3d1..b67f862d 100644 --- a/web/html/rss.php +++ b/web/html/rss.php @@ -50,6 +50,7 @@ foreach ($packages as $indx => $row) { $item->date = intval($row["SubmittedTS"]); $item->source = "{$protocol}://{$host}"; $item->author = username_from_id($row["MaintainerUID"]); + $item->guid = $item->link; $rss->addItem($item); } From d5d333005eafb338fc5aefff9d4426cc23dbd84c Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Thu, 19 Nov 2020 23:27:21 +0100 Subject: [PATCH 050/844] RSS: Decrease cache time and increase item count I think after 10-15 years we might want to adjust those values. With a 30min cache and 20 items per creation I would bet some new AUR packages might be swept under the carpet. Signed-off-by: Lukas Fleischer --- web/html/rss.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/html/rss.php b/web/html/rss.php index b67f862d..1e6335cf 100644 --- a/web/html/rss.php +++ b/web/html/rss.php @@ -40,7 +40,7 @@ $image->description = "AUR Newest Packages Feed"; $rss->image = $image; #Get the latest packages and add items for them -$packages = latest_pkgs(20); +$packages = latest_pkgs(100); foreach ($packages as $indx => $row) { $item = new FeedItem(); @@ -56,6 +56,6 @@ foreach ($packages as $indx => $row) { #save it so that useCached() can find it $feedContent = $rss->createFeed(); -set_cache_value($feed_key, $feedContent, 1800); +set_cache_value($feed_key, $feedContent, 600); echo $feedContent; ?> From 62b413f6b76fd852abf728897f9929c4bf7024ea Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Sun, 27 Dec 2020 19:38:48 -0500 Subject: [PATCH 051/844] .gitignore: add test/trash directory* Signed-off-by: Lukas Fleischer --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7c9fa60b..4d961d13 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ aur.git/ __pycache__/ *.py[cod] test/test-results/ +test/trash directory* schema/aur-schema-sqlite.sql From 933d2705f935011035ee661e4a7acac37cd7aca3 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Mon, 28 Dec 2020 13:03:24 -0500 Subject: [PATCH 052/844] Fetch Transifex image from https://www.transifex.com Fixes GitLab issue #3. Signed-off-by: Lukas Fleischer --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f7285a51..77f3b13d 100644 --- a/README.md +++ b/README.md @@ -45,4 +45,4 @@ Translations Translations are welcome via our Transifex project at https://www.transifex.com/lfleischer/aurweb; see `doc/i18n.txt` for details. -![Transifex](http://www.transifex.net/projects/p/aurweb/chart/image_png) +![Transifex](https://www.transifex.com/projects/p/aurweb/chart/image_png) From 21c457817fe8ec5db60a10e2270f9fcf951aca5f Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Sat, 23 Jan 2021 00:43:57 +0800 Subject: [PATCH 053/844] Use jsDelivr instead of Google CDN for jquery jsdelivr is another free CDN service for open source projects. The main motivation for this change is that it is the only one that works fairly well across the globe. The Google CDN service is known to be hardly accessible in mainland China, unfortunately. Signed-off-by: Lukas Fleischer --- web/html/home.php | 2 +- web/html/packages.php | 2 +- web/html/pkgmerge.php | 2 +- web/template/pkgreq_form.php | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/web/html/home.php b/web/html/home.php index 0ce89f40..8fb05246 100644 --- a/web/html/home.php +++ b/web/html/home.php @@ -202,7 +202,7 @@ if (isset($_COOKIE["AURSID"])) {
- + + + + ", ""); EOD - "$RENDERCOMMENT" 3 && + cover "$RENDERCOMMENT" 3 && cat <<-EOD >expected && <script>alert("XSS!");</script> EOD @@ -61,7 +61,7 @@ test_expect_success 'Test link conversion.' ' [arch]: https://www.archlinux.org/ ", ""); EOD - "$RENDERCOMMENT" 4 && + cover "$RENDERCOMMENT" 4 && cat <<-EOD >expected &&

Visit https://www.archlinux.org/#_test_. Visit https://www.archlinux.org/. @@ -89,7 +89,7 @@ test_expect_success 'Test Git commit linkification.' ' http://example.com/$oid ", ""); EOD - "$RENDERCOMMENT" 5 && + cover "$RENDERCOMMENT" 5 && cat <<-EOD >expected &&

${oid:0:12} ${oid:0:7} @@ -116,7 +116,7 @@ test_expect_success 'Test Flyspray issue linkification.' ' https://archlinux.org/?test=FS#1234 ", ""); EOD - "$RENDERCOMMENT" 6 && + cover "$RENDERCOMMENT" 6 && cat <<-EOD >expected &&

FS#1234567. FS#1234 @@ -142,7 +142,7 @@ test_expect_success 'Test headings lowering.' ' ###### Six ", ""); EOD - "$RENDERCOMMENT" 7 && + cover "$RENDERCOMMENT" 7 && cat <<-EOD >expected &&

One
Two
diff --git a/test/t2700-usermaint.t b/test/t2700-usermaint.t index f0bb449b..c119e3f4 100755 --- a/test/t2700-usermaint.t +++ b/test/t2700-usermaint.t @@ -16,7 +16,7 @@ test_expect_success 'Test removal of login IP addresses.' ' UPDATE Users SET LastLogin = 0, LastLoginIPAddress = "5.6.7.8" WHERE ID = 5; UPDATE Users SET LastLogin = $tendaysago, LastLoginIPAddress = "6.7.8.9" WHERE ID = 6; EOD - "$USERMAINT" && + cover "$USERMAINT" && cat <<-EOD >expected && 1.2.3.4 3.4.5.6 @@ -37,7 +37,7 @@ test_expect_success 'Test removal of SSH login IP addresses.' ' UPDATE Users SET LastSSHLogin = 0, LastSSHLoginIPAddress = "5.6.7.8" WHERE ID = 5; UPDATE Users SET LastSSHLogin = $tendaysago, LastSSHLoginIPAddress = "6.7.8.9" WHERE ID = 6; EOD - "$USERMAINT" && + cover "$USERMAINT" && cat <<-EOD >expected && 1.2.3.4 2.3.4.5 From 4b7609681deb8ce6627919b4c43f879601c49d30 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 26 Dec 2020 18:26:32 -0800 Subject: [PATCH 069/844] add test_exceptions.py This helps gain coverage over aurweb.exceptions regardless of their actual use in the testing base. Signed-off-by: Kevin Morris --- test/test_exceptions.py | 102 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 test/test_exceptions.py diff --git a/test/test_exceptions.py b/test/test_exceptions.py new file mode 100644 index 00000000..feac2656 --- /dev/null +++ b/test/test_exceptions.py @@ -0,0 +1,102 @@ +from aurweb.exceptions import (AlreadyVotedException, AurwebException, BannedException, BrokenUpdateHookException, + InvalidArgumentsException, InvalidCommentException, InvalidPackageBaseException, + InvalidReasonException, InvalidRepositoryNameException, InvalidUserException, + MaintenanceException, NotVotedException, PackageBaseExistsException, PermissionDeniedException) + + +def test_aurweb_exception(): + try: + raise AurwebException("test") + except AurwebException as exc: + assert str(exc) == "test" + + +def test_maintenance_exception(): + try: + raise MaintenanceException("test") + except MaintenanceException as exc: + assert str(exc) == "test" + + +def test_banned_exception(): + try: + raise BannedException("test") + except BannedException as exc: + assert str(exc) == "test" + + +def test_already_voted_exception(): + try: + raise AlreadyVotedException("test") + except AlreadyVotedException as exc: + assert str(exc) == "already voted for package base: test" + + +def test_broken_update_hook_exception(): + try: + raise BrokenUpdateHookException("test") + except BrokenUpdateHookException as exc: + assert str(exc) == "broken update hook: test" + + +def test_invalid_arguments_exception(): + try: + raise InvalidArgumentsException("test") + except InvalidArgumentsException as exc: + assert str(exc) == "test" + + +def test_invalid_packagebase_exception(): + try: + raise InvalidPackageBaseException("test") + except InvalidPackageBaseException as exc: + assert str(exc) == "package base not found: test" + + +def test_invalid_comment_exception(): + try: + raise InvalidCommentException("test") + except InvalidCommentException as exc: + assert str(exc) == "comment is too short: test" + + +def test_invalid_reason_exception(): + try: + raise InvalidReasonException("test") + except InvalidReasonException as exc: + assert str(exc) == "invalid reason: test" + + +def test_invalid_user_exception(): + try: + raise InvalidUserException("test") + except InvalidUserException as exc: + assert str(exc) == "unknown user: test" + + +def test_not_voted_exception(): + try: + raise NotVotedException("test") + except NotVotedException as exc: + assert str(exc) == "missing vote for package base: test" + + +def test_packagebase_exists_exception(): + try: + raise PackageBaseExistsException("test") + except PackageBaseExistsException as exc: + assert str(exc) == "package base already exists: test" + + +def test_permission_denied_exception(): + try: + raise PermissionDeniedException("test") + except PermissionDeniedException as exc: + assert str(exc) == "permission denied: test" + + +def test_repository_name_exception(): + try: + raise InvalidRepositoryNameException("test") + except InvalidRepositoryNameException as exc: + assert str(exc) == "invalid repository name: test" From 6d08789ac1f759b66006ab2ec225513863308f91 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 31 Dec 2020 22:00:15 -0800 Subject: [PATCH 070/844] add test_popupdate.py We had no coverage over aurweb.scripts.popupdate. This test covers all of its functionality. Signed-off-by: Kevin Morris --- aurweb/db.py | 3 +++ aurweb/scripts/popupdate.py | 1 - test/test_popupdate.py | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 test/test_popupdate.py diff --git a/aurweb/db.py b/aurweb/db.py index 8ca32165..04b40f43 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -1,3 +1,5 @@ +import math + try: import mysql.connector except ImportError: @@ -95,6 +97,7 @@ class Connection: elif aur_db_backend == 'sqlite': aur_db_name = aurweb.config.get('database', 'name') self._conn = sqlite3.connect(aur_db_name) + self._conn.create_function("POWER", 2, math.pow) self._paramstyle = sqlite3.paramstyle else: raise ValueError('unsupported database backend') diff --git a/aurweb/scripts/popupdate.py b/aurweb/scripts/popupdate.py index b64deedb..b1e70403 100755 --- a/aurweb/scripts/popupdate.py +++ b/aurweb/scripts/popupdate.py @@ -7,7 +7,6 @@ import aurweb.db def main(): conn = aurweb.db.Connection() - conn.execute("UPDATE PackageBases SET NumVotes = (" + "SELECT COUNT(*) FROM PackageVotes " + "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") diff --git a/test/test_popupdate.py b/test/test_popupdate.py new file mode 100644 index 00000000..93f86f10 --- /dev/null +++ b/test/test_popupdate.py @@ -0,0 +1,5 @@ +from aurweb.scripts import popupdate + + +def test_popupdate(): + popupdate.main() From 57c11ae13fea0276359cf552ab59455fe2c579a3 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 20 Jan 2021 14:08:39 -0800 Subject: [PATCH 071/844] install aurweb package & init db on GitLab CI Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 71c14457..6db573d2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,5 +18,8 @@ before_script: test: script: + - python setup.py install + - sed -r "s;YOUR_AUR_ROOT;$(pwd);g" conf/config.dev > conf/config + - AUR_CONFIG=conf/config python -m aurweb.initdb - make -C test - coverage report --include='aurweb/*' From 52ab056e1876fa68c70d3ff1b1d63ea9f74d6e31 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 21 Dec 2020 22:33:46 -0800 Subject: [PATCH 072/844] update documentation for FastAPI tests and deps. Additionally, we now ask for two more favors from contributors: 1. All source modified or added within a patchset **must** maintain equivalent or increased coverage by providing tests that use the functionality. 2. Please keep your source within an 80 column width. PS: Sneak a few test Makefile and gitlab fixes. Signed-off-by: Kevin Morris --- CONTRIBUTING.md | 9 +++++++++ INSTALL | 4 ++-- README.md | 9 ++++++++- test/README.md | 53 +++++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 68 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7b9ff466..1d57d742 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,3 +8,12 @@ You can add a git hook to do this by installing `python-pre-commit` and running `pre-commit install`. [1] https://lists.archlinux.org/listinfo/aur-dev + +### Coding Guidelines + +1. All source modified or added within a patchset **must** maintain equivalent + or increased coverage by providing tests that use the functionality. + +2. Please keep your source within an 80 column width. + +Test patches that increase coverage in the codebase are always welcome. diff --git a/INSTALL b/INSTALL index a32d6f5a..8607b07f 100644 --- a/INSTALL +++ b/INSTALL @@ -48,8 +48,8 @@ read the instructions below. 4) Install Python modules and dependencies: # pacman -S python-mysql-connector python-pygit2 python-srcinfo python-sqlalchemy \ - python-bleach python-markdown python-alembic python-jinja \ - python-itsdangerous python-authlib python-httpx hypercorn + python-bleach python-markdown python-alembic hypercorn \ + python-itsdangerous python-authlib python-httpx # python3 setup.py install 5) Create a new MySQL database and a user and import the aurweb SQL schema: diff --git a/README.md b/README.md index 77f3b13d..7521b4d9 100644 --- a/README.md +++ b/README.md @@ -19,12 +19,14 @@ Directory Layout * `aurweb`: aurweb Python modules, Git interface and maintenance scripts * `conf`: configuration and configuration templates +* `static`: static resource files +* `templates`: jinja2 template collection * `doc`: project documentation * `po`: translation files for strings in the aurweb interface * `schema`: schema for the SQL database * `test`: test suite and test cases * `upgrading`: instructions for upgrading setups from one release to another -* `web`: web interface for the AUR +* `web`: PHP-based web interface for the AUR Links ----- @@ -46,3 +48,8 @@ Translations are welcome via our Transifex project at https://www.transifex.com/lfleischer/aurweb; see `doc/i18n.txt` for details. ![Transifex](https://www.transifex.com/projects/p/aurweb/chart/image_png) + +Testing +------- + +See [test/README.md](test/README.md) for details on dependencies and testing. diff --git a/test/README.md b/test/README.md index de7eff18..3261899b 100644 --- a/test/README.md +++ b/test/README.md @@ -1,10 +1,11 @@ Running tests ------------- -To run all the tests, you may run `make check` under `test/`. +To run all tests, you may run `make check` under `test/` (alternative targets: +`make pytest`, `make sh`). -For more control, you may use the `prove` command, which receives a directory -or a list of files to run, and produces a report. +For more control, you may use the `prove` or `pytest` command, which receives a +directory or a list of files to run, and produces a report. Each test script is standalone, so you may run them individually. Some tests may receive command-line options to help debugging. See for example sharness's @@ -22,16 +23,60 @@ For all the test to run, the following Arch packages should be installed: - python-pygit2 - python-sqlalchemy - python-srcinfo +- python-coverage +- python-pytest +- python-pytest-cov +- python-pytest-asyncio + +Running tests +------------- + +Recommended method of running tests: `make check`. + +First, setup the test configuration: + + $ sed -r 's;YOUR_AUR_ROOT;$(pwd);g' conf/config.dev > conf/config + +With those installed, one can run Python tests manually with any AUR config +specified by `AUR_CONFIG`: + + $ AUR_CONFIG=conf/config coverage run --append /usr/bin/pytest test/ + +After tests are run (particularly, with `coverage run` included), one can +produce coverage reports. + + # Print out a CLI coverage report. + $ coverage report + # Produce an HTML-based coverage report. + $ coverage html + +When running `make -C test`, all tests ran will produce coverage data via +`coverage run --append`. After running `make -C test`, one can continue with +coverage reporting steps above. Running tests through `make` will test and +cover code ran by both aurweb scripts and our pytest collection. Writing tests ------------- -Test scripts must follow the Test Anything Protocol specification: +Shell test scripts must follow the Test Anything Protocol specification: http://testanything.org/tap-specification.html +Python tests must be compatible with `pytest` and included in `pytest test/` +execution after setting up a configuration. + Tests must support being run from any directory. They may use $0 to determine their location. Python scripts should expect aurweb to be installed and importable without toying with os.path or PYTHONPATH. Tests written in shell should use sharness. In general, new tests should be consistent with existing tests unless they have a good reason not to. + +Debugging tests +--------------- + +By default, `make -C test` is quiet and does not print out verbose information +about tests being run. If a test is failing, one can look into verbose details +of sharness tests by executing them with the `--verbose` flag. Example: +`./t1100_git_auth.t --verbose`. This is particularly useful when tests happen +to fail in a remote continuous integration environment, where the reader does +not have complete access to the runner. From cd3e880264339ffd73e507775b4fb7e9d740a24d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 23 Dec 2020 16:11:40 -0800 Subject: [PATCH 073/844] add Dockerfile This docker file downloads deps, sets up some things beforehand and finishes with running our entire collection of tests. Signed-off-by: Kevin Morris Signed-off-by: Lukas Fleischer --- Dockerfile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7e981340 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +FROM archlinux +COPY . /aurweb +WORKDIR /aurweb + +# Install dependencies. +RUN pacman -Syu --noconfirm base-devel git gpgme protobuf pyalpm \ + python-mysql-connector python-pygit2 python-srcinfo python-bleach \ + python-markdown python-sqlalchemy python-alembic python-pytest \ + python-werkzeug python-pytest-tap python-fastapi nginx python-authlib \ + python-itsdangerous python-httpx python-jinja python-pytest-cov \ + python-requests python-aiofiles python-python-multipart \ + python-pytest-asyncio python-coverage hypercorn + +# Remove aurweb.sqlite3 if it was copied over via COPY. +RUN rm -fv aurweb.sqlite3 + +# Setup our test config. +RUN sed -r "s;YOUR_AUR_ROOT;/aurweb;g" conf/config.dev > conf/config + +# Install translations. +RUN AUR_CONFIG=conf/config make -C po all install + +# Initialize the database. +RUN AUR_CONFIG=conf/config python -m aurweb.initdb + +# Test everything! +RUN make -C test + +# Produce a coverage report. +RUN coverage report --include='aurweb/*' From 21140e28a8d5aa6ccf7a3366ca0d5fe7de8d2364 Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Wed, 7 Apr 2021 09:56:16 +0100 Subject: [PATCH 074/844] Filter out current username from co-maintainers list. Closes: #8 Signed-off-by: Leonidas Spyropoulos Signed-off-by: Eli Schwartz --- web/lib/pkgbasefuncs.inc.php | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/web/lib/pkgbasefuncs.inc.php b/web/lib/pkgbasefuncs.inc.php index 4a49898c..a053962e 100644 --- a/web/lib/pkgbasefuncs.inc.php +++ b/web/lib/pkgbasefuncs.inc.php @@ -1189,7 +1189,8 @@ function pkgbase_get_comaintainer_uids($base_ids) { * @return array Tuple of success/failure indicator and error message */ function pkgbase_set_comaintainers($base_id, $users, $override=false) { - if (!$override && !has_credential(CRED_PKGBASE_EDIT_COMAINTAINERS, array(pkgbase_maintainer_uid($base_id)))) { + $maintainer_uid = pkgbase_maintainer_uid($base_id); + if (!$override && !has_credential(CRED_PKGBASE_EDIT_COMAINTAINERS, array($maintainer_uid))) { return array(false, __("You are not allowed to manage co-maintainers of this package base.")); } @@ -1207,9 +1208,12 @@ function pkgbase_set_comaintainers($base_id, $users, $override=false) { if (!$uid) { return array(false, __("Invalid user name: %s", $user)); + } elseif ($uid == $maintainer_uid) { + // silently ignore when maintainer == co-maintainer + continue; + } else { + $uids_new[] = $uid; } - - $uids_new[] = $uid; } $q = sprintf("SELECT UsersID FROM PackageComaintainers WHERE PackageBaseID = %d", $base_id); From c1e29e90ca2a7e58faef573c975ddf47f108e048 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Mar 2021 15:17:17 -0700 Subject: [PATCH 075/844] aurweb: Globalize a Translator instance, add more utility + Added SUPPORTED_LANGUAGES, a global constant dictionary of language => display pairs for languages we support. + Add Translator.get_translator, a function used to retrieve a translator after initializing it (if needed). Use `fallback=True` while creating languages, in case we setup a language that we don't have a translation for, it will noop the translation. This is particularly useful for "en," since we do not translate it, but doing this will allow us to go through our normal translation flow in any case. + Added typing. + Added get_request_language, a function that grabs the language for a request session, defaulting to aurweb.config [options] default_lang. + Added get_raw_translator_for_request, a function that retrieves the concrete translation object for a given language. + Added tr, a jinja2 contextfilter that can be used to inline translate strings in jinja2 templates. + Added `python-jinja` dep to .gitlab-ci.yml. This needs to be included in documentation before this set is merged in. + Introduce pytest units (test_l10n.py) in `test` along with __init__.py, which marks `test` as a test package. + Additionally, fix up notify.py to use the global translator. Also reduce its source width to <= 80 by newlining some code. + Additionally, prepare locale in .gitlab-ci.yml and add aurweb.config [options] localedir to config.dev with YOUR_AUR_ROOT like others. Signed-off-by: Kevin Morris Signed-off-by: Lukas Fleischer --- .gitlab-ci.yml | 1 + aurweb/l10n.py | 79 +++++++++++-- aurweb/scripts/notify.py | 238 ++++++++++++++++++++------------------- conf/config.dev | 1 + test/__init__.py | 0 test/test_l10n.py | 44 ++++++++ 6 files changed, 240 insertions(+), 123 deletions(-) create mode 100644 test/__init__.py create mode 100644 test/test_l10n.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6db573d2..1e287748 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,7 @@ test: script: - python setup.py install - sed -r "s;YOUR_AUR_ROOT;$(pwd);g" conf/config.dev > conf/config + - AUR_CONFIG=conf/config make -C po all install - AUR_CONFIG=conf/config python -m aurweb.initdb - make -C test - coverage report --include='aurweb/*' diff --git a/aurweb/l10n.py b/aurweb/l10n.py index a476ecd8..030ab274 100644 --- a/aurweb/l10n.py +++ b/aurweb/l10n.py @@ -1,24 +1,79 @@ import gettext +import typing + +from collections import OrderedDict + +from fastapi import Request +from jinja2 import contextfilter import aurweb.config +SUPPORTED_LANGUAGES = OrderedDict({ + "ar": "العربية", + "ast": "Asturianu", + "ca": "Català", + "cs": "Český", + "da": "Dansk", + "de": "Deutsch", + "el": "Ελληνικά", + "en": "English", + "es": "Español", + "es_419": "Español (Latinoamérica)", + "fi": "Suomi", + "fr": "Français", + "he": "עברית", + "hr": "Hrvatski", + "hu": "Magyar", + "it": "Italiano", + "ja": "日本語", + "nb": "Norsk", + "nl": "Nederlands", + "pl": "Polski", + "pt_BR": "Português (Brasil)", + "pt_PT": "Português (Portugal)", + "ro": "Română", + "ru": "Русский", + "sk": "Slovenčina", + "sr": "Srpski", + "tr": "Türkçe", + "uk": "Українська", + "zh_CN": "简体中文", + "zh_TW": "正體中文" +}) + class Translator: def __init__(self): self._localedir = aurweb.config.get('options', 'localedir') self._translator = {} - def translate(self, s, lang): - if lang == 'en': - return s + def get_translator(self, lang: str): if lang not in self._translator: self._translator[lang] = gettext.translation("aurweb", self._localedir, - languages=[lang]) - return self._translator[lang].gettext(s) + languages=[lang], + fallback=True) + return self._translator.get(lang) + + def translate(self, s: str, lang: str): + return self.get_translator(lang).gettext(s) -def get_translator_for_request(request): +# Global translator object. +translator = Translator() + + +def get_request_language(request: Request): + return request.cookies.get("AURLANG", + aurweb.config.get("options", "default_lang")) + + +def get_raw_translator_for_request(request: Request): + lang = get_request_language(request) + return translator.get_translator(lang) + + +def get_translator_for_request(request: Request): """ Determine the preferred language from a FastAPI request object and build a translator function for it. @@ -29,12 +84,16 @@ def get_translator_for_request(request): print(_("Hello")) ``` """ - lang = request.cookies.get("AURLANG") - if lang is None: - lang = aurweb.config.get("options", "default_lang") - translator = Translator() + lang = get_request_language(request) def translate(message): return translator.translate(message, lang) return translate + + +@contextfilter +def tr(context: typing.Any, value: str): + """ A translation filter; example: {{ "Hello" | tr("de") }}. """ + _ = get_translator_for_request(context.get("request")) + return _(value) diff --git a/aurweb/scripts/notify.py b/aurweb/scripts/notify.py index 7f8e7168..1df0175a 100755 --- a/aurweb/scripts/notify.py +++ b/aurweb/scripts/notify.py @@ -40,9 +40,6 @@ def pkgbase_from_pkgreq(conn, reqid): class Notification: - def __init__(self): - self._l10n = aurweb.l10n.Translator() - def get_refs(self): return () @@ -97,9 +94,12 @@ class Notification: else: # send email using smtplib; no local MTA required server_addr = aurweb.config.get('notifications', 'smtp-server') - server_port = aurweb.config.getint('notifications', 'smtp-port') - use_ssl = aurweb.config.getboolean('notifications', 'smtp-use-ssl') - use_starttls = aurweb.config.getboolean('notifications', 'smtp-use-starttls') + server_port = aurweb.config.getint('notifications', + 'smtp-port') + use_ssl = aurweb.config.getboolean('notifications', + 'smtp-use-ssl') + use_starttls = aurweb.config.getboolean('notifications', + 'smtp-use-starttls') user = aurweb.config.get('notifications', 'smtp-user') passwd = aurweb.config.get('notifications', 'smtp-password') @@ -127,7 +127,8 @@ class ResetKeyNotification(Notification): cur = conn.execute('SELECT UserName, Email, BackupEmail, ' + 'LangPreference, ResetKey ' + 'FROM Users WHERE ID = ? AND Suspended = 0', [uid]) - self._username, self._to, self._backup, self._lang, self._resetkey = cur.fetchone() + self._username, self._to, self._backup, self._lang, self._resetkey = \ + cur.fetchone() super().__init__() def get_recipients(self): @@ -137,15 +138,15 @@ class ResetKeyNotification(Notification): return [(self._to, self._lang)] def get_subject(self, lang): - return self._l10n.translate('AUR Password Reset', lang) + return aurweb.l10n.translator.translate('AUR Password Reset', lang) def get_body(self, lang): - return self._l10n.translate( - 'A password reset request was submitted for the account ' - '{user} associated with your email address. If you wish to ' - 'reset your password follow the link [1] below, otherwise ' - 'ignore this message and nothing will happen.', - lang).format(user=self._username) + return aurweb.l10n.translator.translate( + 'A password reset request was submitted for the account ' + '{user} associated with your email address. If you wish to ' + 'reset your password follow the link [1] below, otherwise ' + 'ignore this message and nothing will happen.', + lang).format(user=self._username) def get_refs(self): return (aur_location + '/passreset/?resetkey=' + self._resetkey,) @@ -153,15 +154,16 @@ class ResetKeyNotification(Notification): class WelcomeNotification(ResetKeyNotification): def get_subject(self, lang): - return self._l10n.translate('Welcome to the Arch User Repository', - lang) + return aurweb.l10n.translator.translate( + 'Welcome to the Arch User Repository', + lang) def get_body(self, lang): - return self._l10n.translate( - 'Welcome to the Arch User Repository! In order to set an ' - 'initial password for your new account, please click the ' - 'link [1] below. If the link does not work, try copying and ' - 'pasting it into your browser.', lang) + return aurweb.l10n.translator.translate( + 'Welcome to the Arch User Repository! In order to set an ' + 'initial password for your new account, please click the ' + 'link [1] below. If the link does not work, try copying and ' + 'pasting it into your browser.', lang) class CommentNotification(Notification): @@ -186,19 +188,21 @@ class CommentNotification(Notification): return self._recipients def get_subject(self, lang): - return self._l10n.translate('AUR Comment for {pkgbase}', - lang).format(pkgbase=self._pkgbase) + return aurweb.l10n.translator.translate( + 'AUR Comment for {pkgbase}', + lang).format(pkgbase=self._pkgbase) def get_body(self, lang): - body = self._l10n.translate( - '{user} [1] added the following comment to {pkgbase} [2]:', - lang).format(user=self._user, pkgbase=self._pkgbase) + body = aurweb.l10n.translator.translate( + '{user} [1] added the following comment to {pkgbase} [2]:', + lang).format(user=self._user, pkgbase=self._pkgbase) body += '\n\n' + self._text + '\n\n-- \n' - dnlabel = self._l10n.translate('Disable notifications', lang) - body += self._l10n.translate( - 'If you no longer wish to receive notifications about this ' - 'package, please go to the package page [2] and select ' - '"{label}".', lang).format(label=dnlabel) + dnlabel = aurweb.l10n.translator.translate( + 'Disable notifications', lang) + body += aurweb.l10n.translator.translate( + 'If you no longer wish to receive notifications about this ' + 'package, please go to the package page [2] and select ' + '"{label}".', lang).format(label=dnlabel) return body def get_refs(self): @@ -231,20 +235,21 @@ class UpdateNotification(Notification): return self._recipients def get_subject(self, lang): - return self._l10n.translate('AUR Package Update: {pkgbase}', - lang).format(pkgbase=self._pkgbase) + return aurweb.l10n.translator.translate( + 'AUR Package Update: {pkgbase}', + lang).format(pkgbase=self._pkgbase) def get_body(self, lang): - body = self._l10n.translate('{user} [1] pushed a new commit to ' - '{pkgbase} [2].', lang).format( - user=self._user, - pkgbase=self._pkgbase) + body = aurweb.l10n.translator.translate( + '{user} [1] pushed a new commit to {pkgbase} [2].', + lang).format(user=self._user, pkgbase=self._pkgbase) body += '\n\n-- \n' - dnlabel = self._l10n.translate('Disable notifications', lang) - body += self._l10n.translate( - 'If you no longer wish to receive notifications about this ' - 'package, please go to the package page [2] and select ' - '"{label}".', lang).format(label=dnlabel) + dnlabel = aurweb.l10n.translator.translate( + 'Disable notifications', lang) + body += aurweb.l10n.translator.translate( + 'If you no longer wish to receive notifications about this ' + 'package, please go to the package page [2] and select ' + '"{label}".', lang).format(label=dnlabel) return body def get_refs(self): @@ -261,15 +266,16 @@ class FlagNotification(Notification): def __init__(self, conn, uid, pkgbase_id): self._user = username_from_id(conn, uid) self._pkgbase = pkgbase_from_id(conn, pkgbase_id) - cur = conn.execute('SELECT DISTINCT Users.Email, ' + - 'Users.LangPreference FROM Users ' + - 'LEFT JOIN PackageComaintainers ' + - 'ON PackageComaintainers.UsersID = Users.ID ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.MaintainerUID = Users.ID OR ' + - 'PackageBases.ID = PackageComaintainers.PackageBaseID ' + - 'WHERE PackageBases.ID = ? AND ' + - 'Users.Suspended = 0', [pkgbase_id]) + cur = conn.execute( + 'SELECT DISTINCT Users.Email, ' + + 'Users.LangPreference FROM Users ' + + 'LEFT JOIN PackageComaintainers ' + + 'ON PackageComaintainers.UsersID = Users.ID ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.MaintainerUID = Users.ID OR ' + + 'PackageBases.ID = PackageComaintainers.PackageBaseID ' + + 'WHERE PackageBases.ID = ? AND ' + + 'Users.Suspended = 0', [pkgbase_id]) self._recipients = cur.fetchall() cur = conn.execute('SELECT FlaggerComment FROM PackageBases WHERE ' + 'ID = ?', [pkgbase_id]) @@ -280,15 +286,15 @@ class FlagNotification(Notification): return self._recipients def get_subject(self, lang): - return self._l10n.translate('AUR Out-of-date Notification for ' - '{pkgbase}', - lang).format(pkgbase=self._pkgbase) + return aurweb.l10n.translator.translate( + 'AUR Out-of-date Notification for {pkgbase}', + lang).format(pkgbase=self._pkgbase) def get_body(self, lang): - body = self._l10n.translate( - 'Your package {pkgbase} [1] has been flagged out-of-date by ' - '{user} [2]:', lang).format(pkgbase=self._pkgbase, - user=self._user) + body = aurweb.l10n.translator.translate( + 'Your package {pkgbase} [1] has been flagged out-of-date by ' + '{user} [2]:', lang).format(pkgbase=self._pkgbase, + user=self._user) body += '\n\n' + self._text return body @@ -320,8 +326,9 @@ class OwnershipEventNotification(Notification): return self._recipients def get_subject(self, lang): - return self._l10n.translate('AUR Ownership Notification for {pkgbase}', - lang).format(pkgbase=self._pkgbase) + return aurweb.l10n.translator.translate( + 'AUR Ownership Notification for {pkgbase}', + lang).format(pkgbase=self._pkgbase) def get_refs(self): return (aur_location + '/pkgbase/' + self._pkgbase + '/', @@ -330,17 +337,17 @@ class OwnershipEventNotification(Notification): class AdoptNotification(OwnershipEventNotification): def get_body(self, lang): - return self._l10n.translate( - 'The package {pkgbase} [1] was adopted by {user} [2].', - lang).format(pkgbase=self._pkgbase, user=self._user) + return aurweb.l10n.translator.translate( + 'The package {pkgbase} [1] was adopted by {user} [2].', + lang).format(pkgbase=self._pkgbase, user=self._user) class DisownNotification(OwnershipEventNotification): def get_body(self, lang): - return self._l10n.translate( - 'The package {pkgbase} [1] was disowned by {user} ' - '[2].', lang).format(pkgbase=self._pkgbase, - user=self._user) + return aurweb.l10n.translator.translate( + 'The package {pkgbase} [1] was disowned by {user} ' + '[2].', lang).format(pkgbase=self._pkgbase, + user=self._user) class ComaintainershipEventNotification(Notification): @@ -355,9 +362,9 @@ class ComaintainershipEventNotification(Notification): return [(self._to, self._lang)] def get_subject(self, lang): - return self._l10n.translate('AUR Co-Maintainer Notification for ' - '{pkgbase}', - lang).format(pkgbase=self._pkgbase) + return aurweb.l10n.translator.translate( + 'AUR Co-Maintainer Notification for {pkgbase}', + lang).format(pkgbase=self._pkgbase) def get_refs(self): return (aur_location + '/pkgbase/' + self._pkgbase + '/',) @@ -365,16 +372,16 @@ class ComaintainershipEventNotification(Notification): class ComaintainerAddNotification(ComaintainershipEventNotification): def get_body(self, lang): - return self._l10n.translate( - 'You were added to the co-maintainer list of {pkgbase} [1].', - lang).format(pkgbase=self._pkgbase) + return aurweb.l10n.translator.translate( + 'You were added to the co-maintainer list of {pkgbase} [1].', + lang).format(pkgbase=self._pkgbase) class ComaintainerRemoveNotification(ComaintainershipEventNotification): def get_body(self, lang): - return self._l10n.translate( - 'You were removed from the co-maintainer list of {pkgbase} ' - '[1].', lang).format(pkgbase=self._pkgbase) + return aurweb.l10n.translator.translate( + 'You were removed from the co-maintainer list of {pkgbase} ' + '[1].', lang).format(pkgbase=self._pkgbase) class DeleteNotification(Notification): @@ -400,25 +407,27 @@ class DeleteNotification(Notification): return self._recipients def get_subject(self, lang): - return self._l10n.translate('AUR Package deleted: {pkgbase}', - lang).format(pkgbase=self._old_pkgbase) + return aurweb.l10n.translator.translate( + 'AUR Package deleted: {pkgbase}', + lang).format(pkgbase=self._old_pkgbase) def get_body(self, lang): if self._new_pkgbase: - dnlabel = self._l10n.translate('Disable notifications', lang) - return self._l10n.translate( - '{user} [1] merged {old} [2] into {new} [3].\n\n' - '-- \n' - 'If you no longer wish receive notifications about the ' - 'new package, please go to [3] and click "{label}".', - lang).format(user=self._user, old=self._old_pkgbase, - new=self._new_pkgbase, label=dnlabel) + dnlabel = aurweb.l10n.translator.translate( + 'Disable notifications', lang) + return aurweb.l10n.translator.translate( + '{user} [1] merged {old} [2] into {new} [3].\n\n' + '-- \n' + 'If you no longer wish receive notifications about the ' + 'new package, please go to [3] and click "{label}".', + lang).format(user=self._user, old=self._old_pkgbase, + new=self._new_pkgbase, label=dnlabel) else: - return self._l10n.translate( - '{user} [1] deleted {pkgbase} [2].\n\n' - 'You will no longer receive notifications about this ' - 'package.', lang).format(user=self._user, - pkgbase=self._old_pkgbase) + return aurweb.l10n.translator.translate( + '{user} [1] deleted {pkgbase} [2].\n\n' + 'You will no longer receive notifications about this ' + 'package.', lang).format(user=self._user, + pkgbase=self._old_pkgbase) def get_refs(self): refs = (aur_location + '/account/' + self._user + '/', @@ -432,14 +441,15 @@ class RequestOpenNotification(Notification): def __init__(self, conn, uid, reqid, reqtype, pkgbase_id, merge_into=None): self._user = username_from_id(conn, uid) self._pkgbase = pkgbase_from_id(conn, pkgbase_id) - cur = conn.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + - 'INNER JOIN Users ' + - 'ON Users.ID = PackageRequests.UsersID ' + - 'OR Users.ID = PackageBases.MaintainerUID ' + - 'WHERE PackageRequests.ID = ? AND ' + - 'Users.Suspended = 0', [reqid]) + cur = conn.execute( + 'SELECT DISTINCT Users.Email FROM PackageRequests ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + + 'INNER JOIN Users ' + + 'ON Users.ID = PackageRequests.UsersID ' + + 'OR Users.ID = PackageBases.MaintainerUID ' + + 'WHERE PackageRequests.ID = ? AND ' + + 'Users.Suspended = 0', [reqid]) self._to = aurweb.config.get('options', 'aur_request_ml') self._cc = [row[0] for row in cur.fetchall()] cur = conn.execute('SELECT Comments FROM PackageRequests WHERE ID = ?', @@ -489,14 +499,15 @@ class RequestOpenNotification(Notification): class RequestCloseNotification(Notification): def __init__(self, conn, uid, reqid, reason): self._user = username_from_id(conn, uid) if int(uid) else None - cur = conn.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + - 'INNER JOIN Users ' + - 'ON Users.ID = PackageRequests.UsersID ' + - 'OR Users.ID = PackageBases.MaintainerUID ' + - 'WHERE PackageRequests.ID = ? AND ' + - 'Users.Suspended = 0', [reqid]) + cur = conn.execute( + 'SELECT DISTINCT Users.Email FROM PackageRequests ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + + 'INNER JOIN Users ' + + 'ON Users.ID = PackageRequests.UsersID ' + + 'OR Users.ID = PackageBases.MaintainerUID ' + + 'WHERE PackageRequests.ID = ? AND ' + + 'Users.Suspended = 0', [reqid]) self._to = aurweb.config.get('options', 'aur_request_ml') self._cc = [row[0] for row in cur.fetchall()] cur = conn.execute('SELECT PackageRequests.ClosureComment, ' + @@ -563,14 +574,15 @@ class TUVoteReminderNotification(Notification): return self._recipients def get_subject(self, lang): - return self._l10n.translate('TU Vote Reminder: Proposal {id}', - lang).format(id=self._vote_id) + return aurweb.l10n.translator.translate( + 'TU Vote Reminder: Proposal {id}', + lang).format(id=self._vote_id) def get_body(self, lang): - return self._l10n.translate( - 'Please remember to cast your vote on proposal {id} [1]. ' - 'The voting period ends in less than 48 hours.', - lang).format(id=self._vote_id) + return aurweb.l10n.translator.translate( + 'Please remember to cast your vote on proposal {id} [1]. ' + 'The voting period ends in less than 48 hours.', + lang).format(id=self._vote_id) def get_refs(self): return (aur_location + '/tu/?id=' + str(self._vote_id),) diff --git a/conf/config.dev b/conf/config.dev index 37f38c45..ef7b5ed7 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -19,6 +19,7 @@ name = YOUR_AUR_ROOT/aurweb.sqlite3 aur_location = http://127.0.0.1:8080 disable_http_login = 0 enable-maintenance = 0 +localedir = YOUR_AUR_ROOT/web/locale ; Single sign-on; see doc/sso.txt. [sso] diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/test_l10n.py b/test/test_l10n.py new file mode 100644 index 00000000..1a1ef3e6 --- /dev/null +++ b/test/test_l10n.py @@ -0,0 +1,44 @@ +""" Test our l10n module. """ +from aurweb import l10n + + +class FakeRequest: + """ A fake Request doppleganger; use this to change request.cookies + easily and with no side-effects. """ + + def __init__(self, *args, **kwargs): + self.cookies = kwargs.pop("cookies", dict()) + + +def test_translator(): + """ Test creating l10n translation tools. """ + de_home = l10n.translator.translate("Home", "de") + assert de_home == "Startseite" + + +def test_get_request_language(): + """ First, tests default_lang, then tests a modified AURLANG cookie. """ + request = FakeRequest() + assert l10n.get_request_language(request) == "en" + + request.cookies["AURLANG"] = "de" + assert l10n.get_request_language(request) == "de" + + +def test_get_raw_translator_for_request(): + """ Make sure that get_raw_translator_for_request is giving us + the translator we expect. """ + request = FakeRequest(cookies={"AURLANG": "de"}) + + translator = l10n.get_raw_translator_for_request(request) + assert translator.gettext("Home") == \ + l10n.translator.translate("Home", "de") + + +def test_get_translator_for_request(): + """ Make sure that get_translator_for_request is giving us back + our expected translation function. """ + request = FakeRequest(cookies={"AURLANG": "de"}) + + translate = l10n.get_translator_for_request(request) + assert translate("Home") == "Startseite" From bda9256ab11101397977282236af6cbc6d664181 Mon Sep 17 00:00:00 2001 From: Marcus Andersson Date: Fri, 7 May 2021 18:50:41 +0200 Subject: [PATCH 076/844] Add error color when package is orphaned Signed-off-by: Eli Schwartz From 1ff822bb1492497adb284f278e0e537ea9be00f3 Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Tue, 11 May 2021 00:01:13 +0200 Subject: [PATCH 077/844] Use the clipboard API for copy paste The Document.execCommand API is deprecated and no longer recommended to be used. It's replacement is the much simpler navigator.clipboard API which is supported in all browsers except internet explorer. Signed-off-by: Eli Schwartz --- web/template/pkg_details.php | 10 +++------- web/template/pkgbase_details.php | 10 +++------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/web/template/pkg_details.php b/web/template/pkg_details.php index c6bb32d8..047de9a7 100644 --- a/web/template/pkg_details.php +++ b/web/template/pkg_details.php @@ -308,14 +308,10 @@ endif; diff --git a/web/template/pkgbase_details.php b/web/template/pkgbase_details.php index a6857c4e..35ad217a 100644 --- a/web/template/pkgbase_details.php +++ b/web/template/pkgbase_details.php @@ -137,14 +137,10 @@ endif; From 2df90ce28087d02e7b1dbd0e8efd5d5f99407793 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Mar 2021 15:20:04 -0700 Subject: [PATCH 078/844] port over base HTML layout from PHP to FastAPI+Jinja2 + Mounted static files (at web/html) to /static. + Added AURWEB_VERSION to aurweb.config (this is used around HTML to refer back to aurweb's release on git.archlinux.org), so we need it easily accessible in the Python codebase. + Implemented basic Jinja2 partials to put together whole aurweb pages. This may be missing some things currently and is a WIP until this set is ready to be merged. + Added config [options] aurwebdir = YOUR_AUR_ROOT; this configuration option should specify the root directory of the aurweb project. It is used by various parts of the FastAPI codebase to target project directories. Added routes via aurweb.routers.html: * POST /language: Set your session language. * GET /favicon.ico: Redirect to /static/images/favicon.ico. * Some browsers always look for $ROOT/favicon.ico to get an icon for the page being loaded, regardless of a specified "shortcut icon" given in a directive. * GET /: Home page; WIP. * Updated aurweb.routers.html.language passes query parameters to its next redirection. When calling aurweb.templates.render_template, the context passed should be formed via the aurweb.templates.make_context. See aurweb.routers.html.index for an example of this. Signed-off-by: Kevin Morris --- .coveragerc | 1 + INSTALL | 4 +- aurweb/asgi.py | 23 ++++++++- aurweb/config.py | 5 ++ aurweb/routers/html.py | 50 +++++++++++++++++++ aurweb/templates.py | 57 +++++++++++++++++++++ conf/config.dev | 1 + templates/index.html | 4 ++ templates/partials/archdev-navbar.html | 8 +++ templates/partials/body.html | 10 ++++ templates/partials/footer.html | 5 ++ templates/partials/head.html | 16 ++++++ templates/partials/layout.html | 10 ++++ templates/partials/meta.html | 1 + templates/partials/navbar.html | 19 +++++++ templates/partials/set_lang.html | 28 +++++++++++ templates/partials/typeahead.html | 30 +++++++++++ test/test_routes.py | 69 ++++++++++++++++++++++++++ 18 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 aurweb/routers/html.py create mode 100644 aurweb/templates.py create mode 100644 templates/index.html create mode 100644 templates/partials/archdev-navbar.html create mode 100644 templates/partials/body.html create mode 100644 templates/partials/footer.html create mode 100644 templates/partials/head.html create mode 100644 templates/partials/layout.html create mode 100644 templates/partials/meta.html create mode 100644 templates/partials/navbar.html create mode 100644 templates/partials/set_lang.html create mode 100644 templates/partials/typeahead.html create mode 100644 test/test_routes.py diff --git a/.coveragerc b/.coveragerc index 144a9f5c..9dcfca18 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,5 +3,6 @@ disable_warnings = already-imported [report] include = aurweb/* +fail_under = 85 exclude_lines = if __name__ == .__main__.: diff --git a/INSTALL b/INSTALL index 8607b07f..e4c52480 100644 --- a/INSTALL +++ b/INSTALL @@ -49,7 +49,9 @@ read the instructions below. # pacman -S python-mysql-connector python-pygit2 python-srcinfo python-sqlalchemy \ python-bleach python-markdown python-alembic hypercorn \ - python-itsdangerous python-authlib python-httpx + python-itsdangerous python-authlib python-httpx \ + python-jinja python-aiofiles python-python-multipart \ + python-requests # python3 setup.py install 5) Create a new MySQL database and a user and import the aurweb SQL schema: diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 9293ed77..00d7c595 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -2,13 +2,26 @@ import http from fastapi import FastAPI, HTTPException from fastapi.responses import HTMLResponse +from fastapi.staticfiles import StaticFiles from starlette.middleware.sessions import SessionMiddleware import aurweb.config -from aurweb.routers import sso +from aurweb.routers import html, sso +routes = set() + +# Setup the FastAPI app. app = FastAPI() +app.mount("/static/css", + StaticFiles(directory="web/html/css"), + name="static_css") +app.mount("/static/js", + StaticFiles(directory="web/html/js"), + name="static_js") +app.mount("/static/images", + StaticFiles(directory="web/html/images"), + name="static_images") session_secret = aurweb.config.get("fastapi", "session_secret") if not session_secret: @@ -17,6 +30,14 @@ if not session_secret: app.add_middleware(SessionMiddleware, secret_key=session_secret) app.include_router(sso.router) +app.include_router(html.router) + +# NOTE: Always keep this dictionary updated with all routes +# that the application contains. We use this to check for +# parameter value verification. +routes = {route.path for route in app.routes} +routes.update({route.path for route in sso.router.routes}) +routes.update({route.path for route in html.router.routes}) @app.exception_handler(HTTPException) diff --git a/aurweb/config.py b/aurweb/config.py index 52ec461e..020c3b80 100644 --- a/aurweb/config.py +++ b/aurweb/config.py @@ -1,6 +1,11 @@ import configparser import os +# Publicly visible version of aurweb. This is used to display +# aurweb versioning in the footer and must be maintained. +# Todo: Make this dynamic/automated. +AURWEB_VERSION = "v5.0.0" + _parser = None diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py new file mode 100644 index 00000000..ae08c764 --- /dev/null +++ b/aurweb/routers/html.py @@ -0,0 +1,50 @@ +""" AURWeb's primary routing module. Define all routes via @app.app.{get,post} +decorators in some way; more complex routes should be defined in their +own modules and imported here. """ +from http import HTTPStatus +from urllib.parse import unquote + +from fastapi import APIRouter, Form, Request +from fastapi.responses import HTMLResponse, RedirectResponse + +from aurweb.templates import make_context, render_template + +router = APIRouter() + + +@router.get("/favicon.ico") +async def favicon(request: Request): + """ Some browsers attempt to find a website's favicon via root uri at + /favicon.ico, so provide a redirection here to our static icon. """ + return RedirectResponse("/static/images/favicon.ico") + + +@router.post("/language", response_class=RedirectResponse) +async def language(request: Request, + set_lang: str = Form(...), + next: str = Form(...), + q: str = Form(default=None)): + """ A POST route used to set a session's language. + + Return a 303 See Other redirect to {next}?next={next}. If we are + setting the language on any page, we want to preserve query + parameters across the redirect. + """ + from aurweb.asgi import routes + if unquote(next) not in routes: + return HTMLResponse( + b"Invalid 'next' parameter.", + status_code=400) + + query_string = "?" + q if q else str() + response = RedirectResponse(url=f"{next}{query_string}", + status_code=int(HTTPStatus.SEE_OTHER)) + response.set_cookie("AURLANG", set_lang) + return response + + +@router.get("/", response_class=HTMLResponse) +async def index(request: Request): + """ Homepage route. """ + context = make_context(request, "Home") + return render_template("index.html", context) diff --git a/aurweb/templates.py b/aurweb/templates.py new file mode 100644 index 00000000..c05dce79 --- /dev/null +++ b/aurweb/templates.py @@ -0,0 +1,57 @@ +import copy +import os + +from datetime import datetime +from http import HTTPStatus + +import jinja2 + +from fastapi import Request +from fastapi.responses import HTMLResponse + +import aurweb.config + +from aurweb import l10n + +# Prepare jinja2 objects. +loader = jinja2.FileSystemLoader(os.path.join( + aurweb.config.get("options", "aurwebdir"), "templates")) +env = jinja2.Environment(loader=loader, autoescape=True, + extensions=["jinja2.ext.i18n"]) + +# Add tr translation filter. +env.filters["tr"] = l10n.tr + + +def make_context(request: Request, title: str, next: str = None): + """ Create a context for a jinja2 TemplateResponse. """ + + return { + "request": request, + "language": l10n.get_request_language(request), + "languages": l10n.SUPPORTED_LANGUAGES, + "title": title, + # The 'now' context variable will not show proper datetimes + # until we've implemented timezone support here. + "now": datetime.now(), + "config": aurweb.config, + "next": next if next else request.url.path + } + + +def render_template(path: str, context: dict, status_code=int(HTTPStatus.OK)): + """ Render a Jinja2 multi-lingual template with some context. """ + + # Create a deep copy of our jinja2 environment. The environment in + # total by itself is 48 bytes large (according to sys.getsizeof). + # This is done so we can install gettext translations on the template + # environment being rendered without installing them into a global + # which is reused in this function. + templates = copy.copy(env) + + translator = l10n.get_raw_translator_for_request(context.get("request")) + templates.install_gettext_translations(translator) + + template = templates.get_template(path) + rendered = template.render(context) + return HTMLResponse(rendered, status_code=status_code) diff --git a/conf/config.dev b/conf/config.dev index ef7b5ed7..ccb01f4f 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -16,6 +16,7 @@ name = YOUR_AUR_ROOT/aurweb.sqlite3 ;password = aur [options] +aurwebdir = YOUR_AUR_ROOT aur_location = http://127.0.0.1:8080 disable_http_login = 0 enable-maintenance = 0 diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 00000000..27d3375d --- /dev/null +++ b/templates/index.html @@ -0,0 +1,4 @@ +{% extends 'partials/layout.html' %} + +{% block pageContent %} +{% endblock %} diff --git a/templates/partials/archdev-navbar.html b/templates/partials/archdev-navbar.html new file mode 100644 index 00000000..55338bc4 --- /dev/null +++ b/templates/partials/archdev-navbar.html @@ -0,0 +1,8 @@ + diff --git a/templates/partials/body.html b/templates/partials/body.html new file mode 100644 index 00000000..ccae0fe3 --- /dev/null +++ b/templates/partials/body.html @@ -0,0 +1,10 @@ +
+ {% include 'partials/set_lang.html' %} + {% include 'partials/archdev-navbar.html' %} + + {% block pageContent %} + + {% endblock %} + + {% include 'partials/footer.html' %} +
diff --git a/templates/partials/footer.html b/templates/partials/footer.html new file mode 100644 index 00000000..0ac4d089 --- /dev/null +++ b/templates/partials/footer.html @@ -0,0 +1,5 @@ + diff --git a/templates/partials/head.html b/templates/partials/head.html new file mode 100644 index 00000000..0351fd6e --- /dev/null +++ b/templates/partials/head.html @@ -0,0 +1,16 @@ + + {% include 'partials/meta.html' %} + + + + + + + + + + + + AUR ({{ language }}) - {{ title | tr }} + diff --git a/templates/partials/layout.html b/templates/partials/layout.html new file mode 100644 index 00000000..d30208a9 --- /dev/null +++ b/templates/partials/layout.html @@ -0,0 +1,10 @@ + + + {% include 'partials/head.html' %} + + + {% include 'partials/navbar.html' %} + {% extends 'partials/body.html' %} + {% include 'partials/typeahead.html' %} + + diff --git a/templates/partials/meta.html b/templates/partials/meta.html new file mode 100644 index 00000000..727100b9 --- /dev/null +++ b/templates/partials/meta.html @@ -0,0 +1 @@ + diff --git a/templates/partials/navbar.html b/templates/partials/navbar.html new file mode 100644 index 00000000..199b2067 --- /dev/null +++ b/templates/partials/navbar.html @@ -0,0 +1,19 @@ + diff --git a/templates/partials/set_lang.html b/templates/partials/set_lang.html new file mode 100644 index 00000000..e9590050 --- /dev/null +++ b/templates/partials/set_lang.html @@ -0,0 +1,28 @@ +
+
+
+
+ + + + + + + + + +
+
+
+
diff --git a/templates/partials/typeahead.html b/templates/partials/typeahead.html new file mode 100644 index 00000000..d943dbc4 --- /dev/null +++ b/templates/partials/typeahead.html @@ -0,0 +1,30 @@ + + + diff --git a/test/test_routes.py b/test/test_routes.py new file mode 100644 index 00000000..46ba39f5 --- /dev/null +++ b/test/test_routes.py @@ -0,0 +1,69 @@ +import urllib.parse + +from http import HTTPStatus + +import pytest + +from fastapi.testclient import TestClient + +from aurweb.asgi import app +from aurweb.testing import setup_test_db + +client = TestClient(app) + + +@pytest.fixture +def setup(): + setup_test_db("Users", "Session") + + +def test_index(): + """ Test the index route at '/'. """ + # Use `with` to trigger FastAPI app events. + with client as req: + response = req.get("/") + assert response.status_code == int(HTTPStatus.OK) + + +def test_favicon(): + """ Test the favicon route at '/favicon.ico'. """ + response1 = client.get("/static/images/favicon.ico") + response2 = client.get("/favicon.ico") + assert response1.status_code == int(HTTPStatus.OK) + assert response1.content == response2.content + + +def test_language(): + """ Test the language post route at '/language'. """ + post_data = { + "set_lang": "de", + "next": "/" + } + with client as req: + response = req.post("/language", data=post_data) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + + +def test_language_invalid_next(): + """ Test an invalid next route at '/language'. """ + post_data = { + "set_lang": "de", + "next": "/BLAHBLAHFAKE" + } + with client as req: + response = req.post("/language", data=post_data) + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + +def test_language_query_params(): + """ Test the language post route with query params. """ + next = urllib.parse.quote_plus("/") + post_data = { + "set_lang": "de", + "next": "/", + "q": f"next={next}" + } + q = post_data.get("q") + with client as req: + response = req.post("/language", data=post_data) + assert response.headers.get("location") == f"/?{q}" + assert response.status_code == int(HTTPStatus.SEE_OTHER) From 7c65604dad8d9c68bee59129b03af05d26db1582 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Mar 2021 15:20:19 -0700 Subject: [PATCH 079/844] move off env.py's active code to __name__ == "__main__" * Moved migrations/env.py's logging initialization and migration execution into a `__name__ == "__main__"` stanza so it doesn't immediately happen when imported by another module. Signed-off-by: Kevin Morris --- migrations/env.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/migrations/env.py b/migrations/env.py index c2ff58c1..23759123 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -11,10 +11,6 @@ import aurweb.schema # access to the values within the .ini file in use. config = context.config -# Interpret the config file for Python logging. -# This line sets up loggers basically. -logging.config.fileConfig(config.config_file_name) - # model MetaData for autogenerating migrations target_metadata = aurweb.schema.metadata @@ -68,7 +64,12 @@ def run_migrations_online(): context.run_migrations() -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() +if __name__ == "__main__": + # Interpret the config file for Python logging. + # This line sets up loggers basically. + logging.config.fileConfig(config.config_file_name) + + if context.is_offline_mode(): + run_migrations_offline() + else: + run_migrations_online() From 4238a9fc6855242121402b392581ba5b695e2f90 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Mar 2021 15:20:23 -0700 Subject: [PATCH 080/844] add aurweb.db.session + Added Session class and global session object to aurweb.db, these are sessions created by sqlalchemy ORM's sessionmaker and will allow us to use declarative/imperative models. Signed-off-by: Kevin Morris --- aurweb/asgi.py | 36 +++++---- aurweb/config.py | 7 ++ aurweb/db.py | 33 ++++----- test/test_asgi.py | 29 ++++++++ test/test_config.py | 13 ++++ test/test_db.py | 174 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 260 insertions(+), 32 deletions(-) create mode 100644 test/test_asgi.py create mode 100644 test/test_config.py create mode 100644 test/test_db.py diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 00d7c595..b6e15582 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -7,30 +7,36 @@ from starlette.middleware.sessions import SessionMiddleware import aurweb.config +from aurweb.db import get_engine from aurweb.routers import html, sso routes = set() # Setup the FastAPI app. app = FastAPI() -app.mount("/static/css", - StaticFiles(directory="web/html/css"), - name="static_css") -app.mount("/static/js", - StaticFiles(directory="web/html/js"), - name="static_js") -app.mount("/static/images", - StaticFiles(directory="web/html/images"), - name="static_images") -session_secret = aurweb.config.get("fastapi", "session_secret") -if not session_secret: - raise Exception("[fastapi] session_secret must not be empty") -app.add_middleware(SessionMiddleware, secret_key=session_secret) +@app.on_event("startup") +async def app_startup(): + session_secret = aurweb.config.get("fastapi", "session_secret") + if not session_secret: + raise Exception("[fastapi] session_secret must not be empty") -app.include_router(sso.router) -app.include_router(html.router) + app.mount("/static/css", + StaticFiles(directory="web/html/css"), + name="static_css") + app.mount("/static/js", + StaticFiles(directory="web/html/js"), + name="static_js") + app.mount("/static/images", + StaticFiles(directory="web/html/images"), + name="static_images") + + app.add_middleware(SessionMiddleware, secret_key=session_secret) + app.include_router(sso.router) + app.include_router(html.router) + + get_engine() # NOTE: Always keep this dictionary updated with all routes # that the application contains. We use this to check for diff --git a/aurweb/config.py b/aurweb/config.py index 020c3b80..49a2765a 100644 --- a/aurweb/config.py +++ b/aurweb/config.py @@ -25,6 +25,13 @@ def _get_parser(): return _parser +def rehash(): + """ Globally rehash the configuration parser. """ + global _parser + _parser = None + _get_parser() + + def get(section, option): return _get_parser().get(section, option) diff --git a/aurweb/db.py b/aurweb/db.py index 04b40f43..7993dfdb 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -1,19 +1,15 @@ import math -try: - import mysql.connector -except ImportError: - pass - -try: - import sqlite3 -except ImportError: - pass - import aurweb.config engine = None # See get_engine +# ORM Session class. +Session = None + +# Global ORM Session object. +session = None + def get_sqlalchemy_url(): """ @@ -49,14 +45,15 @@ def get_engine(): `engine` global variable for the next calls. """ from sqlalchemy import create_engine - global engine + from sqlalchemy.orm import sessionmaker + + global engine, session, Session + if engine is None: - connect_args = dict() - if aurweb.config.get("database", "backend") == "sqlite": - # check_same_thread is for a SQLite technicality - # https://fastapi.tiangolo.com/tutorial/sql-databases/#note - connect_args["check_same_thread"] = False - engine = create_engine(get_sqlalchemy_url(), connect_args=connect_args) + engine = create_engine(get_sqlalchemy_url(), + # check_same_thread is for a SQLite technicality + # https://fastapi.tiangolo.com/tutorial/sql-databases/#note + connect_args={"check_same_thread": False}) Session = sessionmaker(autocommit=False, autoflush=False, bind=engine) session = Session() @@ -82,6 +79,7 @@ class Connection: aur_db_backend = aurweb.config.get('database', 'backend') if aur_db_backend == 'mysql': + import mysql.connector aur_db_host = aurweb.config.get('database', 'host') aur_db_name = aurweb.config.get('database', 'name') aur_db_user = aurweb.config.get('database', 'user') @@ -95,6 +93,7 @@ class Connection: buffered=True) self._paramstyle = mysql.connector.paramstyle elif aur_db_backend == 'sqlite': + import sqlite3 aur_db_name = aurweb.config.get('database', 'name') self._conn = sqlite3.connect(aur_db_name) self._conn.create_function("POWER", 2, math.pow) diff --git a/test/test_asgi.py b/test/test_asgi.py new file mode 100644 index 00000000..79b34daf --- /dev/null +++ b/test/test_asgi.py @@ -0,0 +1,29 @@ +import http +import os + +from unittest import mock + +import pytest + +from fastapi import HTTPException + +import aurweb.asgi +import aurweb.config + + +@pytest.mark.asyncio +async def test_asgi_startup_exception(monkeypatch): + with mock.patch.dict(os.environ, {"AUR_CONFIG": "conf/config.defaults"}): + aurweb.config.rehash() + with pytest.raises(Exception): + await aurweb.asgi.app_startup() + aurweb.config.rehash() + + +@pytest.mark.asyncio +async def test_asgi_http_exception_handler(): + exc = HTTPException(status_code=422, detail="EXCEPTION!") + phrase = http.HTTPStatus(exc.status_code).phrase + response = await aurweb.asgi.http_exception_handler(None, exc) + assert response.body.decode() == \ + f"

{exc.status_code} {phrase}

{exc.detail}

" diff --git a/test/test_config.py b/test/test_config.py new file mode 100644 index 00000000..4f10b60d --- /dev/null +++ b/test/test_config.py @@ -0,0 +1,13 @@ +from aurweb import config + + +def test_get(): + assert config.get("options", "disable_http_login") == "0" + + +def test_getboolean(): + assert not config.getboolean("options", "disable_http_login") + + +def test_getint(): + assert config.getint("options", "disable_http_login") == 0 diff --git a/test/test_db.py b/test/test_db.py new file mode 100644 index 00000000..0a134541 --- /dev/null +++ b/test/test_db.py @@ -0,0 +1,174 @@ +import os +import re +import sqlite3 +import tempfile + +from unittest import mock + +import mysql.connector +import pytest + +import aurweb.config + +from aurweb import db +from aurweb.testing import setup_test_db + + +class DBCursor: + """ A fake database cursor object used in tests. """ + items = [] + + def execute(self, *args, **kwargs): + self.items = list(args) + return self + + def fetchall(self): + return self.items + + +class DBConnection: + """ A fake database connection object used in tests. """ + @staticmethod + def cursor(): + return DBCursor() + + @staticmethod + def create_function(name, num_args, func): + pass + + +@pytest.fixture(autouse=True) +def setup_db(): + setup_test_db() + + +def test_sqlalchemy_sqlite_url(): + with mock.patch.dict(os.environ, {"AUR_CONFIG": "conf/config.dev"}): + aurweb.config.rehash() + assert db.get_sqlalchemy_url() + aurweb.config.rehash() + + +def test_sqlalchemy_mysql_url(): + with mock.patch.dict(os.environ, {"AUR_CONFIG": "conf/config.defaults"}): + aurweb.config.rehash() + assert db.get_sqlalchemy_url() + aurweb.config.rehash() + + +def make_temp_config(backend): + if not os.path.isdir("/tmp"): + os.mkdir("/tmp") + tmpdir = tempfile.mkdtemp() + tmp = os.path.join(tmpdir, "config.tmp") + with open("conf/config") as f: + config = re.sub(r'backend = sqlite', f'backend = {backend}', f.read()) + with open(tmp, "w") as o: + o.write(config) + return (tmpdir, tmp) + + +def test_sqlalchemy_unknown_backend(): + tmpdir, tmp = make_temp_config("blah") + + with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): + aurweb.config.rehash() + with pytest.raises(ValueError): + db.get_sqlalchemy_url() + aurweb.config.rehash() + + os.remove(tmp) + os.removedirs(tmpdir) + + +def test_db_connects_without_fail(): + db.connect() + assert db.engine is not None + + +def test_connection_class_without_fail(): + conn = db.Connection() + + cur = conn.execute( + "SELECT AccountType FROM AccountTypes WHERE ID = ?", (1,)) + account_type = cur.fetchone()[0] + + assert account_type == "User" + + +def test_connection_class_unsupported_backend(): + tmpdir, tmp = make_temp_config("blah") + + with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): + aurweb.config.rehash() + with pytest.raises(ValueError): + db.Connection() + aurweb.config.rehash() + + os.remove(tmp) + os.removedirs(tmpdir) + + +@mock.patch("mysql.connector.connect", mock.MagicMock(return_value=True)) +@mock.patch.object(mysql.connector, "paramstyle", "qmark") +def test_connection_mysql(): + tmpdir, tmp = make_temp_config("mysql") + with mock.patch.dict(os.environ, { + "AUR_CONFIG": tmp, + "AUR_CONFIG_DEFAULTS": "conf/config.defaults" + }): + aurweb.config.rehash() + db.Connection() + aurweb.config.rehash() + + os.remove(tmp) + os.removedirs(tmpdir) + + +@mock.patch("sqlite3.connect", mock.MagicMock(return_value=DBConnection())) +@mock.patch.object(sqlite3, "paramstyle", "qmark") +def test_connection_sqlite(): + db.Connection() + + +@mock.patch("sqlite3.connect", mock.MagicMock(return_value=DBConnection())) +@mock.patch.object(sqlite3, "paramstyle", "format") +def test_connection_execute_paramstyle_format(): + conn = db.Connection() + + # First, test ? to %s format replacement. + account_types = conn\ + .execute("SELECT * FROM AccountTypes WHERE AccountType = ?", ["User"])\ + .fetchall() + assert account_types == \ + ["SELECT * FROM AccountTypes WHERE AccountType = %s", ["User"]] + + # Test other format replacement. + account_types = conn\ + .execute("SELECT * FROM AccountTypes WHERE AccountType = %", ["User"])\ + .fetchall() + assert account_types == \ + ["SELECT * FROM AccountTypes WHERE AccountType = %%", ["User"]] + + +@mock.patch("sqlite3.connect", mock.MagicMock(return_value=DBConnection())) +@mock.patch.object(sqlite3, "paramstyle", "qmark") +def test_connection_execute_paramstyle_qmark(): + conn = db.Connection() + # We don't modify anything when using qmark, so test equality. + account_types = conn\ + .execute("SELECT * FROM AccountTypes WHERE AccountType = ?", ["User"])\ + .fetchall() + assert account_types == \ + ["SELECT * FROM AccountTypes WHERE AccountType = ?", ["User"]] + + +@mock.patch("sqlite3.connect", mock.MagicMock(return_value=DBConnection())) +@mock.patch.object(sqlite3, "paramstyle", "unsupported") +def test_connection_execute_paramstyle_unsupported(): + conn = db.Connection() + with pytest.raises(ValueError, match="unsupported paramstyle"): + conn.execute( + "SELECT * FROM AccountTypes WHERE AccountType = ?", + ["User"] + ).fetchall() From 32f289309579554d85a9971be59ccc24c973840c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Mar 2021 15:20:26 -0700 Subject: [PATCH 081/844] add aurweb.models.account_type.AccountType Signed-off-by: Kevin Morris --- aurweb/models/__init__.py | 0 aurweb/models/account_type.py | 20 ++++++++++++++++++++ test/test_account_type.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 aurweb/models/__init__.py create mode 100644 aurweb/models/account_type.py create mode 100644 test/test_account_type.py diff --git a/aurweb/models/__init__.py b/aurweb/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/aurweb/models/account_type.py b/aurweb/models/account_type.py new file mode 100644 index 00000000..44225e35 --- /dev/null +++ b/aurweb/models/account_type.py @@ -0,0 +1,20 @@ +from sqlalchemy.orm import mapper + +from aurweb.schema import AccountTypes + + +class AccountType: + """ An ORM model of a single AccountTypes record. """ + + def __init__(self, **kwargs): + self.AccountType = kwargs.pop("AccountType") + + def __str__(self): + return str(self.AccountType) + + def __repr__(self): + return "" % ( + self.ID, str(self)) + + +mapper(AccountType, AccountTypes, confirm_deleted_rows=False) diff --git a/test/test_account_type.py b/test/test_account_type.py new file mode 100644 index 00000000..b6a12363 --- /dev/null +++ b/test/test_account_type.py @@ -0,0 +1,28 @@ +import pytest + +from aurweb.models.account_type import AccountType +from aurweb.testing import setup_test_db + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db() + + +def test_account_type(): + """ Test creating an AccountType, and reading its columns. """ + from aurweb.db import session + account_type = AccountType(AccountType="TestUser") + session.add(account_type) + session.commit() + + # Make sure it got created and was given an ID. + assert bool(account_type.ID) + + # Next, test our string functions. + assert str(account_type) == "TestUser" + assert repr(account_type) == \ + "" % ( + account_type.ID) + + session.delete(account_type) From e860d828b6f9babaa5829db92951087de3774b78 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Mar 2021 15:20:28 -0700 Subject: [PATCH 082/844] add aurweb.testing, a module with testing utilities Signed-off-by: Kevin Morris --- aurweb/testing.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 aurweb/testing.py diff --git a/aurweb/testing.py b/aurweb/testing.py new file mode 100644 index 00000000..7516d918 --- /dev/null +++ b/aurweb/testing.py @@ -0,0 +1,30 @@ +from aurweb.db import get_engine + + +def setup_test_db(*args): + """ This function is to be used to setup a test database before + using it. It takes a variable number of table strings, and for + each table in that set of table strings, it deletes all records. + + The primary goal of this method is to configure empty tables + that tests can use from scratch. This means that tests using + this function should make sure they do not depend on external + records and keep their logic self-contained. + + Generally used inside of pytest fixtures, this function + can be used anywhere, but keep in mind its functionality when + doing so. + + Examples: + setup_test_db("Users", "Sessions") + + test_tables = ["Users", "Sessions"]; + setup_test_db(*test_tables) + """ + engine = get_engine() + conn = engine.connect() + + tables = list(args) + for table in tables: + conn.execute(f"DELETE FROM {table}") + conn.close() From 8a47afd2ea8ce56b17aba95503d7c97f22023dff Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Mar 2021 15:20:32 -0700 Subject: [PATCH 083/844] add aurweb.models.user.User + Added aurweb.models.user.User class. This is the first example of an sqlalchemy ORM model. We can search for users via for example: `session.query(User).filter(User.ID==1).first()`, where `session` is a configured `aurweb.db.session` object. + Along with the User class, defined the AccountType class. Each User maintains a relationship to its AccountType via User.AccountType. + Added AccountType.users backref. Signed-off-by: Kevin Morris --- aurweb/models/user.py | 43 +++++++++++++++++++++++++++++++++++ test/test_user.py | 52 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 aurweb/models/user.py create mode 100644 test/test_user.py diff --git a/aurweb/models/user.py b/aurweb/models/user.py new file mode 100644 index 00000000..ba91c439 --- /dev/null +++ b/aurweb/models/user.py @@ -0,0 +1,43 @@ +from sqlalchemy.orm import backref, mapper, relationship + +from aurweb.models.account_type import AccountType +from aurweb.schema import Users + + +class User: + """ An ORM model of a single Users record. """ + + def __init__(self, **kwargs): + self.AccountTypeID = kwargs.get("AccountTypeID") + + account_type = kwargs.get("AccountType") + if account_type: + self.AccountType = account_type + + self.Username = kwargs.get("Username") + self.Email = kwargs.get("Email") + self.BackupEmail = kwargs.get("BackupEmail") + self.Passwd = kwargs.get("Passwd") + self.Salt = kwargs.get("Salt") + self.RealName = kwargs.get("RealName") + self.LangPreference = kwargs.get("LangPreference") + self.Timezone = kwargs.get("Timezone") + self.Homepage = kwargs.get("Homepage") + self.IRCNick = kwargs.get("IRCNick") + self.PGPKey = kwargs.get("PGPKey") + self.RegistrationTS = kwargs.get("RegistrationTS") + self.CommentNotify = kwargs.get("CommentNotify") + self.UpdateNotify = kwargs.get("UpdateNotify") + self.OwnershipNotify = kwargs.get("OwnershipNotify") + self.SSOAccountID = kwargs.get("SSOAccountID") + + def __repr__(self): + return "" % ( + self.ID, str(self.AccountType), self.Username) + + +# Map schema.Users to User and give it some relationships. +mapper(User, Users, properties={ + "AccountType": relationship(AccountType, + backref=backref("users", lazy="dynamic")) +}) diff --git a/test/test_user.py b/test/test_user.py new file mode 100644 index 00000000..8ac9b00b --- /dev/null +++ b/test/test_user.py @@ -0,0 +1,52 @@ +import pytest + +from aurweb.models.account_type import AccountType +from aurweb.models.user import User +from aurweb.testing import setup_test_db + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db("Users") + + +def test_user(): + """ Test creating a user and reading its columns. """ + from aurweb.db import session + + # First, grab our target AccountType. + account_type = session.query(AccountType).filter( + AccountType.AccountType == "User").first() + + user = User( + AccountType=account_type, + RealName="Test User", Username="test", + Email="test@example.org", Passwd="abcd", + IRCNick="tester", + Salt="efgh", ResetKey="blahblah") + session.add(user) + session.commit() + + assert user in account_type.users + + # Make sure the user was created and given an ID. + assert bool(user.ID) + + # Search for the user via query API. + result = session.query(User).filter(User.ID == user.ID).first() + + # Compare the result and our original user. + assert result.ID == user.ID + assert result.AccountType.ID == user.AccountType.ID + assert result.Username == user.Username + assert result.Email == user.Email + + # Ensure we've got the correct account type. + assert user.AccountType.ID == account_type.ID + assert user.AccountType.AccountType == account_type.AccountType + + # Test out user string functions. + assert repr(user) == f"" + + session.delete(user) From 02311eab7604d29de7b70721cb1e10329178cfc7 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Mar 2021 15:20:34 -0700 Subject: [PATCH 084/844] add test_initdb.py IMPORTANT: This test completely wipes out the database it's using. Make sure you've got AUR_CONFIG set to a test database configuration! Signed-off-by: Kevin Morris --- test/test_initdb.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 test/test_initdb.py diff --git a/test/test_initdb.py b/test/test_initdb.py new file mode 100644 index 00000000..ff089b63 --- /dev/null +++ b/test/test_initdb.py @@ -0,0 +1,27 @@ +import pytest + +import aurweb.config +import aurweb.db +import aurweb.initdb + +from aurweb.models.account_type import AccountType +from aurweb.schema import metadata +from aurweb.testing import setup_test_db + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db() + + tables = metadata.tables.keys() + for table in tables: + aurweb.db.session.execute(f"DROP TABLE IF EXISTS {table}") + + +def test_run(): + class Args: + use_alembic = True + verbose = False + aurweb.initdb.run(Args()) + assert aurweb.db.session.query(AccountType).filter( + AccountType.AccountType == "User").first() is not None From 81856f3b646ad2bf27008a97b1604a5325eea03c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 13 May 2021 21:05:34 -0700 Subject: [PATCH 085/844] Fix incorrect construction of MySQL SQLAlchemy URL Signed-off-by: Kevin Morris From 82f3871a83bf234983f8d63d2f1861d876a52fb1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 13 May 2021 21:11:57 -0700 Subject: [PATCH 086/844] Support SQLAlchemy 1.4 URL.create recommendation This fixes a deprecating warning when using SQLAlchemy 1.4. Signed-off-by: Kevin Morris --- .coveragerc | 1 + aurweb/db.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index 9dcfca18..69c153ce 100644 --- a/.coveragerc +++ b/.coveragerc @@ -6,3 +6,4 @@ include = aurweb/* fail_under = 85 exclude_lines = if __name__ == .__main__.: + pragma: no cover diff --git a/aurweb/db.py b/aurweb/db.py index 7993dfdb..49e0abd2 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -16,9 +16,18 @@ def get_sqlalchemy_url(): Build an SQLAlchemy for use with create_engine based on the aurweb configuration. """ import sqlalchemy + + constructor = sqlalchemy.engine.url.URL + + parts = sqlalchemy.__version__.split('.') + major = int(parts[0]) + minor = int(parts[1]) + if major == 1 and minor >= 4: # pragma: no cover + constructor = sqlalchemy.engine.url.URL.create + aur_db_backend = aurweb.config.get('database', 'backend') if aur_db_backend == 'mysql': - return sqlalchemy.engine.url.URL( + return constructor( 'mysql+mysqlconnector', username=aurweb.config.get('database', 'user'), password=aurweb.config.get('database', 'password'), @@ -29,7 +38,7 @@ def get_sqlalchemy_url(): }, ) elif aur_db_backend == 'sqlite': - return sqlalchemy.engine.url.URL( + return constructor( 'sqlite', database=aurweb.config.get('database', 'name'), ) From cdf75ced9abe855753e33b48cc869c4ac64506ba Mon Sep 17 00:00:00 2001 From: Marcus Andersson Date: Thu, 13 May 2021 00:08:15 +0200 Subject: [PATCH 087/844] Adding error 404 catcher --- aurweb/routers/errors.py | 14 ++++++++++++++ templates/errors/404.html | 8 ++++++++ 2 files changed, 22 insertions(+) create mode 100644 aurweb/routers/errors.py create mode 100644 templates/errors/404.html diff --git a/aurweb/routers/errors.py b/aurweb/routers/errors.py new file mode 100644 index 00000000..5d4ca4ce --- /dev/null +++ b/aurweb/routers/errors.py @@ -0,0 +1,14 @@ +from aurweb.templates import make_context, render_template +from aurweb import l10n + + +async def not_found(request, exc): + _ = l10n.get_translator_for_request(request) + context = make_context(request, f"404 - {_('Page Not Found')}") + return render_template("errors/404.html", context) + + +# Maps HTTP errors to functions +exceptions = { + 404: not_found, +} diff --git a/templates/errors/404.html b/templates/errors/404.html new file mode 100644 index 00000000..0afdd2fa --- /dev/null +++ b/templates/errors/404.html @@ -0,0 +1,8 @@ +{% extends 'partials/layout.html' %} + +{% block pageContent %} +
+

404 - {% trans %}Page Not Found{% endtrans %}

+

{% trans %}Sorry, the page you've requested does not exist.{% endtrans %}

+
+ {% endblock %} From f6744d3e39fd044f797f9a702d8c9883fe40c527 Mon Sep 17 00:00:00 2001 From: Marcus Andersson Date: Thu, 13 May 2021 00:08:15 +0200 Subject: [PATCH 088/844] Adding error 503 catcher --- aurweb/asgi.py | 4 ++-- aurweb/routers/errors.py | 6 ++++++ templates/errors/503.html | 8 ++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 templates/errors/503.html diff --git a/aurweb/asgi.py b/aurweb/asgi.py index b6e15582..c03a00f7 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -8,12 +8,12 @@ from starlette.middleware.sessions import SessionMiddleware import aurweb.config from aurweb.db import get_engine -from aurweb.routers import html, sso +from aurweb.routers import html, sso, errors routes = set() # Setup the FastAPI app. -app = FastAPI() +app = FastAPI(exception_handlers=errors.exceptions) @app.on_event("startup") diff --git a/aurweb/routers/errors.py b/aurweb/routers/errors.py index 5d4ca4ce..3bdaeb9d 100644 --- a/aurweb/routers/errors.py +++ b/aurweb/routers/errors.py @@ -8,7 +8,13 @@ async def not_found(request, exc): return render_template("errors/404.html", context) +async def service_unavailable(request, exc): + _ = l10n.get_translator_for_request(request) + context = make_context(request, "503 - {_('Service Unavailable')}") + return render_template("errors/503.html", context) + # Maps HTTP errors to functions exceptions = { 404: not_found, + 503: service_unavailable } diff --git a/templates/errors/503.html b/templates/errors/503.html new file mode 100644 index 00000000..d31666a1 --- /dev/null +++ b/templates/errors/503.html @@ -0,0 +1,8 @@ +{% extends 'partials/layout.html' %} + +{% block pageContent %} +
+

503 - {% trans %}Service Unavailable{% endtrans %}

+

{% trans %}Don't panic! This site is down due to maintenance. We will be back soon.{% endtrans %}

+
+{% endblock %} From 1d5827007f205c8972d07eee138372e8f9303684 Mon Sep 17 00:00:00 2001 From: Marcus Andersson Date: Thu, 13 May 2021 22:02:50 +0200 Subject: [PATCH 089/844] Adding route tests Removing status code from 404 title Removing status code from 503 title Adding id to 503 error box Indatation fix --- aurweb/routers/errors.py | 11 ++++------- aurweb/routers/html.py | 8 +++++++- templates/errors/404.html | 4 ++-- templates/errors/503.html | 2 +- test/test_routes.py | 7 +++++++ 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/aurweb/routers/errors.py b/aurweb/routers/errors.py index 3bdaeb9d..111d802a 100644 --- a/aurweb/routers/errors.py +++ b/aurweb/routers/errors.py @@ -1,17 +1,14 @@ from aurweb.templates import make_context, render_template -from aurweb import l10n async def not_found(request, exc): - _ = l10n.get_translator_for_request(request) - context = make_context(request, f"404 - {_('Page Not Found')}") - return render_template("errors/404.html", context) + context = make_context(request, "Page Not Found") + return render_template("errors/404.html", context, 404) async def service_unavailable(request, exc): - _ = l10n.get_translator_for_request(request) - context = make_context(request, "503 - {_('Service Unavailable')}") - return render_template("errors/503.html", context) + context = make_context(request, "Service Unavailable") + return render_template("errors/503.html", context, 503) # Maps HTTP errors to functions exceptions = { diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index ae08c764..50b62450 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -4,7 +4,7 @@ own modules and imported here. """ from http import HTTPStatus from urllib.parse import unquote -from fastapi import APIRouter, Form, Request +from fastapi import APIRouter, Form, Request, HTTPException from fastapi.responses import HTMLResponse, RedirectResponse from aurweb.templates import make_context, render_template @@ -48,3 +48,9 @@ async def index(request: Request): """ Homepage route. """ context = make_context(request, "Home") return render_template("index.html", context) + + +# A route that returns a error 503. For testing purposes. +@router.get("/raisefivethree", response_class=HTMLResponse) +async def raise_service_unavailable(request: Request): + raise HTTPException(status_code=503) diff --git a/templates/errors/404.html b/templates/errors/404.html index 0afdd2fa..4926aff6 100644 --- a/templates/errors/404.html +++ b/templates/errors/404.html @@ -1,8 +1,8 @@ {% extends 'partials/layout.html' %} {% block pageContent %} -
+

404 - {% trans %}Page Not Found{% endtrans %}

{% trans %}Sorry, the page you've requested does not exist.{% endtrans %}

- {% endblock %} +{% endblock %} diff --git a/templates/errors/503.html b/templates/errors/503.html index d31666a1..9a0ed56a 100644 --- a/templates/errors/503.html +++ b/templates/errors/503.html @@ -1,7 +1,7 @@ {% extends 'partials/layout.html' %} {% block pageContent %} -
+

503 - {% trans %}Service Unavailable{% endtrans %}

{% trans %}Don't panic! This site is down due to maintenance. We will be back soon.{% endtrans %}

diff --git a/test/test_routes.py b/test/test_routes.py index 46ba39f5..86221108 100644 --- a/test/test_routes.py +++ b/test/test_routes.py @@ -67,3 +67,10 @@ def test_language_query_params(): response = req.post("/language", data=post_data) assert response.headers.get("location") == f"/?{q}" assert response.status_code == int(HTTPStatus.SEE_OTHER) + + +def test_error_messages(): + response1 = client.get("/thisroutedoesnotexist") + response2 = client.get("/raisefivethree") + assert response1.status_code == int(HTTPStatus.NOT_FOUND) + assert response2.status_code == int(HTTPStatus.SERVICE_UNAVAILABLE) \ No newline at end of file From e0eb6b0e76311bc4e2df02e083edea28c42c178c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 16 May 2021 06:26:38 -0700 Subject: [PATCH 090/844] test_db: remove use of mkdtemp and os.removedirs Signed-off-by: Kevin Morris --- test/test_db.py | 52 +++++++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/test/test_db.py b/test/test_db.py index 0a134541..f5902e4c 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -57,10 +57,8 @@ def test_sqlalchemy_mysql_url(): def make_temp_config(backend): - if not os.path.isdir("/tmp"): - os.mkdir("/tmp") - tmpdir = tempfile.mkdtemp() - tmp = os.path.join(tmpdir, "config.tmp") + tmpdir = tempfile.TemporaryDirectory() + tmp = os.path.join(tmpdir.name, "config.tmp") with open("conf/config") as f: config = re.sub(r'backend = sqlite', f'backend = {backend}', f.read()) with open(tmp, "w") as o: @@ -69,16 +67,14 @@ def make_temp_config(backend): def test_sqlalchemy_unknown_backend(): - tmpdir, tmp = make_temp_config("blah") + tmpctx, tmp = make_temp_config("blah") - with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): + with tmpctx: + with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): + aurweb.config.rehash() + with pytest.raises(ValueError): + db.get_sqlalchemy_url() aurweb.config.rehash() - with pytest.raises(ValueError): - db.get_sqlalchemy_url() - aurweb.config.rehash() - - os.remove(tmp) - os.removedirs(tmpdir) def test_db_connects_without_fail(): @@ -97,32 +93,28 @@ def test_connection_class_without_fail(): def test_connection_class_unsupported_backend(): - tmpdir, tmp = make_temp_config("blah") + tmpctx, tmp = make_temp_config("blah") - with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): + with tmpctx: + with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): + aurweb.config.rehash() + with pytest.raises(ValueError): + db.Connection() aurweb.config.rehash() - with pytest.raises(ValueError): - db.Connection() - aurweb.config.rehash() - - os.remove(tmp) - os.removedirs(tmpdir) @mock.patch("mysql.connector.connect", mock.MagicMock(return_value=True)) @mock.patch.object(mysql.connector, "paramstyle", "qmark") def test_connection_mysql(): - tmpdir, tmp = make_temp_config("mysql") - with mock.patch.dict(os.environ, { - "AUR_CONFIG": tmp, - "AUR_CONFIG_DEFAULTS": "conf/config.defaults" - }): + tmpctx, tmp = make_temp_config("mysql") + with tmpctx: + with mock.patch.dict(os.environ, { + "AUR_CONFIG": tmp, + "AUR_CONFIG_DEFAULTS": "conf/config.defaults" + }): + aurweb.config.rehash() + db.Connection() aurweb.config.rehash() - db.Connection() - aurweb.config.rehash() - - os.remove(tmp) - os.removedirs(tmpdir) @mock.patch("sqlite3.connect", mock.MagicMock(return_value=DBConnection())) From 3f1f03e03c904f1c8202a692bc18ca153c529f50 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 16 May 2021 01:40:19 -0700 Subject: [PATCH 091/844] aurweb.db: only pass check_same_thread with sqlite Signed-off-by: Kevin Morris --- aurweb/db.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/aurweb/db.py b/aurweb/db.py index 49e0abd2..f5530bcf 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -59,10 +59,12 @@ def get_engine(): global engine, session, Session if engine is None: - engine = create_engine(get_sqlalchemy_url(), - # check_same_thread is for a SQLite technicality - # https://fastapi.tiangolo.com/tutorial/sql-databases/#note - connect_args={"check_same_thread": False}) + connect_args = dict() + if aurweb.config.get("database", "backend") == "sqlite": + # check_same_thread is for a SQLite technicality + # https://fastapi.tiangolo.com/tutorial/sql-databases/#note + connect_args["check_same_thread"] = False + engine = create_engine(get_sqlalchemy_url(), connect_args=connect_args) Session = sessionmaker(autocommit=False, autoflush=False, bind=engine) session = Session() From 66189c4460d1516344b13ad6a381aca6a7a0786e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 18 May 2021 02:46:56 -0700 Subject: [PATCH 092/844] alembic: restore logging, fix pytest conflicts In this case, when running pytests, we do not allow alembic to configure loggers. Signed-off-by: Kevin Morris --- aurweb/initdb.py | 1 + migrations/env.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/aurweb/initdb.py b/aurweb/initdb.py index c8d0b2ae..5f55bfc9 100644 --- a/aurweb/initdb.py +++ b/aurweb/initdb.py @@ -40,6 +40,7 @@ def run(args): if args.use_alembic: alembic_config = alembic.config.Config('alembic.ini') alembic_config.get_main_option('script_location') + alembic_config.attributes["configure_logger"] = False engine = sqlalchemy.create_engine(aurweb.db.get_sqlalchemy_url(), echo=(args.verbose >= 1)) diff --git a/migrations/env.py b/migrations/env.py index 23759123..dfe14804 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -20,6 +20,12 @@ target_metadata = aurweb.schema.metadata # ... etc. +# If configure_logger is either True or not specified, +# configure the logger via fileConfig. +if config.attributes.get("configure_logger", True): + logging.config.fileConfig(config.config_file_name) + + def run_migrations_offline(): """Run migrations in 'offline' mode. @@ -64,12 +70,7 @@ def run_migrations_online(): context.run_migrations() -if __name__ == "__main__": - # Interpret the config file for Python logging. - # This line sets up loggers basically. - logging.config.fileConfig(config.config_file_name) - - if context.is_offline_mode(): - run_migrations_offline() - else: - run_migrations_online() +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() From 72f755817c012a5c38255175522f32a059f976c0 Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Mon, 17 May 2021 19:12:05 +0100 Subject: [PATCH 093/844] Adds Alembic migration for DB/Tables conversion to utf8mb4 MySql defaults to `utf8` and case insensitive collation so migrate these to case sensitive and `utf8mb4` Closes #21 Signed-off-by: Leonidas Spyropoulos From 7b7c3abbe2dab35bf4a7bb67c7ab4121a0ee7566 Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Tue, 18 May 2021 13:04:20 +0100 Subject: [PATCH 094/844] Conditionally apply SSOAccountId migration to support misaligned databases Closes: #34 Signed-off-by: Leonidas Spyropoulos From ac31f520ea25d31f675dd4e2ac236078418c2f69 Mon Sep 17 00:00:00 2001 From: Kristian Klausen Date: Mon, 10 May 2021 22:34:19 +0200 Subject: [PATCH 095/844] Add coverage report for "Test Coverage Visualization"[1] [1] https://docs.gitlab.com/ee/user/project/merge_requests/test_coverage_visualization.html --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e287748..ca3055ad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,3 +24,7 @@ test: - AUR_CONFIG=conf/config python -m aurweb.initdb - make -C test - coverage report --include='aurweb/*' + - coverage xml --include='aurweb/*' + artifacts: + reports: + cobertura: coverage.xml From 64bc93926f87aa9a0a29b4f014af2374e527fc8a Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Tue, 18 May 2021 13:15:47 +0100 Subject: [PATCH 096/844] Add support for configuring database with port instead of socket Signed-off-by: Leonidas Spyropoulos --- .gitignore | 1 + aurweb/config.py | 4 ++++ aurweb/db.py | 11 ++++++++--- conf/config.defaults | 1 + conf/config.dev | 5 ++++- test/test_db.py | 24 +++++++++++++++++------- 6 files changed, 35 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 372fa105..35b571d7 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ fastapi_aw/ .vim/ .pylintrc .coverage +.idea diff --git a/aurweb/config.py b/aurweb/config.py index 49a2765a..2a6cfc3e 100644 --- a/aurweb/config.py +++ b/aurweb/config.py @@ -32,6 +32,10 @@ def rehash(): _get_parser() +def get_with_fallback(section, option, fallback): + return _get_parser().get(section, option, fallback=fallback) + + def get(section, option): return _get_parser().get(section, option) diff --git a/aurweb/db.py b/aurweb/db.py index f5530bcf..491ce9e2 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -27,15 +27,20 @@ def get_sqlalchemy_url(): aur_db_backend = aurweb.config.get('database', 'backend') if aur_db_backend == 'mysql': + if aurweb.config.get_with_fallback('database', 'port', fallback=None): + port = aurweb.config.get('database', 'port') + param_query = None + else: + port = None + param_query = {'unix_socket': aurweb.config.get('database', 'socket')} return constructor( 'mysql+mysqlconnector', username=aurweb.config.get('database', 'user'), password=aurweb.config.get('database', 'password'), host=aurweb.config.get('database', 'host'), database=aurweb.config.get('database', 'name'), - query={ - 'unix_socket': aurweb.config.get('database', 'socket'), - }, + port=port, + query=param_query ) elif aur_db_backend == 'sqlite': return constructor( diff --git a/conf/config.defaults b/conf/config.defaults index 98e033b7..c05648d5 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -2,6 +2,7 @@ backend = mysql host = localhost socket = /var/run/mysqld/mysqld.sock +;port = 3306 name = AUR user = aur password = aur diff --git a/conf/config.dev b/conf/config.dev index ccb01f4f..194a3bf8 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -9,11 +9,14 @@ backend = sqlite name = YOUR_AUR_ROOT/aurweb.sqlite3 -; Alternative MySQL configuration +; Alternative MySQL configuration (Use either port of socket, if both defined port takes priority) ;backend = mysql ;name = aurweb ;user = aur ;password = aur +;host = localhost +;port = 3306 +;socket = /var/run/mysqld/mysqld.sock [options] aurwebdir = YOUR_AUR_ROOT diff --git a/test/test_db.py b/test/test_db.py index f5902e4c..41936321 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -56,18 +56,28 @@ def test_sqlalchemy_mysql_url(): aurweb.config.rehash() -def make_temp_config(backend): +def test_sqlalchemy_mysql_port_url(): + tmpctx, tmp = make_temp_config("conf/config.defaults", ";port = 3306", "port = 3306") + + with tmpctx: + with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): + aurweb.config.rehash() + assert db.get_sqlalchemy_url() + aurweb.config.rehash() + + +def make_temp_config(config_file, src_str, replace_with): tmpdir = tempfile.TemporaryDirectory() tmp = os.path.join(tmpdir.name, "config.tmp") - with open("conf/config") as f: - config = re.sub(r'backend = sqlite', f'backend = {backend}', f.read()) + with open(config_file) as f: + config = re.sub(src_str, f'{replace_with}', f.read()) with open(tmp, "w") as o: o.write(config) - return (tmpdir, tmp) + return tmpdir, tmp def test_sqlalchemy_unknown_backend(): - tmpctx, tmp = make_temp_config("blah") + tmpctx, tmp = make_temp_config("conf/config", "backend = sqlite", "backend = blah") with tmpctx: with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): @@ -93,7 +103,7 @@ def test_connection_class_without_fail(): def test_connection_class_unsupported_backend(): - tmpctx, tmp = make_temp_config("blah") + tmpctx, tmp = make_temp_config("conf/config", "backend = sqlite", "backend = blah") with tmpctx: with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): @@ -106,7 +116,7 @@ def test_connection_class_unsupported_backend(): @mock.patch("mysql.connector.connect", mock.MagicMock(return_value=True)) @mock.patch.object(mysql.connector, "paramstyle", "qmark") def test_connection_mysql(): - tmpctx, tmp = make_temp_config("mysql") + tmpctx, tmp = make_temp_config("conf/config", "backend = sqlite", "backend = mysql") with tmpctx: with mock.patch.dict(os.environ, { "AUR_CONFIG": tmp, From 5185df629ee7d2190fac7f0268935e3f4477d114 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 24 Dec 2020 20:48:35 -0800 Subject: [PATCH 097/844] move aurweb.testing to its own package + Added aurweb.testing.setup_test_db(*tables) + Added aurweb.testing.models.make_user(**kwargs) + Added aurweb.testing.models.make_session(**kwargs) + Added aurweb.testing.requests.Client + Added aurweb.testing.requests.Request * Updated test_l10n.py to use our new Request Signed-off-by: Kevin Morris --- aurweb/{testing.py => testing/__init__.py} | 4 ++-- aurweb/testing/models.py | 25 ++++++++++++++++++++++ aurweb/testing/requests.py | 8 +++++++ test/test_l10n.py | 18 ++++++---------- 4 files changed, 41 insertions(+), 14 deletions(-) rename aurweb/{testing.py => testing/__init__.py} (93%) create mode 100644 aurweb/testing/models.py create mode 100644 aurweb/testing/requests.py diff --git a/aurweb/testing.py b/aurweb/testing/__init__.py similarity index 93% rename from aurweb/testing.py rename to aurweb/testing/__init__.py index 7516d918..0a807b40 100644 --- a/aurweb/testing.py +++ b/aurweb/testing/__init__.py @@ -1,4 +1,4 @@ -from aurweb.db import get_engine +import aurweb.db def setup_test_db(*args): @@ -21,7 +21,7 @@ def setup_test_db(*args): test_tables = ["Users", "Sessions"]; setup_test_db(*test_tables) """ - engine = get_engine() + engine = aurweb.db.get_engine() conn = engine.connect() tables = list(args) diff --git a/aurweb/testing/models.py b/aurweb/testing/models.py new file mode 100644 index 00000000..8a27c409 --- /dev/null +++ b/aurweb/testing/models.py @@ -0,0 +1,25 @@ +import warnings + +from sqlalchemy import exc + +import aurweb.db + + +def make_user(**kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", exc.SAWarning) + from aurweb.models.user import User + user = User(**kwargs) + aurweb.db.session.add(user) + aurweb.db.session.commit() + return user + + +def make_session(**kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", exc.SAWarning) + from aurweb.models.session import Session + session = Session(**kwargs) + aurweb.db.session.add(session) + aurweb.db.session.commit() + return session diff --git a/aurweb/testing/requests.py b/aurweb/testing/requests.py new file mode 100644 index 00000000..2839c93f --- /dev/null +++ b/aurweb/testing/requests.py @@ -0,0 +1,8 @@ +class Client: + host = "127.0.0.1" + + +class Request: + client = Client() + cookies = dict() + headers = dict() diff --git a/test/test_l10n.py b/test/test_l10n.py index 1a1ef3e6..e833cd44 100644 --- a/test/test_l10n.py +++ b/test/test_l10n.py @@ -1,13 +1,6 @@ """ Test our l10n module. """ from aurweb import l10n - - -class FakeRequest: - """ A fake Request doppleganger; use this to change request.cookies - easily and with no side-effects. """ - - def __init__(self, *args, **kwargs): - self.cookies = kwargs.pop("cookies", dict()) +from aurweb.testing.requests import Request def test_translator(): @@ -18,7 +11,7 @@ def test_translator(): def test_get_request_language(): """ First, tests default_lang, then tests a modified AURLANG cookie. """ - request = FakeRequest() + request = Request() assert l10n.get_request_language(request) == "en" request.cookies["AURLANG"] = "de" @@ -28,8 +21,8 @@ def test_get_request_language(): def test_get_raw_translator_for_request(): """ Make sure that get_raw_translator_for_request is giving us the translator we expect. """ - request = FakeRequest(cookies={"AURLANG": "de"}) - + request = Request() + request.cookies["AURLANG"] = "de" translator = l10n.get_raw_translator_for_request(request) assert translator.gettext("Home") == \ l10n.translator.translate("Home", "de") @@ -38,7 +31,8 @@ def test_get_raw_translator_for_request(): def test_get_translator_for_request(): """ Make sure that get_translator_for_request is giving us back our expected translation function. """ - request = FakeRequest(cookies={"AURLANG": "de"}) + request = Request() + request.cookies["AURLANG"] = "de" translate = l10n.get_translator_for_request(request) assert translate("Home") == "Startseite" From a836892cde9a8f89fb7cb9e159bc8d4711f88439 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 14 Jan 2021 21:06:41 -0800 Subject: [PATCH 098/844] aurweb.db: add query, create, delete helpers Takes sqlalchemy kwargs or stanzas: query(Model, Model.Column == value) query(Model, and_(Model.Column == value, Model.Column != "BAD!")) Updated tests to reflect the new utility and a comment about upcoming function deprecation is added to get_account_type(). From here on, phase out the use of get_account_type(). + aurweb.db: Added create utility function + aurweb.db: Added delete utility function The `delete` function can be used to delete a record by search kwargs directly. Example: delete(User, User.ID == 6) All three functions added in this commit are typically useful to perform these operations without having to import aurweb.db.session. Removes a bit of redundancy overall. Signed-off-by: Kevin Morris --- aurweb/db.py | 18 ++++++++++++++++++ test/test_account_type.py | 40 ++++++++++++++++++++++++++++++++++----- test/test_db.py | 13 ++++++++++++- test/test_routes.py | 18 ++++++++++++++++-- test/test_user.py | 4 +++- 5 files changed, 84 insertions(+), 9 deletions(-) diff --git a/aurweb/db.py b/aurweb/db.py index 491ce9e2..9ca51de2 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -11,6 +11,24 @@ Session = None session = None +def query(model, *args, **kwargs): + return session.query(model).filter(*args, **kwargs) + + +def create(model, *args, **kwargs): + instance = model(*args, **kwargs) + session.add(instance) + session.commit() + return instance + + +def delete(model, *args, **kwargs): + instance = session.query(model).filter(*args, **kwargs) + for record in instance: + session.delete(record) + session.commit() + + def get_sqlalchemy_url(): """ Build an SQLAlchemy for use with create_engine based on the aurweb configuration. diff --git a/test/test_account_type.py b/test/test_account_type.py index b6a12363..9419970c 100644 --- a/test/test_account_type.py +++ b/test/test_account_type.py @@ -1,20 +1,34 @@ import pytest from aurweb.models.account_type import AccountType +from aurweb.models.user import User from aurweb.testing import setup_test_db +from aurweb.testing.models import make_user + +account_type = None @pytest.fixture(autouse=True) def setup(): - setup_test_db() + setup_test_db("Users") + + from aurweb.db import session + + global account_type + + account_type = AccountType(AccountType="TestUser") + session.add(account_type) + session.commit() + + yield account_type + + session.delete(account_type) + session.commit() def test_account_type(): """ Test creating an AccountType, and reading its columns. """ from aurweb.db import session - account_type = AccountType(AccountType="TestUser") - session.add(account_type) - session.commit() # Make sure it got created and was given an ID. assert bool(account_type.ID) @@ -25,4 +39,20 @@ def test_account_type(): "" % ( account_type.ID) - session.delete(account_type) + record = session.query(AccountType).filter( + AccountType.AccountType == "TestUser").first() + assert account_type == record + + +def test_user_account_type_relationship(): + from aurweb.db import session + + user = make_user(Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + + assert user.AccountType == account_type + assert account_type.users.filter(User.ID == user.ID).first() + + session.delete(user) + session.commit() diff --git a/test/test_db.py b/test/test_db.py index 41936321..1eb0dc28 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -3,6 +3,7 @@ import re import sqlite3 import tempfile +from datetime import datetime from unittest import mock import mysql.connector @@ -11,6 +12,7 @@ import pytest import aurweb.config from aurweb import db +from aurweb.models.account_type import AccountType from aurweb.testing import setup_test_db @@ -39,7 +41,7 @@ class DBConnection: @pytest.fixture(autouse=True) def setup_db(): - setup_test_db() + setup_test_db("Bans") def test_sqlalchemy_sqlite_url(): @@ -174,3 +176,12 @@ def test_connection_execute_paramstyle_unsupported(): "SELECT * FROM AccountTypes WHERE AccountType = ?", ["User"] ).fetchall() + + +def test_create_delete(): + db.create(AccountType, AccountType="test") + record = db.query(AccountType, AccountType.AccountType == "test").first() + assert record is not None + db.delete(AccountType, AccountType.AccountType == "test") + record = db.query(AccountType, AccountType.AccountType == "test").first() + assert record is None diff --git a/test/test_routes.py b/test/test_routes.py index 86221108..950d9b71 100644 --- a/test/test_routes.py +++ b/test/test_routes.py @@ -7,14 +7,26 @@ import pytest from fastapi.testclient import TestClient from aurweb.asgi import app +from aurweb.db import query +from aurweb.models.account_type import AccountType from aurweb.testing import setup_test_db client = TestClient(app) +user = None + @pytest.fixture def setup(): - setup_test_db("Users", "Session") + global user + + setup_test_db("Users", "Sessions") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = make_user(Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) def test_index(): @@ -54,6 +66,7 @@ def test_language_invalid_next(): response = req.post("/language", data=post_data) assert response.status_code == int(HTTPStatus.BAD_REQUEST) + def test_language_query_params(): """ Test the language post route with query params. """ next = urllib.parse.quote_plus("/") @@ -73,4 +86,5 @@ def test_error_messages(): response1 = client.get("/thisroutedoesnotexist") response2 = client.get("/raisefivethree") assert response1.status_code == int(HTTPStatus.NOT_FOUND) - assert response2.status_code == int(HTTPStatus.SERVICE_UNAVAILABLE) \ No newline at end of file + assert response2.status_code == int(HTTPStatus.SERVICE_UNAVAILABLE) + diff --git a/test/test_user.py b/test/test_user.py index 8ac9b00b..5a56a035 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -1,5 +1,8 @@ import pytest +import aurweb.config + +from aurweb.db import query from aurweb.models.account_type import AccountType from aurweb.models.user import User from aurweb.testing import setup_test_db @@ -26,7 +29,6 @@ def test_user(): Salt="efgh", ResetKey="blahblah") session.add(user) session.commit() - assert user in account_type.users # Make sure the user was created and given an ID. From adc9fccb7d0e984cd780cd1a785911e36a6316b1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 26 Dec 2020 19:29:19 -0800 Subject: [PATCH 099/844] add aurweb.models.ban.Ban ORM mapping Signed-off-by: Kevin Morris --- aurweb/models/ban.py | 19 +++++++++++++ test/test_ban.py | 63 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 aurweb/models/ban.py create mode 100644 test/test_ban.py diff --git a/aurweb/models/ban.py b/aurweb/models/ban.py new file mode 100644 index 00000000..be030380 --- /dev/null +++ b/aurweb/models/ban.py @@ -0,0 +1,19 @@ +from fastapi import Request +from sqlalchemy.orm import mapper + +from aurweb.schema import Bans + + +class Ban: + def __init__(self, **kwargs): + self.IPAddress = kwargs.get("IPAddress") + self.BanTS = kwargs.get("BanTS") + + +def is_banned(request: Request): + from aurweb.db import session + ip = request.client.host + return session.query(Ban).filter(Ban.IPAddress == ip).first() is not None + + +mapper(Ban, Bans) diff --git a/test/test_ban.py b/test/test_ban.py new file mode 100644 index 00000000..de4f5b1b --- /dev/null +++ b/test/test_ban.py @@ -0,0 +1,63 @@ +import warnings + +from datetime import datetime, timedelta + +import pytest + +from sqlalchemy import exc as sa_exc + +from aurweb.models.ban import Ban, is_banned +from aurweb.testing import setup_test_db +from aurweb.testing.requests import Request + +ban = None + +request = Request() + + +@pytest.fixture(autouse=True) +def setup(): + from aurweb.db import session + + global ban + + setup_test_db("Bans") + + ban = Ban(IPAddress="127.0.0.1", + BanTS=datetime.utcnow() + timedelta(seconds=30)) + session.add(ban) + session.commit() + + +def test_ban(): + assert ban.IPAddress == "127.0.0.1" + assert bool(ban.BanTS) + + +def test_invalid_ban(): + from aurweb.db import session + + with pytest.raises(sa_exc.IntegrityError, + match="NOT NULL constraint failed: Bans.IPAddress"): + bad_ban = Ban(BanTS=datetime.utcnow()) + session.add(bad_ban) + + # We're adding a ban with no primary key; this causes an + # SQLAlchemy warnings when committing to the DB. + # Ignore them. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", sa_exc.SAWarning) + session.commit() + + # Since we got a transaction failure, we need to rollback. + session.rollback() + + +def test_banned(): + request.client.host = "127.0.0.1" + assert is_banned(request) + + +def test_not_banned(): + request.client.host = "192.168.0.1" + assert not is_banned(request) From 1922e5380d819501b1ee3f9b50ff69bc583dbf6c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 25 Dec 2020 20:55:43 -0800 Subject: [PATCH 100/844] add aurweb.models.session.Session ORM database object + Added aurweb.util module. - Added make_random_string function. + Added aurweb.db.make_random_value function. - Takes a model and a column and introspects them to figure out the proper column length to create a random string for; then creates a unique string for that column. Signed-off-by: Kevin Morris --- aurweb/db.py | 42 +++++++++++++++++++++++++++++- aurweb/models/session.py | 25 ++++++++++++++++++ aurweb/util.py | 7 +++++ test/test_session.py | 56 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 aurweb/models/session.py create mode 100644 aurweb/util.py create mode 100644 test/test_session.py diff --git a/aurweb/db.py b/aurweb/db.py index 9ca51de2..3f5731a9 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -1,8 +1,10 @@ import math import aurweb.config +import aurweb.util -engine = None # See get_engine +# See get_engine. +engine = None # ORM Session class. Session = None @@ -10,6 +12,44 @@ Session = None # Global ORM Session object. session = None +# Global introspected object memo. +introspected = dict() + + +def make_random_value(table: str, column: str): + """ Generate a unique, random value for a string column in a table. + + This can be used to generate for example, session IDs that + align with the properties of the database column with regards + to size. + + Internally, we use SQLAlchemy introspection to look at column + to decide which length to use for random string generation. + + :return: A unique string that is not in the database + """ + global introspected + + # Make sure column is converted to a string for memo interaction. + scolumn = str(column) + + # If the target column is not yet introspected, store its introspection + # object into our global `introspected` memo. + if scolumn not in introspected: + from sqlalchemy import inspect + target_column = scolumn.split('.')[-1] + col = list(filter(lambda c: c.name == target_column, + inspect(table).columns))[0] + introspected[scolumn] = col + + col = introspected.get(scolumn) + length = col.type.length + + string = aurweb.util.make_random_string(length) + while session.query(table).filter(column == string).first(): + string = aurweb.util.make_random_string(length) + return string + def query(model, *args, **kwargs): return session.query(model).filter(*args, **kwargs) diff --git a/aurweb/models/session.py b/aurweb/models/session.py new file mode 100644 index 00000000..60749303 --- /dev/null +++ b/aurweb/models/session.py @@ -0,0 +1,25 @@ +from sqlalchemy import Column, Integer +from sqlalchemy.orm import backref, mapper, relationship + +from aurweb.db import make_random_value +from aurweb.models.user import User +from aurweb.schema import Sessions + + +class Session: + UsersID = Column(Integer, nullable=True) + + def __init__(self, **kwargs): + self.UsersID = kwargs.get("UsersID") + self.SessionID = kwargs.get("SessionID") + self.LastUpdateTS = kwargs.get("LastUpdateTS") + + +mapper(Session, Sessions, primary_key=[Sessions.c.SessionID], properties={ + "User": relationship(User, backref=backref("session", + uselist=False)) +}) + + +def generate_unique_sid(): + return make_random_value(Session, Session.SessionID) diff --git a/aurweb/util.py b/aurweb/util.py new file mode 100644 index 00000000..65f18a4c --- /dev/null +++ b/aurweb/util.py @@ -0,0 +1,7 @@ +import random +import string + + +def make_random_string(length): + return ''.join(random.choices(string.ascii_lowercase + + string.digits, k=length)) diff --git a/test/test_session.py b/test/test_session.py new file mode 100644 index 00000000..560f628c --- /dev/null +++ b/test/test_session.py @@ -0,0 +1,56 @@ +""" Test our Session model. """ +from datetime import datetime +from unittest import mock + +import pytest + +from aurweb.models.account_type import AccountType +from aurweb.models.session import generate_unique_sid +from aurweb.testing import setup_test_db +from aurweb.testing.models import make_session, make_user + +user, _session = None, None + + +@pytest.fixture(autouse=True) +def setup(): + from aurweb.db import session + + global user, _session + + setup_test_db("Users", "Sessions") + + account_type = session.query(AccountType).filter( + AccountType.AccountType == "User").first() + user = make_user(Username="test", Email="test@example.org", + ResetKey="testReset", Passwd="testPassword", + AccountType=account_type) + _session = make_session(UsersID=user.ID, SessionID="testSession", + LastUpdateTS=datetime.utcnow()) + + +def test_session(): + assert _session.SessionID == "testSession" + assert _session.UsersID == user.ID + + +def test_session_user_association(): + # Make sure that the Session user attribute is correct. + assert _session.User == user + + +def test_generate_unique_sid(): + # Mock up aurweb.models.session.generate_sid by returning + # sids[i % 2] from 0 .. n. This will swap between each sid + # between each call. + sids = ["testSession", "realSession"] + i = 0 + + def mock_generate_sid(length): + nonlocal i + sid = sids[i % 2] + i += 1 + return sid + + with mock.patch("aurweb.util.make_random_string", mock_generate_sid): + assert generate_unique_sid() == "realSession" From 137c050f99e29f3d039c42f3b693dd9ef7ed4bd1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 30 Jan 2021 16:43:35 -0800 Subject: [PATCH 101/844] add python-bcrypt dependency Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 2 +- Dockerfile | 2 +- INSTALL | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ca3055ad..4ad97393 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,7 @@ before_script: python-pytest-tap python-fastapi hypercorn nginx python-authlib python-itsdangerous python-httpx python-jinja python-pytest-cov python-requests python-aiofiles python-python-multipart - python-pytest-asyncio python-coverage + python-pytest-asyncio python-coverage python-bcrypt test: script: diff --git a/Dockerfile b/Dockerfile index 7e981340..6638f9a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN pacman -Syu --noconfirm base-devel git gpgme protobuf pyalpm \ python-werkzeug python-pytest-tap python-fastapi nginx python-authlib \ python-itsdangerous python-httpx python-jinja python-pytest-cov \ python-requests python-aiofiles python-python-multipart \ - python-pytest-asyncio python-coverage hypercorn + python-pytest-asyncio python-coverage hypercorn python-bcrypt # Remove aurweb.sqlite3 if it was copied over via COPY. RUN rm -fv aurweb.sqlite3 diff --git a/INSTALL b/INSTALL index e4c52480..6c43fec8 100644 --- a/INSTALL +++ b/INSTALL @@ -51,7 +51,7 @@ read the instructions below. python-bleach python-markdown python-alembic hypercorn \ python-itsdangerous python-authlib python-httpx \ python-jinja python-aiofiles python-python-multipart \ - python-requests + python-requests hypercorn python-bcrypt # python3 setup.py install 5) Create a new MySQL database and a user and import the aurweb SQL schema: From 56f2798279f3cbde46389aa65a27fb58bfb0bcfc Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 25 Dec 2020 20:54:53 -0800 Subject: [PATCH 102/844] add aurweb.auth and authentication to User + Added aurweb.auth.AnonymousUser * An instance of this model is returned as the request user when the request is not authenticated + Added aurweb.auth.BasicAuthBackend + Add starlette's AuthenticationMiddleware to app middleware, which uses our BasicAuthBackend facility + Added User.is_authenticated() + Added User.authenticate(password) + Added User.login(request, password) + Added User.logout(request) + Added repr(User(...)) representation + Added aurweb.auth.auth_required decorator. This change uses the same AURSID logic in the PHP implementation. Additionally, introduce a few helpers for authentication, one of which being `User.update_password(password, rounds = 12)` where `rounds` is a configurable number of salt rounds. Signed-off-by: Kevin Morris --- aurweb/asgi.py | 8 +++ aurweb/auth.py | 77 +++++++++++++++++++++++ aurweb/models/user.py | 125 ++++++++++++++++++++++++++++++++++++- test/test_auth.py | 80 ++++++++++++++++++++++++ test/test_user.py | 142 +++++++++++++++++++++++++++++++++++++----- 5 files changed, 412 insertions(+), 20 deletions(-) create mode 100644 aurweb/auth.py create mode 100644 test/test_auth.py diff --git a/aurweb/asgi.py b/aurweb/asgi.py index c03a00f7..4d21ad03 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -1,12 +1,15 @@ import http +import os from fastapi import FastAPI, HTTPException from fastapi.responses import HTMLResponse from fastapi.staticfiles import StaticFiles +from starlette.middleware.authentication import AuthenticationMiddleware from starlette.middleware.sessions import SessionMiddleware import aurweb.config +from aurweb.auth import BasicAuthBackend from aurweb.db import get_engine from aurweb.routers import html, sso, errors @@ -32,10 +35,15 @@ async def app_startup(): StaticFiles(directory="web/html/images"), name="static_images") + # Add application middlewares. + app.add_middleware(AuthenticationMiddleware, backend=BasicAuthBackend()) app.add_middleware(SessionMiddleware, secret_key=session_secret) + + # Add application routes. app.include_router(sso.router) app.include_router(html.router) + # Initialize the database engine and ORM. get_engine() # NOTE: Always keep this dictionary updated with all routes diff --git a/aurweb/auth.py b/aurweb/auth.py new file mode 100644 index 00000000..8608a82a --- /dev/null +++ b/aurweb/auth.py @@ -0,0 +1,77 @@ +import functools + +from datetime import datetime +from http import HTTPStatus + +from fastapi.responses import RedirectResponse +from starlette.authentication import AuthCredentials, AuthenticationBackend, AuthenticationError +from starlette.requests import HTTPConnection + +from aurweb.models.session import Session +from aurweb.models.user import User +from aurweb.templates import make_context, render_template + + +class AnonymousUser: + @staticmethod + def is_authenticated(): + return False + + +class BasicAuthBackend(AuthenticationBackend): + async def authenticate(self, conn: HTTPConnection): + from aurweb.db import session + + sid = conn.cookies.get("AURSID") + if not sid: + return None, AnonymousUser() + + now_ts = datetime.utcnow().timestamp() + record = session.query(Session).filter( + Session.SessionID == sid, Session.LastUpdateTS >= now_ts).first() + if not record: + return None, AnonymousUser() + + user = session.query(User).filter(User.ID == record.UsersID).first() + if not user: + raise AuthenticationError(f"Invalid User ID: {record.UsersID}") + + user.authenticated = True + return AuthCredentials(["authenticated"]), user + + +def auth_required(is_required: bool = True, + redirect: str = "/", + template: tuple = None): + """ Authentication route decorator. + + If redirect is given, the user will be redirected if the auth state + does not match is_required. + + If template is given, it will be rendered with Unauthorized if + is_required does not match and take priority over redirect. + + :param is_required: A boolean indicating whether the function requires auth + :param redirect: Path to redirect to if is_required isn't True + :param template: A template tuple: ("template.html", "Template Page") + """ + + def decorator(func): + @functools.wraps(func) + async def wrapper(request, *args, **kwargs): + if request.user.is_authenticated() != is_required: + status_code = int(HTTPStatus.UNAUTHORIZED) + url = "/" + if redirect: + status_code = int(HTTPStatus.SEE_OTHER) + url = redirect + if template: + path, title = template + context = make_context(request, title) + return render_template(request, path, context, + status_code=int(HTTPStatus.UNAUTHORIZED)) + return RedirectResponse(url=url, status_code=status_code) + return await func(request, *args, **kwargs) + return wrapper + + return decorator diff --git a/aurweb/models/user.py b/aurweb/models/user.py index ba91c439..aff4ce6b 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -1,13 +1,25 @@ +import hashlib + +from datetime import datetime + +import bcrypt + +from fastapi import Request from sqlalchemy.orm import backref, mapper, relationship +import aurweb.config + from aurweb.models.account_type import AccountType +from aurweb.models.ban import is_banned from aurweb.schema import Users class User: """ An ORM model of a single Users record. """ + authenticated = False def __init__(self, **kwargs): + # Set AccountTypeID if it was passed. self.AccountTypeID = kwargs.get("AccountTypeID") account_type = kwargs.get("AccountType") @@ -15,22 +27,129 @@ class User: self.AccountType = account_type self.Username = kwargs.get("Username") + + self.ResetKey = kwargs.get("ResetKey") self.Email = kwargs.get("Email") self.BackupEmail = kwargs.get("BackupEmail") - self.Passwd = kwargs.get("Passwd") - self.Salt = kwargs.get("Salt") self.RealName = kwargs.get("RealName") self.LangPreference = kwargs.get("LangPreference") self.Timezone = kwargs.get("Timezone") self.Homepage = kwargs.get("Homepage") self.IRCNick = kwargs.get("IRCNick") self.PGPKey = kwargs.get("PGPKey") - self.RegistrationTS = kwargs.get("RegistrationTS") + self.RegistrationTS = datetime.utcnow() self.CommentNotify = kwargs.get("CommentNotify") self.UpdateNotify = kwargs.get("UpdateNotify") self.OwnershipNotify = kwargs.get("OwnershipNotify") self.SSOAccountID = kwargs.get("SSOAccountID") + self.Salt = None + self.Passwd = str() + + passwd = kwargs.get("Passwd") + if passwd: + self.update_password(passwd) + + def update_password(self, password, salt_rounds=12): + from aurweb.db import session + self.Passwd = bcrypt.hashpw( + password.encode(), + bcrypt.gensalt(rounds=salt_rounds)).decode() + session.commit() + + @staticmethod + def minimum_passwd_length(): + return aurweb.config.getint("options", "passwd_min_len") + + def is_authenticated(self): + """ Return internal authenticated state. """ + return self.authenticated + + def valid_password(self, password: str): + """ Check authentication against a given password. """ + from aurweb.db import session + + if password is None: + return False + + password_is_valid = False + + try: + password_is_valid = bcrypt.checkpw(password.encode(), + self.Passwd.encode()) + except ValueError: + pass + + # If our Salt column is not empty, we're using a legacy password. + if not password_is_valid and self.Salt != str(): + # Try to login with legacy method. + password_is_valid = hashlib.md5( + f"{self.Salt}{password}".encode() + ).hexdigest() == self.Passwd + + # We got here, we passed the legacy authentication. + # Update the password to our modern hash style. + if password_is_valid: + self.update_password(password) + + return password_is_valid + + def _login_approved(self, request: Request): + return not is_banned(request) and not self.Suspended + + def login(self, request: Request, password: str, session_time=0): + """ Login and authenticate a request. """ + + from aurweb.db import session + from aurweb.models.session import Session, generate_unique_sid + + if not self._login_approved(request): + return None + + self.authenticated = self.valid_password(password) + if not self.authenticated: + return None + + self.LastLogin = now_ts = datetime.utcnow().timestamp() + self.LastLoginIPAddress = request.client.host + session.commit() + + session_ts = now_ts + ( + session_time if session_time + else aurweb.config.getint("options", "login_timeout") + ) + + sid = None + + if not self.session: + sid = generate_unique_sid() + self.session = Session(UsersID=self.ID, SessionID=sid, + LastUpdateTS=session_ts) + session.add(self.session) + else: + last_updated = self.session.LastUpdateTS + if last_updated and last_updated < now_ts: + self.session.SessionID = sid = generate_unique_sid() + else: + # Session is still valid; retrieve the current SID. + sid = self.session.SessionID + + self.session.LastUpdateTS = session_ts + + session.commit() + + request.cookies["AURSID"] = self.session.SessionID + return self.session.SessionID + + def logout(self, request): + from aurweb.db import session + + del request.cookies["AURSID"] + self.authenticated = False + if self.session: + session.delete(self.session) + session.commit() + def __repr__(self): return "" % ( self.ID, str(self.AccountType), self.Username) diff --git a/test/test_auth.py b/test/test_auth.py new file mode 100644 index 00000000..d2251de4 --- /dev/null +++ b/test/test_auth.py @@ -0,0 +1,80 @@ +from datetime import datetime + +import pytest + +from starlette.authentication import AuthenticationError + +from aurweb.db import query +from aurweb.auth import BasicAuthBackend +from aurweb.models.account_type import AccountType +from aurweb.testing import setup_test_db +from aurweb.testing.models import make_session, make_user +from aurweb.testing.requests import Request + +# Persistent user object, initialized in our setup fixture. +user = None +backend = None +request = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, backend, request + + setup_test_db("Users", "Sessions") + + from aurweb.db import session + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = make_user(Username="test", Email="test@example.com", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + + session.add(user) + session.commit() + + backend = BasicAuthBackend() + request = Request() + + +@pytest.mark.asyncio +async def test_auth_backend_missing_sid(): + # The request has no AURSID cookie, so authentication fails, and + # AnonymousUser is returned. + _, result = await backend.authenticate(request) + assert not result.is_authenticated() + + +@pytest.mark.asyncio +async def test_auth_backend_invalid_sid(): + # Provide a fake AURSID that won't be found in the database. + # This results in our path going down the invalid sid route, + # which gives us an AnonymousUser. + request.cookies["AURSID"] = "fake" + _, result = await backend.authenticate(request) + assert not result.is_authenticated() + + +@pytest.mark.asyncio +async def test_auth_backend_invalid_user_id(): + # Create a new session with a fake user id. + now_ts = datetime.utcnow().timestamp() + make_session(UsersID=666, SessionID="realSession", + LastUpdateTS=now_ts + 5) + + # Here, we specify a real SID; but it's user is not there. + request.cookies["AURSID"] = "realSession" + with pytest.raises(AuthenticationError, match="Invalid User ID: 666"): + await backend.authenticate(request) + + +@pytest.mark.asyncio +async def test_basic_auth_backend(): + # This time, everything matches up. We expect the user to + # equal the real_user. + now_ts = datetime.utcnow().timestamp() + make_session(UsersID=user.ID, SessionID="realSession", + LastUpdateTS=now_ts + 5) + _, result = await backend.authenticate(request) + assert result == user diff --git a/test/test_user.py b/test/test_user.py index 5a56a035..b8d4248a 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -1,48 +1,86 @@ +import hashlib + +from datetime import datetime, timedelta + +import bcrypt import pytest +import aurweb.auth import aurweb.config from aurweb.db import query from aurweb.models.account_type import AccountType +from aurweb.models.ban import Ban +from aurweb.models.session import Session from aurweb.models.user import User from aurweb.testing import setup_test_db +from aurweb.testing.models import make_session, make_user +from aurweb.testing.requests import Request + +account_type, user = None, None @pytest.fixture(autouse=True) def setup(): - setup_test_db("Users") - - -def test_user(): - """ Test creating a user and reading its columns. """ from aurweb.db import session - # First, grab our target AccountType. + global account_type, user + + setup_test_db("Users", "Sessions", "Bans") + account_type = session.query(AccountType).filter( AccountType.AccountType == "User").first() - user = User( - AccountType=account_type, - RealName="Test User", Username="test", - Email="test@example.org", Passwd="abcd", - IRCNick="tester", - Salt="efgh", ResetKey="blahblah") - session.add(user) - session.commit() + user = make_user(Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + + +def test_user_login_logout(): + """ Test creating a user and reading its columns. """ + from aurweb.db import session + + # Assert that make_user created a valid user. + assert bool(user.ID) + + # Test authentication. + assert user.valid_password("testPassword") + assert not user.valid_password("badPassword") + assert user in account_type.users - # Make sure the user was created and given an ID. - assert bool(user.ID) + # Make a raw request. + request = Request() + assert not user.login(request, "badPassword") + assert not user.is_authenticated() + + sid = user.login(request, "testPassword") + assert sid is not None + assert user.is_authenticated() + assert "AURSID" in request.cookies + + # Expect that User session relationships work right. + user_session = session.query(Session).filter( + Session.UsersID == user.ID).first() + assert user_session == user.session + assert user.session.SessionID == sid + assert user.session.User == user # Search for the user via query API. result = session.query(User).filter(User.ID == user.ID).first() # Compare the result and our original user. + assert result == user assert result.ID == user.ID assert result.AccountType.ID == user.AccountType.ID assert result.Username == user.Username assert result.Email == user.Email + # Test result authenticate methods to ensure they work the same. + assert not result.valid_password("badPassword") + assert result.valid_password("testPassword") + assert result.is_authenticated() + # Ensure we've got the correct account type. assert user.AccountType.ID == account_type.ID assert user.AccountType.AccountType == account_type.AccountType @@ -51,4 +89,74 @@ def test_user(): assert repr(user) == f"" - session.delete(user) + # Test logout. + user.logout(request) + assert "AURSID" not in request.cookies + assert not user.is_authenticated() + + +def test_user_login_twice(): + request = Request() + assert user.login(request, "testPassword") + assert user.login(request, "testPassword") + + +def test_user_login_banned(): + from aurweb.db import session + + # Add ban for the next 30 seconds. + banned_timestamp = datetime.utcnow() + timedelta(seconds=30) + ban = Ban(IPAddress="127.0.0.1", BanTS=banned_timestamp) + session.add(ban) + session.commit() + + request = Request() + request.client.host = "127.0.0.1" + assert not user.login(request, "testPassword") + + +def test_user_login_suspended(): + from aurweb.db import session + user.Suspended = True + session.commit() + assert not user.login(Request(), "testPassword") + + +def test_legacy_user_authentication(): + from aurweb.db import session + + user.Salt = bcrypt.gensalt().decode() + user.Passwd = hashlib.md5(f"{user.Salt}testPassword".encode()).hexdigest() + session.commit() + + assert not user.valid_password("badPassword") + assert user.valid_password("testPassword") + + # Test by passing a password of None value in. + assert not user.valid_password(None) + + +def test_user_login_with_outdated_sid(): + from aurweb.db import session + + # Make a session with a LastUpdateTS 5 seconds ago, causing + # user.login to update it with a new sid. + _session = make_session(UsersID=user.ID, SessionID="stub", + LastUpdateTS=datetime.utcnow().timestamp() - 5) + sid = user.login(Request(), "testPassword") + assert sid and user.is_authenticated() + assert sid != "stub" + + session.delete(_session) + session.commit() + + +def test_user_update_password(): + user.update_password("secondPassword") + assert not user.valid_password("testPassword") + assert user.valid_password("secondPassword") + + +def test_user_minimum_passwd_length(): + passwd_min_len = aurweb.config.getint("options", "passwd_min_len") + assert User.minimum_passwd_length() == passwd_min_len From 5d4a5deddf59806a691cda8d6933c7049b84db53 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 31 Dec 2020 20:44:59 -0800 Subject: [PATCH 103/844] implement login + logout routes and templates + Added route: GET `/login` via `aurweb.routers.auth.login_get` + Added route: POST `/login` via `aurweb.routers.auth.login_post` + Added route: GET `/logout` via `aurweb.routers.auth.logout` + Added route: POST `/logout` via `aurweb.routers.auth.logout_post` * Modify archdev-navbar.html template to toggle displays on auth state + Added login.html template Signed-off-by: Kevin Morris --- aurweb/asgi.py | 3 +- aurweb/routers/auth.py | 85 +++++++++++++++++ templates/login.html | 84 +++++++++++++++++ templates/partials/archdev-navbar.html | 18 +++- test/test_auth_routes.py | 126 +++++++++++++++++++++++++ 5 files changed, 313 insertions(+), 3 deletions(-) create mode 100644 aurweb/routers/auth.py create mode 100644 templates/login.html create mode 100644 test/test_auth_routes.py diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 4d21ad03..b15e5874 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -11,7 +11,7 @@ import aurweb.config from aurweb.auth import BasicAuthBackend from aurweb.db import get_engine -from aurweb.routers import html, sso, errors +from aurweb.routers import auth, html, sso, errors routes = set() @@ -42,6 +42,7 @@ async def app_startup(): # Add application routes. app.include_router(sso.router) app.include_router(html.router) + app.include_router(auth.router) # Initialize the database engine and ORM. get_engine() diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py new file mode 100644 index 00000000..24f5d4e3 --- /dev/null +++ b/aurweb/routers/auth.py @@ -0,0 +1,85 @@ +from datetime import datetime +from http import HTTPStatus + +from fastapi import APIRouter, Form, Request +from fastapi.responses import HTMLResponse, RedirectResponse + +import aurweb.config + +from aurweb.models.user import User +from aurweb.templates import make_context, render_template + +router = APIRouter() + + +def login_template(request: Request, next: str, errors: list = None): + """ Provide login-specific template context to render_template. """ + context = make_context(request, "Login", next) + context["errors"] = errors + context["url_base"] = f"{request.url.scheme}://{request.url.netloc}" + return render_template("login.html", context) + + +@router.get("/login", response_class=HTMLResponse) +async def login_get(request: Request, next: str = "/"): + """ Homepage route. """ + return login_template(request, next) + + +@router.post("/login", response_class=HTMLResponse) +async def login_post(request: Request, + next: str = Form(...), + user: str = Form(default=str()), + passwd: str = Form(default=str()), + remember_me: bool = Form(default=False)): + from aurweb.db import session + + user = session.query(User).filter(User.Username == user).first() + if not user: + return login_template(request, next, + errors=["Bad username or password."]) + + cookie_timeout = 0 + + if remember_me: + cookie_timeout = aurweb.config.getint( + "options", "persistent_cookie_timeout") + + _, sid = user.login(request, passwd, cookie_timeout) + if not _: + return login_template(request, next, + errors=["Bad username or password."]) + + login_timeout = aurweb.config.getint("options", "login_timeout") + + expires_at = int(datetime.utcnow().timestamp() + + max(cookie_timeout, login_timeout)) + + response = RedirectResponse(url=next, + status_code=int(HTTPStatus.SEE_OTHER)) + response.set_cookie("AURSID", sid, expires=expires_at) + return response + + +@router.get("/logout") +async def logout(request: Request, next: str = "/"): + """ A GET and POST route for logging out. + + @param request FastAPI request + @param next Route to redirect to + """ + if request.user.is_authenticated(): + request.user.logout(request) + + # Use 303 since we may be handling a post request, that'll get it + # to redirect to a get request. + response = RedirectResponse(url=next, + status_code=int(HTTPStatus.SEE_OTHER)) + response.delete_cookie("AURSID") + response.delete_cookie("AURTZ") + return response + + +@router.post("/logout") +async def logout_post(request: Request, next: str = "/"): + return await logout(request=request, next=next) diff --git a/templates/login.html b/templates/login.html new file mode 100644 index 00000000..da7bd722 --- /dev/null +++ b/templates/login.html @@ -0,0 +1,84 @@ +{% extends 'partials/layout.html' %} + +{% block pageContent %} + +
+

AUR {% trans %}Login{% endtrans %}

+ + {% if request.url.scheme == "http" and config.getboolean("options", "disable_http_login") %} + {% set https_login = url_base.replace("http://", "https://") + "/login/" %} +

+ {{ "HTTP login is disabled. Please %sswitch to HTTPs%s if you want to login." + | tr + | format( + '' | format(https_login), + "") + | safe + }} +

+ {% else %} + {% if request.user.is_authenticated() %} +

+ {{ "Logged-in as: %s" + | tr + | format("%s" | format(request.user.Username)) + | safe + }} + [{% trans %}Logout{% endtrans %}] +

+ {% else %} +
+
+ {% trans %}Enter login credentials{% endtrans %} + + {% if errors %} +
    + {% for error in errors %} +
  • {{ error }}
  • + {% endfor %} +
+ {% endif %} + +

+ + + +

+ +

+ + +

+ +

+ + +

+ +

+ + + [{% trans %}Forgot Password{% endtrans %}] + + + +

+ +
+
+ {% endif %} + {% endif %} + +
+ +{% endblock %} diff --git a/templates/partials/archdev-navbar.html b/templates/partials/archdev-navbar.html index 55338bc4..7662e3a4 100644 --- a/templates/partials/archdev-navbar.html +++ b/templates/partials/archdev-navbar.html @@ -1,8 +1,22 @@ diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py new file mode 100644 index 00000000..adf75329 --- /dev/null +++ b/test/test_auth_routes.py @@ -0,0 +1,126 @@ +from datetime import datetime +from http import HTTPStatus + +import pytest + +from fastapi.testclient import TestClient + +import aurweb.config + +from aurweb.asgi import app +from aurweb.db import query +from aurweb.models.account_type import AccountType +from aurweb.models.session import Session +from aurweb.testing import setup_test_db +from aurweb.testing.models import make_user + +client = TestClient(app) + +user = None + + +@pytest.fixture(autouse=True) +def setup(): + global user + + setup_test_db("Users", "Sessions", "Bans") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = make_user(Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + + +def test_login_logout(): + post_data = { + "user": "test", + "passwd": "testPassword", + "next": "/" + } + + with client as request: + response = client.get("/login") + assert response.status_code == int(HTTPStatus.OK) + + response = request.post("/login", data=post_data, + allow_redirects=False) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + + response = request.get(response.headers.get("location"), cookies={ + "AURSID": response.cookies.get("AURSID") + }) + assert response.status_code == int(HTTPStatus.OK) + + response = request.post("/logout", data=post_data, + allow_redirects=False) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + + response = request.post("/logout", data=post_data, cookies={ + "AURSID": response.cookies.get("AURSID") + }, allow_redirects=False) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert "AURSID" not in response.cookies + + +def test_login_missing_username(): + post_data = { + "passwd": "testPassword", + "next": "/" + } + + with client as request: + response = request.post("/login", data=post_data) + assert "AURSID" not in response.cookies + + +def test_login_remember_me(): + from aurweb.db import session + + post_data = { + "user": "test", + "passwd": "testPassword", + "next": "/", + "remember_me": True + } + + with client as request: + response = request.post("/login", data=post_data, + allow_redirects=False) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert "AURSID" in response.cookies + + cookie_timeout = aurweb.config.getint( + "options", "persistent_cookie_timeout") + expected_ts = datetime.utcnow().timestamp() + cookie_timeout + + _session = session.query(Session).filter( + Session.UsersID == user.ID).first() + + # Expect that LastUpdateTS was within 5 seconds of the expected_ts, + # which is equal to the current timestamp + persistent_cookie_timeout. + assert _session.LastUpdateTS > expected_ts - 5 + assert _session.LastUpdateTS < expected_ts + 5 + + +def test_login_missing_password(): + post_data = { + "user": "test", + "next": "/" + } + + with client as request: + response = request.post("/login", data=post_data) + assert "AURSID" not in response.cookies + + +def test_login_incorrect_password(): + post_data = { + "user": "test", + "passwd": "badPassword", + "next": "/" + } + + with client as request: + response = request.post("/login", data=post_data) + assert "AURSID" not in response.cookies From 4423326cec91dbfc9cd90294fc09ca40e917bc63 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 8 Jan 2021 20:24:22 -0800 Subject: [PATCH 104/844] add the request parameter to render_template This allows us to inspect things about the request we're rendering from. * Use render_template(request, ...) in aurweb.routers.auth Signed-off-by: Kevin Morris --- aurweb/routers/auth.py | 2 +- aurweb/routers/errors.py | 4 ++-- aurweb/routers/html.py | 2 +- aurweb/templates.py | 10 ++++++++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index 24f5d4e3..3a1c7192 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -17,7 +17,7 @@ def login_template(request: Request, next: str, errors: list = None): context = make_context(request, "Login", next) context["errors"] = errors context["url_base"] = f"{request.url.scheme}://{request.url.netloc}" - return render_template("login.html", context) + return render_template(request, "login.html", context) @router.get("/login", response_class=HTMLResponse) diff --git a/aurweb/routers/errors.py b/aurweb/routers/errors.py index 111d802a..eb935b57 100644 --- a/aurweb/routers/errors.py +++ b/aurweb/routers/errors.py @@ -3,12 +3,12 @@ from aurweb.templates import make_context, render_template async def not_found(request, exc): context = make_context(request, "Page Not Found") - return render_template("errors/404.html", context, 404) + return render_template(request, "errors/404.html", context, 404) async def service_unavailable(request, exc): context = make_context(request, "Service Unavailable") - return render_template("errors/503.html", context, 503) + return render_template(request, "errors/503.html", context, 503) # Maps HTTP errors to functions exceptions = { diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index 50b62450..32a7e630 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -47,7 +47,7 @@ async def language(request: Request, async def index(request: Request): """ Homepage route. """ context = make_context(request, "Home") - return render_template("index.html", context) + return render_template(request, "index.html", context) # A route that returns a error 503. For testing purposes. diff --git a/aurweb/templates.py b/aurweb/templates.py index c05dce79..c5f378b8 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -39,7 +39,10 @@ def make_context(request: Request, title: str, next: str = None): } -def render_template(path: str, context: dict, status_code=int(HTTPStatus.OK)): +def render_template(request: Request, + path: str, + context: dict, + status_code=int(HTTPStatus.OK)): """ Render a Jinja2 multi-lingual template with some context. """ # Create a deep copy of our jinja2 environment. The environment in @@ -54,4 +57,7 @@ def render_template(path: str, context: dict, status_code=int(HTTPStatus.OK)): template = templates.get_template(path) rendered = template.render(context) - return HTMLResponse(rendered, status_code=status_code) + + response = HTMLResponse(rendered, status_code=status_code) + response.set_cookie("AURLANG", context.get("language")) + return response From a33d076d8bae9ab2e988aaa5bc3ab5d8eabd44d3 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 6 Jan 2021 21:00:12 -0800 Subject: [PATCH 105/844] add passreset routes Introduced `get|post` `/passreset` routes. These routes mimic the behavior of the existing PHP implementation, with the exception of HTTP status code returns. Routes added: GET /passreset POST /passreset Routers added: aurweb.routers.accounts * On an unknown user or mismatched resetkey (where resetkey must == user.resetkey), return HTTP status NOT_FOUND (404). * On another error in the request, return HTTP status BAD_REQUEST (400). Both `get|post` routes requires that the current user is **not** authenticated, hence `@auth_required(False, redirect="/")`. + Added auth_required decorator to aurweb.auth. + Added some more utility to aurweb.models.user.User. + Added `partials/error.html` template. + Added `passreset.html` template. + Added aurweb.db.ConnectionExecutor functor for paramstyle logic. Decoupling the executor logic from the database connection logic is needed for us to easily use the same logic with a fastapi database session, when we need to use aurweb.scripts modules. At this point, notification configuration is now required to complete tests involved with notifications properly, like passreset. `conf/config.dev` has been modified to include [notifications] sendmail, sender and reply-to overrides. Dockerfile and .gitlab-ci.yml have been updated to setup /etc/hosts and start postfix before running tests. * setup.cfg: ignore E741, C901 in aurweb.routers.accounts These two warnings (shown in the commit) are not dangerous and a bi-product of maintaining compatibility with our current code flow. Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 2 + aurweb/asgi.py | 4 +- aurweb/db.py | 70 +++++++---- aurweb/routers/accounts.py | 102 ++++++++++++++++ aurweb/routers/auth.py | 10 +- conf/config.dev | 6 + setup.cfg | 13 ++ templates/partials/error.html | 15 +++ templates/passreset.html | 76 ++++++++++++ test/README.md | 5 + test/test_accounts_routes.py | 218 ++++++++++++++++++++++++++++++++++ test/test_auth_routes.py | 51 ++++++-- test/test_db.py | 13 +- test/test_user.py | 6 +- util/sendmail | 2 + 15 files changed, 552 insertions(+), 41 deletions(-) create mode 100644 aurweb/routers/accounts.py create mode 100644 templates/partials/error.html create mode 100644 templates/passreset.html create mode 100644 test/test_accounts_routes.py create mode 100755 util/sendmail diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4ad97393..db7dec9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,8 @@ before_script: python-itsdangerous python-httpx python-jinja python-pytest-cov python-requests python-aiofiles python-python-multipart python-pytest-asyncio python-coverage python-bcrypt + - bash -c "echo '127.0.0.1' > /etc/hosts" + - bash -c "echo '::1' >> /etc/hosts" test: script: diff --git a/aurweb/asgi.py b/aurweb/asgi.py index b15e5874..1a61b1f4 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -1,5 +1,4 @@ import http -import os from fastapi import FastAPI, HTTPException from fastapi.responses import HTMLResponse @@ -11,7 +10,7 @@ import aurweb.config from aurweb.auth import BasicAuthBackend from aurweb.db import get_engine -from aurweb.routers import auth, html, sso, errors +from aurweb.routers import accounts, auth, errors, html, sso routes = set() @@ -43,6 +42,7 @@ async def app_startup(): app.include_router(sso.router) app.include_router(html.router) app.include_router(auth.router) + app.include_router(accounts.router) # Initialize the database engine and ORM. get_engine() diff --git a/aurweb/db.py b/aurweb/db.py index 3f5731a9..7dab6c4a 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -145,35 +145,21 @@ def connect(): return get_engine().connect() -class Connection: +class ConnectionExecutor: _conn = None _paramstyle = None - def __init__(self): - aur_db_backend = aurweb.config.get('database', 'backend') - - if aur_db_backend == 'mysql': + def __init__(self, conn, backend=aurweb.config.get("database", "backend")): + self._conn = conn + if backend == "mysql": import mysql.connector - aur_db_host = aurweb.config.get('database', 'host') - aur_db_name = aurweb.config.get('database', 'name') - aur_db_user = aurweb.config.get('database', 'user') - aur_db_pass = aurweb.config.get('database', 'password') - aur_db_socket = aurweb.config.get('database', 'socket') - self._conn = mysql.connector.connect(host=aur_db_host, - user=aur_db_user, - passwd=aur_db_pass, - db=aur_db_name, - unix_socket=aur_db_socket, - buffered=True) self._paramstyle = mysql.connector.paramstyle - elif aur_db_backend == 'sqlite': + elif backend == "sqlite": import sqlite3 - aur_db_name = aurweb.config.get('database', 'name') - self._conn = sqlite3.connect(aur_db_name) - self._conn.create_function("POWER", 2, math.pow) self._paramstyle = sqlite3.paramstyle - else: - raise ValueError('unsupported database backend') + + def paramstyle(self): + return self._paramstyle def execute(self, query, params=()): if self._paramstyle in ('format', 'pyformat'): @@ -193,3 +179,43 @@ class Connection: def close(self): self._conn.close() + + +class Connection: + _executor = None + _conn = None + + def __init__(self): + aur_db_backend = aurweb.config.get('database', 'backend') + + if aur_db_backend == 'mysql': + import mysql.connector + aur_db_host = aurweb.config.get('database', 'host') + aur_db_name = aurweb.config.get('database', 'name') + aur_db_user = aurweb.config.get('database', 'user') + aur_db_pass = aurweb.config.get('database', 'password') + aur_db_socket = aurweb.config.get('database', 'socket') + self._conn = mysql.connector.connect(host=aur_db_host, + user=aur_db_user, + passwd=aur_db_pass, + db=aur_db_name, + unix_socket=aur_db_socket, + buffered=True) + elif aur_db_backend == 'sqlite': + import sqlite3 + aur_db_name = aurweb.config.get('database', 'name') + self._conn = sqlite3.connect(aur_db_name) + self._conn.create_function("POWER", 2, math.pow) + else: + raise ValueError('unsupported database backend') + + self._conn = ConnectionExecutor(self._conn) + + def execute(self, query, params=()): + return self._conn.execute(query, params) + + def commit(self): + self._conn.commit() + + def close(self): + self._conn.close() diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py new file mode 100644 index 00000000..0839f64e --- /dev/null +++ b/aurweb/routers/accounts.py @@ -0,0 +1,102 @@ +from http import HTTPStatus + +from fastapi import APIRouter, Form, Request +from fastapi.responses import HTMLResponse, RedirectResponse +from sqlalchemy import or_ + +from aurweb import db +from aurweb.auth import auth_required +from aurweb.l10n import get_translator_for_request +from aurweb.models.user import User +from aurweb.scripts.notify import ResetKeyNotification +from aurweb.templates import make_context, render_template + +router = APIRouter() + + +@router.get("/passreset", response_class=HTMLResponse) +@auth_required(False) +async def passreset(request: Request): + context = make_context(request, "Password Reset") + + for k, v in request.query_params.items(): + context[k] = v + + return render_template(request, "passreset.html", context) + + +@router.post("/passreset", response_class=HTMLResponse) +@auth_required(False) +async def passreset_post(request: Request, + user: str = Form(...), + resetkey: str = Form(default=None), + password: str = Form(default=None), + confirm: str = Form(default=None)): + from aurweb.db import session + + context = make_context(request, "Password Reset") + + for k, v in dict(await request.form()).items(): + context[k] = v + + # The user parameter being required, we can match against + user = db.query(User, or_(User.Username == user, + User.Email == user)).first() + if not user: + context["errors"] = ["Invalid e-mail."] + return render_template(request, "passreset.html", context, + status_code=int(HTTPStatus.NOT_FOUND)) + + if resetkey: + context["resetkey"] = resetkey + + if not user.ResetKey or resetkey != user.ResetKey: + context["errors"] = ["Invalid e-mail."] + return render_template(request, "passreset.html", context, + status_code=int(HTTPStatus.NOT_FOUND)) + + if not user or not password: + context["errors"] = ["Missing a required field."] + return render_template(request, "passreset.html", context, + status_code=int(HTTPStatus.BAD_REQUEST)) + + if password != confirm: + # If the provided password does not match the provided confirm. + context["errors"] = ["Password fields do not match."] + return render_template(request, "passreset.html", context, + status_code=int(HTTPStatus.BAD_REQUEST)) + + if len(password) < User.minimum_passwd_length(): + # Translate the error here, which simplifies error output + # in the jinja2 template. + _ = get_translator_for_request(request) + context["errors"] = [_( + "Your password must be at least %s characters.") % ( + str(User.minimum_passwd_length()))] + return render_template(request, "passreset.html", context, + status_code=int(HTTPStatus.BAD_REQUEST)) + + # We got to this point; everything matched up. Update the password + # and remove the ResetKey. + user.ResetKey = str() + user.update_password(password) + + if user.session: + session.delete(user.session) + session.commit() + + # Render ?step=complete. + return RedirectResponse(url="/passreset?step=complete", + status_code=int(HTTPStatus.SEE_OTHER)) + + # If we got here, we continue with issuing a resetkey for the user. + resetkey = db.make_random_value(User, User.ResetKey) + user.ResetKey = resetkey + session.commit() + + executor = db.ConnectionExecutor(db.get_engine().raw_connection()) + ResetKeyNotification(executor, user.ID).send() + + # Render ?step=confirm. + return RedirectResponse(url="/passreset?step=confirm", + status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index 3a1c7192..e4864424 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -6,6 +6,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse import aurweb.config +from aurweb.auth import auth_required from aurweb.models.user import User from aurweb.templates import make_context, render_template @@ -21,12 +22,13 @@ def login_template(request: Request, next: str, errors: list = None): @router.get("/login", response_class=HTMLResponse) +@auth_required(False) async def login_get(request: Request, next: str = "/"): - """ Homepage route. """ return login_template(request, next) @router.post("/login", response_class=HTMLResponse) +@auth_required(False) async def login_post(request: Request, next: str = Form(...), user: str = Form(default=str()), @@ -45,8 +47,8 @@ async def login_post(request: Request, cookie_timeout = aurweb.config.getint( "options", "persistent_cookie_timeout") - _, sid = user.login(request, passwd, cookie_timeout) - if not _: + sid = user.login(request, passwd, cookie_timeout) + if not sid: return login_template(request, next, errors=["Bad username or password."]) @@ -62,6 +64,7 @@ async def login_post(request: Request, @router.get("/logout") +@auth_required() async def logout(request: Request, next: str = "/"): """ A GET and POST route for logging out. @@ -81,5 +84,6 @@ async def logout(request: Request, next: str = "/"): @router.post("/logout") +@auth_required() async def logout_post(request: Request, next: str = "/"): return await logout(request=request, next=next) diff --git a/conf/config.dev b/conf/config.dev index 194a3bf8..94775a92 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -25,6 +25,12 @@ disable_http_login = 0 enable-maintenance = 0 localedir = YOUR_AUR_ROOT/web/locale +[notifications] +; For development/testing, use /usr/bin/sendmail +sendmail = YOUR_AUR_ROOT/util/sendmail +sender = notify@localhost +reply-to = noreply@localhost + ; Single sign-on; see doc/sso.txt. [sso] openid_configuration = http://127.0.0.1:8083/auth/realms/aurweb/.well-known/openid-configuration diff --git a/setup.cfg b/setup.cfg index b868c096..98261651 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,6 +2,19 @@ max-line-length = 127 max-complexity = 10 +# Ignore some unavoidable flake8 warnings; we know this is against +# pycodestyle, but some of the existing codebase uses `I` variables, +# so specifically silence warnings about it in pre-defined files. +# In E741, the 'I', 'O', 'l' are ambiguous variable names. +# Our current implementation uses these variables through HTTP +# and the FastAPI form specification wants them named as such. +# In C901's case, our process_account_form function is way too +# complex for PEP (too many if statements). However, we need to +# process these anyways, and making it any more complex would +# just add confusion to the implementation. +per-file-ignores = + aurweb/routers/accounts.py:E741,C901 + [isort] line_length = 127 lines_between_types = 1 diff --git a/templates/partials/error.html b/templates/partials/error.html new file mode 100644 index 00000000..6043dfd1 --- /dev/null +++ b/templates/partials/error.html @@ -0,0 +1,15 @@ +{% if errors %} +
    + {% for error in errors %} + {% if error is string %} +
  • {{ error | tr | safe }}
  • + {% elif error is iterable %} +
      + {% for e in error %} +
    • {{ e | tr | safe }}
    • + {% endfor %} +
    + {% endif %} + {% endfor %} +
+{% endif %} diff --git a/templates/passreset.html b/templates/passreset.html new file mode 100644 index 00000000..d2c3c2ee --- /dev/null +++ b/templates/passreset.html @@ -0,0 +1,76 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} +
+

{% trans %}Password Reset{% endtrans %}

+ +

+ {% if step == "confirm" %} + {% trans %}Check your e-mail for the confirmation link.{% endtrans %} + {% elif step == "complete" %} + {% trans %}Your password has been reset successfully.{% endtrans %} + {% elif resetkey %} + + {% include "partials/error.html" %} + +

+ + + + + + + + + + + + + + + +
{% trans %}Confirm your user name or primary e-mail address:{% endtrans %} + +
{% trans %}Enter your new password:{% endtrans %} + +
{% trans %}Confirm your new password:{% endtrans %} + +
+
+ + +
+ {% else %} + + {% set url = "https://mailman.archlinux.org/mailman/listinfo/aur-general" %} + {{ "If you have forgotten the user name and the primary e-mail " + "address you used to register, please send a message to the " + "%saur-general%s mailing list." + | tr + | format( + '' | format(url), + "") + | safe + }} +

+ + {% include "partials/error.html" %} + +
+

+ {% trans %}Enter your user name or your primary e-mail address:{% endtrans %} + +

+ +
+ {% endif %} +

+
+{% endblock %} diff --git a/test/README.md b/test/README.md index 3261899b..872d980b 100644 --- a/test/README.md +++ b/test/README.md @@ -27,6 +27,7 @@ For all the test to run, the following Arch packages should be installed: - python-pytest - python-pytest-cov - python-pytest-asyncio +- postfix Running tests ------------- @@ -37,6 +38,10 @@ First, setup the test configuration: $ sed -r 's;YOUR_AUR_ROOT;$(pwd);g' conf/config.dev > conf/config +You'll need to make sure that emails can be sent out by aurweb.scripts.notify. +If you don't have anything setup, just install postfix and start it before +running tests. + With those installed, one can run Python tests manually with any AUR config specified by `AUR_CONFIG`: diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py new file mode 100644 index 00000000..0f548805 --- /dev/null +++ b/test/test_accounts_routes.py @@ -0,0 +1,218 @@ +from http import HTTPStatus + +import pytest + +from fastapi.testclient import TestClient + +from aurweb.asgi import app +from aurweb.db import query +from aurweb.models.account_type import AccountType +from aurweb.models.session import Session +from aurweb.models.user import User +from aurweb.testing import setup_test_db +from aurweb.testing.models import make_user +from aurweb.testing.requests import Request + +# Some test global constants. +TEST_USERNAME = "test" +TEST_EMAIL = "test@example.org" + +# Global mutables. +client = TestClient(app) +user = None + + +@pytest.fixture(autouse=True) +def setup(): + global user + + setup_test_db("Users", "Sessions", "Bans") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = make_user(Username=TEST_USERNAME, Email=TEST_EMAIL, + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + + +def test_get_passreset_authed_redirects(): + sid = user.login(Request(), "testPassword") + assert sid is not None + + with client as request: + response = request.get("/passreset", cookies={"AURSID": sid}, + allow_redirects=False) + + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.headers.get("location") == "/" + + +def test_get_passreset(): + with client as request: + response = request.get("/passreset") + assert response.status_code == int(HTTPStatus.OK) + + +def test_get_passreset_translation(): + # Test that translation works. + with client as request: + response = request.get("/passreset", cookies={"AURLANG": "de"}) + + # The header title should be translated. + assert "Passwort zurücksetzen".encode("utf-8") in response.content + + # The form input label should be translated. + assert "Benutzername oder primäre E-Mail-Adresse eingeben:".encode( + "utf-8") in response.content + + # And the button. + assert "Weiter".encode("utf-8") in response.content + + +def test_get_passreset_with_resetkey(): + with client as request: + response = request.get("/passreset", data={"resetkey": "abcd"}) + assert response.status_code == int(HTTPStatus.OK) + + +def test_post_passreset_authed_redirects(): + sid = user.login(Request(), "testPassword") + assert sid is not None + + with client as request: + response = request.post("/passreset", + cookies={"AURSID": sid}, + data={"user": "blah"}, + allow_redirects=False) + + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.headers.get("location") == "/" + + +def test_post_passreset_user(): + # With username. + with client as request: + response = request.post("/passreset", data={"user": TEST_USERNAME}) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.headers.get("location") == "/passreset?step=confirm" + + # With e-mail. + with client as request: + response = request.post("/passreset", data={"user": TEST_EMAIL}) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.headers.get("location") == "/passreset?step=confirm" + + +def test_post_passreset_resetkey(): + from aurweb.db import session + + user.session = Session(UsersID=user.ID, SessionID="blah", + LastUpdateTS=datetime.utcnow().timestamp()) + session.commit() + + # Prepare a password reset. + with client as request: + response = request.post("/passreset", data={"user": TEST_USERNAME}) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.headers.get("location") == "/passreset?step=confirm" + + # Now that we've prepared the password reset, prepare a POST + # request with the user's ResetKey. + resetkey = user.ResetKey + post_data = { + "user": TEST_USERNAME, + "resetkey": resetkey, + "password": "abcd1234", + "confirm": "abcd1234" + } + + with client as request: + response = request.post("/passreset", data=post_data) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.headers.get("location") == "/passreset?step=complete" + + +def test_post_passreset_error_invalid_email(): + # First, test with a user that doesn't even exist. + with client as request: + response = request.post("/passreset", data={"user": "invalid"}) + assert response.status_code == int(HTTPStatus.NOT_FOUND) + + error = "Invalid e-mail." + assert error in response.content.decode("utf-8") + + # Then, test with an invalid resetkey for a real user. + _ = make_resetkey() + post_data = make_passreset_data("fake") + post_data["password"] = "abcd1234" + post_data["confirm"] = "abcd1234" + + with client as request: + response = request.post("/passreset", data=post_data) + assert response.status_code == int(HTTPStatus.NOT_FOUND) + assert error in response.content.decode("utf-8") + + +def make_resetkey(): + with client as request: + response = request.post("/passreset", data={"user": TEST_USERNAME}) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.headers.get("location") == "/passreset?step=confirm" + return user.ResetKey + + +def make_passreset_data(resetkey): + return { + "user": user.Username, + "resetkey": resetkey + } + + +def test_post_passreset_error_missing_field(): + # Now that we've prepared the password reset, prepare a POST + # request with the user's ResetKey. + resetkey = make_resetkey() + post_data = make_passreset_data(resetkey) + + with client as request: + response = request.post("/passreset", data=post_data) + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + error = "Missing a required field." + assert error in response.content.decode("utf-8") + + +def test_post_passreset_error_password_mismatch(): + resetkey = make_resetkey() + post_data = make_passreset_data(resetkey) + + post_data["password"] = "abcd1234" + post_data["confirm"] = "mismatched" + + with client as request: + response = request.post("/passreset", data=post_data) + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + error = "Password fields do not match." + assert error in response.content.decode("utf-8") + + +def test_post_passreset_error_password_requirements(): + resetkey = make_resetkey() + post_data = make_passreset_data(resetkey) + + passwd_min_len = User.minimum_passwd_length() + assert passwd_min_len >= 4 + + post_data["password"] = "x" + post_data["confirm"] = "x" + + with client as request: + response = request.post("/passreset", data=post_data) + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + error = f"Your password must be at least {passwd_min_len} characters." + assert error in response.content.decode("utf-8") diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index adf75329..ff8a08e9 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -14,8 +14,12 @@ from aurweb.models.session import Session from aurweb.testing import setup_test_db from aurweb.testing.models import make_user -client = TestClient(app) +# Some test global constants. +TEST_USERNAME = "test" +TEST_EMAIL = "test@example.org" +# Global mutables. +client = TestClient(app) user = None @@ -27,7 +31,8 @@ def setup(): account_type = query(AccountType, AccountType.AccountType == "User").first() - user = make_user(Username="test", Email="test@example.org", + + user = make_user(Username=TEST_USERNAME, Email=TEST_EMAIL, RealName="Test User", Passwd="testPassword", AccountType=account_type) @@ -40,16 +45,16 @@ def test_login_logout(): } with client as request: - response = client.get("/login") + # First, let's test get /login. + response = request.get("/login") assert response.status_code == int(HTTPStatus.OK) response = request.post("/login", data=post_data, allow_redirects=False) assert response.status_code == int(HTTPStatus.SEE_OTHER) - response = request.get(response.headers.get("location"), cookies={ - "AURSID": response.cookies.get("AURSID") - }) + # Simulate following the redirect location from above's response. + response = request.get(response.headers.get("location")) assert response.status_code == int(HTTPStatus.OK) response = request.post("/logout", data=post_data, @@ -60,9 +65,37 @@ def test_login_logout(): "AURSID": response.cookies.get("AURSID") }, allow_redirects=False) assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert "AURSID" not in response.cookies +def test_authenticated_login_forbidden(): + post_data = { + "user": "test", + "passwd": "testPassword", + "next": "/" + } + + with client as request: + # Login. + response = request.post("/login", data=post_data, + allow_redirects=False) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + + # Now, let's verify that we receive 403 Forbidden when we + # try to get /login as an authenticated user. + response = request.get("/login", allow_redirects=False) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + + +def test_unauthenticated_logout_unauthorized(): + with client as request: + # Alright, let's verify that attempting to /logout when not + # authenticated returns 401 Unauthorized. + response = request.get("/logout", allow_redirects=False) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + + def test_login_missing_username(): post_data = { "passwd": "testPassword", @@ -75,8 +108,6 @@ def test_login_missing_username(): def test_login_remember_me(): - from aurweb.db import session - post_data = { "user": "test", "passwd": "testPassword", @@ -94,8 +125,8 @@ def test_login_remember_me(): "options", "persistent_cookie_timeout") expected_ts = datetime.utcnow().timestamp() + cookie_timeout - _session = session.query(Session).filter( - Session.UsersID == user.ID).first() + _session = query(Session, + Session.UsersID == user.ID).first() # Expect that LastUpdateTS was within 5 seconds of the expected_ts, # which is equal to the current timestamp + persistent_cookie_timeout. diff --git a/test/test_db.py b/test/test_db.py index 1eb0dc28..e0946ed5 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -3,7 +3,6 @@ import re import sqlite3 import tempfile -from datetime import datetime from unittest import mock import mysql.connector @@ -185,3 +184,15 @@ def test_create_delete(): db.delete(AccountType, AccountType.AccountType == "test") record = db.query(AccountType, AccountType.AccountType == "test").first() assert record is None + + +@mock.patch("mysql.connector.paramstyle", "qmark") +def test_connection_executor_mysql_paramstyle(): + executor = db.ConnectionExecutor(None, backend="mysql") + assert executor.paramstyle() == "qmark" + + +@mock.patch("sqlite3.paramstyle", "pyformat") +def test_connection_executor_sqlite_paramstyle(): + executor = db.ConnectionExecutor(None, backend="sqlite") + assert executor.paramstyle() == "pyformat" diff --git a/test/test_user.py b/test/test_user.py index b8d4248a..4f144819 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -28,8 +28,8 @@ def setup(): setup_test_db("Users", "Sessions", "Bans") - account_type = session.query(AccountType).filter( - AccountType.AccountType == "User").first() + account_type = query(AccountType, + AccountType.AccountType == "User").first() user = make_user(Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", @@ -67,7 +67,7 @@ def test_user_login_logout(): assert user.session.User == user # Search for the user via query API. - result = session.query(User).filter(User.ID == user.ID).first() + result = query(User, User.ID == user.ID).first() # Compare the result and our original user. assert result == user diff --git a/util/sendmail b/util/sendmail new file mode 100755 index 00000000..06bd9865 --- /dev/null +++ b/util/sendmail @@ -0,0 +1,2 @@ +#!/bin/bash +exit 0 From 9fdbe3f775a3d13e92a02a11e5eb4830e6daf875 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 8 Jan 2021 20:10:45 -0800 Subject: [PATCH 106/844] add authenticated User LangPreference tracking + Use User.LangPreference when there is no set AURSID if request.user.is_authenticated is true. + Updated post /language to update LangPreference when request.user.is_authenticated. + Restore language during test where we change it. + Added the user attribute to aurweb.testing.requests.Request. Signed-off-by: Kevin Morris --- aurweb/l10n.py | 12 ++++-------- aurweb/routers/html.py | 11 ++++++++++- aurweb/testing/requests.py | 19 +++++++++++++++++++ test/test_accounts_routes.py | 6 +++++- test/test_routes.py | 25 +++++++++++++++++++++---- 5 files changed, 59 insertions(+), 14 deletions(-) diff --git a/aurweb/l10n.py b/aurweb/l10n.py index 030ab274..4a5c1a46 100644 --- a/aurweb/l10n.py +++ b/aurweb/l10n.py @@ -64,8 +64,10 @@ translator = Translator() def get_request_language(request: Request): - return request.cookies.get("AURLANG", - aurweb.config.get("options", "default_lang")) + if request.user.is_authenticated(): + return request.user.LangPreference + default_lang = aurweb.config.get("options", "default_lang") + return request.cookies.get("AURLANG", default_lang) def get_raw_translator_for_request(request: Request): @@ -77,12 +79,6 @@ def get_translator_for_request(request: Request): """ Determine the preferred language from a FastAPI request object and build a translator function for it. - - Example: - ```python - _ = get_translator_for_request(request) - print(_("Hello")) - ``` """ lang = get_request_language(request) diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index 32a7e630..e947d213 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -24,12 +24,14 @@ async def language(request: Request, set_lang: str = Form(...), next: str = Form(...), q: str = Form(default=None)): - """ A POST route used to set a session's language. + """ + A POST route used to set a session's language. Return a 303 See Other redirect to {next}?next={next}. If we are setting the language on any page, we want to preserve query parameters across the redirect. """ + from aurweb.db import session from aurweb.asgi import routes if unquote(next) not in routes: return HTMLResponse( @@ -37,6 +39,13 @@ async def language(request: Request, status_code=400) query_string = "?" + q if q else str() + + # If the user is authenticated, update the user's LangPreference. + if request.user.is_authenticated(): + request.user.LangPreference = set_lang + session.commit() + + # In any case, set the response's AURLANG cookie that never expires. response = RedirectResponse(url=f"{next}{query_string}", status_code=int(HTTPStatus.SEE_OTHER)) response.set_cookie("AURLANG", set_lang) diff --git a/aurweb/testing/requests.py b/aurweb/testing/requests.py index 2839c93f..2e64fd3d 100644 --- a/aurweb/testing/requests.py +++ b/aurweb/testing/requests.py @@ -1,8 +1,27 @@ +import aurweb.config + + +class User: + """ A fake User model. """ + # Fake columns. + LangPreference = aurweb.config.get("options", "default_lang") + + # A fake authenticated flag. + authenticated = False + + def is_authenticated(self): + return self.authenticated + + class Client: + """ A fake FastAPI Request.client object. """ + # A fake host. host = "127.0.0.1" class Request: + """ A fake Request object which mimics a FastAPI Request for tests. """ client = Client() cookies = dict() headers = dict() + user = User() diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index 0f548805..69896a0f 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -54,7 +54,7 @@ def test_get_passreset(): def test_get_passreset_translation(): - # Test that translation works. + # Test that translation works; set it to de. with client as request: response = request.get("/passreset", cookies={"AURLANG": "de"}) @@ -68,6 +68,10 @@ def test_get_passreset_translation(): # And the button. assert "Weiter".encode("utf-8") in response.content + # Restore english. + with client as request: + response = request.get("/passreset", cookies={"AURLANG": "en"}) + def test_get_passreset_with_resetkey(): with client as request: diff --git a/test/test_routes.py b/test/test_routes.py index 950d9b71..d512a172 100644 --- a/test/test_routes.py +++ b/test/test_routes.py @@ -10,13 +10,14 @@ from aurweb.asgi import app from aurweb.db import query from aurweb.models.account_type import AccountType from aurweb.testing import setup_test_db +from aurweb.testing.models import make_user +from aurweb.testing.requests import Request client = TestClient(app) - user = None -@pytest.fixture +@pytest.fixture(autouse=True) def setup(): global user @@ -46,7 +47,7 @@ def test_favicon(): def test_language(): - """ Test the language post route at '/language'. """ + """ Test the language post route as a guest user. """ post_data = { "set_lang": "de", "next": "/" @@ -67,6 +68,23 @@ def test_language_invalid_next(): assert response.status_code == int(HTTPStatus.BAD_REQUEST) +def test_user_language(): + """ Test the language post route as an authenticated user. """ + post_data = { + "set_lang": "de", + "next": "/" + } + + sid = user.login(Request(), "testPassword") + assert sid is not None + + with client as req: + response = req.post("/language", data=post_data, + cookies={"AURSID": sid}) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert user.LangPreference == "de" + + def test_language_query_params(): """ Test the language post route with query params. """ next = urllib.parse.quote_plus("/") @@ -87,4 +105,3 @@ def test_error_messages(): response2 = client.get("/raisefivethree") assert response1.status_code == int(HTTPStatus.NOT_FOUND) assert response2.status_code == int(HTTPStatus.SERVICE_UNAVAILABLE) - From 670f711b593ed6c040ed3facb6212d327a3c38af Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 13 Jan 2021 06:46:30 -0800 Subject: [PATCH 107/844] add SSHPubKey ORM model Includes `aurweb.models.ssh_pub_key.get_fingerprint(pubkey)` helper. Signed-off-by: Kevin Morris --- aurweb/models/ssh_pub_key.py | 41 +++++++++++++++++++++++++ test/test_ssh_pub_key.py | 58 ++++++++++++++++++++++++++++++++++++ test/test_user.py | 20 ++++++++++++- 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 aurweb/models/ssh_pub_key.py create mode 100644 test/test_ssh_pub_key.py diff --git a/aurweb/models/ssh_pub_key.py b/aurweb/models/ssh_pub_key.py new file mode 100644 index 00000000..01ff558e --- /dev/null +++ b/aurweb/models/ssh_pub_key.py @@ -0,0 +1,41 @@ +import os +import tempfile + +from subprocess import PIPE, Popen + +from sqlalchemy.orm import backref, mapper, relationship + +from aurweb.models.user import User +from aurweb.schema import SSHPubKeys + + +class SSHPubKey: + def __init__(self, **kwargs): + self.UserID = kwargs.get("UserID") + self.Fingerprint = kwargs.get("Fingerprint") + self.PubKey = kwargs.get("PubKey") + + +def get_fingerprint(pubkey): + with tempfile.TemporaryDirectory() as tmpdir: + pk = os.path.join(tmpdir, "ssh.pub") + + with open(pk, "w") as f: + f.write(pubkey) + + proc = Popen(["ssh-keygen", "-l", "-f", pk], stdout=PIPE, stderr=PIPE) + out, err = proc.communicate() + + # Invalid SSH Public Key. Return None to the caller. + if proc.returncode != 0: + return None + + parts = out.decode().split() + fp = parts[1].replace("SHA256:", "") + + return fp + + +mapper(SSHPubKey, SSHPubKeys, properties={ + "User": relationship(User, backref=backref("ssh_pub_key", uselist=False)) +}) diff --git a/test/test_ssh_pub_key.py b/test/test_ssh_pub_key.py new file mode 100644 index 00000000..fe9df047 --- /dev/null +++ b/test/test_ssh_pub_key.py @@ -0,0 +1,58 @@ +import pytest + +from aurweb.db import query +from aurweb.models.account_type import AccountType +from aurweb.models.ssh_pub_key import SSHPubKey, get_fingerprint +from aurweb.testing import setup_test_db +from aurweb.testing.models import make_user + +TEST_SSH_PUBKEY = """ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCycoCi5yGCvSclH2wmNBUuwsYEzRZZBJaQquRc4ysl+Tg+/jiDkR3Zn9fIznC4KnFoyrIHzkKuePZ3bNDYwkZxkJKoWBCh4hXKDXSm87FMN0+VDC+1QxF/z0XaAGr/P6f4XukabyddypBdnHcZiplbw+YOSqcAE2TCqOlSXwNMOcF9U89UsR/Q9i9I52hlvU0q8+fZVGhou1KCowFSnHYtrr5KYJ04CXkJ13DkVf3+pjQWyrByvBcf1hGEaczlgfobrrv/y96jDhgfXucxliNKLdufDPPkii3LhhsNcDmmI1VZ3v0irKvd9WZuauqloobY84zEFcDTyjn0hxGjVeYFejm4fBnvjga0yZXORuWksdNfXWLDxFk6MDDd1jF0ExRbP+OxDuU4IVyIuDL7S3cnbf2YjGhkms/8voYT2OBE7FwNlfv98Kr0NUp51zpf55Arxn9j0Rz9xTA7FiODQgCn6iQ0SDtzUNL0IKTCw26xJY5gzMxbfpvzPQGeulx/ioM= kevr@volcano +""" + +user, ssh_pub_key = None, None + + +@pytest.fixture(autouse=True) +def setup(): + from aurweb.db import session + + global user, ssh_pub_key + + setup_test_db("Users", "SSHPubKeys") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = make_user(Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + + assert account_type == user.AccountType + assert account_type.ID == user.AccountTypeID + + ssh_pub_key = SSHPubKey(UserID=user.ID, + Fingerprint="testFingerprint", + PubKey="testPubKey") + + session.add(ssh_pub_key) + session.commit() + + yield ssh_pub_key + + session.delete(ssh_pub_key) + session.commit() + + +def test_ssh_pub_key(): + assert ssh_pub_key.UserID == user.ID + assert ssh_pub_key.User == user + assert ssh_pub_key.Fingerprint == "testFingerprint" + assert ssh_pub_key.PubKey == "testPubKey" + + +def test_ssh_pub_key_fingerprint(): + assert get_fingerprint(TEST_SSH_PUBKEY) is not None + + +def test_ssh_pub_key_invalid_fingerprint(): + assert get_fingerprint("ssh-rsa fake and invalid") is None diff --git a/test/test_user.py b/test/test_user.py index 4f144819..473b035a 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -12,6 +12,7 @@ from aurweb.db import query from aurweb.models.account_type import AccountType from aurweb.models.ban import Ban from aurweb.models.session import Session +from aurweb.models.ssh_pub_key import SSHPubKey from aurweb.models.user import User from aurweb.testing import setup_test_db from aurweb.testing.models import make_session, make_user @@ -26,7 +27,7 @@ def setup(): global account_type, user - setup_test_db("Users", "Sessions", "Bans") + setup_test_db("Users", "Sessions", "Bans", "SSHPubKeys") account_type = query(AccountType, AccountType.AccountType == "User").first() @@ -160,3 +161,20 @@ def test_user_update_password(): def test_user_minimum_passwd_length(): passwd_min_len = aurweb.config.getint("options", "passwd_min_len") assert User.minimum_passwd_length() == passwd_min_len + + +def test_user_ssh_pub_key(): + from aurweb.db import session + + assert user.ssh_pub_key is None + + ssh_pub_key = SSHPubKey(UserID=user.ID, + Fingerprint="testFingerprint", + PubKey="testPubKey") + session.add(ssh_pub_key) + session.commit() + + assert user.ssh_pub_key == ssh_pub_key + + session.delete(ssh_pub_key) + session.commit() From 07d5907ecda5e93ebe44bd591a7f0ce87fb73cc2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 25 Jan 2021 16:30:47 -0800 Subject: [PATCH 108/844] aurweb.auth: add user credentials and matcher functions This clones the behavior already present in the PHP implementation, but it uses a global dict with credential constant keys to validation functions to determine if a given user has a credential. Signed-off-by: Kevin Morris --- aurweb/auth.py | 101 ++++++++++++++++++++++++++++++++++++++++++ aurweb/models/user.py | 5 +++ test/test_auth.py | 7 ++- test/test_user.py | 42 ++++++++++++++++++ 4 files changed, 154 insertions(+), 1 deletion(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index 8608a82a..53c853de 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -17,6 +17,10 @@ class AnonymousUser: def is_authenticated(): return False + @staticmethod + def has_credential(credential): + return False + class BasicAuthBackend(AuthenticationBackend): async def authenticate(self, conn: HTTPConnection): @@ -75,3 +79,100 @@ def auth_required(is_required: bool = True, return wrapper return decorator + + +CRED_ACCOUNT_CHANGE_TYPE = 1 +CRED_ACCOUNT_EDIT = 2 +CRED_ACCOUNT_EDIT_DEV = 3 +CRED_ACCOUNT_LAST_LOGIN = 4 +CRED_ACCOUNT_SEARCH = 5 +CRED_ACCOUNT_LIST_COMMENTS = 28 +CRED_COMMENT_DELETE = 6 +CRED_COMMENT_UNDELETE = 27 +CRED_COMMENT_VIEW_DELETED = 22 +CRED_COMMENT_EDIT = 25 +CRED_COMMENT_PIN = 26 +CRED_PKGBASE_ADOPT = 7 +CRED_PKGBASE_SET_KEYWORDS = 8 +CRED_PKGBASE_DELETE = 9 +CRED_PKGBASE_DISOWN = 10 +CRED_PKGBASE_EDIT_COMAINTAINERS = 24 +CRED_PKGBASE_FLAG = 11 +CRED_PKGBASE_LIST_VOTERS = 12 +CRED_PKGBASE_NOTIFY = 13 +CRED_PKGBASE_UNFLAG = 15 +CRED_PKGBASE_VOTE = 16 +CRED_PKGREQ_FILE = 23 +CRED_PKGREQ_CLOSE = 17 +CRED_PKGREQ_LIST = 18 +CRED_TU_ADD_VOTE = 19 +CRED_TU_LIST_VOTES = 20 +CRED_TU_VOTE = 21 + + +def has_any(user, *account_types): + return str(user.AccountType) in set(account_types) + + +def user_developer_or_trusted_user(user): + return has_any(user, "User", "Trusted User", "Developer", + "Trusted User & Developer") + + +def trusted_user(user): + return has_any(user, "Trusted User", "Trusted User & Developer") + + +def developer(user): + return has_any(user, "Developer", "Trusted User & Developer") + + +def trusted_user_or_dev(user): + return has_any(user, "Trusted User", "Developer", + "Trusted User & Developer") + + +# A mapping of functions that users must pass to have credentials. +cred_filters = { + CRED_PKGBASE_FLAG: user_developer_or_trusted_user, + CRED_PKGBASE_NOTIFY: user_developer_or_trusted_user, + CRED_PKGBASE_VOTE: user_developer_or_trusted_user, + CRED_PKGREQ_FILE: user_developer_or_trusted_user, + CRED_ACCOUNT_CHANGE_TYPE: trusted_user_or_dev, + CRED_ACCOUNT_EDIT: trusted_user_or_dev, + CRED_ACCOUNT_LAST_LOGIN: trusted_user_or_dev, + CRED_ACCOUNT_LIST_COMMENTS: trusted_user_or_dev, + CRED_ACCOUNT_SEARCH: trusted_user_or_dev, + CRED_COMMENT_DELETE: trusted_user_or_dev, + CRED_COMMENT_UNDELETE: trusted_user_or_dev, + CRED_COMMENT_VIEW_DELETED: trusted_user_or_dev, + CRED_COMMENT_EDIT: trusted_user_or_dev, + CRED_COMMENT_PIN: trusted_user_or_dev, + CRED_PKGBASE_ADOPT: trusted_user_or_dev, + CRED_PKGBASE_SET_KEYWORDS: trusted_user_or_dev, + CRED_PKGBASE_DELETE: trusted_user_or_dev, + CRED_PKGBASE_EDIT_COMAINTAINERS: trusted_user_or_dev, + CRED_PKGBASE_DISOWN: trusted_user_or_dev, + CRED_PKGBASE_LIST_VOTERS: trusted_user_or_dev, + CRED_PKGBASE_UNFLAG: trusted_user_or_dev, + CRED_PKGREQ_CLOSE: trusted_user_or_dev, + CRED_PKGREQ_LIST: trusted_user_or_dev, + CRED_TU_ADD_VOTE: trusted_user, + CRED_TU_LIST_VOTES: trusted_user, + CRED_TU_VOTE: trusted_user, + CRED_ACCOUNT_EDIT_DEV: developer, +} + + +def has_credential(user: User, + credential: int, + approved_users: list = tuple()): + + if user in approved_users: + return True + + if credential in cred_filters: + cred_filter = cred_filters.get(credential) + return cred_filter(user) + + return False diff --git a/aurweb/models/user.py b/aurweb/models/user.py index aff4ce6b..3983e098 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -141,6 +141,11 @@ class User: request.cookies["AURSID"] = self.session.SessionID return self.session.SessionID + def has_credential(self, credential: str, approved: list = tuple()): + import aurweb.auth + cred = getattr(aurweb.auth, credential) + return aurweb.auth.has_credential(self, cred, approved) + def logout(self, request): from aurweb.db import session diff --git a/test/test_auth.py b/test/test_auth.py index d2251de4..d43459cd 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -4,8 +4,8 @@ import pytest from starlette.authentication import AuthenticationError +from aurweb.auth import BasicAuthBackend, has_credential from aurweb.db import query -from aurweb.auth import BasicAuthBackend from aurweb.models.account_type import AccountType from aurweb.testing import setup_test_db from aurweb.testing.models import make_session, make_user @@ -78,3 +78,8 @@ async def test_basic_auth_backend(): LastUpdateTS=now_ts + 5) _, result = await backend.authenticate(request) assert result == user + + +def test_has_fake_credential_fails(): + # Fake credential 666 does not exist. + assert not has_credential(user, 666) diff --git a/test/test_user.py b/test/test_user.py index 473b035a..e8056681 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -163,6 +163,11 @@ def test_user_minimum_passwd_length(): assert User.minimum_passwd_length() == passwd_min_len +def test_user_has_credential(): + assert user.has_credential("CRED_PKGBASE_FLAG") + assert not user.has_credential("CRED_ACCOUNT_CHANGE_TYPE") + + def test_user_ssh_pub_key(): from aurweb.db import session @@ -178,3 +183,40 @@ def test_user_ssh_pub_key(): session.delete(ssh_pub_key) session.commit() + + +def test_user_credential_types(): + from aurweb.db import session + + assert aurweb.auth.user_developer_or_trusted_user(user) + assert not aurweb.auth.trusted_user(user) + assert not aurweb.auth.developer(user) + assert not aurweb.auth.trusted_user_or_dev(user) + + trusted_user_type = query(AccountType, + AccountType.AccountType == "Trusted User")\ + .first() + user.AccountType = trusted_user_type + session.commit() + + assert aurweb.auth.trusted_user(user) + assert aurweb.auth.trusted_user_or_dev(user) + + developer_type = query(AccountType, + AccountType.AccountType == "Developer")\ + .first() + user.AccountType = developer_type + session.commit() + + assert aurweb.auth.developer(user) + assert aurweb.auth.trusted_user_or_dev(user) + + type_str = "Trusted User & Developer" + elevated_type = query(AccountType, + AccountType.AccountType == type_str).first() + user.AccountType = elevated_type + session.commit() + + assert aurweb.auth.trusted_user(user) + assert aurweb.auth.developer(user) + assert aurweb.auth.trusted_user_or_dev(user) From 9052688ed247bccda516ffa84183b7ed442f0a04 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 25 Jan 2021 16:52:14 -0800 Subject: [PATCH 109/844] add aurweb.time module This module includes timezone-based utilities for a FastAPI request. This commit introduces use of the AURTZ cookie within get_request_timezone. This cookie should be set to the user or session's timezone. * `make_context` has been modified to parse the request's timezone and include the "timezone" and "timezones" variables, along with a timezone specified "now" date. + Added `Timezone` attribute to aurweb.testing.requests.Request.user. Signed-off-by: Kevin Morris --- aurweb/templates.py | 11 ++++--- aurweb/testing/requests.py | 1 + aurweb/time.py | 63 ++++++++++++++++++++++++++++++++++++++ test/test_time.py | 33 ++++++++++++++++++++ 4 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 aurweb/time.py create mode 100644 test/test_time.py diff --git a/aurweb/templates.py b/aurweb/templates.py index c5f378b8..564f3149 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -1,5 +1,6 @@ import copy import os +import zoneinfo from datetime import datetime from http import HTTPStatus @@ -11,7 +12,7 @@ from fastapi.responses import HTMLResponse import aurweb.config -from aurweb import l10n +from aurweb import l10n, time # Prepare jinja2 objects. loader = jinja2.FileSystemLoader(os.path.join( @@ -26,14 +27,15 @@ env.filters["tr"] = l10n.tr def make_context(request: Request, title: str, next: str = None): """ Create a context for a jinja2 TemplateResponse. """ + timezone = time.get_request_timezone(request) return { "request": request, "language": l10n.get_request_language(request), "languages": l10n.SUPPORTED_LANGUAGES, + "timezone": timezone, + "timezones": time.SUPPORTED_TIMEZONES, "title": title, - # The 'now' context variable will not show proper datetimes - # until we've implemented timezone support here. - "now": datetime.now(), + "now": datetime.now(tz=zoneinfo.ZoneInfo(timezone)), "config": aurweb.config, "next": next if next else request.url.path } @@ -60,4 +62,5 @@ def render_template(request: Request, response = HTMLResponse(rendered, status_code=status_code) response.set_cookie("AURLANG", context.get("language")) + response.set_cookie("AURTZ", context.get("timezone")) return response diff --git a/aurweb/testing/requests.py b/aurweb/testing/requests.py index 2e64fd3d..9976b6fb 100644 --- a/aurweb/testing/requests.py +++ b/aurweb/testing/requests.py @@ -5,6 +5,7 @@ class User: """ A fake User model. """ # Fake columns. LangPreference = aurweb.config.get("options", "default_lang") + Timezone = aurweb.config.get("options", "default_timezone") # A fake authenticated flag. authenticated = False diff --git a/aurweb/time.py b/aurweb/time.py new file mode 100644 index 00000000..0b1dff11 --- /dev/null +++ b/aurweb/time.py @@ -0,0 +1,63 @@ +import zoneinfo + +from collections import OrderedDict +from datetime import datetime + +from fastapi import Request + +import aurweb.config + + +def tz_offset(name: str): + """ Get a timezone offset in the form "+00:00" by its name. + + Example: tz_offset('America/Los_Angeles') + + :param name: Timezone name + :return: UTC offset in the form "+00:00" + """ + dt = datetime.now(tz=zoneinfo.ZoneInfo(name)) + + # Our offset in hours. + offset = dt.utcoffset().total_seconds() / 60 / 60 + + # Prefix the offset string with a - or +. + offset_string = '-' if offset < 0 else '+' + + # Remove any negativity from the offset. We want a good offset. :) + offset = abs(offset) + + # Truncate the floating point digits, giving the hours. + hours = int(offset) + + # Subtract hours from the offset, and multiply the remaining fraction + # (0 - 0.99[repeated]) with 60 minutes to get the number of minutes + # remaining in the hour. + minutes = int((offset - hours) * 60) + + # Pad the hours and minutes by two places. + offset_string += "{:0>2}:{:0>2}".format(hours, minutes) + return offset_string + + +SUPPORTED_TIMEZONES = OrderedDict({ + # Flatten out the list of tuples into an OrderedDict. + timezone: offset for timezone, offset in sorted([ + # Comprehend a list of tuples (timezone, offset display string) + # and sort them by (offset, timezone). + (tz, "(UTC%s) %s" % (tz_offset(tz), tz)) + for tz in zoneinfo.available_timezones() + ], key=lambda element: (tz_offset(element[0]), element[0])) +}) + + +def get_request_timezone(request: Request): + """ Get a request's timezone by its AURTZ cookie. We use the + configuration's [options] default_timezone otherwise. + + @param request FastAPI request + """ + if request.user.is_authenticated(): + return request.user.Timezone + default_tz = aurweb.config.get("options", "default_timezone") + return request.cookies.get("AURTZ", default_tz) diff --git a/test/test_time.py b/test/test_time.py new file mode 100644 index 00000000..2134d217 --- /dev/null +++ b/test/test_time.py @@ -0,0 +1,33 @@ +import aurweb.config + +from aurweb.testing.requests import Request +from aurweb.time import get_request_timezone, tz_offset + + +def test_tz_offset_utc(): + offset = tz_offset("UTC") + assert offset == "+00:00" + + +def test_tz_offset_mst(): + offset = tz_offset("MST") + assert offset == "-07:00" + + +def test_request_timezone(): + request = Request() + tz = get_request_timezone(request) + assert tz == aurweb.config.get("options", "default_timezone") + + +def test_authenticated_request_timezone(): + # Modify a fake request to be authenticated with the + # America/Los_Angeles timezone. + request = Request() + request.user.authenticated = True + request.user.Timezone = "America/Los_Angeles" + + # Get the request's timezone, it should be America/Los_Angeles. + tz = get_request_timezone(request) + assert tz == request.user.Timezone + assert tz == "America/Los_Angeles" From a5be6fc9beaa8a195b9c8e382d25b5bb51225412 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 27 Jan 2021 18:30:57 -0800 Subject: [PATCH 110/844] aurweb.templates: add make_variable_context A new make_context wrapper which additionally includes either query parameters (get) or form data (post) in the context. Use this to simplify setting context variables for form data in particular. Signed-off-by: Kevin Morris --- aurweb/routers/accounts.py | 13 +++---------- aurweb/templates.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 0839f64e..db23bc3a 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -9,7 +9,7 @@ from aurweb.auth import auth_required from aurweb.l10n import get_translator_for_request from aurweb.models.user import User from aurweb.scripts.notify import ResetKeyNotification -from aurweb.templates import make_context, render_template +from aurweb.templates import make_variable_context, render_template router = APIRouter() @@ -17,11 +17,7 @@ router = APIRouter() @router.get("/passreset", response_class=HTMLResponse) @auth_required(False) async def passreset(request: Request): - context = make_context(request, "Password Reset") - - for k, v in request.query_params.items(): - context[k] = v - + context = await make_variable_context(request, "Password Reset") return render_template(request, "passreset.html", context) @@ -34,10 +30,7 @@ async def passreset_post(request: Request, confirm: str = Form(default=None)): from aurweb.db import session - context = make_context(request, "Password Reset") - - for k, v in dict(await request.form()).items(): - context[k] = v + context = await make_variable_context(request, "Password Reset") # The user parameter being required, we can match against user = db.query(User, or_(User.Username == user, diff --git a/aurweb/templates.py b/aurweb/templates.py index 564f3149..4ea74a62 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -41,6 +41,20 @@ def make_context(request: Request, title: str, next: str = None): } +async def make_variable_context(request: Request, title: str, next: str = None): + """ Make a context with variables provided by the user + (query params via GET or form data via POST). """ + context = make_context(request, title, next) + to_copy = dict(request.query_params) \ + if request.method.lower() == "get" \ + else dict(await request.form()) + + for k, v in to_copy.items(): + context[k] = v + + return context + + def render_template(request: Request, path: str, context: dict, From 7a6a38592e63db1ca8a5a1748458afe659d5be3f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 27 Jan 2021 18:36:06 -0800 Subject: [PATCH 111/844] add python-email-validator dependency Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 1 + Dockerfile | 3 ++- INSTALL | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index db7dec9b..f1fe5e6f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,7 @@ before_script: python-itsdangerous python-httpx python-jinja python-pytest-cov python-requests python-aiofiles python-python-multipart python-pytest-asyncio python-coverage python-bcrypt + python-email-validator - bash -c "echo '127.0.0.1' > /etc/hosts" - bash -c "echo '::1' >> /etc/hosts" diff --git a/Dockerfile b/Dockerfile index 6638f9a2..f65acc7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,8 @@ RUN pacman -Syu --noconfirm base-devel git gpgme protobuf pyalpm \ python-werkzeug python-pytest-tap python-fastapi nginx python-authlib \ python-itsdangerous python-httpx python-jinja python-pytest-cov \ python-requests python-aiofiles python-python-multipart \ - python-pytest-asyncio python-coverage hypercorn python-bcrypt + python-pytest-asyncio python-coverage hypercorn python-bcrypt \ + python-email-validator # Remove aurweb.sqlite3 if it was copied over via COPY. RUN rm -fv aurweb.sqlite3 diff --git a/INSTALL b/INSTALL index 6c43fec8..04ccd69e 100644 --- a/INSTALL +++ b/INSTALL @@ -51,7 +51,7 @@ read the instructions below. python-bleach python-markdown python-alembic hypercorn \ python-itsdangerous python-authlib python-httpx \ python-jinja python-aiofiles python-python-multipart \ - python-requests hypercorn python-bcrypt + python-requests hypercorn python-bcrypt python-email-validator # python3 setup.py install 5) Create a new MySQL database and a user and import the aurweb SQL schema: From df0a637d2b5da4a2fc6dae0c7f07bcd7f50e4828 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 28 Jan 2021 16:52:56 -0800 Subject: [PATCH 112/844] add aurweb.captcha, a CAPTCHA utility module This CAPTCHA workflow is the same workflow used by our current PHP implementation of account registration. Signed-off-by: Kevin Morris --- aurweb/captcha.py | 54 +++++++++++++++++++++++++++++++++++++++ aurweb/templates.py | 6 ++++- test/test_captcha.py | 60 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 aurweb/captcha.py create mode 100644 test/test_captcha.py diff --git a/aurweb/captcha.py b/aurweb/captcha.py new file mode 100644 index 00000000..5475d85f --- /dev/null +++ b/aurweb/captcha.py @@ -0,0 +1,54 @@ +""" This module consists of aurweb's CAPTCHA utility functions and filters. """ +import hashlib + +import jinja2 + +from aurweb.db import query +from aurweb.models.user import User + + +def get_captcha_salts(): + """ Produce salts based on the current user count. """ + count = query(User).count() + salts = [] + for i in range(0, 6): + salts.append(f"aurweb-{count - i}") + return salts + + +def get_captcha_token(salt): + """ Produce a token for the CAPTCHA salt. """ + return hashlib.md5(salt.encode()).hexdigest()[:3] + + +def get_captcha_challenge(salt): + """ Get a CAPTCHA challenge string (shell command) for a salt. """ + token = get_captcha_token(salt) + return f"LC_ALL=C pacman -V|sed -r 's#[0-9]+#{token}#g'|md5sum|cut -c1-6" + + +def get_captcha_answer(token): + """ Compute the answer via md5 of the real template text, return the + first six digits of the hexadecimal hash. """ + text = r""" + .--. Pacman v%s.%s.%s - libalpm v%s.%s.%s +/ _.-' .-. .-. .-. Copyright (C) %s-%s Pacman Development Team +\ '-. '-' '-' '-' Copyright (C) %s-%s Judd Vinet + '--' + This program may be freely redistributed under + the terms of the GNU General Public License. +""" % tuple([token] * 10) + return hashlib.md5((text + "\n").encode()).hexdigest()[:6] + + +@jinja2.contextfilter +def captcha_salt_filter(context): + """ Returns the most recent CAPTCHA salt in the list of salts. """ + salts = get_captcha_salts() + return salts[0] + + +@jinja2.contextfilter +def captcha_cmdline_filter(context, salt): + """ Returns a CAPTCHA challenge for a given salt. """ + return get_captcha_challenge(salt) diff --git a/aurweb/templates.py b/aurweb/templates.py index 4ea74a62..d548e92b 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -12,7 +12,7 @@ from fastapi.responses import HTMLResponse import aurweb.config -from aurweb import l10n, time +from aurweb import captcha, l10n, time # Prepare jinja2 objects. loader = jinja2.FileSystemLoader(os.path.join( @@ -23,6 +23,10 @@ env = jinja2.Environment(loader=loader, autoescape=True, # Add tr translation filter. env.filters["tr"] = l10n.tr +# Add captcha filters. +env.filters["captcha_salt"] = captcha.captcha_salt_filter +env.filters["captcha_cmdline"] = captcha.captcha_cmdline_filter + def make_context(request: Request, title: str, next: str = None): """ Create a context for a jinja2 TemplateResponse. """ diff --git a/test/test_captcha.py b/test/test_captcha.py new file mode 100644 index 00000000..ec19dee9 --- /dev/null +++ b/test/test_captcha.py @@ -0,0 +1,60 @@ +import hashlib + +from aurweb import captcha + + +def test_captcha_salts(): + """ Make sure we can get some captcha salts. """ + salts = captcha.get_captcha_salts() + assert len(salts) == 6 + + +def test_captcha_token(): + """ Make sure getting a captcha salt's token matches up against + the first three digits of the md5 hash of the salt. """ + salts = captcha.get_captcha_salts() + salt = salts[0] + + token1 = captcha.get_captcha_token(salt) + token2 = hashlib.md5(salt.encode()).hexdigest()[:3] + + assert token1 == token2 + + +def test_captcha_challenge_answer(): + """ Make sure that executing the captcha challenge via shell + produces the correct result by comparing it against a straight + up token conversion. """ + salts = captcha.get_captcha_salts() + salt = salts[0] + + challenge = captcha.get_captcha_challenge(salt) + + token = captcha.get_captcha_token(salt) + challenge2 = f"LC_ALL=C pacman -V|sed -r 's#[0-9]+#{token}#g'|md5sum|cut -c1-6" + + assert challenge == challenge2 + + +def test_captcha_salt_filter(): + """ Make sure captcha_salt_filter returns the first salt from + get_captcha_salts(). + + Example usage: + + """ + salt = captcha.captcha_salt_filter(None) + assert salt == captcha.get_captcha_salts()[0] + + +def test_captcha_cmdline_filter(): + """ Make sure that the captcha_cmdline filter gives us the + same challenge that get_captcha_challenge does. + + Example usage: + {{ captcha_salt | captcha_cmdline }} + """ + salt = captcha.captcha_salt_filter(None) + display1 = captcha.captcha_cmdline_filter(None, salt) + display2 = captcha.get_captcha_challenge(salt) + assert display1 == display2 From 19b4a896f111c34fcf57a5d6fb0b40cd9ad43e51 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 30 Jan 2021 02:08:59 -0800 Subject: [PATCH 113/844] add openssh to test dependencies Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 2 +- Dockerfile | 2 +- test/README.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f1fe5e6f..58fb9fed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,7 @@ before_script: python-itsdangerous python-httpx python-jinja python-pytest-cov python-requests python-aiofiles python-python-multipart python-pytest-asyncio python-coverage python-bcrypt - python-email-validator + python-email-validator openssh - bash -c "echo '127.0.0.1' > /etc/hosts" - bash -c "echo '::1' >> /etc/hosts" diff --git a/Dockerfile b/Dockerfile index f65acc7c..cf54a13c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN pacman -Syu --noconfirm base-devel git gpgme protobuf pyalpm \ python-itsdangerous python-httpx python-jinja python-pytest-cov \ python-requests python-aiofiles python-python-multipart \ python-pytest-asyncio python-coverage hypercorn python-bcrypt \ - python-email-validator + python-email-validator openssh # Remove aurweb.sqlite3 if it was copied over via COPY. RUN rm -fv aurweb.sqlite3 diff --git a/test/README.md b/test/README.md index 872d980b..0f3f4cbd 100644 --- a/test/README.md +++ b/test/README.md @@ -28,6 +28,7 @@ For all the test to run, the following Arch packages should be installed: - python-pytest-cov - python-pytest-asyncio - postfix +- openssh Running tests ------------- From c94793b0b11b372a299fb6d23e39562066b7531b Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 28 Jan 2021 20:26:34 -0800 Subject: [PATCH 114/844] add user registration routes * Added /register get and post routes. + Added default attributes to AnonymousUser, including a new AnonymousList which behaves like an sqlalchemy relationship list. + aurweb.util: Added validation functions for various user fields used throughout registration. + test_accounts_routes: Added get|post register route tests. Signed-off-by: Kevin Morris --- aurweb/auth.py | 10 + aurweb/routers/accounts.py | 320 +++++++++++++++++++++++- aurweb/util.py | 84 +++++++ templates/partials/account_form.html | 343 ++++++++++++++++++++++++++ templates/register.html | 30 +++ test/test_accounts_routes.py | 356 ++++++++++++++++++++++++++- 6 files changed, 1140 insertions(+), 3 deletions(-) create mode 100644 templates/partials/account_form.html create mode 100644 templates/register.html diff --git a/aurweb/auth.py b/aurweb/auth.py index 53c853de..a4ff2167 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -7,12 +7,22 @@ from fastapi.responses import RedirectResponse from starlette.authentication import AuthCredentials, AuthenticationBackend, AuthenticationError from starlette.requests import HTTPConnection +import aurweb.config + from aurweb.models.session import Session from aurweb.models.user import User from aurweb.templates import make_context, render_template class AnonymousUser: + # Stub attributes used to mimic a real user. + ID = 0 + LangPreference = aurweb.config.get("options", "default_lang") + Timezone = aurweb.config.get("options", "default_timezone") + + # A stub ssh_pub_key relationship. + ssh_pub_key = None + @staticmethod def is_authenticated(): return False diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index db23bc3a..a43ba9f7 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -1,12 +1,20 @@ +import copy + from http import HTTPStatus from fastapi import APIRouter, Form, Request from fastapi.responses import HTMLResponse, RedirectResponse -from sqlalchemy import or_ +from sqlalchemy import and_, func, or_ -from aurweb import db +import aurweb.config + +from aurweb import db, l10n, time, util from aurweb.auth import auth_required +from aurweb.captcha import get_captcha_answer, get_captcha_salts, get_captcha_token from aurweb.l10n import get_translator_for_request +from aurweb.models.account_type import AccountType +from aurweb.models.ban import Ban +from aurweb.models.ssh_pub_key import SSHPubKey, get_fingerprint from aurweb.models.user import User from aurweb.scripts.notify import ResetKeyNotification from aurweb.templates import make_variable_context, render_template @@ -93,3 +101,311 @@ async def passreset_post(request: Request, # Render ?step=confirm. return RedirectResponse(url="/passreset?step=confirm", status_code=int(HTTPStatus.SEE_OTHER)) + + +def process_account_form(request: Request, user: User, args: dict): + """ Process an account form. All fields are optional and only checks + requirements in the case they are present. + + ``` + context = await make_variable_context(request, "Accounts") + ok, errors = process_account_form(request, user, **kwargs) + if not ok: + context["errors"] = errors + return render_template(request, "some_account_template.html", context) + ``` + + :param request: An incoming FastAPI request + :param user: The user model of the account being processed + :param args: A dictionary of arguments generated via request.form() + :return: A (passed processing boolean, list of errors) tuple + """ + + # Get a local translator. + _ = get_translator_for_request(request) + + host = request.client.host + ban = db.query(Ban, Ban.IPAddress == host).first() + if ban: + return False, [ + "Account registration has been disabled for your " + + "IP address, probably due to sustained spam attacks. " + + "Sorry for the inconvenience." + ] + + if request.user.is_authenticated(): + if not request.user.valid_password(args.get("passwd", None)): + return False, ["Invalid password."] + + email = args.get("E", None) + username = args.get("U", None) + + if not email or not username: + return False, ["Missing a required field."] + + username_min_len = aurweb.config.getint("options", "username_min_len") + username_max_len = aurweb.config.getint("options", "username_max_len") + if not util.valid_username(args.get("U")): + return False, [ + "The username is invalid.", + [ + _("It must be between %s and %s characters long") % ( + username_min_len, username_max_len), + "Start and end with a letter or number", + "Can contain only one period, underscore or hyphen.", + ] + ] + + password = args.get("P", None) + if password: + confirmation = args.get("C", None) + if not util.valid_password(password): + return False, [ + _("Your password must be at least %s characters.") % ( + username_min_len) + ] + elif not confirmation: + return False, ["Please confirm your new password."] + elif password != confirmation: + return False, ["Password fields do not match."] + + backup_email = args.get("BE", None) + homepage = args.get("HP", None) + pgp_key = args.get("K", None) + ssh_pubkey = args.get("PK", None) + language = args.get("L", None) + timezone = args.get("TZ", None) + + def username_exists(username): + return and_(User.ID != user.ID, + func.lower(User.Username) == username.lower()) + + def email_exists(email): + return and_(User.ID != user.ID, + func.lower(User.Email) == email.lower()) + + if not util.valid_email(email): + return False, ["The email address is invalid."] + elif backup_email and not util.valid_email(backup_email): + return False, ["The backup email address is invalid."] + elif homepage and not util.valid_homepage(homepage): + return False, [ + "The home page is invalid, please specify the full HTTP(s) URL."] + elif pgp_key and not util.valid_pgp_fingerprint(pgp_key): + return False, ["The PGP key fingerprint is invalid."] + elif ssh_pubkey and not util.valid_ssh_pubkey(ssh_pubkey): + return False, ["The SSH public key is invalid."] + elif language and language not in l10n.SUPPORTED_LANGUAGES: + return False, ["Language is not currently supported."] + elif timezone and timezone not in time.SUPPORTED_TIMEZONES: + return False, ["Timezone is not currently supported."] + elif db.query(User, username_exists(username)).first(): + # If the username already exists... + return False, [ + _("The username, %s%s%s, is already in use.") % ( + "", username, "") + ] + elif db.query(User, email_exists(email)).first(): + # If the email already exists... + return False, [ + _("The address, %s%s%s, is already in use.") % ( + "", email, "") + ] + + def ssh_fingerprint_exists(fingerprint): + return and_(SSHPubKey.UserID != user.ID, + SSHPubKey.Fingerprint == fingerprint) + + if ssh_pubkey: + fingerprint = get_fingerprint(ssh_pubkey.strip().rstrip()) + if fingerprint is None: + return False, ["The SSH public key is invalid."] + + if db.query(SSHPubKey, ssh_fingerprint_exists(fingerprint)).first(): + return False, [ + _("The SSH public key, %s%s%s, is already in use.") % ( + "", fingerprint, "") + ] + + captcha_salt = args.get("captcha_salt", None) + if captcha_salt and captcha_salt not in get_captcha_salts(): + return False, ["This CAPTCHA has expired. Please try again."] + + captcha = args.get("captcha", None) + if captcha: + answer = get_captcha_answer(get_captcha_token(captcha_salt)) + if captcha != answer: + return False, ["The entered CAPTCHA answer is invalid."] + + return True, [] + + +def make_account_form_context(context: dict, + request: Request, + user: User, + args: dict): + """ Modify a FastAPI context and add attributes for the account form. + + :param context: FastAPI context + :param request: FastAPI request + :param user: Target user + :param args: Persistent arguments: request.form() + :return: FastAPI context adjusted for account form + """ + # Do not modify the original context. + context = copy.copy(context) + + context["account_types"] = [ + (1, "Normal User"), + (2, "Trusted User") + ] + + user_account_type_id = context.get("account_types")[0][0] + + if request.user.has_credential("CRED_ACCOUNT_EDIT_DEV"): + context["account_types"].append((3, "Developer")) + context["account_types"].append((4, "Trusted User & Developer")) + + if request.user.is_authenticated(): + context["username"] = args.get("U", user.Username) + context["account_type"] = args.get("T", user.AccountType.ID) + context["suspended"] = args.get("S", user.Suspended) + context["email"] = args.get("E", user.Email) + context["hide_email"] = args.get("H", user.HideEmail) + context["backup_email"] = args.get("BE", user.BackupEmail) + context["realname"] = args.get("R", user.RealName) + context["homepage"] = args.get("HP", user.Homepage or str()) + context["ircnick"] = args.get("I", user.IRCNick) + context["pgp"] = args.get("K", user.PGPKey or str()) + context["lang"] = args.get("L", user.LangPreference) + context["tz"] = args.get("TZ", user.Timezone) + ssh_pk = user.ssh_pub_key.PubKey if user.ssh_pub_key else str() + context["ssh_pk"] = args.get("PK", ssh_pk) + context["cn"] = args.get("CN", user.CommentNotify) + context["un"] = args.get("UN", user.UpdateNotify) + context["on"] = args.get("ON", user.OwnershipNotify) + else: + context["username"] = args.get("U", str()) + context["account_type"] = args.get("T", user_account_type_id) + context["suspended"] = args.get("S", False) + context["email"] = args.get("E", str()) + context["hide_email"] = args.get("H", False) + context["backup_email"] = args.get("BE", str()) + context["realname"] = args.get("R", str()) + context["homepage"] = args.get("HP", str()) + context["ircnick"] = args.get("I", str()) + context["pgp"] = args.get("K", str()) + context["lang"] = args.get("L", context.get("language")) + context["tz"] = args.get("TZ", context.get("timezone")) + context["ssh_pk"] = args.get("PK", str()) + context["cn"] = args.get("CN", True) + context["un"] = args.get("UN", False) + context["on"] = args.get("ON", True) + + context["password"] = args.get("P", str()) + context["confirm"] = args.get("C", str()) + + return context + + +@router.get("/register", response_class=HTMLResponse) +@auth_required(False) +async def account_register(request: Request, + U: str = Form(default=str()), # Username + E: str = Form(default=str()), # Email + H: str = Form(default=False), # Hide Email + BE: str = Form(default=None), # Backup Email + R: str = Form(default=None), # Real Name + HP: str = Form(default=None), # Homepage + I: str = Form(default=None), # IRC Nick + K: str = Form(default=None), # PGP Key FP + L: str = Form(default=aurweb.config.get( + "options", "default_lang")), + TZ: str = Form(default=aurweb.config.get( + "options", "default_timezone")), + PK: str = Form(default=None), + CN: bool = Form(default=False), # Comment Notify + CU: bool = Form(default=False), # Update Notify + CO: bool = Form(default=False), # Owner Notify + captcha: str = Form(default=str())): + context = await make_variable_context(request, "Register") + context["captcha_salt"] = get_captcha_salts()[0] + context = make_account_form_context(context, request, None, dict()) + return render_template(request, "register.html", context) + + +@router.post("/register", response_class=HTMLResponse) +@auth_required(False) +async def account_register_post(request: Request, + U: str = Form(default=str()), # Username + E: str = Form(default=str()), # Email + H: str = Form(default=False), # Hide Email + BE: str = Form(default=None), # Backup Email + R: str = Form(default=''), # Real Name + HP: str = Form(default=None), # Homepage + I: str = Form(default=None), # IRC Nick + K: str = Form(default=None), # PGP Key + L: str = Form(default=aurweb.config.get( + "options", "default_lang")), + TZ: str = Form(default=aurweb.config.get( + "options", "default_timezone")), + PK: str = Form(default=None), # SSH PubKey + CN: bool = Form(default=False), + UN: bool = Form(default=False), + ON: bool = Form(default=False), + captcha: str = Form(default=None), + captcha_salt: str = Form(...)): + from aurweb.db import session + + context = await make_variable_context(request, "Register") + + args = dict(await request.form()) + context = make_account_form_context(context, request, None, args) + + ok, errors = process_account_form(request, request.user, args) + + if not ok: + # If the field values given do not meet the requirements, + # return HTTP 400 with an error. + context["errors"] = errors + return render_template(request, "register.html", context, + status_code=int(HTTPStatus.BAD_REQUEST)) + + if not captcha: + context["errors"] = ["The CAPTCHA is missing."] + return render_template(request, "register.html", context, + status_code=int(HTTPStatus.BAD_REQUEST)) + + # Create a user with no password with a resetkey, then send + # an email off about it. + resetkey = db.make_random_value(User, User.ResetKey) + + # By default, we grab the User account type to associate with. + account_type = db.query(AccountType, + AccountType.AccountType == "User").first() + + # Create a user given all parameters available. + user = db.create(User, Username=U, Email=E, HideEmail=H, BackupEmail=BE, + RealName=R, Homepage=HP, IRCNick=I, PGPKey=K, + LangPreference=L, Timezone=TZ, CommentNotify=CN, + UpdateNotify=UN, OwnershipNotify=ON, ResetKey=resetkey, + AccountType=account_type) + + # If a PK was given and either one does not exist or the given + # PK mismatches the existing user's SSHPubKey.PubKey. + if PK: + # Get the second element in the PK, which is the actual key. + pubkey = PK.strip().rstrip() + fingerprint = get_fingerprint(pubkey) + user.ssh_pub_key = SSHPubKey(UserID=user.ID, + PubKey=pubkey, + Fingerprint=fingerprint) + session.commit() + + # Send a reset key notification to the new user. + executor = db.ConnectionExecutor(db.get_engine().raw_connection()) + ResetKeyNotification(executor, user.ID).send() + + context["complete"] = True + context["user"] = user + return render_template(request, "register.html", context) diff --git a/aurweb/util.py b/aurweb/util.py index 65f18a4c..5e1717bd 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -1,7 +1,91 @@ +import base64 import random +import re import string +from urllib.parse import urlparse + +import jinja2 + +from email_validator import EmailNotValidError, EmailUndeliverableError, validate_email + +import aurweb.config + def make_random_string(length): return ''.join(random.choices(string.ascii_lowercase + string.digits, k=length)) + + +def valid_username(username): + min_len = aurweb.config.getint("options", "username_min_len") + max_len = aurweb.config.getint("options", "username_max_len") + if not (min_len <= len(username) <= max_len): + return False + + # Check that username contains: one or more alphanumeric + # characters, an optional separator of '.', '-' or '_', followed + # by alphanumeric characters. + return re.match(r'^[a-zA-Z0-9]+[.\-_]?[a-zA-Z0-9]+$', username) + + +def valid_email(email): + try: + validate_email(email) + except EmailUndeliverableError: + return False + except EmailNotValidError: + return False + return True + + +def valid_homepage(homepage): + parts = urlparse(homepage) + return parts.scheme in ("http", "https") and bool(parts.netloc) + + +def valid_password(password): + min_len = aurweb.config.getint("options", "passwd_min_len") + return len(password) >= min_len + + +def valid_pgp_fingerprint(fp): + fp = fp.replace(" ", "") + try: + # Attempt to convert the fingerprint to an int via base16. + # If it can't, it's not a hex string. + int(fp, 16) + except ValueError: + return False + + # Check the length; must be 40 hexadecimal digits. + return len(fp) == 40 + + +def valid_ssh_pubkey(pk): + valid_prefixes = ("ssh-rsa", "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521", + "ssh-ed25519") + + has_valid_prefix = False + for prefix in valid_prefixes: + if "%s " % prefix in pk: + has_valid_prefix = True + break + if not has_valid_prefix: + return False + + tokens = pk.strip().rstrip().split(" ") + if len(tokens) < 2: + return False + + return base64.b64encode(base64.b64decode(tokens[1])).decode() == tokens[1] + + +@jinja2.contextfilter +def account_url(context, user): + request = context.get("request") + base = f"{request.url.scheme}://{request.url.hostname}" + if request.url.scheme == "http" and request.url.port != 80: + base += f":{request.url.port}" + return f"{base}/account/{user.Username}" diff --git a/templates/partials/account_form.html b/templates/partials/account_form.html new file mode 100644 index 00000000..3af13368 --- /dev/null +++ b/templates/partials/account_form.html @@ -0,0 +1,343 @@ + +
+
+ +
+
+ +

+ + + + ({% trans %}required{% endtrans %}) +

+

+ {{ "Your user name is the name you will use to login. " + "It is visible to the general public, even if your " + "account is inactive." | tr }} +

+ + {% if request.user.has_credential("CRED_ACCOUNT_CHANGE_TYPE") %} +

+ + +

+ +

+ + + +

+ {% endif %} + + +

+ + + + ({% trans %}required{% endtrans %}) +

+

+ {{ "Please ensure you correctly entered your email " + "address, otherwise you will be locked out." | tr }} +

+ + +

+ + + +

+

+ {{ "If you do not hide your email address, it is " + "visible to all registered AUR users. If you hide your " + "email address, it is visible to members of the Arch " + "Linux staff only." | tr }} +

+ + +

+ + + +

+

+ + {{ "Optionally provide a secondary email address that " + "can be used to restore your account in case you lose " + "access to your primary email address." | tr }} + {{ "Password reset links are always sent to both your " + "primary and your backup email address." | tr }} + {{ "Your backup email address is always only visible to " + "members of the Arch Linux staff, independent of the %s " + "setting." | tr + | format("%s" | format("Hide Email Address" | tr)) + | safe }} + +

+ + +

+ + + +

+ + +

+ + + +

+ + +

+ + + +

+ + +

+ + + +

+ + +

+ + + +

+ + +

+ + + +

+ +
+ + {% if form_type == "UpdateAccount" %} +
+ + {{ + "If you want to change the password, enter a new password " + "and confirm the new password by entering it again." | tr + }} + +

+ + +

+ +

+ + + +

+
+ {% endif %} + +
+ + {{ + "The following information is only required if you " + "want to submit packages to the Arch User Repository." | tr + }} + +

+ + + + +

+
+ +
+ {% trans%}Notification settings{% endtrans %}: +

+ + + +

+

+ + + +

+

+ + + +

+
+ +
+ {% if form_type == "UpdateAccount" %} + + {{ "To confirm the profile changes, please enter " + "your current password:" | tr }} + +

+ + +

+ {% else %} + + + {{ "To protect the AUR against automated account creation, " + "we kindly ask you to provide the output of the following " + "command:" | tr }} + + {{ captcha_salt | captcha_cmdline }} + + +

+ + + ({% trans %}required{% endtrans %}) + + +

+ {% endif %} +
+ +
+

+ + {% if form_type == "UpdateAccount" %} +   + {% else %} +   + {% endif %} + +

+
+
diff --git a/templates/register.html b/templates/register.html new file mode 100644 index 00000000..a15971a1 --- /dev/null +++ b/templates/register.html @@ -0,0 +1,30 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} +
+

{% trans %}Register{% endtrans %}

+ + {% if complete %} + {{ + "The account, %s%s%s, has been successfully created." + | tr + | format("", "'" + user.Username + "'", "") + | safe + }} +

+ {% trans %}A password reset key has been sent to your e-mail address.{% endtrans %} +

+ {% else %} + {% if errors %} + {% include "partials/error.html" %} + {% else %} +

+ {% trans %}Use this form to create an account.{% endtrans %} +

+ {% endif %} + + {% set form_type = "NewAccount" %} + {% include "partials/account_form.html" %} + {% endif %} +
+{% endblock %} diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index 69896a0f..d79137bf 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -1,13 +1,21 @@ +import re +import tempfile + +from datetime import datetime from http import HTTPStatus +from subprocess import Popen import pytest from fastapi.testclient import TestClient +from aurweb import captcha from aurweb.asgi import app -from aurweb.db import query +from aurweb.db import create, delete, query from aurweb.models.account_type import AccountType +from aurweb.models.ban import Ban from aurweb.models.session import Session +from aurweb.models.ssh_pub_key import SSHPubKey, get_fingerprint from aurweb.models.user import User from aurweb.testing import setup_test_db from aurweb.testing.models import make_user @@ -220,3 +228,349 @@ def test_post_passreset_error_password_requirements(): error = f"Your password must be at least {passwd_min_len} characters." assert error in response.content.decode("utf-8") + + +def test_get_register(): + with client as request: + response = request.get("/register") + assert response.status_code == int(HTTPStatus.OK) + + +def post_register(request, **kwargs): + """ A simple helper that allows overrides to test defaults. """ + salt = captcha.get_captcha_salts()[0] + token = captcha.get_captcha_token(salt) + answer = captcha.get_captcha_answer(token) + + data = { + "U": "newUser", + "E": "newUser@email.org", + "P": "newUserPassword", + "C": "newUserPassword", + "L": "en", + "TZ": "UTC", + "captcha": answer, + "captcha_salt": salt + } + + # For any kwargs given, override their k:v pairs in data. + args = dict(kwargs) + for k, v in args.items(): + data[k] = v + + return request.post("/register", data=data, allow_redirects=False) + + +def test_post_register(): + with client as request: + response = post_register(request) + assert response.status_code == int(HTTPStatus.OK) + + expected = "The account, 'newUser', " + expected += "has been successfully created." + assert expected in response.content.decode() + + +def test_post_register_rejects_case_insensitive_spoof(): + with client as request: + response = post_register(request, U="newUser", E="newUser@example.org") + assert response.status_code == int(HTTPStatus.OK) + + with client as request: + response = post_register(request, U="NEWUSER", E="BLAH@GMAIL.COM") + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + expected = "The username, NEWUSER, is already in use." + assert expected in response.content.decode() + + with client as request: + response = post_register(request, U="BLAH", E="NEWUSER@EXAMPLE.ORG") + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + expected = "The address, NEWUSER@EXAMPLE.ORG, " + expected += "is already in use." + assert expected in response.content.decode() + + +def test_post_register_error_expired_captcha(): + with client as request: + response = post_register(request, captcha_salt="invalid-salt") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "This CAPTCHA has expired. Please try again." in content + + +def test_post_register_error_missing_captcha(): + with client as request: + response = post_register(request, captcha=None) + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "The CAPTCHA is missing." in content + + +def test_post_register_error_invalid_captcha(): + with client as request: + response = post_register(request, captcha="invalid blah blah") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "The entered CAPTCHA answer is invalid." in content + + +def test_post_register_error_ip_banned(): + # 'testclient' is used as request.client.host via FastAPI TestClient. + create(Ban, IPAddress="testclient", BanTS=datetime.utcnow()) + + with client as request: + response = post_register(request) + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert ("Account registration has been disabled for your IP address, " + + "probably due to sustained spam attacks. Sorry for the " + + "inconvenience.") in content + + +def test_post_register_error_missing_username(): + with client as request: + response = post_register(request, U="") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "Missing a required field." in content + + +def test_post_register_error_missing_email(): + with client as request: + response = post_register(request, E="") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "Missing a required field." in content + + +def test_post_register_error_invalid_username(): + with client as request: + # Our test config requires at least three characters for a + # valid username, so test against two characters: 'ba'. + response = post_register(request, U="ba") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "The username is invalid." in content + + +def test_post_register_invalid_password(): + with client as request: + response = post_register(request, P="abc", C="abc") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + expected = r"Your password must be at least \d+ characters." + assert re.search(expected, content) + + +def test_post_register_error_missing_confirm(): + with client as request: + response = post_register(request, C=None) + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "Please confirm your new password." in content + + +def test_post_register_error_mismatched_confirm(): + with client as request: + response = post_register(request, C="mismatched") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "Password fields do not match." in content + + +def test_post_register_error_invalid_email(): + with client as request: + response = post_register(request, E="bad@email") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "The email address is invalid." in content + + +def test_post_register_error_undeliverable_email(): + with client as request: + # At the time of writing, webchat.freenode.net does not contain + # mx records; if it ever does, it'll break this test. + response = post_register(request, E="email@bad.c") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "The email address is invalid." in content + + +def test_post_register_invalid_backup_email(): + with client as request: + response = post_register(request, BE="bad@email") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "The backup email address is invalid." in content + + +def test_post_register_error_invalid_homepage(): + with client as request: + response = post_register(request, HP="bad") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + expected = "The home page is invalid, please specify the full HTTP(s) URL." + assert expected in content + + +def test_post_register_error_invalid_pgp_fingerprints(): + with client as request: + response = post_register(request, K="bad") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + expected = "The PGP key fingerprint is invalid." + assert expected in content + + pk = 'z' + ('a' * 39) + with client as request: + response = post_register(request, K=pk) + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + expected = "The PGP key fingerprint is invalid." + assert expected in content + + +def test_post_register_error_invalid_ssh_pubkeys(): + with client as request: + response = post_register(request, PK="bad") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "The SSH public key is invalid." in content + + with client as request: + response = post_register(request, PK="ssh-rsa ") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "The SSH public key is invalid." in content + + +def test_post_register_error_unsupported_language(): + with client as request: + response = post_register(request, L="bad") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + expected = "Language is not currently supported." + assert expected in content + + +def test_post_register_error_unsupported_timezone(): + with client as request: + response = post_register(request, TZ="ABCDEFGH") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + expected = "Timezone is not currently supported." + assert expected in content + + +def test_post_register_error_username_taken(): + with client as request: + response = post_register(request, U="test") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + expected = r"The username, .*, is already in use." + assert re.search(expected, content) + + +def test_post_register_error_email_taken(): + with client as request: + response = post_register(request, E="test@example.org") + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + expected = r"The address, .*, is already in use." + assert re.search(expected, content) + + +def test_post_register_error_ssh_pubkey_taken(): + pk = str() + + # Create a public key with ssh-keygen (this adds ssh-keygen as a + # dependency to passing this test). + with tempfile.TemporaryDirectory() as tmpdir: + with open("/dev/null", "w") as null: + proc = Popen(["ssh-keygen", "-f", f"{tmpdir}/test.ssh", "-N", ""], + stdout=null, stderr=null) + proc.wait() + assert proc.returncode == 0 + + # Read in the public key, then delete the temp dir we made. + pk = open(f"{tmpdir}/test.ssh.pub").read().rstrip() + + # Take the sha256 fingerprint of the ssh public key, create it. + fp = get_fingerprint(pk) + create(SSHPubKey, UserID=user.ID, PubKey=pk, Fingerprint=fp) + + with client as request: + response = post_register(request, PK=pk) + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + expected = r"The SSH public key, .*, is already in use." + assert re.search(expected, content) + + +def test_post_register_with_ssh_pubkey(): + pk = str() + + # Create a public key with ssh-keygen (this adds ssh-keygen as a + # dependency to passing this test). + with tempfile.TemporaryDirectory() as tmpdir: + with open("/dev/null", "w") as null: + proc = Popen(["ssh-keygen", "-f", f"{tmpdir}/test.ssh", "-N", ""], + stdout=null, stderr=null) + proc.wait() + assert proc.returncode == 0 + + # Read in the public key, then delete the temp dir we made. + pk = open(f"{tmpdir}/test.ssh.pub").read().rstrip() + + with client as request: + response = post_register(request, PK=pk) + + assert response.status_code == int(HTTPStatus.OK) From d323c1f95b666eb5d607919c14c719884b5e1457 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 30 Jan 2021 02:09:34 -0800 Subject: [PATCH 115/844] add python-lxml to dependencies Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 2 +- Dockerfile | 2 +- INSTALL | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58fb9fed..5bdf427c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,7 @@ before_script: python-itsdangerous python-httpx python-jinja python-pytest-cov python-requests python-aiofiles python-python-multipart python-pytest-asyncio python-coverage python-bcrypt - python-email-validator openssh + python-email-validator openssh python-lxml - bash -c "echo '127.0.0.1' > /etc/hosts" - bash -c "echo '::1' >> /etc/hosts" diff --git a/Dockerfile b/Dockerfile index cf54a13c..c432f73f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN pacman -Syu --noconfirm base-devel git gpgme protobuf pyalpm \ python-itsdangerous python-httpx python-jinja python-pytest-cov \ python-requests python-aiofiles python-python-multipart \ python-pytest-asyncio python-coverage hypercorn python-bcrypt \ - python-email-validator openssh + python-email-validator openssh python-lxml # Remove aurweb.sqlite3 if it was copied over via COPY. RUN rm -fv aurweb.sqlite3 diff --git a/INSTALL b/INSTALL index 04ccd69e..3381daf5 100644 --- a/INSTALL +++ b/INSTALL @@ -51,7 +51,8 @@ read the instructions below. python-bleach python-markdown python-alembic hypercorn \ python-itsdangerous python-authlib python-httpx \ python-jinja python-aiofiles python-python-multipart \ - python-requests hypercorn python-bcrypt python-email-validator + python-requests hypercorn python-bcrypt python-email-validator \ + python-lxml # python3 setup.py install 5) Create a new MySQL database and a user and import the aurweb SQL schema: From 4e9ef6fb00211378ca7373b0e41ee29479c9aa44 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 28 Jan 2021 20:34:27 -0800 Subject: [PATCH 116/844] add account edit (settings) routes * Added account_url filter to jinja2 environment. This produces a path to the user's account url (/account/{username}). * Updated archdev-navbar to link to new edit route. + Added migrate_cookies(request, response) to aurweb.util, a function that simply migrates the request cookies to response and returns it. + Added account_edit tests to test_accounts_routes.py. Signed-off-by: Kevin Morris --- aurweb/routers/accounts.py | 145 ++++++++++++ aurweb/templates.py | 5 +- aurweb/util.py | 6 + templates/account/edit.html | 46 ++++ templates/partials/account_form.html | 9 + templates/partials/archdev-navbar.html | 20 +- test/test_accounts_routes.py | 296 +++++++++++++++++++++++++ 7 files changed, 522 insertions(+), 5 deletions(-) create mode 100644 templates/account/edit.html diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index a43ba9f7..689f7f58 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -1,5 +1,6 @@ import copy +from datetime import datetime from http import HTTPStatus from fastapi import APIRouter, Form, Request @@ -284,6 +285,7 @@ def make_account_form_context(context: dict, context["cn"] = args.get("CN", user.CommentNotify) context["un"] = args.get("UN", user.UpdateNotify) context["on"] = args.get("ON", user.OwnershipNotify) + context["inactive"] = args.get("J", user.InactivityTS != 0) else: context["username"] = args.get("U", str()) context["account_type"] = args.get("T", user_account_type_id) @@ -301,6 +303,7 @@ def make_account_form_context(context: dict, context["cn"] = args.get("CN", True) context["un"] = args.get("UN", False) context["on"] = args.get("ON", True) + context["inactive"] = args.get("J", False) context["password"] = args.get("P", str()) context["confirm"] = args.get("C", str()) @@ -409,3 +412,145 @@ async def account_register_post(request: Request, context["complete"] = True context["user"] = user return render_template(request, "register.html", context) + + +def cannot_edit(request, user): + """ Return a 401 HTMLResponse if the request user doesn't + have authorization, otherwise None. """ + has_dev_cred = request.user.has_credential("CRED_ACCOUNT_EDIT_DEV", + approved=[user]) + if not has_dev_cred: + return HTMLResponse(status_code=int(HTTPStatus.UNAUTHORIZED)) + return None + + +@router.get("/account/{username}/edit", response_class=HTMLResponse) +@auth_required(True) +async def account_edit(request: Request, + username: str): + user = db.query(User, User.Username == username).first() + response = cannot_edit(request, user) + if response: + return response + + context = await make_variable_context(request, "Accounts") + context["user"] = user + + context = make_account_form_context(context, request, user, dict()) + return render_template(request, "account/edit.html", context) + + +@router.post("/account/{username}/edit", response_class=HTMLResponse) +@auth_required(True) +async def account_edit_post(request: Request, + username: str, + U: str = Form(default=str()), # Username + J: bool = Form(default=False), + E: str = Form(default=str()), # Email + H: str = Form(default=False), # Hide Email + BE: str = Form(default=None), # Backup Email + R: str = Form(default=None), # Real Name + HP: str = Form(default=None), # Homepage + I: str = Form(default=None), # IRC Nick + K: str = Form(default=None), # PGP Key + L: str = Form(aurweb.config.get( + "options", "default_lang")), + TZ: str = Form(aurweb.config.get( + "options", "default_timezone")), + P: str = Form(default=str()), # New Password + C: str = Form(default=None), # Password Confirm + PK: str = Form(default=None), # PubKey + CN: bool = Form(default=False), # Comment Notify + UN: bool = Form(default=False), # Update Notify + ON: bool = Form(default=False), # Owner Notify + passwd: str = Form(default=str())): + from aurweb.db import session + + user = session.query(User).filter(User.Username == username).first() + response = cannot_edit(request, user) + if response: + return response + + context = await make_variable_context(request, "Accounts") + context["user"] = user + + if not passwd: + context["errors"] = ["Invalid password."] + return render_template(request, "account/edit.html", context, + status_code=int(HTTPStatus.BAD_REQUEST)) + + args = dict(await request.form()) + context = make_account_form_context(context, request, user, args) + ok, errors = process_account_form(request, user, args) + + if not ok: + context["errors"] = errors + return render_template(request, "account/edit.html", context, + status_code=int(HTTPStatus.BAD_REQUEST)) + + # Set all updated fields as needed. + user.Username = U or user.Username + user.Email = E or user.Email + user.HideEmail = bool(H) + user.BackupEmail = BE or user.BackupEmail + user.RealName = R or user.RealName + user.Homepage = HP or user.Homepage + user.IRCNick = I or user.IRCNick + user.PGPKey = K or user.PGPKey + user.InactivityTS = datetime.utcnow().timestamp() if J else 0 + + # If we update the language, update the cookie as well. + if L and L != user.LangPreference: + request.cookies["AURLANG"] = L + user.LangPreference = L + context["language"] = L + + # If we update the timezone, also update the cookie. + if TZ and TZ != user.Timezone: + user.Timezone = TZ + request.cookies["AURTZ"] = TZ + context["timezone"] = TZ + + user.CommentNotify = bool(CN) + user.UpdateNotify = bool(UN) + user.OwnershipNotify = bool(ON) + + # If a PK is given, compare it against the target user's PK. + if PK: + # Get the second token in the public key, which is the actual key. + pubkey = PK.strip().rstrip() + fingerprint = get_fingerprint(pubkey) + if not user.ssh_pub_key: + # No public key exists, create one. + user.ssh_pub_key = SSHPubKey(UserID=user.ID, + PubKey=PK, + Fingerprint=fingerprint) + elif user.ssh_pub_key.Fingerprint != fingerprint: + # A public key already exists, update it. + user.ssh_pub_key.PubKey = PK + user.ssh_pub_key.Fingerprint = fingerprint + elif user.ssh_pub_key: + # Else, if the user has a public key already, delete it. + session.delete(user.ssh_pub_key) + + # Commit changes, if any. + session.commit() + + if P and not user.valid_password(P): + # Remove the fields we consumed for passwords. + context["P"] = context["C"] = str() + + # If a password was given and it doesn't match the user's, update it. + user.update_password(P) + if user == request.user: + # If the target user is the request user, login with + # the updated password and update AURSID. + request.cookies["AURSID"] = user.login(request, P) + + if not errors: + context["complete"] = True + + # Update cookies with requests, in case they were changed. + response = render_template(request, "account/edit.html", context) + return util.migrate_cookies(request, response) +>>>>>> > dddd1137... add account edit(settings) routes diff --git a/aurweb/templates.py b/aurweb/templates.py index d548e92b..c0472b2e 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -12,7 +12,7 @@ from fastapi.responses import HTMLResponse import aurweb.config -from aurweb import captcha, l10n, time +from aurweb import captcha, l10n, time, util # Prepare jinja2 objects. loader = jinja2.FileSystemLoader(os.path.join( @@ -27,6 +27,9 @@ env.filters["tr"] = l10n.tr env.filters["captcha_salt"] = captcha.captcha_salt_filter env.filters["captcha_cmdline"] = captcha.captcha_cmdline_filter +# Add account utility filters. +env.filters["account_url"] = util.account_url + def make_context(request: Request, title: str, next: str = None): """ Create a context for a jinja2 TemplateResponse. """ diff --git a/aurweb/util.py b/aurweb/util.py index 5e1717bd..8b6ddbe7 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -82,6 +82,12 @@ def valid_ssh_pubkey(pk): return base64.b64encode(base64.b64decode(tokens[1])).decode() == tokens[1] +def migrate_cookies(request, response): + for k, v in request.cookies.items(): + response.set_cookie(k, v) + return response + + @jinja2.contextfilter def account_url(context, user): request = context.get("request") diff --git a/templates/account/edit.html b/templates/account/edit.html new file mode 100644 index 00000000..f8895d92 --- /dev/null +++ b/templates/account/edit.html @@ -0,0 +1,46 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} +
+

{% trans %}Accounts{% endtrans %}

+ + {% if complete %} + + {{ + "The account, %s%s%s, has been successfully modified." + | tr + | format("", user.Username, "") + | safe + }} + + {% else %} + {% if errors %} + {% include "partials/error.html" %} + {% else %} +

+ {{ "Click %shere%s if you want to permanently delete this account." + | tr + | format('' | format(user | account_url), + "") + | safe + }} + {{ "Click %shere%s for user details." + | tr + | format('' | format(user | account_url), + "") + | safe + }} + {{ "Click %shere%s to list the comments made by this account." + | tr + | format('' | format(user | account_url), + "") + | safe + }} +

+ {% endif %} + + {% set form_type = "UpdateAccount" %} + {% include "partials/account_form.html" %} + {% endif %} +
+{% endblock %} diff --git a/templates/partials/account_form.html b/templates/partials/account_form.html index 3af13368..5ae18131 100644 --- a/templates/partials/account_form.html +++ b/templates/partials/account_form.html @@ -42,6 +42,15 @@ "account is inactive." | tr }}

+

+ + +

+ {% if request.user.has_credential("CRED_ACCOUNT_CHANGE_TYPE") %}

  • AUR {% trans %}Home{% endtrans %}
  • {% endif %}
  • {% trans %}Packages{% endtrans %}
  • -
  • {% trans %}Register{% endtrans %}
  • -
  • - {% if request.user.is_authenticated() %} + {% if request.user.is_authenticated() %} +
  • + + {% trans %} My Account{% endtrans %} + +
  • +
  • {% trans %}Logout{% endtrans %} - {% else %} +
  • + {% else %} +
  • + + {% trans %}Register{% endtrans %} + +
  • +
  • {% trans %}Login{% endtrans %} +
  • {% endif %} diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index d79137bf..540adde7 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -5,6 +5,7 @@ from datetime import datetime from http import HTTPStatus from subprocess import Popen +import lxml.html import pytest from fastapi.testclient import TestClient @@ -574,3 +575,298 @@ def test_post_register_with_ssh_pubkey(): response = post_register(request, PK=pk) assert response.status_code == int(HTTPStatus.OK) + + +def test_get_account_edit(): + request = Request() + sid = user.login(request, "testPassword") + + with client as request: + response = request.get("/account/test/edit", cookies={ + "AURSID": sid + }, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.OK) + + +def test_get_account_edit_unauthorized(): + request = Request() + sid = user.login(request, "testPassword") + + create(User, Username="test2", Email="test2@example.org", + Passwd="testPassword") + + with client as request: + # Try to edit `test2` while authenticated as `test`. + response = request.get("/account/test2/edit", cookies={ + "AURSID": sid + }, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.UNAUTHORIZED) + + +def test_post_account_edit(): + request = Request() + sid = user.login(request, "testPassword") + + post_data = { + "U": "test", + "E": "test666@example.org", + "passwd": "testPassword" + } + + with client as request: + response = request.post("/account/test/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.OK) + + expected = "The account, test, " + expected += "has been successfully modified." + assert expected in response.content.decode() + + +def test_post_account_edit_dev(): + from aurweb.db import session + + # Modify our user to be a "Trusted User & Developer" + name = "Trusted User & Developer" + tu_or_dev = query(AccountType, AccountType.AccountType == name).first() + user.AccountType = tu_or_dev + session.commit() + + request = Request() + sid = user.login(request, "testPassword") + + post_data = { + "U": "test", + "E": "test666@example.org", + "passwd": "testPassword" + } + + with client as request: + response = request.post("/account/test/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.OK) + + expected = "The account, test, " + expected += "has been successfully modified." + assert expected in response.content.decode() + + +def test_post_account_edit_language(): + request = Request() + sid = user.login(request, "testPassword") + + post_data = { + "U": "test", + "E": "test@example.org", + "L": "de", # German + "passwd": "testPassword" + } + + with client as request: + response = request.post("/account/test/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.OK) + + # Parse the response content html into an lxml root, then make + # sure we see a 'de' option selected on the page. + content = response.content.decode() + root = lxml.html.fromstring(content) + lang_nodes = root.xpath('//option[@value="de"]/@selected') + assert lang_nodes and len(lang_nodes) != 0 + assert lang_nodes[0] == "selected" + + +def test_post_account_edit_timezone(): + request = Request() + sid = user.login(request, "testPassword") + + post_data = { + "U": "test", + "E": "test@example.org", + "TZ": "CET", + "passwd": "testPassword" + } + + with client as request: + response = request.post("/account/test/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.OK) + + +def test_post_account_edit_error_missing_password(): + request = Request() + sid = user.login(request, "testPassword") + + post_data = { + "U": "test", + "E": "test@example.org", + "TZ": "CET", + "passwd": "" + } + + with client as request: + response = request.post("/account/test/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "Invalid password." in content + + +def test_post_account_edit_error_invalid_password(): + request = Request() + sid = user.login(request, "testPassword") + + post_data = { + "U": "test", + "E": "test@example.org", + "TZ": "CET", + "passwd": "invalid" + } + + with client as request: + response = request.post("/account/test/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + content = response.content.decode() + assert "Invalid password." in content + + +def test_post_account_edit_error_unauthorized(): + request = Request() + sid = user.login(request, "testPassword") + + test2 = create(User, Username="test2", Email="test2@example.org", + Passwd="testPassword") + + post_data = { + "U": "test", + "E": "test@example.org", + "TZ": "CET", + "passwd": "testPassword" + } + + with client as request: + # Attempt to edit 'test2' while logged in as 'test'. + response = request.post("/account/test2/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.UNAUTHORIZED) + + +def test_post_account_edit_ssh_pub_key(): + pk = str() + + # Create a public key with ssh-keygen (this adds ssh-keygen as a + # dependency to passing this test). + with tempfile.TemporaryDirectory() as tmpdir: + with open("/dev/null", "w") as null: + proc = Popen(["ssh-keygen", "-f", f"{tmpdir}/test.ssh", "-N", ""], + stdout=null, stderr=null) + proc.wait() + assert proc.returncode == 0 + + # Read in the public key, then delete the temp dir we made. + pk = open(f"{tmpdir}/test.ssh.pub").read().rstrip() + + request = Request() + sid = user.login(request, "testPassword") + + post_data = { + "U": "test", + "E": "test@example.org", + "PK": pk, + "passwd": "testPassword" + } + + with client as request: + response = request.post("/account/test/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.OK) + + # Now let's update what's already there to gain coverage over that path. + pk = str() + with tempfile.TemporaryDirectory() as tmpdir: + with open("/dev/null", "w") as null: + proc = Popen(["ssh-keygen", "-f", f"{tmpdir}/test.ssh", "-N", ""], + stdout=null, stderr=null) + proc.wait() + assert proc.returncode == 0 + + # Read in the public key, then delete the temp dir we made. + pk = open(f"{tmpdir}/test.ssh.pub").read().rstrip() + + post_data["PK"] = pk + + with client as request: + response = request.post("/account/test/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.OK) + + +def test_post_account_edit_invalid_ssh_pubkey(): + pubkey = "ssh-rsa fake key" + + request = Request() + sid = user.login(request, "testPassword") + + post_data = { + "U": "test", + "E": "test@example.org", + "P": "newPassword", + "C": "newPassword", + "PK": pubkey, + "passwd": "testPassword" + } + + with client as request: + response = request.post("/account/test/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + +def test_post_account_edit_password(): + request = Request() + sid = user.login(request, "testPassword") + + post_data = { + "U": "test", + "E": "test@example.org", + "P": "newPassword", + "C": "newPassword", + "passwd": "testPassword" + } + + with client as request: + response = request.post("/account/test/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.OK) + + assert user.valid_password("newPassword") + + +>>>>>> > dddd1137... add account edit(settings) routes From 4f928b45770b3b8fd6013473b57feb223679f884 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 29 Jan 2021 23:40:38 -0800 Subject: [PATCH 117/844] add account (view) route + Added get /account/{username} route. + Added account/show.html template which shows a single use Signed-off-by: Kevin Morris --- aurweb/routers/accounts.py | 18 ++++++- templates/account/show.html | 96 ++++++++++++++++++++++++++++++++++++ test/test_accounts_routes.py | 31 +++++++++++- 3 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 templates/account/show.html diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 689f7f58..c7c96003 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -3,7 +3,7 @@ import copy from datetime import datetime from http import HTTPStatus -from fastapi import APIRouter, Form, Request +from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse from sqlalchemy import and_, func, or_ @@ -553,4 +553,18 @@ async def account_edit_post(request: Request, # Update cookies with requests, in case they were changed. response = render_template(request, "account/edit.html", context) return util.migrate_cookies(request, response) ->>>>>> > dddd1137... add account edit(settings) routes + + +@router.get("/account/{username}") +@auth_required(True, template=("account/show.html", "Accounts")) +async def account(request: Request, username: str): + user = db.query(User, User.Username == username).first() + + context = await make_variable_context(request, "Accounts") + + if not user: + raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) + + context["user"] = user + + return render_template(request, "account/show.html", context) diff --git a/templates/account/show.html b/templates/account/show.html new file mode 100644 index 00000000..139ff1f5 --- /dev/null +++ b/templates/account/show.html @@ -0,0 +1,96 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} +
    +

    {% trans %}Accounts{% endtrans %}

    + + {% if not request.user.is_authenticated() %} + {% trans %}You must log in to view user information.{% endtrans %} + {% else %} + + + + + +
    +

    {{ user.Username }}

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    {% trans %}Username{% endtrans %}:{{ user.Username }}
    {% trans %}Account Type{% endtrans %}:{{ user.AccountType }}
    {% trans %}Email Address{% endtrans %}: + {{ user.Email }} +
    {% trans %}Real Name{% endtrans %}:{{ user.RealName }}
    {% trans %}Homepage{% endtrans %}: + {% if user.Homepage %} + {{ user.Homepage }} + {% endif %} +
    {% trans %}IRC Nick{% endtrans %}:{{ user.IRCNick }}
    {% trans %}PGP Key Fingerprint{% endtrans %}:{{ user.PGPKey or '' }}
    {% trans %}Status{% endtrans %}:{{ "Active" if not user.Suspended else "Suspended" | tr }}
    {% trans %}Registration date{% endtrans %}: + {{ user.RegistrationTS.strftime("%Y-%m-%d") }} +
    {% trans %}Links{% endtrans %}: + +
    +
    + {% endif %} +
    +{% endblock %} diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index 540adde7..c42736fa 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -869,4 +869,33 @@ def test_post_account_edit_password(): assert user.valid_password("newPassword") ->>>>>> > dddd1137... add account edit(settings) routes +def test_get_account(): + request = Request() + sid = user.login(request, "testPassword") + + with client as request: + response = request.get("/account/test", cookies={"AURSID": sid}, + allow_redirects=False) + + assert response.status_code == int(HTTPStatus.OK) + + +def test_get_account_not_found(): + request = Request() + sid = user.login(request, "testPassword") + + with client as request: + response = request.get("/account/not_found", cookies={"AURSID": sid}, + allow_redirects=False) + + assert response.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_get_account_unauthenticated(): + with client as request: + response = request.get("/account/test", allow_redirects=False) + + assert response.status_code == int(HTTPStatus.UNAUTHORIZED) + + content = response.content.decode() + assert "You must log in to view user information." in content From 32abdbafaed74a0a9dbf3c75401dfa1002f62ba6 Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Mon, 24 May 2021 12:42:57 +0100 Subject: [PATCH 118/844] fastapi: Jinja contextfilter renamed to pass_context Closes: #23 Signed-off-by: Leonidas Spyropoulos --- aurweb/captcha.py | 6 +++--- aurweb/l10n.py | 4 ++-- aurweb/util.py | 5 ++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/aurweb/captcha.py b/aurweb/captcha.py index 5475d85f..9451f42c 100644 --- a/aurweb/captcha.py +++ b/aurweb/captcha.py @@ -1,7 +1,7 @@ """ This module consists of aurweb's CAPTCHA utility functions and filters. """ import hashlib -import jinja2 +from jinja2 import pass_context from aurweb.db import query from aurweb.models.user import User @@ -41,14 +41,14 @@ def get_captcha_answer(token): return hashlib.md5((text + "\n").encode()).hexdigest()[:6] -@jinja2.contextfilter +@pass_context def captcha_salt_filter(context): """ Returns the most recent CAPTCHA salt in the list of salts. """ salts = get_captcha_salts() return salts[0] -@jinja2.contextfilter +@pass_context def captcha_cmdline_filter(context, salt): """ Returns a CAPTCHA challenge for a given salt. """ return get_captcha_challenge(salt) diff --git a/aurweb/l10n.py b/aurweb/l10n.py index 4a5c1a46..9270f3ce 100644 --- a/aurweb/l10n.py +++ b/aurweb/l10n.py @@ -4,7 +4,7 @@ import typing from collections import OrderedDict from fastapi import Request -from jinja2 import contextfilter +from jinja2 import pass_context import aurweb.config @@ -88,7 +88,7 @@ def get_translator_for_request(request: Request): return translate -@contextfilter +@pass_context def tr(context: typing.Any, value: str): """ A translation filter; example: {{ "Hello" | tr("de") }}. """ _ = get_translator_for_request(context.get("request")) diff --git a/aurweb/util.py b/aurweb/util.py index 8b6ddbe7..8e4b291d 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -5,9 +5,8 @@ import string from urllib.parse import urlparse -import jinja2 - from email_validator import EmailNotValidError, EmailUndeliverableError, validate_email +from jinja2 import pass_context import aurweb.config @@ -88,7 +87,7 @@ def migrate_cookies(request, response): return response -@jinja2.contextfilter +@pass_context def account_url(context, user): request = context.get("request") base = f"{request.url.scheme}://{request.url.hostname}" From 822905be7d41fbd52790de58ba20f0d82ce69efc Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 24 May 2021 05:19:57 -0700 Subject: [PATCH 119/844] bugfix: relax `next` verification AUR renders its own 404 Not Found page when a bad route is encountered. Introducing the previous verification caused an error in this case when setting a language while viewing the Not Found page. So, instead of checking through routes, just make sure that the next parameter starts with a '/' character, which removes the possibility of any cross attacks. + Removed aurweb.asgi.routes; no longer needed. Signed-off-by: Kevin Morris --- aurweb/asgi.py | 9 --------- aurweb/routers/html.py | 8 +++----- test/test_routes.py | 2 +- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 1a61b1f4..861f6056 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -12,8 +12,6 @@ from aurweb.auth import BasicAuthBackend from aurweb.db import get_engine from aurweb.routers import accounts, auth, errors, html, sso -routes = set() - # Setup the FastAPI app. app = FastAPI(exception_handlers=errors.exceptions) @@ -47,13 +45,6 @@ async def app_startup(): # Initialize the database engine and ORM. get_engine() -# NOTE: Always keep this dictionary updated with all routes -# that the application contains. We use this to check for -# parameter value verification. -routes = {route.path for route in app.routes} -routes.update({route.path for route in sso.router.routes}) -routes.update({route.path for route in html.router.routes}) - @app.exception_handler(HTTPException) async def http_exception_handler(request, exc): diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index e947d213..8f89e05c 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -32,11 +32,9 @@ async def language(request: Request, parameters across the redirect. """ from aurweb.db import session - from aurweb.asgi import routes - if unquote(next) not in routes: - return HTMLResponse( - b"Invalid 'next' parameter.", - status_code=400) + + if next[0] != '/': + return HTMLResponse(b"Invalid 'next' parameter.", status_code=400) query_string = "?" + q if q else str() diff --git a/test/test_routes.py b/test/test_routes.py index d512a172..e4816231 100644 --- a/test/test_routes.py +++ b/test/test_routes.py @@ -61,7 +61,7 @@ def test_language_invalid_next(): """ Test an invalid next route at '/language'. """ post_data = { "set_lang": "de", - "next": "/BLAHBLAHFAKE" + "next": "https://evil.net" } with client as req: response = req.post("/language", data=post_data) From a7e5498197ebac1986b7b05c4acb6026d9c6f24d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 30 May 2021 16:38:16 -0700 Subject: [PATCH 120/844] add PackageBase SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/db.py | 7 +++++ aurweb/models/package_base.py | 39 ++++++++++++++++++++++++ test/test_package_base.py | 57 +++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 aurweb/models/package_base.py create mode 100644 test/test_package_base.py diff --git a/aurweb/db.py b/aurweb/db.py index 7dab6c4a..bb58c0c8 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -1,5 +1,7 @@ import math +from sqlalchemy.orm import backref, relationship + import aurweb.config import aurweb.util @@ -51,6 +53,11 @@ def make_random_value(table: str, column: str): return string +def make_relationship(model, foreign_key, backref_): + return relationship(model, foreign_keys=[foreign_key], + backref=backref(backref_, lazy="dynamic")) + + def query(model, *args, **kwargs): return session.query(model).filter(*args, **kwargs) diff --git a/aurweb/models/package_base.py b/aurweb/models/package_base.py new file mode 100644 index 00000000..57e5a46b --- /dev/null +++ b/aurweb/models/package_base.py @@ -0,0 +1,39 @@ +from datetime import datetime + +from sqlalchemy.orm import mapper + +from aurweb.db import make_relationship +from aurweb.models.user import User +from aurweb.schema import PackageBases + + +class PackageBase: + def __init__(self, Name: str = None, Flagger: User = None, + Maintainer: User = None, Submitter: User = None, + Packager: User = None, **kwargs): + self.Name = Name + self.Flagger = Flagger + self.Maintainer = Maintainer + self.Submitter = Submitter + self.Packager = Packager + + self.NumVotes = kwargs.get("NumVotes") + self.Popularity = kwargs.get("Popularity") + self.OutOfDateTS = kwargs.get("OutOfDateTS") + self.FlaggerComment = kwargs.get("FlaggerComment", str()) + self.SubmittedTS = kwargs.get("SubmittedTS", + datetime.utcnow().timestamp()) + self.ModifiedTS = kwargs.get("ModifiedTS", + datetime.utcnow().timestamp()) + + +mapper(PackageBase, PackageBases, properties={ + "Flagger": make_relationship(User, PackageBases.c.FlaggerUID, + "flagged_bases"), + "Submitter": make_relationship(User, PackageBases.c.SubmitterUID, + "submitted_bases"), + "Maintainer": make_relationship(User, PackageBases.c.MaintainerUID, + "maintained_bases"), + "Packager": make_relationship(User, PackageBases.c.PackagerUID, + "package_bases") +}) diff --git a/test/test_package_base.py b/test/test_package_base.py new file mode 100644 index 00000000..dcb0eb9e --- /dev/null +++ b/test/test_package_base.py @@ -0,0 +1,57 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query +from aurweb.models.account_type import AccountType +from aurweb.models.package_base import PackageBase +from aurweb.testing import setup_test_db +from aurweb.testing.models import make_user + +user = None + + +@pytest.fixture(autouse=True) +def setup(): + global user + + setup_test_db("Users", "PackageBases") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = make_user(Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + yield user + + +def test_package_base(): + pkgbase = create(PackageBase, + Name="beautiful-package", + Maintainer=user) + assert pkgbase in user.maintained_bases + + assert not pkgbase.OutOfDateTS + assert pkgbase.SubmittedTS > 0 + assert pkgbase.ModifiedTS > 0 + + +def test_package_base_relationships(): + pkgbase = create(PackageBase, + Name="beautiful-package", + Flagger=user, + Maintainer=user, + Submitter=user, + Packager=user) + assert pkgbase in user.flagged_bases + assert pkgbase in user.maintained_bases + assert pkgbase in user.submitted_bases + assert pkgbase in user.package_bases + + +def test_package_base_null_name_raises_exception(): + from aurweb.db import session + + with pytest.raises(IntegrityError): + create(PackageBase) + session.rollback() From fb210158113307d2fb17cd6377eee8e27a2cec05 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 30 May 2021 18:12:46 -0700 Subject: [PATCH 121/844] add PackageKeyword SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/package_keyword.py | 20 ++++++++++++ test/test_package_keyword.py | 54 ++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 aurweb/models/package_keyword.py create mode 100644 test/test_package_keyword.py diff --git a/aurweb/models/package_keyword.py b/aurweb/models/package_keyword.py new file mode 100644 index 00000000..87d97558 --- /dev/null +++ b/aurweb/models/package_keyword.py @@ -0,0 +1,20 @@ +from sqlalchemy.orm import mapper + +from aurweb.db import make_relationship +from aurweb.models.package_base import PackageBase +from aurweb.schema import PackageKeywords + + +class PackageKeyword: + def __init__(self, + PackageBase: PackageBase = None, + Keyword: str = None): + self.PackageBase = PackageBase + self.Keyword = Keyword + + +mapper(PackageKeyword, PackageKeywords, properties={ + "PackageBase": make_relationship(PackageBase, + PackageKeywords.c.PackageBaseID, + "keywords") +}) diff --git a/test/test_package_keyword.py b/test/test_package_keyword.py new file mode 100644 index 00000000..6e2df344 --- /dev/null +++ b/test/test_package_keyword.py @@ -0,0 +1,54 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query +from aurweb.models.account_type import AccountType +from aurweb.models.package_base import PackageBase +from aurweb.models.package_keyword import PackageKeyword +from aurweb.testing import setup_test_db +from aurweb.testing.models import make_user + +user, pkgbase = None, None + + +@pytest.fixture(autouse=True) +def setup(): + global user, pkgbase + + setup_test_db("Users", "PackageBases", "PackageKeywords") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = make_user(Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + pkgbase = create(PackageBase, + Name="beautiful-package", + Maintainer=user) + + yield pkgbase + + from aurweb.db import session + session.delete(pkgbase) + session.commit() + + +def test_package_keyword(): + from aurweb.db import session + pkg_keyword = create(PackageKeyword, + PackageBase=pkgbase, + Keyword="test") + assert pkg_keyword in pkgbase.keywords + assert pkgbase == pkg_keyword.PackageBase + session.delete(pkg_keyword) + session.commit() + + +def test_package_keyword_null_pkgbase_raises_exception(): + from aurweb.db import session + + with pytest.raises(IntegrityError): + create(PackageKeyword, + Keyword="test") + session.rollback() From 29db2ee5139baaa7cf4e9989e5916afca6f98bf1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 30 May 2021 22:28:43 -0700 Subject: [PATCH 122/844] add Term SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/term.py | 15 +++++++++++++++ test/test_term.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 aurweb/models/term.py create mode 100644 test/test_term.py diff --git a/aurweb/models/term.py b/aurweb/models/term.py new file mode 100644 index 00000000..1b4902f7 --- /dev/null +++ b/aurweb/models/term.py @@ -0,0 +1,15 @@ +from sqlalchemy.orm import mapper + +from aurweb.schema import Terms + + +class Term: + def __init__(self, + Description: str = None, URL: str = None, + Revision: int = None): + self.Description = Description + self.URL = URL + self.Revision = Revision + + +mapper(Term, Terms) diff --git a/test/test_term.py b/test/test_term.py new file mode 100644 index 00000000..4ae1e1cd --- /dev/null +++ b/test/test_term.py @@ -0,0 +1,30 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, delete +from aurweb.models.term import Term + + +def test_term_creation(): + term = create(Term, Description="Term description", + URL="https://fake_url.io") + assert bool(term.ID) + assert term.Description == "Term description" + assert term.URL == "https://fake_url.io" + assert term.Revision == 1 + delete(Term, Term.ID == term.ID) + + +def test_term_null_description_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(Term, URL="https://fake_url.io") + session.rollback() + + +def test_term_null_url_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(Term, Description="Term description") + session.rollback() From 718fa48a5cb0be18cea315bfb1742ef95f30da98 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 30 May 2021 22:29:01 -0700 Subject: [PATCH 123/844] add AcceptedTerm SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/accepted_term.py | 24 ++++++++++++++ test/test_accepted_term.py | 57 ++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 aurweb/models/accepted_term.py create mode 100644 test/test_accepted_term.py diff --git a/aurweb/models/accepted_term.py b/aurweb/models/accepted_term.py new file mode 100644 index 00000000..6e8ffe99 --- /dev/null +++ b/aurweb/models/accepted_term.py @@ -0,0 +1,24 @@ +from sqlalchemy.orm import mapper + +from aurweb.db import make_relationship +from aurweb.models.term import Term +from aurweb.models.user import User +from aurweb.schema import AcceptedTerms + + +class AcceptedTerm: + def __init__(self, + User: User = None, Term: Term = None, + Revision: int = None): + self.User = User + self.Term = Term + self.Revision = Revision + + +properties = { + "User": make_relationship(User, AcceptedTerms.c.UsersID, "accepted_terms"), + "Term": make_relationship(Term, AcceptedTerms.c.TermsID, "accepted") +} + +mapper(AcceptedTerm, AcceptedTerms, properties=properties, + primary_key=[AcceptedTerms.c.UsersID, AcceptedTerms.c.TermsID]) diff --git a/test/test_accepted_term.py b/test/test_accepted_term.py new file mode 100644 index 00000000..4dd8a5ca --- /dev/null +++ b/test/test_accepted_term.py @@ -0,0 +1,57 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, delete, query +from aurweb.models.accepted_term import AcceptedTerm +from aurweb.models.account_type import AccountType +from aurweb.models.term import Term +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user, term, accepted_term = None, None, None + + +@pytest.fixture(autouse=True) +def setup(): + global user, term, accepted_term + + setup_test_db("Users", "AcceptedTerms", "Terms") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + account_type=account_type) + + term = create(Term, Description="Test term", URL="https://test.term") + + yield term + + delete(Term, Term.ID == term.ID) + delete(User, User.ID == user.ID) + + +def test_accepted_term(): + accepted_term = create(AcceptedTerm, User=user, Term=term) + + # Make sure our AcceptedTerm relationships got initialized properly. + assert accepted_term.User == user + assert accepted_term in user.accepted_terms + assert accepted_term in term.accepted + + delete(AcceptedTerm, AcceptedTerm.User == user, AcceptedTerm.Term == term) + + +def test_accepted_term_null_user_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(AcceptedTerm, Term=term) + session.rollback() + + +def test_accepted_term_null_term_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(AcceptedTerm, User=user) + session.rollback() From e1ab02c2bf88e2a2fcf2048da212f586c6e3a389 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 30 May 2021 23:06:33 -0700 Subject: [PATCH 124/844] Fix database initialization in test_term.py Signed-off-by: Kevin Morris --- test/test_term.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_term.py b/test/test_term.py index 4ae1e1cd..00397b33 100644 --- a/test/test_term.py +++ b/test/test_term.py @@ -2,10 +2,15 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, delete +from aurweb.db import create, delete, get_engine from aurweb.models.term import Term +@pytest.fixture(autouse=True) +def setup(): + get_engine() + + def test_term_creation(): term = create(Term, Description="Term description", URL="https://fake_url.io") From b692b11f62efffd8554fce95c7a0f2a2cdb9014b Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 30 May 2021 23:05:16 -0700 Subject: [PATCH 125/844] add Group SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/group.py | 11 +++++++++++ test/test_group.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 aurweb/models/group.py create mode 100644 test/test_group.py diff --git a/aurweb/models/group.py b/aurweb/models/group.py new file mode 100644 index 00000000..5d4f3834 --- /dev/null +++ b/aurweb/models/group.py @@ -0,0 +1,11 @@ +from sqlalchemy.orm import mapper + +from aurweb.schema import Groups + + +class Group: + def __init__(self, Name: str = None): + self.Name = Name + + +mapper(Group, Groups) diff --git a/test/test_group.py b/test/test_group.py new file mode 100644 index 00000000..bbb774b9 --- /dev/null +++ b/test/test_group.py @@ -0,0 +1,21 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, delete, get_engine +from aurweb.models.group import Group + + +def test_group_creation(): + get_engine() + group = create(Group, Name="Test Group") + assert bool(group.ID) + assert group.Name == "Test Group" + delete(Group, Group.ID == group.ID) + + +def test_group_null_name_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(Group) + session.rollback() From 794868b20f700461fd8978be9c162c708db43c57 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 31 May 2021 22:41:06 -0700 Subject: [PATCH 126/844] aurweb.testing.setup_test_db: Expunge objects This is needed to avoid redundant objects in SQLAlchemy's IdentityMap, since we pass a direct .execute to delete the tables passed in. Additionally, remove our engine.connect() call in favor of relying on the already-established Session. Signed-off-by: Kevin Morris --- aurweb/testing/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/aurweb/testing/__init__.py b/aurweb/testing/__init__.py index 0a807b40..02c21a4c 100644 --- a/aurweb/testing/__init__.py +++ b/aurweb/testing/__init__.py @@ -21,10 +21,12 @@ def setup_test_db(*args): test_tables = ["Users", "Sessions"]; setup_test_db(*test_tables) """ - engine = aurweb.db.get_engine() - conn = engine.connect() + # Make sure that we've grabbed the engine before using the session. + aurweb.db.get_engine() tables = list(args) for table in tables: - conn.execute(f"DELETE FROM {table}") - conn.close() + aurweb.db.session.execute(f"DELETE FROM {table}") + + # Expunge all objects from SQLAlchemy's IdentityMap. + aurweb.db.session.expunge_all() From f8a6049de24a1b92b6d1c3456570c47f464aaf21 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 31 May 2021 22:43:28 -0700 Subject: [PATCH 127/844] aurweb.db.session: Use autoflush=True for Sessions We'd like SQLAlchemy to automatically maintain flushes for us. Signed-off-by: Kevin Morris --- aurweb/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aurweb/db.py b/aurweb/db.py index bb58c0c8..ca5ce412 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -135,7 +135,7 @@ def get_engine(): # https://fastapi.tiangolo.com/tutorial/sql-databases/#note connect_args["check_same_thread"] = False engine = create_engine(get_sqlalchemy_url(), connect_args=connect_args) - Session = sessionmaker(autocommit=False, autoflush=False, bind=engine) + Session = sessionmaker(autocommit=False, autoflush=True, bind=engine) session = Session() return engine From 621e459dfbd3d6e8e0d7a790dae4e14b092078ad Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 31 May 2021 22:45:30 -0700 Subject: [PATCH 128/844] aurweb.models.user: Remove session.commit() from construction We don't want to do this on construction. We only want to do this when we want to actually add the user to the database (or modify it). Signed-off-by: Kevin Morris --- aurweb/models/user.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 3983e098..6c5c6e21 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -51,11 +51,9 @@ class User: self.update_password(passwd) def update_password(self, password, salt_rounds=12): - from aurweb.db import session self.Passwd = bcrypt.hashpw( password.encode(), bcrypt.gensalt(rounds=salt_rounds)).decode() - session.commit() @staticmethod def minimum_passwd_length(): From 15b1332656a7f2bb0aa2abe108af4ede0629b749 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 1 Jun 2021 00:25:49 -0700 Subject: [PATCH 129/844] add Package SQLAlchemy ORM model Additionally, add an optional **kwargs passing via make_relationship. This allows us to use things like `uselist=False`, which was needed for test/test_package.py. Signed-off-by: Kevin Morris --- aurweb/db.py | 5 ++- aurweb/models/package.py | 24 ++++++++++++ test/test_package.py | 79 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 aurweb/models/package.py create mode 100644 test/test_package.py diff --git a/aurweb/db.py b/aurweb/db.py index ca5ce412..500cf95a 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -53,9 +53,10 @@ def make_random_value(table: str, column: str): return string -def make_relationship(model, foreign_key, backref_): +def make_relationship(model, foreign_key: str, backref_: str, **kwargs): return relationship(model, foreign_keys=[foreign_key], - backref=backref(backref_, lazy="dynamic")) + backref=backref(backref_, lazy="dynamic"), + **kwargs) def query(model, *args, **kwargs): diff --git a/aurweb/models/package.py b/aurweb/models/package.py new file mode 100644 index 00000000..fa82bb74 --- /dev/null +++ b/aurweb/models/package.py @@ -0,0 +1,24 @@ +from sqlalchemy.orm import mapper + +from aurweb.db import make_relationship +from aurweb.models.package_base import PackageBase +from aurweb.schema import Packages + + +class Package: + def __init__(self, + PackageBase: PackageBase = None, + Name: str = None, Version: str = None, + Description: str = None, URL: str = None): + self.PackageBase = PackageBase + self.Name = Name + self.Version = Version + self.Description = Description + self.URL = URL + + +mapper(Package, Packages, properties={ + "PackageBase": make_relationship(PackageBase, + Packages.c.PackageBaseID, + "package", uselist=False) +}) diff --git a/test/test_package.py b/test/test_package.py new file mode 100644 index 00000000..1d670087 --- /dev/null +++ b/test/test_package.py @@ -0,0 +1,79 @@ +import pytest + +from sqlalchemy import and_ +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query +from aurweb.models.account_type import AccountType +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = pkgbase = package = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, pkgbase, package + + setup_test_db("Users", "PackageBases", "Packages") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + + pkgbase = create(PackageBase, + Name="beautiful-package", + Maintainer=user) + package = create(Package, + PackageBase=pkgbase, + Name=pkgbase.Name, + Description="Test description.", + URL="https://test.package") + + yield package + + +def test_package(): + from aurweb.db import session + + assert pkgbase == package.PackageBase + assert package.Name == "beautiful-package" + assert package.Description == "Test description." + assert package.Version == str() # Default version. + assert package.URL == "https://test.package" + + # Update package Version. + package.Version = "1.2.3" + session.commit() + + # Make sure it got updated in the database. + record = query(Package, + and_(Package.ID == package.ID, + Package.Version == "1.2.3")).first() + assert record is not None + + +def test_package_null_pkgbase_raises_exception(): + from aurweb.db import session + + with pytest.raises(IntegrityError): + create(Package, + Name="some-package", + Description="Some description.", + URL="https://some.package") + session.rollback() + + +def test_package_null_name_raises_exception(): + from aurweb.db import session + + with pytest.raises(IntegrityError): + create(Package, + PackageBase=pkgbase, + Description="Some description.", + URL="https://some.package") + session.rollback() From f2121fb833d2279c5fb6b5863988209e45176fd0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 1 Jun 2021 00:26:32 -0700 Subject: [PATCH 130/844] simplify test_package_keyword.py We no longer need to delete records like this; in fact, it causes errors now. Fix this by removing the deletions and allow setup_test_db to do it's job. We'll need to do this for other tests as well. Signed-off-by: Kevin Morris --- test/test_package_keyword.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/test_package_keyword.py b/test/test_package_keyword.py index 6e2df344..f110b123 100644 --- a/test/test_package_keyword.py +++ b/test/test_package_keyword.py @@ -29,20 +29,13 @@ def setup(): yield pkgbase - from aurweb.db import session - session.delete(pkgbase) - session.commit() - def test_package_keyword(): - from aurweb.db import session pkg_keyword = create(PackageKeyword, PackageBase=pkgbase, Keyword="test") assert pkg_keyword in pkgbase.keywords assert pkgbase == pkg_keyword.PackageBase - session.delete(pkg_keyword) - session.commit() def test_package_keyword_null_pkgbase_raises_exception(): From 38dc2bb99dcbab372c4c7fcd3716f7f532c22ee0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 1 Jun 2021 03:26:14 -0700 Subject: [PATCH 131/844] Sanitize and modernize pytests Some of these tests were written before some of our convenient tooling existed. Additionally, some of the tests were not cooperating with PEP-8 guidelines or isorted. This commit does the following: - Replaces all calls to make_(user|session) with aurweb.db.create(Model, ...). - Replace calls to session.add(...) + session.commit() with aurweb.db.create. - Removes the majority of calls to (session|aurweb.db).delete(...). - Replaces session.query calls with aurweb.db.query. - Initializes all mutable globals in pytest fixture setup(). - Makes mutable global declarations more concise: `var1, var2 = None, None` -> `var1 = var2 = None` - Defines a warning exclusion for test/test_ssh_pub_key.py. - Removes the aurweb.testing.models module. - Removes some useless pytest.fixture yielding. As of this commit, developers should use the following guidelines when writing tests: - Always use aurweb.db.(create|delete|query) for database operations, where possible. - Always define mutable globals in the style: `var1 = var2 = None`. - `yield` the most dependent model in pytest setup fixture **iff** you must delete records after test runs to maintain database integrity. Example: test/test_account_type.py. This all makes the test code look and behave much cleaner. Previously, aurweb.testing.setup_test_db was buggy and leaving objects around in SQLAlchemy's IdentityMap. Signed-off-by: Kevin Morris --- aurweb/testing/models.py | 25 --------------- setup.cfg | 28 +++++++++++------ test/test_accepted_term.py | 11 ++----- test/test_account_type.py | 28 ++++++----------- test/test_accounts_routes.py | 13 ++++---- test/test_auth.py | 29 +++++++---------- test/test_auth_routes.py | 17 +++++----- test/test_ban.py | 16 ++++------ test/test_exceptions.py | 61 +++++++++++++++++------------------- test/test_group.py | 10 ++++-- test/test_initdb.py | 5 +-- test/test_package.py | 2 -- test/test_package_base.py | 9 +++--- test/test_package_keyword.py | 12 +++---- test/test_routes.py | 17 +++++----- test/test_session.py | 31 +++++++++--------- test/test_ssh_pub_key.py | 29 ++++++----------- test/test_term.py | 6 ++-- test/test_user.py | 50 +++++++++-------------------- 19 files changed, 160 insertions(+), 239 deletions(-) delete mode 100644 aurweb/testing/models.py diff --git a/aurweb/testing/models.py b/aurweb/testing/models.py deleted file mode 100644 index 8a27c409..00000000 --- a/aurweb/testing/models.py +++ /dev/null @@ -1,25 +0,0 @@ -import warnings - -from sqlalchemy import exc - -import aurweb.db - - -def make_user(**kwargs): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", exc.SAWarning) - from aurweb.models.user import User - user = User(**kwargs) - aurweb.db.session.add(user) - aurweb.db.session.commit() - return user - - -def make_session(**kwargs): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", exc.SAWarning) - from aurweb.models.session import Session - session = Session(**kwargs) - aurweb.db.session.add(session) - aurweb.db.session.commit() - return session diff --git a/setup.cfg b/setup.cfg index 98261651..31a0eb8a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,18 +2,26 @@ max-line-length = 127 max-complexity = 10 -# Ignore some unavoidable flake8 warnings; we know this is against -# pycodestyle, but some of the existing codebase uses `I` variables, -# so specifically silence warnings about it in pre-defined files. -# In E741, the 'I', 'O', 'l' are ambiguous variable names. -# Our current implementation uses these variables through HTTP -# and the FastAPI form specification wants them named as such. -# In C901's case, our process_account_form function is way too -# complex for PEP (too many if statements). However, we need to -# process these anyways, and making it any more complex would -# just add confusion to the implementation. +# aurweb/routers/accounts.py +# Ignore some unavoidable flake8 warnings; we know this is against +# pycodestyle, but some of the existing codebase uses `I` variables, +# so specifically silence warnings about it in pre-defined files. +# In E741, the 'I', 'O', 'l' are ambiguous variable names. +# Our current implementation uses these variables through HTTP +# and the FastAPI form specification wants them named as such. +# In C901's case, our process_account_form function is way too +# complex for PEP (too many if statements). However, we need to +# process these anyways, and making it any more complex would +# just add confusion to the implementation. +# +# test/test_ssh_pub_key.py +# E501 is detected due to our >127 width test constant. Ignore it. +# Due to this, line width should _always_ be looked at in code reviews. +# Anything like this should be questioned. +# per-file-ignores = aurweb/routers/accounts.py:E741,C901 + test/test_ssh_pub_key.py:E501 [isort] line_length = 127 diff --git a/test/test_accepted_term.py b/test/test_accepted_term.py index 4dd8a5ca..4ddf1fc3 100644 --- a/test/test_accepted_term.py +++ b/test/test_accepted_term.py @@ -2,14 +2,14 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, delete, query +from aurweb.db import create, query from aurweb.models.accepted_term import AcceptedTerm from aurweb.models.account_type import AccountType from aurweb.models.term import Term from aurweb.models.user import User from aurweb.testing import setup_test_db -user, term, accepted_term = None, None, None +user = term = accepted_term = None @pytest.fixture(autouse=True) @@ -26,11 +26,6 @@ def setup(): term = create(Term, Description="Test term", URL="https://test.term") - yield term - - delete(Term, Term.ID == term.ID) - delete(User, User.ID == user.ID) - def test_accepted_term(): accepted_term = create(AcceptedTerm, User=user, Term=term) @@ -40,8 +35,6 @@ def test_accepted_term(): assert accepted_term in user.accepted_terms assert accepted_term in term.accepted - delete(AcceptedTerm, AcceptedTerm.User == user, AcceptedTerm.Term == term) - def test_accepted_term_null_user_raises_exception(): from aurweb.db import session diff --git a/test/test_account_type.py b/test/test_account_type.py index 9419970c..3bd76d1e 100644 --- a/test/test_account_type.py +++ b/test/test_account_type.py @@ -1,9 +1,9 @@ import pytest +from aurweb.db import create, delete, query from aurweb.models.account_type import AccountType from aurweb.models.user import User from aurweb.testing import setup_test_db -from aurweb.testing.models import make_user account_type = None @@ -12,24 +12,17 @@ account_type = None def setup(): setup_test_db("Users") - from aurweb.db import session - global account_type - account_type = AccountType(AccountType="TestUser") - session.add(account_type) - session.commit() + account_type = create(AccountType, AccountType="TestUser") yield account_type - session.delete(account_type) - session.commit() + delete(AccountType, AccountType.ID == account_type.ID) def test_account_type(): """ Test creating an AccountType, and reading its columns. """ - from aurweb.db import session - # Make sure it got created and was given an ID. assert bool(account_type.ID) @@ -39,20 +32,17 @@ def test_account_type(): "" % ( account_type.ID) - record = session.query(AccountType).filter( - AccountType.AccountType == "TestUser").first() + record = query(AccountType, + AccountType.AccountType == "TestUser").first() assert account_type == record def test_user_account_type_relationship(): - from aurweb.db import session - - user = make_user(Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) assert user.AccountType == account_type assert account_type.users.filter(User.ID == user.ID).first() - session.delete(user) - session.commit() + delete(User, User.ID == user.ID) diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index c42736fa..0f813823 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -12,14 +12,13 @@ from fastapi.testclient import TestClient from aurweb import captcha from aurweb.asgi import app -from aurweb.db import create, delete, query +from aurweb.db import create, query from aurweb.models.account_type import AccountType from aurweb.models.ban import Ban from aurweb.models.session import Session from aurweb.models.ssh_pub_key import SSHPubKey, get_fingerprint from aurweb.models.user import User from aurweb.testing import setup_test_db -from aurweb.testing.models import make_user from aurweb.testing.requests import Request # Some test global constants. @@ -39,9 +38,9 @@ def setup(): account_type = query(AccountType, AccountType.AccountType == "User").first() - user = make_user(Username=TEST_USERNAME, Email=TEST_EMAIL, - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + user = create(User, Username=TEST_USERNAME, Email=TEST_EMAIL, + RealName="Test User", Passwd="testPassword", + AccountType=account_type) def test_get_passreset_authed_redirects(): @@ -751,8 +750,8 @@ def test_post_account_edit_error_unauthorized(): request = Request() sid = user.login(request, "testPassword") - test2 = create(User, Username="test2", Email="test2@example.org", - Passwd="testPassword") + create(User, Username="test2", + Email="test2@example.org", Passwd="testPassword") post_data = { "U": "test", diff --git a/test/test_auth.py b/test/test_auth.py index d43459cd..7837e7f7 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -5,16 +5,14 @@ import pytest from starlette.authentication import AuthenticationError from aurweb.auth import BasicAuthBackend, has_credential -from aurweb.db import query +from aurweb.db import create, query from aurweb.models.account_type import AccountType +from aurweb.models.session import Session +from aurweb.models.user import User from aurweb.testing import setup_test_db -from aurweb.testing.models import make_session, make_user from aurweb.testing.requests import Request -# Persistent user object, initialized in our setup fixture. -user = None -backend = None -request = None +user = backend = request = None @pytest.fixture(autouse=True) @@ -23,16 +21,11 @@ def setup(): setup_test_db("Users", "Sessions") - from aurweb.db import session - account_type = query(AccountType, AccountType.AccountType == "User").first() - user = make_user(Username="test", Email="test@example.com", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - - session.add(user) - session.commit() + user = create(User, Username="test", Email="test@example.com", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) backend = BasicAuthBackend() request = Request() @@ -60,8 +53,8 @@ async def test_auth_backend_invalid_sid(): async def test_auth_backend_invalid_user_id(): # Create a new session with a fake user id. now_ts = datetime.utcnow().timestamp() - make_session(UsersID=666, SessionID="realSession", - LastUpdateTS=now_ts + 5) + create(Session, UsersID=666, SessionID="realSession", + LastUpdateTS=now_ts + 5) # Here, we specify a real SID; but it's user is not there. request.cookies["AURSID"] = "realSession" @@ -74,8 +67,8 @@ async def test_basic_auth_backend(): # This time, everything matches up. We expect the user to # equal the real_user. now_ts = datetime.utcnow().timestamp() - make_session(UsersID=user.ID, SessionID="realSession", - LastUpdateTS=now_ts + 5) + create(Session, UsersID=user.ID, SessionID="realSession", + LastUpdateTS=now_ts + 5) _, result = await backend.authenticate(request) assert result == user diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index ff8a08e9..360b48cc 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -8,33 +8,34 @@ from fastapi.testclient import TestClient import aurweb.config from aurweb.asgi import app -from aurweb.db import query +from aurweb.db import create, query from aurweb.models.account_type import AccountType from aurweb.models.session import Session +from aurweb.models.user import User from aurweb.testing import setup_test_db -from aurweb.testing.models import make_user # Some test global constants. TEST_USERNAME = "test" TEST_EMAIL = "test@example.org" # Global mutables. -client = TestClient(app) -user = None +user = client = None @pytest.fixture(autouse=True) def setup(): - global user + global user, client setup_test_db("Users", "Sessions", "Bans") account_type = query(AccountType, AccountType.AccountType == "User").first() - user = make_user(Username=TEST_USERNAME, Email=TEST_EMAIL, - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + user = create(User, Username=TEST_USERNAME, Email=TEST_EMAIL, + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + + client = TestClient(app) def test_login_logout(): diff --git a/test/test_ban.py b/test/test_ban.py index de4f5b1b..a4fa5a28 100644 --- a/test/test_ban.py +++ b/test/test_ban.py @@ -6,27 +6,23 @@ import pytest from sqlalchemy import exc as sa_exc +from aurweb.db import create from aurweb.models.ban import Ban, is_banned from aurweb.testing import setup_test_db from aurweb.testing.requests import Request -ban = None - -request = Request() +ban = request = None @pytest.fixture(autouse=True) def setup(): - from aurweb.db import session - - global ban + global ban, request setup_test_db("Bans") - ban = Ban(IPAddress="127.0.0.1", - BanTS=datetime.utcnow() + timedelta(seconds=30)) - session.add(ban) - session.commit() + ts = datetime.utcnow() + timedelta(seconds=30) + ban = create(Ban, IPAddress="127.0.0.1", BanTS=ts) + request = Request() def test_ban(): diff --git a/test/test_exceptions.py b/test/test_exceptions.py index feac2656..7247106b 100644 --- a/test/test_exceptions.py +++ b/test/test_exceptions.py @@ -1,102 +1,99 @@ -from aurweb.exceptions import (AlreadyVotedException, AurwebException, BannedException, BrokenUpdateHookException, - InvalidArgumentsException, InvalidCommentException, InvalidPackageBaseException, - InvalidReasonException, InvalidRepositoryNameException, InvalidUserException, - MaintenanceException, NotVotedException, PackageBaseExistsException, PermissionDeniedException) +from aurweb import exceptions def test_aurweb_exception(): try: - raise AurwebException("test") - except AurwebException as exc: + raise exceptions.AurwebException("test") + except exceptions.AurwebException as exc: assert str(exc) == "test" def test_maintenance_exception(): try: - raise MaintenanceException("test") - except MaintenanceException as exc: + raise exceptions.MaintenanceException("test") + except exceptions.MaintenanceException as exc: assert str(exc) == "test" def test_banned_exception(): try: - raise BannedException("test") - except BannedException as exc: + raise exceptions.BannedException("test") + except exceptions.BannedException as exc: assert str(exc) == "test" def test_already_voted_exception(): try: - raise AlreadyVotedException("test") - except AlreadyVotedException as exc: + raise exceptions.AlreadyVotedException("test") + except exceptions.AlreadyVotedException as exc: assert str(exc) == "already voted for package base: test" def test_broken_update_hook_exception(): try: - raise BrokenUpdateHookException("test") - except BrokenUpdateHookException as exc: + raise exceptions.BrokenUpdateHookException("test") + except exceptions.BrokenUpdateHookException as exc: assert str(exc) == "broken update hook: test" def test_invalid_arguments_exception(): try: - raise InvalidArgumentsException("test") - except InvalidArgumentsException as exc: + raise exceptions.InvalidArgumentsException("test") + except exceptions.InvalidArgumentsException as exc: assert str(exc) == "test" def test_invalid_packagebase_exception(): try: - raise InvalidPackageBaseException("test") - except InvalidPackageBaseException as exc: + raise exceptions.InvalidPackageBaseException("test") + except exceptions.InvalidPackageBaseException as exc: assert str(exc) == "package base not found: test" def test_invalid_comment_exception(): try: - raise InvalidCommentException("test") - except InvalidCommentException as exc: + raise exceptions.InvalidCommentException("test") + except exceptions.InvalidCommentException as exc: assert str(exc) == "comment is too short: test" def test_invalid_reason_exception(): try: - raise InvalidReasonException("test") - except InvalidReasonException as exc: + raise exceptions.InvalidReasonException("test") + except exceptions.InvalidReasonException as exc: assert str(exc) == "invalid reason: test" def test_invalid_user_exception(): try: - raise InvalidUserException("test") - except InvalidUserException as exc: + raise exceptions.InvalidUserException("test") + except exceptions.InvalidUserException as exc: assert str(exc) == "unknown user: test" def test_not_voted_exception(): try: - raise NotVotedException("test") - except NotVotedException as exc: + raise exceptions.NotVotedException("test") + except exceptions.NotVotedException as exc: assert str(exc) == "missing vote for package base: test" def test_packagebase_exists_exception(): try: - raise PackageBaseExistsException("test") - except PackageBaseExistsException as exc: + raise exceptions.PackageBaseExistsException("test") + except exceptions.PackageBaseExistsException as exc: assert str(exc) == "package base already exists: test" def test_permission_denied_exception(): try: - raise PermissionDeniedException("test") - except PermissionDeniedException as exc: + raise exceptions.PermissionDeniedException("test") + except exceptions.PermissionDeniedException as exc: assert str(exc) == "permission denied: test" def test_repository_name_exception(): try: - raise InvalidRepositoryNameException("test") - except InvalidRepositoryNameException as exc: + raise exceptions.InvalidRepositoryNameException("test") + except exceptions.InvalidRepositoryNameException as exc: assert str(exc) == "invalid repository name: test" diff --git a/test/test_group.py b/test/test_group.py index bbb774b9..da017a96 100644 --- a/test/test_group.py +++ b/test/test_group.py @@ -2,16 +2,20 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, delete, get_engine +from aurweb.db import create from aurweb.models.group import Group +from aurweb.testing import setup_test_db + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db("Groups") def test_group_creation(): - get_engine() group = create(Group, Name="Test Group") assert bool(group.ID) assert group.Name == "Test Group" - delete(Group, Group.ID == group.ID) def test_group_null_name_raises_exception(): diff --git a/test/test_initdb.py b/test/test_initdb.py index ff089b63..eae33007 100644 --- a/test/test_initdb.py +++ b/test/test_initdb.py @@ -23,5 +23,6 @@ def test_run(): use_alembic = True verbose = False aurweb.initdb.run(Args()) - assert aurweb.db.session.query(AccountType).filter( - AccountType.AccountType == "User").first() is not None + record = aurweb.db.query(AccountType, + AccountType.AccountType == "User").first() + assert record is not None diff --git a/test/test_package.py b/test/test_package.py index 1d670087..66d557f3 100644 --- a/test/test_package.py +++ b/test/test_package.py @@ -34,8 +34,6 @@ def setup(): Description="Test description.", URL="https://test.package") - yield package - def test_package(): from aurweb.db import session diff --git a/test/test_package_base.py b/test/test_package_base.py index dcb0eb9e..e0359f4f 100644 --- a/test/test_package_base.py +++ b/test/test_package_base.py @@ -5,8 +5,8 @@ from sqlalchemy.exc import IntegrityError from aurweb.db import create, query from aurweb.models.account_type import AccountType from aurweb.models.package_base import PackageBase +from aurweb.models.user import User from aurweb.testing import setup_test_db -from aurweb.testing.models import make_user user = None @@ -19,10 +19,9 @@ def setup(): account_type = query(AccountType, AccountType.AccountType == "User").first() - user = make_user(Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - yield user + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) def test_package_base(): diff --git a/test/test_package_keyword.py b/test/test_package_keyword.py index f110b123..316e7ca8 100644 --- a/test/test_package_keyword.py +++ b/test/test_package_keyword.py @@ -6,10 +6,10 @@ from aurweb.db import create, query from aurweb.models.account_type import AccountType from aurweb.models.package_base import PackageBase from aurweb.models.package_keyword import PackageKeyword +from aurweb.models.user import User from aurweb.testing import setup_test_db -from aurweb.testing.models import make_user -user, pkgbase = None, None +user = pkgbase = None @pytest.fixture(autouse=True) @@ -20,15 +20,13 @@ def setup(): account_type = query(AccountType, AccountType.AccountType == "User").first() - user = make_user(Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) pkgbase = create(PackageBase, Name="beautiful-package", Maintainer=user) - yield pkgbase - def test_package_keyword(): pkg_keyword = create(PackageKeyword, diff --git a/test/test_routes.py b/test/test_routes.py index e4816231..f4bb063f 100644 --- a/test/test_routes.py +++ b/test/test_routes.py @@ -7,27 +7,28 @@ import pytest from fastapi.testclient import TestClient from aurweb.asgi import app -from aurweb.db import query +from aurweb.db import create, query from aurweb.models.account_type import AccountType +from aurweb.models.user import User from aurweb.testing import setup_test_db -from aurweb.testing.models import make_user from aurweb.testing.requests import Request -client = TestClient(app) -user = None +user = client = None @pytest.fixture(autouse=True) def setup(): - global user + global user, client setup_test_db("Users", "Sessions") account_type = query(AccountType, AccountType.AccountType == "User").first() - user = make_user(Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + + client = TestClient(app) def test_index(): diff --git a/test/test_session.py b/test/test_session.py index 560f628c..2877ea7f 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -4,39 +4,38 @@ from unittest import mock import pytest +from aurweb.db import create, query from aurweb.models.account_type import AccountType -from aurweb.models.session import generate_unique_sid +from aurweb.models.session import Session, generate_unique_sid +from aurweb.models.user import User from aurweb.testing import setup_test_db -from aurweb.testing.models import make_session, make_user -user, _session = None, None +user = session = None @pytest.fixture(autouse=True) def setup(): - from aurweb.db import session - - global user, _session + global user, session setup_test_db("Users", "Sessions") - account_type = session.query(AccountType).filter( - AccountType.AccountType == "User").first() - user = make_user(Username="test", Email="test@example.org", - ResetKey="testReset", Passwd="testPassword", - AccountType=account_type) - _session = make_session(UsersID=user.ID, SessionID="testSession", - LastUpdateTS=datetime.utcnow()) + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = create(User, Username="test", Email="test@example.org", + ResetKey="testReset", Passwd="testPassword", + AccountType=account_type) + session = create(Session, UsersID=user.ID, SessionID="testSession", + LastUpdateTS=datetime.utcnow()) def test_session(): - assert _session.SessionID == "testSession" - assert _session.UsersID == user.ID + assert session.SessionID == "testSession" + assert session.UsersID == user.ID def test_session_user_association(): # Make sure that the Session user attribute is correct. - assert _session.User == user + assert session.User == user def test_generate_unique_sid(): diff --git a/test/test_ssh_pub_key.py b/test/test_ssh_pub_key.py index fe9df047..4072549e 100644 --- a/test/test_ssh_pub_key.py +++ b/test/test_ssh_pub_key.py @@ -1,46 +1,37 @@ import pytest -from aurweb.db import query +from aurweb.db import create, query from aurweb.models.account_type import AccountType from aurweb.models.ssh_pub_key import SSHPubKey, get_fingerprint +from aurweb.models.user import User from aurweb.testing import setup_test_db -from aurweb.testing.models import make_user TEST_SSH_PUBKEY = """ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCycoCi5yGCvSclH2wmNBUuwsYEzRZZBJaQquRc4ysl+Tg+/jiDkR3Zn9fIznC4KnFoyrIHzkKuePZ3bNDYwkZxkJKoWBCh4hXKDXSm87FMN0+VDC+1QxF/z0XaAGr/P6f4XukabyddypBdnHcZiplbw+YOSqcAE2TCqOlSXwNMOcF9U89UsR/Q9i9I52hlvU0q8+fZVGhou1KCowFSnHYtrr5KYJ04CXkJ13DkVf3+pjQWyrByvBcf1hGEaczlgfobrrv/y96jDhgfXucxliNKLdufDPPkii3LhhsNcDmmI1VZ3v0irKvd9WZuauqloobY84zEFcDTyjn0hxGjVeYFejm4fBnvjga0yZXORuWksdNfXWLDxFk6MDDd1jF0ExRbP+OxDuU4IVyIuDL7S3cnbf2YjGhkms/8voYT2OBE7FwNlfv98Kr0NUp51zpf55Arxn9j0Rz9xTA7FiODQgCn6iQ0SDtzUNL0IKTCw26xJY5gzMxbfpvzPQGeulx/ioM= kevr@volcano """ -user, ssh_pub_key = None, None +user = ssh_pub_key = None @pytest.fixture(autouse=True) def setup(): - from aurweb.db import session - global user, ssh_pub_key setup_test_db("Users", "SSHPubKeys") account_type = query(AccountType, AccountType.AccountType == "User").first() - user = make_user(Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) assert account_type == user.AccountType assert account_type.ID == user.AccountTypeID - ssh_pub_key = SSHPubKey(UserID=user.ID, - Fingerprint="testFingerprint", - PubKey="testPubKey") - - session.add(ssh_pub_key) - session.commit() - - yield ssh_pub_key - - session.delete(ssh_pub_key) - session.commit() + ssh_pub_key = create(SSHPubKey, + UserID=user.ID, + Fingerprint="testFingerprint", + PubKey="testPubKey") def test_ssh_pub_key(): diff --git a/test/test_term.py b/test/test_term.py index 00397b33..aa1dfcc6 100644 --- a/test/test_term.py +++ b/test/test_term.py @@ -2,13 +2,14 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, delete, get_engine +from aurweb.db import create from aurweb.models.term import Term +from aurweb.testing import setup_test_db @pytest.fixture(autouse=True) def setup(): - get_engine() + setup_test_db("Terms") def test_term_creation(): @@ -18,7 +19,6 @@ def test_term_creation(): assert term.Description == "Term description" assert term.URL == "https://fake_url.io" assert term.Revision == 1 - delete(Term, Term.ID == term.ID) def test_term_null_description_raises_exception(): diff --git a/test/test_user.py b/test/test_user.py index e8056681..8b4da61e 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -8,23 +8,20 @@ import pytest import aurweb.auth import aurweb.config -from aurweb.db import query +from aurweb.db import create, query from aurweb.models.account_type import AccountType from aurweb.models.ban import Ban from aurweb.models.session import Session from aurweb.models.ssh_pub_key import SSHPubKey from aurweb.models.user import User from aurweb.testing import setup_test_db -from aurweb.testing.models import make_session, make_user from aurweb.testing.requests import Request -account_type, user = None, None +account_type = user = None @pytest.fixture(autouse=True) def setup(): - from aurweb.db import session - global account_type, user setup_test_db("Users", "Sessions", "Bans", "SSHPubKeys") @@ -32,15 +29,13 @@ def setup(): account_type = query(AccountType, AccountType.AccountType == "User").first() - user = make_user(Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) def test_user_login_logout(): """ Test creating a user and reading its columns. """ - from aurweb.db import session - # Assert that make_user created a valid user. assert bool(user.ID) @@ -61,8 +56,8 @@ def test_user_login_logout(): assert "AURSID" in request.cookies # Expect that User session relationships work right. - user_session = session.query(Session).filter( - Session.UsersID == user.ID).first() + user_session = query(Session, + Session.UsersID == user.ID).first() assert user_session == user.session assert user.session.SessionID == sid assert user.session.User == user @@ -103,13 +98,9 @@ def test_user_login_twice(): def test_user_login_banned(): - from aurweb.db import session - # Add ban for the next 30 seconds. banned_timestamp = datetime.utcnow() + timedelta(seconds=30) - ban = Ban(IPAddress="127.0.0.1", BanTS=banned_timestamp) - session.add(ban) - session.commit() + create(Ban, IPAddress="127.0.0.1", BanTS=banned_timestamp) request = Request() request.client.host = "127.0.0.1" @@ -138,19 +129,14 @@ def test_legacy_user_authentication(): def test_user_login_with_outdated_sid(): - from aurweb.db import session - # Make a session with a LastUpdateTS 5 seconds ago, causing # user.login to update it with a new sid. - _session = make_session(UsersID=user.ID, SessionID="stub", - LastUpdateTS=datetime.utcnow().timestamp() - 5) + create(Session, UsersID=user.ID, SessionID="stub", + LastUpdateTS=datetime.utcnow().timestamp() - 5) sid = user.login(Request(), "testPassword") assert sid and user.is_authenticated() assert sid != "stub" - session.delete(_session) - session.commit() - def test_user_update_password(): user.update_password("secondPassword") @@ -169,21 +155,14 @@ def test_user_has_credential(): def test_user_ssh_pub_key(): - from aurweb.db import session - assert user.ssh_pub_key is None - ssh_pub_key = SSHPubKey(UserID=user.ID, - Fingerprint="testFingerprint", - PubKey="testPubKey") - session.add(ssh_pub_key) - session.commit() + ssh_pub_key = create(SSHPubKey, UserID=user.ID, + Fingerprint="testFingerprint", + PubKey="testPubKey") assert user.ssh_pub_key == ssh_pub_key - session.delete(ssh_pub_key) - session.commit() - def test_user_credential_types(): from aurweb.db import session @@ -203,8 +182,7 @@ def test_user_credential_types(): assert aurweb.auth.trusted_user_or_dev(user) developer_type = query(AccountType, - AccountType.AccountType == "Developer")\ - .first() + AccountType.AccountType == "Developer").first() user.AccountType = developer_type session.commit() From 943d97efac1f6fca6c823e0edb416b3c300f4b3d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 1 Jun 2021 04:48:49 -0700 Subject: [PATCH 132/844] add License SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/license.py | 11 +++++++++++ test/test_license.py | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 aurweb/models/license.py create mode 100644 test/test_license.py diff --git a/aurweb/models/license.py b/aurweb/models/license.py new file mode 100644 index 00000000..1c174925 --- /dev/null +++ b/aurweb/models/license.py @@ -0,0 +1,11 @@ +from sqlalchemy.orm import mapper + +from aurweb.schema import Licenses + + +class License: + def __init__(self, Name: str = None): + self.Name = Name + + +mapper(License, Licenses) diff --git a/test/test_license.py b/test/test_license.py new file mode 100644 index 00000000..feb7a396 --- /dev/null +++ b/test/test_license.py @@ -0,0 +1,25 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create +from aurweb.models.license import License +from aurweb.testing import setup_test_db + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db("Licenses") + + +def test_license_creation(): + license = create(License, Name="Test License") + assert bool(license.ID) + assert license.Name == "Test License" + + +def test_license_null_name_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(License) + session.rollback() From 75cc0be189271ed3583486c0b66463f8e43605f4 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 1 Jun 2021 05:06:38 -0700 Subject: [PATCH 133/844] add PackageLicense SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/package_license.py | 27 +++++++++++++++++ test/test_package_license.py | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 aurweb/models/package_license.py create mode 100644 test/test_package_license.py diff --git a/aurweb/models/package_license.py b/aurweb/models/package_license.py new file mode 100644 index 00000000..187b113e --- /dev/null +++ b/aurweb/models/package_license.py @@ -0,0 +1,27 @@ +from sqlalchemy.orm import mapper + +from aurweb.db import make_relationship +from aurweb.models.license import License +from aurweb.models.package import Package +from aurweb.schema import PackageLicenses + + +class PackageLicense: + def __init__(self, Package: Package = None, License: License = None): + self.Package = Package + self.License = License + + +properties = { + "Package": make_relationship(Package, + PackageLicenses.c.PackageID, + "package_license", + uselist=False), + "License": make_relationship(License, + PackageLicenses.c.LicenseID, + "package_license", + uselist=False) +} + +mapper(PackageLicense, PackageLicenses, properties=properties, + primary_key=[PackageLicenses.c.PackageID, PackageLicenses.c.LicenseID]) diff --git a/test/test_package_license.py b/test/test_package_license.py new file mode 100644 index 00000000..72eb3681 --- /dev/null +++ b/test/test_package_license.py @@ -0,0 +1,52 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query +from aurweb.models.account_type import AccountType +from aurweb.models.license import License +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.package_license import PackageLicense +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = license = pkgbase = package = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, license, pkgbase, package + + setup_test_db("Users", "PackageBases", "Packages", + "Licenses", "PackageLicenses") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + account_type=account_type) + + license = create(License, Name="Test License") + pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + package = create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + + +def test_package_license(): + package_license = create(PackageLicense, Package=package, License=license) + assert package_license.License == license + assert package_license.Package == package + + +def test_package_license_null_package_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(PackageLicense, License=license) + session.rollback() + + +def test_package_license_null_license_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(PackageLicense, Package=package) + session.rollback() From a8a9c28783d606863b57066103e15b64d75fdb69 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 1 Jun 2021 05:21:01 -0700 Subject: [PATCH 134/844] Jinja bugfix: add xmlns + xml:lang to This was not brought over during the initial commit involving partisl/layout.html. Signed-off-by: Kevin Morris --- templates/partials/layout.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/partials/layout.html b/templates/partials/layout.html index d30208a9..019ebff7 100644 --- a/templates/partials/layout.html +++ b/templates/partials/layout.html @@ -1,5 +1,6 @@ - + {% include 'partials/head.html' %} From 4201348dea2b74bfc172573209561f76b1a36597 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 1 Jun 2021 05:34:27 -0700 Subject: [PATCH 135/844] add PackageGroup SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/package_group.py | 27 ++++++++++++++++++ test/test_package_group.py | 52 ++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 aurweb/models/package_group.py create mode 100644 test/test_package_group.py diff --git a/aurweb/models/package_group.py b/aurweb/models/package_group.py new file mode 100644 index 00000000..8a32c00b --- /dev/null +++ b/aurweb/models/package_group.py @@ -0,0 +1,27 @@ +from sqlalchemy.orm import mapper + +from aurweb.db import make_relationship +from aurweb.models.group import Group +from aurweb.models.package import Package +from aurweb.schema import PackageGroups + + +class PackageGroup: + def __init__(self, Package: Package = None, Group: Group = None): + self.Package = Package + self.Group = Group + + +properties = { + "Package": make_relationship(Package, + PackageGroups.c.PackageID, + "package_group", + uselist=False), + "Group": make_relationship(Group, + PackageGroups.c.GroupID, + "package_group", + uselist=False) +} + +mapper(PackageGroup, PackageGroups, properties=properties, + primary_key=[PackageGroups.c.PackageID, PackageGroups.c.GroupID]) diff --git a/test/test_package_group.py b/test/test_package_group.py new file mode 100644 index 00000000..28047a7f --- /dev/null +++ b/test/test_package_group.py @@ -0,0 +1,52 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query +from aurweb.models.account_type import AccountType +from aurweb.models.group import Group +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.package_group import PackageGroup +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = group = pkgbase = package = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, group, pkgbase, package + + setup_test_db("Users", "PackageBases", "Packages", + "Groups", "PackageGroups") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + account_type=account_type) + + group = create(Group, Name="Test Group") + pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + package = create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + + +def test_package_group(): + package_group = create(PackageGroup, Package=package, Group=group) + assert package_group.Group == group + assert package_group.Package == package + + +def test_package_group_null_package_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(PackageGroup, Group=group) + session.rollback() + + +def test_package_group_null_group_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(PackageGroup, Package=package) + session.rollback() From 068c8ba638dd032df917d82af8fe6ffe70264ab3 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 1 Jun 2021 06:44:24 -0700 Subject: [PATCH 136/844] add DependencyType SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/dependency_type.py | 11 +++++++++++ test/test_dependency_type.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 aurweb/models/dependency_type.py create mode 100644 test/test_dependency_type.py diff --git a/aurweb/models/dependency_type.py b/aurweb/models/dependency_type.py new file mode 100644 index 00000000..87b38069 --- /dev/null +++ b/aurweb/models/dependency_type.py @@ -0,0 +1,11 @@ +from sqlalchemy.orm import mapper + +from aurweb.schema import DependencyTypes + + +class DependencyType: + def __init__(self, Name: str = None): + self.Name = Name + + +mapper(DependencyType, DependencyTypes) diff --git a/test/test_dependency_type.py b/test/test_dependency_type.py new file mode 100644 index 00000000..6c37cc58 --- /dev/null +++ b/test/test_dependency_type.py @@ -0,0 +1,31 @@ +import pytest + +from aurweb.db import create, delete, query +from aurweb.models.dependency_type import DependencyType +from aurweb.testing import setup_test_db + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db() + + +def test_dependency_types(): + dep_types = ["depends", "makedepends", "checkdepends", "optdepends"] + for dep_type in dep_types: + dependency_type = query(DependencyType, + DependencyType.Name == dep_type).first() + assert dependency_type is not None + + +def test_dependency_type_creation(): + dependency_type = create(DependencyType, Name="Test Type") + assert bool(dependency_type.ID) + assert dependency_type.Name == "Test Type" + delete(DependencyType, DependencyType.ID == dependency_type.ID) + + +def test_dependency_type_null_name_uses_default(): + dependency_type = create(DependencyType) + assert dependency_type.Name == str() + delete(DependencyType, DependencyType.ID == dependency_type.ID) From e401b92acb82a52f62441f8decb209448ce457a1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 1 Jun 2021 07:21:54 -0700 Subject: [PATCH 137/844] add PackageDependency (PackageDepends) ORM model Signed-off-by: Kevin Morris --- aurweb/models/package_dependency.py | 31 ++++++++ test/test_package_dependency.py | 113 ++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 aurweb/models/package_dependency.py create mode 100644 test/test_package_dependency.py diff --git a/aurweb/models/package_dependency.py b/aurweb/models/package_dependency.py new file mode 100644 index 00000000..ae6ae62a --- /dev/null +++ b/aurweb/models/package_dependency.py @@ -0,0 +1,31 @@ +from sqlalchemy.orm import mapper + +from aurweb.db import make_relationship +from aurweb.models.dependency_type import DependencyType +from aurweb.models.package import Package +from aurweb.schema import PackageDepends + + +class PackageDependency: + def __init__(self, Package: Package = None, + DependencyType: DependencyType = None, + DepName: str = None, DepDesc: str = None, + DepCondition: str = None, DepArch: str = None): + self.Package = Package + self.DependencyType = DependencyType + self.DepName = DepName # nullable=False + self.DepDesc = DepDesc + self.DepCondition = DepCondition + self.DepArch = DepArch + + +properties = { + "Package": make_relationship(Package, PackageDepends.c.PackageID, + "package_dependencies"), + "DependencyType": make_relationship(DependencyType, + PackageDepends.c.DepTypeID, + "package_dependencies") +} + +mapper(PackageDependency, PackageDepends, properties=properties, + primary_key=[PackageDepends.c.PackageID, PackageDepends.c.DepTypeID]) diff --git a/test/test_package_dependency.py b/test/test_package_dependency.py new file mode 100644 index 00000000..fc21a08c --- /dev/null +++ b/test/test_package_dependency.py @@ -0,0 +1,113 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query +from aurweb.models.account_type import AccountType +from aurweb.models.dependency_type import DependencyType +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.package_dependency import PackageDependency +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = pkgbase = package = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, pkgbase, package + + setup_test_db("Users", "PackageBases", "Packages", "PackageDepends") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + + pkgbase = create(PackageBase, + Name="test-package", + Maintainer=user) + package = create(Package, + PackageBase=pkgbase, + Name=pkgbase.Name, + Description="Test description.", + URL="https://test.package") + + +def test_package_dependencies(): + depends = query(DependencyType, DependencyType.Name == "depends").first() + pkgdep = create(PackageDependency, Package=package, + DependencyType=depends, + DepName="test-dep") + assert pkgdep.DepName == "test-dep" + assert pkgdep.Package == package + assert pkgdep.DependencyType == depends + assert pkgdep in depends.package_dependencies + assert pkgdep in package.package_dependencies + + makedepends = query(DependencyType, + DependencyType.Name == "makedepends").first() + pkgdep = create(PackageDependency, Package=package, + DependencyType=makedepends, + DepName="test-dep") + assert pkgdep.DepName == "test-dep" + assert pkgdep.Package == package + assert pkgdep.DependencyType == makedepends + assert pkgdep in makedepends.package_dependencies + assert pkgdep in package.package_dependencies + + checkdepends = query(DependencyType, + DependencyType.Name == "checkdepends").first() + pkgdep = create(PackageDependency, Package=package, + DependencyType=checkdepends, + DepName="test-dep") + assert pkgdep.DepName == "test-dep" + assert pkgdep.Package == package + assert pkgdep.DependencyType == checkdepends + assert pkgdep in checkdepends.package_dependencies + assert pkgdep in package.package_dependencies + + optdepends = query(DependencyType, + DependencyType.Name == "optdepends").first() + pkgdep = create(PackageDependency, Package=package, + DependencyType=optdepends, + DepName="test-dep") + assert pkgdep.DepName == "test-dep" + assert pkgdep.Package == package + assert pkgdep.DependencyType == optdepends + assert pkgdep in optdepends.package_dependencies + assert pkgdep in package.package_dependencies + + +def test_package_dependencies_null_package_raises_exception(): + from aurweb.db import session + + depends = query(DependencyType, DependencyType.Name == "depends").first() + with pytest.raises(IntegrityError): + create(PackageDependency, + DependencyType=depends, + DepName="test-dep") + session.rollback() + + +def test_package_dependencies_null_dependency_type_raises_exception(): + from aurweb.db import session + + with pytest.raises(IntegrityError): + create(PackageDependency, + Package=package, + DepName="test-dep") + session.rollback() + + +def test_package_dependencies_null_depname_raises_exception(): + from aurweb.db import session + + depends = query(DependencyType, DependencyType.Name == "depends").first() + with pytest.raises(IntegrityError): + create(PackageDependency, + Package=package, + DependencyType=depends) + session.rollback() From a9cfbce11e3c16c22d168f8fc55238f17ea78273 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 1 Jun 2021 07:37:05 -0700 Subject: [PATCH 138/844] add RelationType SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/relation_type.py | 11 +++++++++++ test/test_relation_type.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 aurweb/models/relation_type.py create mode 100644 test/test_relation_type.py diff --git a/aurweb/models/relation_type.py b/aurweb/models/relation_type.py new file mode 100644 index 00000000..b4d1efbc --- /dev/null +++ b/aurweb/models/relation_type.py @@ -0,0 +1,11 @@ +from sqlalchemy.orm import mapper + +from aurweb.schema import RelationTypes + + +class RelationType: + def __init__(self, Name: str = None): + self.Name = Name + + +mapper(RelationType, RelationTypes) diff --git a/test/test_relation_type.py b/test/test_relation_type.py new file mode 100644 index 00000000..bf23505c --- /dev/null +++ b/test/test_relation_type.py @@ -0,0 +1,32 @@ +import pytest + +from aurweb.db import create, delete, query +from aurweb.models.relation_type import RelationType +from aurweb.testing import setup_test_db + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db() + + +def test_relation_type_creation(): + relation_type = create(RelationType, Name="test-relation") + assert bool(relation_type.ID) + assert relation_type.Name == "test-relation" + + delete(RelationType, RelationType.ID == relation_type.ID) + + +def test_relation_types(): + conflicts = query(RelationType, RelationType.Name == "conflicts").first() + assert conflicts is not None + assert conflicts.Name == "conflicts" + + provides = query(RelationType, RelationType.Name == "provides").first() + assert provides is not None + assert provides.Name == "provides" + + replaces = query(RelationType, RelationType.Name == "replaces").first() + assert replaces is not None + assert replaces.Name == "replaces" From 2b83d2fb6bb8b5066053220b5929d5d67333f9dd Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 1 Jun 2021 07:52:22 -0700 Subject: [PATCH 139/844] add PackageRelation SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/package_relation.py | 33 ++++++++++ test/test_package_relation.py | 100 ++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 aurweb/models/package_relation.py create mode 100644 test/test_package_relation.py diff --git a/aurweb/models/package_relation.py b/aurweb/models/package_relation.py new file mode 100644 index 00000000..196f1dee --- /dev/null +++ b/aurweb/models/package_relation.py @@ -0,0 +1,33 @@ +from sqlalchemy.orm import mapper + +from aurweb.db import make_relationship +from aurweb.models.package import Package +from aurweb.models.relation_type import RelationType +from aurweb.schema import PackageRelations + + +class PackageRelation: + def __init__(self, Package: Package = None, + RelationType: RelationType = None, + RelName: str = None, RelCondition: str = None, + RelArch: str = None): + self.Package = Package + self.RelationType = RelationType + self.RelName = RelName # nullable=False + self.RelCondition = RelCondition + self.RelArch = RelArch + + +properties = { + "Package": make_relationship(Package, PackageRelations.c.PackageID, + "package_relations"), + "RelationType": make_relationship(RelationType, + PackageRelations.c.RelTypeID, + "package_relations") +} + +mapper(PackageRelation, PackageRelations, properties=properties, + primary_key=[ + PackageRelations.c.PackageID, + PackageRelations.c.RelTypeID + ]) diff --git a/test/test_package_relation.py b/test/test_package_relation.py new file mode 100644 index 00000000..dd0455cd --- /dev/null +++ b/test/test_package_relation.py @@ -0,0 +1,100 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query +from aurweb.models.account_type import AccountType +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.package_relation import PackageRelation +from aurweb.models.relation_type import RelationType +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = pkgbase = package = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, pkgbase, package + + setup_test_db("Users", "PackageBases", "Packages", "PackageRelations") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + + pkgbase = create(PackageBase, + Name="test-package", + Maintainer=user) + package = create(Package, + PackageBase=pkgbase, + Name=pkgbase.Name, + Description="Test description.", + URL="https://test.package") + + +def test_package_dependencies(): + conflicts = query(RelationType, RelationType.Name == "conflicts").first() + pkgrel = create(PackageRelation, Package=package, + RelationType=conflicts, + RelName="test-relation") + assert pkgrel.RelName == "test-relation" + assert pkgrel.Package == package + assert pkgrel.RelationType == conflicts + assert pkgrel in conflicts.package_relations + assert pkgrel in package.package_relations + + provides = query(RelationType, RelationType.Name == "provides").first() + pkgrel = create(PackageRelation, Package=package, + RelationType=provides, + RelName="test-relation") + assert pkgrel.RelName == "test-relation" + assert pkgrel.Package == package + assert pkgrel.RelationType == provides + assert pkgrel in provides.package_relations + assert pkgrel in package.package_relations + + replaces = query(RelationType, RelationType.Name == "replaces").first() + pkgrel = create(PackageRelation, Package=package, + RelationType=replaces, + RelName="test-relation") + assert pkgrel.RelName == "test-relation" + assert pkgrel.Package == package + assert pkgrel.RelationType == replaces + assert pkgrel in replaces.package_relations + assert pkgrel in package.package_relations + + +def test_package_dependencies_null_package_raises_exception(): + from aurweb.db import session + + conflicts = query(RelationType, RelationType.Name == "conflicts").first() + with pytest.raises(IntegrityError): + create(PackageRelation, + RelationType=conflicts, + RelName="test-relation") + session.rollback() + + +def test_package_dependencies_null_dependency_type_raises_exception(): + from aurweb.db import session + + with pytest.raises(IntegrityError): + create(PackageRelation, + Package=package, + RelName="test-relation") + session.rollback() + + +def test_package_dependencies_null_depname_raises_exception(): + from aurweb.db import session + + depends = query(RelationType, RelationType.Name == "depends").first() + with pytest.raises(IntegrityError): + create(PackageRelation, + Package=package, + RelationType=depends) + session.rollback() From a65a60604ab09e83f18cec58afb7a807b1eb2b30 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 3 Jun 2021 10:51:46 -0700 Subject: [PATCH 140/844] add ApiRateLimit SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/api_rate_limit.py | 15 +++++++++++++ test/test_api_rate_limit.py | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 aurweb/models/api_rate_limit.py create mode 100644 test/test_api_rate_limit.py diff --git a/aurweb/models/api_rate_limit.py b/aurweb/models/api_rate_limit.py new file mode 100644 index 00000000..44e7a463 --- /dev/null +++ b/aurweb/models/api_rate_limit.py @@ -0,0 +1,15 @@ +from sqlalchemy.orm import mapper + +from aurweb.schema import ApiRateLimit as _ApiRateLimit + + +class ApiRateLimit: + def __init__(self, IP: str = None, + Requests: int = None, + WindowStart: int = None): + self.IP = IP + self.Requests = Requests + self.WindowStart = WindowStart + + +mapper(ApiRateLimit, _ApiRateLimit, primary_key=[_ApiRateLimit.c.IP]) diff --git a/test/test_api_rate_limit.py b/test/test_api_rate_limit.py new file mode 100644 index 00000000..91ab5854 --- /dev/null +++ b/test/test_api_rate_limit.py @@ -0,0 +1,40 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create +from aurweb.models.api_rate_limit import ApiRateLimit +from aurweb.testing import setup_test_db + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db("ApiRateLimit") + + +def test_api_rate_key_creation(): + rate = create(ApiRateLimit, IP="127.0.0.1", Requests=10, WindowStart=1) + assert rate.IP == "127.0.0.1" + assert rate.Requests == 10 + assert rate.WindowStart == 1 + + +def test_api_rate_key_null_ip_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(ApiRateLimit, Requests=10, WindowStart=1) + session.rollback() + + +def test_api_rate_key_null_requests_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(ApiRateLimit, IP="127.0.0.1", WindowStart=1) + session.rollback() + + +def test_api_rate_key_null_window_start_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(ApiRateLimit, IP="127.0.0.1", WindowStart=1) + session.rollback() From e5df083d4553440af309be89c5dcdd11d68dc7d5 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 3 Jun 2021 22:56:47 -0700 Subject: [PATCH 141/844] use String(max_len) for DECIMAL types with sqlite This solves an issue where DECIMAL is not native to sqlite by using a string to store values and converting them to float in user code. Signed-off-by: Kevin Morris --- aurweb/schema.py | 10 ++++++++-- web/html/addvote.php | 7 +++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/aurweb/schema.py b/aurweb/schema.py index f0162045..0d40e272 100644 --- a/aurweb/schema.py +++ b/aurweb/schema.py @@ -107,7 +107,10 @@ PackageBases = Table( Column('ID', INTEGER(unsigned=True), primary_key=True), Column('Name', String(255), nullable=False, unique=True), Column('NumVotes', INTEGER(unsigned=True), nullable=False, server_default=text("0")), - Column('Popularity', DECIMAL(10, 6, unsigned=True), nullable=False, server_default=text("0")), + Column('Popularity', + DECIMAL(10, 6, unsigned=True) + if db_backend == "mysql" else String(16), # Stubbed out to test. + nullable=False, server_default=text("0")), Column('OutOfDateTS', BIGINT(unsigned=True)), Column('FlaggerComment', Text, nullable=False), Column('SubmittedTS', BIGINT(unsigned=True), nullable=False), @@ -383,7 +386,10 @@ TU_VoteInfo = Table( Column('User', String(32), nullable=False), Column('Submitted', BIGINT(unsigned=True), nullable=False), Column('End', BIGINT(unsigned=True), nullable=False), - Column('Quorum', DECIMAL(2, 2, unsigned=True), nullable=False), + Column('Quorum', + DECIMAL(2, 2, unsigned=True) + if db_backend == "mysql" else String(4), + nullable=False), Column('SubmitterID', ForeignKey('Users.ID', ondelete='CASCADE'), nullable=False), Column('Yes', TINYINT(3, unsigned=True), nullable=False, server_default=text("'0'")), Column('No', TINYINT(3, unsigned=True), nullable=False, server_default=text("'0'")), diff --git a/web/html/addvote.php b/web/html/addvote.php index 3672c031..70280cfd 100644 --- a/web/html/addvote.php +++ b/web/html/addvote.php @@ -67,8 +67,11 @@ if (has_credential(CRED_TU_ADD_VOTE)) { } } - if (!empty($_POST['addVote']) && empty($error)) { - add_tu_proposal($_POST['agenda'], $_POST['user'], $len, $quorum, $uid); + if (!empty($_POST['addVote']) && empty($error)) { + // Convert $quorum to a String of maximum length "12.34" (5). + $quorum_str = substr(strval($quorum), min(5, strlen($quorum)); + add_tu_proposal($_POST['agenda'], $_POST['user'], + $len, $quorum_str, $uid); print "

    " . __("New proposal submitted.") . "

    \n"; } else { From d7481b96499f06be0d9ca983bb9efc578b38eb97 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 4 Jun 2021 00:31:15 -0700 Subject: [PATCH 142/844] modify schema primary keys to be nullable+defaulted This fixes SQLAlchemy warnings related to primary keys not having an auto_increment or nullable. We've done this by making all foreign primary keys nullable. In ApiRateLimit's case, we can set a default str to act as a null, which seems a bit more sensible. Signed-off-by: Kevin Morris --- aurweb/models/package_group.py | 12 ++++++++++++ aurweb/models/package_keyword.py | 7 +++++++ aurweb/models/package_license.py | 14 ++++++++++++++ aurweb/schema.py | 12 ++++++------ test/test_api_rate_limit.py | 8 +++----- 5 files changed, 42 insertions(+), 11 deletions(-) diff --git a/aurweb/models/package_group.py b/aurweb/models/package_group.py index 8a32c00b..c155fe00 100644 --- a/aurweb/models/package_group.py +++ b/aurweb/models/package_group.py @@ -1,4 +1,5 @@ from sqlalchemy.orm import mapper +from sqlalchemy.exc import IntegrityError from aurweb.db import make_relationship from aurweb.models.group import Group @@ -9,7 +10,18 @@ from aurweb.schema import PackageGroups class PackageGroup: def __init__(self, Package: Package = None, Group: Group = None): self.Package = Package + if not self.Package: + raise IntegrityError( + statement="Primary key PackageID cannot be null.", + orig="PackageGroups.PackageID", + params=("NULL")) + self.Group = Group + if not self.Group: + raise IntegrityError( + statement="Primary key GroupID cannot be null.", + orig="PackageGroups.GroupID", + params=("NULL")) properties = { diff --git a/aurweb/models/package_keyword.py b/aurweb/models/package_keyword.py index 87d97558..4a66f38e 100644 --- a/aurweb/models/package_keyword.py +++ b/aurweb/models/package_keyword.py @@ -1,4 +1,5 @@ from sqlalchemy.orm import mapper +from sqlalchemy.exc import IntegrityError from aurweb.db import make_relationship from aurweb.models.package_base import PackageBase @@ -10,6 +11,12 @@ class PackageKeyword: PackageBase: PackageBase = None, Keyword: str = None): self.PackageBase = PackageBase + if not self.PackageBase: + raise IntegrityError( + statement="Primary key PackageBaseID cannot be null.", + orig="PackageKeywords.PackageBaseID", + params=("NULL")) + self.Keyword = Keyword diff --git a/aurweb/models/package_license.py b/aurweb/models/package_license.py index 187b113e..6f23f84a 100644 --- a/aurweb/models/package_license.py +++ b/aurweb/models/package_license.py @@ -1,4 +1,5 @@ from sqlalchemy.orm import mapper +from sqlalchemy.exc import IntegrityError from aurweb.db import make_relationship from aurweb.models.license import License @@ -9,7 +10,18 @@ from aurweb.schema import PackageLicenses class PackageLicense: def __init__(self, Package: Package = None, License: License = None): self.Package = Package + if not self.Package: + raise IntegrityError( + statement="Primary key PackageID cannot be null.", + orig="PackageLicenses.PackageID", + params=("NULL")) + self.License = License + if not self.License: + raise IntegrityError( + statement="Primary key LicenseID cannot be null.", + orig="PackageLicenses.LicenseID", + params=("NULL")) properties = { @@ -21,6 +33,8 @@ properties = { PackageLicenses.c.LicenseID, "package_license", uselist=False) + + } mapper(PackageLicense, PackageLicenses, properties=properties, diff --git a/aurweb/schema.py b/aurweb/schema.py index 0d40e272..fa8923a3 100644 --- a/aurweb/schema.py +++ b/aurweb/schema.py @@ -133,7 +133,7 @@ PackageBases = Table( # Keywords of package bases PackageKeywords = Table( 'PackageKeywords', metadata, - Column('PackageBaseID', ForeignKey('PackageBases.ID', ondelete='CASCADE'), primary_key=True, nullable=False), + Column('PackageBaseID', ForeignKey('PackageBases.ID', ondelete='CASCADE'), primary_key=True, nullable=True), Column('Keyword', String(255), primary_key=True, nullable=False, server_default=text("''")), mysql_engine='InnoDB', mysql_charset='utf8mb4', @@ -170,8 +170,8 @@ Licenses = Table( # Information about package-license-relations PackageLicenses = Table( 'PackageLicenses', metadata, - Column('PackageID', ForeignKey('Packages.ID', ondelete='CASCADE'), primary_key=True, nullable=False), - Column('LicenseID', ForeignKey('Licenses.ID', ondelete='CASCADE'), primary_key=True, nullable=False), + Column('PackageID', ForeignKey('Packages.ID', ondelete='CASCADE'), primary_key=True, nullable=True), + Column('LicenseID', ForeignKey('Licenses.ID', ondelete='CASCADE'), primary_key=True, nullable=True), mysql_engine='InnoDB', ) @@ -190,8 +190,8 @@ Groups = Table( # Information about package-group-relations PackageGroups = Table( 'PackageGroups', metadata, - Column('PackageID', ForeignKey('Packages.ID', ondelete='CASCADE'), primary_key=True, nullable=False), - Column('GroupID', ForeignKey('Groups.ID', ondelete='CASCADE'), primary_key=True, nullable=False), + Column('PackageID', ForeignKey('Packages.ID', ondelete='CASCADE'), primary_key=True, nullable=True), + Column('GroupID', ForeignKey('Groups.ID', ondelete='CASCADE'), primary_key=True, nullable=True), mysql_engine='InnoDB', ) @@ -445,7 +445,7 @@ AcceptedTerms = Table( # Rate limits for API ApiRateLimit = Table( 'ApiRateLimit', metadata, - Column('IP', String(45), primary_key=True), + Column('IP', String(45), primary_key=True, unique=True, default=str()), Column('Requests', INTEGER(11), nullable=False), Column('WindowStart', BIGINT(20), nullable=False), Index('ApiRateLimitWindowStart', 'WindowStart'), diff --git a/test/test_api_rate_limit.py b/test/test_api_rate_limit.py index 91ab5854..c599ddcf 100644 --- a/test/test_api_rate_limit.py +++ b/test/test_api_rate_limit.py @@ -19,11 +19,9 @@ def test_api_rate_key_creation(): assert rate.WindowStart == 1 -def test_api_rate_key_null_ip_raises_exception(): - from aurweb.db import session - with pytest.raises(IntegrityError): - create(ApiRateLimit, Requests=10, WindowStart=1) - session.rollback() +def test_api_rate_key_ip_default(): + api_rate_limit = create(ApiRateLimit, Requests=10, WindowStart=1) + assert api_rate_limit.IP == str() def test_api_rate_key_null_requests_raises_exception(): From aecb64947354283a9b2dd357a67e997c78f4adac Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 4 Jun 2021 00:43:57 -0700 Subject: [PATCH 143/844] use mysql backend in config.dev First off: This commit changes the default development database backend to mysql. sqlite, however, is still completely supported with the caveat that a user must now modify config.dev to use the sqlite backend. While looking into this, it was discovered that our SQLAlchemy backend for mysql (mysql-connector) completely broke model attributes when we switched to utf8mb4_bin (binary) -- it does not correct the correct conversion to and from binary utf8mb4. The new, replacement dependency mysqlclient does. mysqlclient is also recommended in SQLAlchemy documentation as the "best" one available. The mysqlclient backend uses a different exception flow then sqlite, and so tests expecting IntegrityError has to be modified to expect OperationalError from sqlalchemy.exc. So, for each model that we define, check keys that can't be NULL and raise sqlalchemy.exc.IntegrityError if we have to. This way we keep our exceptions uniform. Signed-off-by: Kevin Morris --- aurweb/db.py | 39 ++++-- aurweb/initdb.py | 6 +- aurweb/models/accepted_term.py | 13 ++ aurweb/models/api_rate_limit.py | 13 ++ aurweb/models/group.py | 6 + aurweb/models/license.py | 6 + aurweb/models/package.py | 13 ++ aurweb/models/package_base.py | 7 + aurweb/models/package_dependency.py | 21 ++- aurweb/models/package_group.py | 2 +- aurweb/models/package_keyword.py | 2 +- aurweb/models/package_license.py | 2 +- aurweb/models/package_relation.py | 19 +++ aurweb/models/session.py | 12 +- aurweb/models/term.py | 13 ++ conf/config.dev | 20 +-- test/Makefile | 2 +- test/test_accounts_routes.py | 42 ++++-- test/test_api_rate_limit.py | 2 +- test/test_auth.py | 13 +- test/test_ban.py | 3 +- test/test_db.py | 202 ++++++++++++++++++++-------- test/test_initdb.py | 20 +-- test/test_package_relation.py | 18 ++- test/test_session.py | 2 +- 25 files changed, 363 insertions(+), 135 deletions(-) diff --git a/aurweb/db.py b/aurweb/db.py index 500cf95a..590712e0 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -98,9 +98,11 @@ def get_sqlalchemy_url(): param_query = None else: port = None - param_query = {'unix_socket': aurweb.config.get('database', 'socket')} + param_query = { + 'unix_socket': aurweb.config.get('database', 'socket') + } return constructor( - 'mysql+mysqlconnector', + 'mysql+mysqldb', username=aurweb.config.get('database', 'user'), password=aurweb.config.get('database', 'password'), host=aurweb.config.get('database', 'host'), @@ -117,7 +119,7 @@ def get_sqlalchemy_url(): raise ValueError('unsupported database backend') -def get_engine(): +def get_engine(echo: bool = False): """ Return the global SQLAlchemy engine. @@ -135,13 +137,24 @@ def get_engine(): # check_same_thread is for a SQLite technicality # https://fastapi.tiangolo.com/tutorial/sql-databases/#note connect_args["check_same_thread"] = False - engine = create_engine(get_sqlalchemy_url(), connect_args=connect_args) + + engine = create_engine(get_sqlalchemy_url(), + connect_args=connect_args, + echo=echo) Session = sessionmaker(autocommit=False, autoflush=True, bind=engine) session = Session() return engine +def kill_engine(): + global engine, Session, session + if engine: + session.close() + engine.dispose() + engine = Session = session = None + + def connect(): """ Return an SQLAlchemy connection. Connections are usually pooled. See @@ -160,8 +173,7 @@ class ConnectionExecutor: def __init__(self, conn, backend=aurweb.config.get("database", "backend")): self._conn = conn if backend == "mysql": - import mysql.connector - self._paramstyle = mysql.connector.paramstyle + self._paramstyle = "format" elif backend == "sqlite": import sqlite3 self._paramstyle = sqlite3.paramstyle @@ -197,18 +209,17 @@ class Connection: aur_db_backend = aurweb.config.get('database', 'backend') if aur_db_backend == 'mysql': - import mysql.connector + import MySQLdb aur_db_host = aurweb.config.get('database', 'host') aur_db_name = aurweb.config.get('database', 'name') aur_db_user = aurweb.config.get('database', 'user') aur_db_pass = aurweb.config.get('database', 'password') aur_db_socket = aurweb.config.get('database', 'socket') - self._conn = mysql.connector.connect(host=aur_db_host, - user=aur_db_user, - passwd=aur_db_pass, - db=aur_db_name, - unix_socket=aur_db_socket, - buffered=True) + self._conn = MySQLdb.connect(host=aur_db_host, + user=aur_db_user, + passwd=aur_db_pass, + db=aur_db_name, + unix_socket=aur_db_socket) elif aur_db_backend == 'sqlite': import sqlite3 aur_db_name = aurweb.config.get('database', 'name') @@ -217,7 +228,7 @@ class Connection: else: raise ValueError('unsupported database backend') - self._conn = ConnectionExecutor(self._conn) + self._conn = ConnectionExecutor(self._conn, aur_db_backend) def execute(self, query, params=()): return self._conn.execute(query, params) diff --git a/aurweb/initdb.py b/aurweb/initdb.py index 5f55bfc9..46f079c0 100644 --- a/aurweb/initdb.py +++ b/aurweb/initdb.py @@ -2,7 +2,6 @@ import argparse import alembic.command import alembic.config -import sqlalchemy import aurweb.db import aurweb.schema @@ -34,6 +33,8 @@ def feed_initial_data(conn): def run(args): + aurweb.config.rehash() + # Ensure Alembic is fine before we do the real work, in order not to fail at # the last step and leave the database in an inconsistent state. The # configuration is loaded lazily, so we query it to force its loading. @@ -42,8 +43,7 @@ def run(args): alembic_config.get_main_option('script_location') alembic_config.attributes["configure_logger"] = False - engine = sqlalchemy.create_engine(aurweb.db.get_sqlalchemy_url(), - echo=(args.verbose >= 1)) + engine = aurweb.db.get_engine(echo=(args.verbose >= 1)) aurweb.schema.metadata.create_all(engine) feed_initial_data(engine.connect()) diff --git a/aurweb/models/accepted_term.py b/aurweb/models/accepted_term.py index 6e8ffe99..483109f1 100644 --- a/aurweb/models/accepted_term.py +++ b/aurweb/models/accepted_term.py @@ -1,3 +1,4 @@ +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import mapper from aurweb.db import make_relationship @@ -11,7 +12,19 @@ class AcceptedTerm: User: User = None, Term: Term = None, Revision: int = None): self.User = User + if not self.User: + raise IntegrityError( + statement="Foreign key UserID cannot be null.", + orig="AcceptedTerms.UserID", + params=("NULL")) + self.Term = Term + if not self.Term: + raise IntegrityError( + statement="Foreign key TermID cannot be null.", + orig="AcceptedTerms.TermID", + params=("NULL")) + self.Revision = Revision diff --git a/aurweb/models/api_rate_limit.py b/aurweb/models/api_rate_limit.py index 44e7a463..8b945b6a 100644 --- a/aurweb/models/api_rate_limit.py +++ b/aurweb/models/api_rate_limit.py @@ -1,3 +1,4 @@ +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import mapper from aurweb.schema import ApiRateLimit as _ApiRateLimit @@ -8,8 +9,20 @@ class ApiRateLimit: Requests: int = None, WindowStart: int = None): self.IP = IP + self.Requests = Requests + if self.Requests is None: + raise IntegrityError( + statement="Column Requests cannot be null.", + orig="ApiRateLimit.Requests", + params=("NULL")) + self.WindowStart = WindowStart + if self.WindowStart is None: + raise IntegrityError( + statement="Column WindowStart cannot be null.", + orig="ApiRateLimit.WindowStart", + params=("NULL")) mapper(ApiRateLimit, _ApiRateLimit, primary_key=[_ApiRateLimit.c.IP]) diff --git a/aurweb/models/group.py b/aurweb/models/group.py index 5d4f3834..c5583eb4 100644 --- a/aurweb/models/group.py +++ b/aurweb/models/group.py @@ -1,3 +1,4 @@ +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import mapper from aurweb.schema import Groups @@ -6,6 +7,11 @@ from aurweb.schema import Groups class Group: def __init__(self, Name: str = None): self.Name = Name + if not self.Name: + raise IntegrityError( + statement="Column Name cannot be null.", + orig="Groups.Name", + params=("NULL")) mapper(Group, Groups) diff --git a/aurweb/models/license.py b/aurweb/models/license.py index 1c174925..bcc02713 100644 --- a/aurweb/models/license.py +++ b/aurweb/models/license.py @@ -1,3 +1,4 @@ +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import mapper from aurweb.schema import Licenses @@ -6,6 +7,11 @@ from aurweb.schema import Licenses class License: def __init__(self, Name: str = None): self.Name = Name + if not self.Name: + raise IntegrityError( + statement="Column Name cannot be null.", + orig="Licenses.Name", + params=("NULL")) mapper(License, Licenses) diff --git a/aurweb/models/package.py b/aurweb/models/package.py index fa82bb74..28a13791 100644 --- a/aurweb/models/package.py +++ b/aurweb/models/package.py @@ -1,3 +1,4 @@ +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import mapper from aurweb.db import make_relationship @@ -11,7 +12,19 @@ class Package: Name: str = None, Version: str = None, Description: str = None, URL: str = None): self.PackageBase = PackageBase + if not self.PackageBase: + raise IntegrityError( + statement="Foreign key UserID cannot be null.", + orig="Packages.PackageBaseID", + params=("NULL")) + self.Name = Name + if not self.Name: + raise IntegrityError( + statement="Column Name cannot be null.", + orig="Packages.Name", + params=("NULL")) + self.Version = Version self.Description = Description self.URL = URL diff --git a/aurweb/models/package_base.py b/aurweb/models/package_base.py index 57e5a46b..699559d5 100644 --- a/aurweb/models/package_base.py +++ b/aurweb/models/package_base.py @@ -1,5 +1,6 @@ from datetime import datetime +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import mapper from aurweb.db import make_relationship @@ -12,6 +13,12 @@ class PackageBase: Maintainer: User = None, Submitter: User = None, Packager: User = None, **kwargs): self.Name = Name + if not self.Name: + raise IntegrityError( + statement="Column Name cannot be null.", + orig="PackageBases.Name", + params=("NULL")) + self.Flagger = Flagger self.Maintainer = Maintainer self.Submitter = Submitter diff --git a/aurweb/models/package_dependency.py b/aurweb/models/package_dependency.py index ae6ae62a..21801802 100644 --- a/aurweb/models/package_dependency.py +++ b/aurweb/models/package_dependency.py @@ -1,3 +1,4 @@ +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import mapper from aurweb.db import make_relationship @@ -12,8 +13,26 @@ class PackageDependency: DepName: str = None, DepDesc: str = None, DepCondition: str = None, DepArch: str = None): self.Package = Package + if not self.Package: + raise IntegrityError( + statement="Foreign key PackageID cannot be null.", + orig="PackageDependencies.PackageID", + params=("NULL")) + self.DependencyType = DependencyType - self.DepName = DepName # nullable=False + if not self.DependencyType: + raise IntegrityError( + statement="Foreign key DepTypeID cannot be null.", + orig="PackageDependencies.DepTypeID", + params=("NULL")) + + self.DepName = DepName + if not self.DepName: + raise IntegrityError( + statement="Column DepName cannot be null.", + orig="PackageDependencies.DepName", + params=("NULL")) + self.DepDesc = DepDesc self.DepCondition = DepCondition self.DepArch = DepArch diff --git a/aurweb/models/package_group.py b/aurweb/models/package_group.py index c155fe00..19a11c80 100644 --- a/aurweb/models/package_group.py +++ b/aurweb/models/package_group.py @@ -1,5 +1,5 @@ -from sqlalchemy.orm import mapper from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import mapper from aurweb.db import make_relationship from aurweb.models.group import Group diff --git a/aurweb/models/package_keyword.py b/aurweb/models/package_keyword.py index 4a66f38e..2bae223c 100644 --- a/aurweb/models/package_keyword.py +++ b/aurweb/models/package_keyword.py @@ -1,5 +1,5 @@ -from sqlalchemy.orm import mapper from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import mapper from aurweb.db import make_relationship from aurweb.models.package_base import PackageBase diff --git a/aurweb/models/package_license.py b/aurweb/models/package_license.py index 6f23f84a..491874a4 100644 --- a/aurweb/models/package_license.py +++ b/aurweb/models/package_license.py @@ -1,5 +1,5 @@ -from sqlalchemy.orm import mapper from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import mapper from aurweb.db import make_relationship from aurweb.models.license import License diff --git a/aurweb/models/package_relation.py b/aurweb/models/package_relation.py index 196f1dee..d9ade727 100644 --- a/aurweb/models/package_relation.py +++ b/aurweb/models/package_relation.py @@ -1,3 +1,4 @@ +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import mapper from aurweb.db import make_relationship @@ -12,8 +13,26 @@ class PackageRelation: RelName: str = None, RelCondition: str = None, RelArch: str = None): self.Package = Package + if not self.Package: + raise IntegrityError( + statement="Foreign key PackageID cannot be null.", + orig="PackageRelations.PackageID", + params=("NULL")) + self.RelationType = RelationType + if not self.RelationType: + raise IntegrityError( + statement="Foreign key RelTypeID cannot be null.", + orig="PackageRelations.RelTypeID", + params=("NULL")) + self.RelName = RelName # nullable=False + if not self.RelName: + raise IntegrityError( + statement="Column RelName cannot be null.", + orig="PackageRelations.RelName", + params=("NULL")) + self.RelCondition = RelCondition self.RelArch = RelArch diff --git a/aurweb/models/session.py b/aurweb/models/session.py index 60749303..f1e0fff5 100644 --- a/aurweb/models/session.py +++ b/aurweb/models/session.py @@ -1,16 +1,20 @@ -from sqlalchemy import Column, Integer +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, mapper, relationship -from aurweb.db import make_random_value +from aurweb.db import make_random_value, query from aurweb.models.user import User from aurweb.schema import Sessions class Session: - UsersID = Column(Integer, nullable=True) - def __init__(self, **kwargs): self.UsersID = kwargs.get("UsersID") + if not query(User, User.ID == self.UsersID).first(): + raise IntegrityError( + statement="Foreign key UsersID cannot be null.", + orig="Sessions.UsersID", + params=("NULL")) + self.SessionID = kwargs.get("SessionID") self.LastUpdateTS = kwargs.get("LastUpdateTS") diff --git a/aurweb/models/term.py b/aurweb/models/term.py index 1b4902f7..1a0780df 100644 --- a/aurweb/models/term.py +++ b/aurweb/models/term.py @@ -1,3 +1,4 @@ +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import mapper from aurweb.schema import Terms @@ -8,7 +9,19 @@ class Term: Description: str = None, URL: str = None, Revision: int = None): self.Description = Description + if not self.Description: + raise IntegrityError( + statement="Column Description cannot be null.", + orig="Terms.Description", + params=("NULL")) + self.URL = URL + if not self.URL: + raise IntegrityError( + statement="Column URL cannot be null.", + orig="Terms.URL", + params=("NULL")) + self.Revision = Revision diff --git a/conf/config.dev b/conf/config.dev index 94775a92..45d940e6 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -6,17 +6,19 @@ ; development-specific options too. [database] -backend = sqlite -name = YOUR_AUR_ROOT/aurweb.sqlite3 +; Options: mysql, sqlite. +backend = mysql -; Alternative MySQL configuration (Use either port of socket, if both defined port takes priority) -;backend = mysql -;name = aurweb -;user = aur -;password = aur -;host = localhost +; If using sqlite, set name to the database file path. +name = aurweb + +; MySQL database information. User defaults to root for containerized +; testing with mysqldb. This should be set to a non-root user. +user = root +;password = non-root-user-password +host = localhost ;port = 3306 -;socket = /var/run/mysqld/mysqld.sock +socket = /var/run/mysqld/mysqld.sock [options] aurwebdir = YOUR_AUR_ROOT diff --git a/test/Makefile b/test/Makefile index 060e57c2..920c7113 100644 --- a/test/Makefile +++ b/test/Makefile @@ -8,7 +8,7 @@ MAKEFLAGS = -j1 check: sh pytest pytest: - cd .. && AUR_CONFIG=conf/config coverage run --append /usr/bin/pytest test + cd .. && coverage run --append /usr/bin/pytest test ifdef PROVE sh: diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index 0f813823..3080a505 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -802,18 +802,40 @@ def test_post_account_edit_ssh_pub_key(): assert response.status_code == int(HTTPStatus.OK) # Now let's update what's already there to gain coverage over that path. - pk = str() - with tempfile.TemporaryDirectory() as tmpdir: - with open("/dev/null", "w") as null: - proc = Popen(["ssh-keygen", "-f", f"{tmpdir}/test.ssh", "-N", ""], - stdout=null, stderr=null) - proc.wait() - assert proc.returncode == 0 + post_data["PK"] = make_ssh_pubkey() - # Read in the public key, then delete the temp dir we made. - pk = open(f"{tmpdir}/test.ssh.pub").read().rstrip() + with client as request: + response = request.post("/account/test/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) - post_data["PK"] = pk + assert response.status_code == int(HTTPStatus.OK) + + +def test_post_account_edit_missing_ssh_pubkey(): + request = Request() + sid = user.login(request, "testPassword") + + post_data = { + "U": user.Username, + "E": user.Email, + "PK": make_ssh_pubkey(), + "passwd": "testPassword" + } + + with client as request: + response = request.post("/account/test/edit", cookies={ + "AURSID": sid + }, data=post_data, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.OK) + + post_data = { + "U": user.Username, + "E": user.Email, + "PK": str(), # Pass an empty string now to walk the delete path. + "passwd": "testPassword" + } with client as request: response = request.post("/account/test/edit", cookies={ diff --git a/test/test_api_rate_limit.py b/test/test_api_rate_limit.py index c599ddcf..536e3841 100644 --- a/test/test_api_rate_limit.py +++ b/test/test_api_rate_limit.py @@ -34,5 +34,5 @@ def test_api_rate_key_null_requests_raises_exception(): def test_api_rate_key_null_window_start_raises_exception(): from aurweb.db import session with pytest.raises(IntegrityError): - create(ApiRateLimit, IP="127.0.0.1", WindowStart=1) + create(ApiRateLimit, IP="127.0.0.1", Requests=1) session.rollback() diff --git a/test/test_auth.py b/test/test_auth.py index 7837e7f7..42eac040 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -4,6 +4,8 @@ import pytest from starlette.authentication import AuthenticationError +import aurweb.config + from aurweb.auth import BasicAuthBackend, has_credential from aurweb.db import create, query from aurweb.models.account_type import AccountType @@ -53,13 +55,12 @@ async def test_auth_backend_invalid_sid(): async def test_auth_backend_invalid_user_id(): # Create a new session with a fake user id. now_ts = datetime.utcnow().timestamp() - create(Session, UsersID=666, SessionID="realSession", - LastUpdateTS=now_ts + 5) + db_backend = aurweb.config.get("database", "backend") + with pytest.raises(IntegrityError): + create(Session, UsersID=666, SessionID="realSession", + LastUpdateTS=now_ts + 5) - # Here, we specify a real SID; but it's user is not there. - request.cookies["AURSID"] = "realSession" - with pytest.raises(AuthenticationError, match="Invalid User ID: 666"): - await backend.authenticate(request) + session.rollback() @pytest.mark.asyncio diff --git a/test/test_ban.py b/test/test_ban.py index a4fa5a28..b728644b 100644 --- a/test/test_ban.py +++ b/test/test_ban.py @@ -33,8 +33,7 @@ def test_ban(): def test_invalid_ban(): from aurweb.db import session - with pytest.raises(sa_exc.IntegrityError, - match="NOT NULL constraint failed: Bans.IPAddress"): + with pytest.raises(sa_exc.IntegrityError): bad_ban = Ban(BanTS=datetime.utcnow()) session.add(bad_ban) diff --git a/test/test_db.py b/test/test_db.py index e0946ed5..3911134f 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -5,16 +5,22 @@ import tempfile from unittest import mock -import mysql.connector import pytest import aurweb.config +import aurweb.initdb from aurweb import db from aurweb.models.account_type import AccountType from aurweb.testing import setup_test_db +class Args: + """ Stub arguments used for running aurweb.initdb. """ + use_alembic = True + verbose = True + + class DBCursor: """ A fake database cursor object used in tests. """ items = [] @@ -38,27 +44,73 @@ class DBConnection: pass +def make_temp_config(config_file, *replacements): + """ Generate a temporary config file with a set of replacements. + + :param *replacements: A variable number of tuple regex replacement pairs + :return: A tuple containing (temp directory, temp config file) + """ + tmpdir = tempfile.TemporaryDirectory() + tmp = os.path.join(tmpdir.name, "config.tmp") + with open(config_file) as f: + config = f.read() + for repl in list(replacements): + config = re.sub(repl[0], repl[1], config) + with open(tmp, "w") as o: + o.write(config) + aurwebdir = aurweb.config.get("options", "aurwebdir") + defaults = os.path.join(aurwebdir, "conf/config.defaults") + with open(defaults) as i: + with open(f"{tmp}.defaults", "w") as o: + o.write(i.read()) + return tmpdir, tmp + + +def make_temp_sqlite_config(config_file): + return make_temp_config(config_file, + (r"backend = .*", "backend = sqlite"), + (r"name = .*", "name = /tmp/aurweb.sqlite3")) + + +def make_temp_mysql_config(config_file): + return make_temp_config(config_file, + (r"backend = .*", "backend = mysql"), + (r"name = .*", "name = aurweb")) + + @pytest.fixture(autouse=True) def setup_db(): - setup_test_db("Bans") + if os.path.exists("/tmp/aurweb.sqlite3"): + os.remove("/tmp/aurweb.sqlite3") + + # In various places in this test, we reinitialize the engine. + # Make sure we kill the previous engine before initializing + # it via setup_test_db(). + aurweb.db.kill_engine() + setup_test_db() def test_sqlalchemy_sqlite_url(): - with mock.patch.dict(os.environ, {"AUR_CONFIG": "conf/config.dev"}): - aurweb.config.rehash() - assert db.get_sqlalchemy_url() + tmpctx, tmp = make_temp_sqlite_config("conf/config") + with tmpctx: + with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): + aurweb.config.rehash() + assert db.get_sqlalchemy_url() aurweb.config.rehash() def test_sqlalchemy_mysql_url(): - with mock.patch.dict(os.environ, {"AUR_CONFIG": "conf/config.defaults"}): - aurweb.config.rehash() - assert db.get_sqlalchemy_url() + tmpctx, tmp = make_temp_mysql_config("conf/config") + with tmpctx: + with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): + aurweb.config.rehash() + assert db.get_sqlalchemy_url() aurweb.config.rehash() def test_sqlalchemy_mysql_port_url(): - tmpctx, tmp = make_temp_config("conf/config.defaults", ";port = 3306", "port = 3306") + tmpctx, tmp = make_temp_config("conf/config", + (r";port = 3306", "port = 3306")) with tmpctx: with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): @@ -67,18 +119,9 @@ def test_sqlalchemy_mysql_port_url(): aurweb.config.rehash() -def make_temp_config(config_file, src_str, replace_with): - tmpdir = tempfile.TemporaryDirectory() - tmp = os.path.join(tmpdir.name, "config.tmp") - with open(config_file) as f: - config = re.sub(src_str, f'{replace_with}', f.read()) - with open(tmp, "w") as o: - o.write(config) - return tmpdir, tmp - - def test_sqlalchemy_unknown_backend(): - tmpctx, tmp = make_temp_config("conf/config", "backend = sqlite", "backend = blah") + tmpctx, tmp = make_temp_config("conf/config", + (r"backend = mysql", "backend = blah")) with tmpctx: with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): @@ -89,22 +132,31 @@ def test_sqlalchemy_unknown_backend(): def test_db_connects_without_fail(): + """ This only tests the actual config supplied to pytest. """ db.connect() assert db.engine is not None -def test_connection_class_without_fail(): - conn = db.Connection() +def test_connection_class_sqlite_without_fail(): + tmpctx, tmp = make_temp_sqlite_config("conf/config") + with tmpctx: + with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): + aurweb.config.rehash() - cur = conn.execute( - "SELECT AccountType FROM AccountTypes WHERE ID = ?", (1,)) - account_type = cur.fetchone()[0] + aurweb.db.kill_engine() + aurweb.initdb.run(Args()) - assert account_type == "User" + conn = db.Connection() + cur = conn.execute( + "SELECT AccountType FROM AccountTypes WHERE ID = ?", (1,)) + account_type = cur.fetchone()[0] + assert account_type == "User" + aurweb.config.rehash() def test_connection_class_unsupported_backend(): - tmpctx, tmp = make_temp_config("conf/config", "backend = sqlite", "backend = blah") + tmpctx, tmp = make_temp_config("conf/config", + (r"backend = mysql", "backend = blah")) with tmpctx: with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): @@ -114,10 +166,9 @@ def test_connection_class_unsupported_backend(): aurweb.config.rehash() -@mock.patch("mysql.connector.connect", mock.MagicMock(return_value=True)) -@mock.patch.object(mysql.connector, "paramstyle", "qmark") +@mock.patch("MySQLdb.connect", mock.MagicMock(return_value=True)) def test_connection_mysql(): - tmpctx, tmp = make_temp_config("conf/config", "backend = sqlite", "backend = mysql") + tmpctx, tmp = make_temp_mysql_config("conf/config") with tmpctx: with mock.patch.dict(os.environ, { "AUR_CONFIG": tmp, @@ -137,44 +188,78 @@ def test_connection_sqlite(): @mock.patch("sqlite3.connect", mock.MagicMock(return_value=DBConnection())) @mock.patch.object(sqlite3, "paramstyle", "format") def test_connection_execute_paramstyle_format(): - conn = db.Connection() + tmpctx, tmp = make_temp_sqlite_config("conf/config") - # First, test ? to %s format replacement. - account_types = conn\ - .execute("SELECT * FROM AccountTypes WHERE AccountType = ?", ["User"])\ - .fetchall() - assert account_types == \ - ["SELECT * FROM AccountTypes WHERE AccountType = %s", ["User"]] + with tmpctx: + with mock.patch.dict(os.environ, { + "AUR_CONFIG": tmp, + "AUR_CONFIG_DEFAULTS": "conf/config.defaults" + }): + aurweb.config.rehash() - # Test other format replacement. - account_types = conn\ - .execute("SELECT * FROM AccountTypes WHERE AccountType = %", ["User"])\ - .fetchall() - assert account_types == \ - ["SELECT * FROM AccountTypes WHERE AccountType = %%", ["User"]] + aurweb.db.kill_engine() + aurweb.initdb.run(Args()) + + conn = db.Connection() + + # First, test ? to %s format replacement. + account_types = conn\ + .execute("SELECT * FROM AccountTypes WHERE AccountType = ?", + ["User"]).fetchall() + assert account_types == \ + ["SELECT * FROM AccountTypes WHERE AccountType = %s", ["User"]] + + # Test other format replacement. + account_types = conn\ + .execute("SELECT * FROM AccountTypes WHERE AccountType = %", + ["User"]).fetchall() + assert account_types == \ + ["SELECT * FROM AccountTypes WHERE AccountType = %%", ["User"]] + aurweb.config.rehash() @mock.patch("sqlite3.connect", mock.MagicMock(return_value=DBConnection())) @mock.patch.object(sqlite3, "paramstyle", "qmark") def test_connection_execute_paramstyle_qmark(): - conn = db.Connection() - # We don't modify anything when using qmark, so test equality. - account_types = conn\ - .execute("SELECT * FROM AccountTypes WHERE AccountType = ?", ["User"])\ - .fetchall() - assert account_types == \ - ["SELECT * FROM AccountTypes WHERE AccountType = ?", ["User"]] + tmpctx, tmp = make_temp_sqlite_config("conf/config") + + with tmpctx: + with mock.patch.dict(os.environ, { + "AUR_CONFIG": tmp, + "AUR_CONFIG_DEFAULTS": "conf/config.defaults" + }): + aurweb.config.rehash() + + aurweb.db.kill_engine() + aurweb.initdb.run(Args()) + + conn = db.Connection() + # We don't modify anything when using qmark, so test equality. + account_types = conn\ + .execute("SELECT * FROM AccountTypes WHERE AccountType = ?", + ["User"]).fetchall() + assert account_types == \ + ["SELECT * FROM AccountTypes WHERE AccountType = ?", ["User"]] + aurweb.config.rehash() @mock.patch("sqlite3.connect", mock.MagicMock(return_value=DBConnection())) @mock.patch.object(sqlite3, "paramstyle", "unsupported") def test_connection_execute_paramstyle_unsupported(): - conn = db.Connection() - with pytest.raises(ValueError, match="unsupported paramstyle"): - conn.execute( - "SELECT * FROM AccountTypes WHERE AccountType = ?", - ["User"] - ).fetchall() + tmpctx, tmp = make_temp_sqlite_config("conf/config") + with tmpctx: + with mock.patch.dict(os.environ, { + "AUR_CONFIG": tmp, + "AUR_CONFIG_DEFAULTS": "conf/config.defaults" + }): + aurweb.config.rehash() + conn = db.Connection() + with pytest.raises(ValueError, match="unsupported paramstyle"): + conn.execute( + "SELECT * FROM AccountTypes WHERE AccountType = ?", + ["User"] + ).fetchall() + aurweb.config.rehash() def test_create_delete(): @@ -186,13 +271,12 @@ def test_create_delete(): assert record is None -@mock.patch("mysql.connector.paramstyle", "qmark") def test_connection_executor_mysql_paramstyle(): executor = db.ConnectionExecutor(None, backend="mysql") - assert executor.paramstyle() == "qmark" + assert executor.paramstyle() == "format" @mock.patch("sqlite3.paramstyle", "pyformat") def test_connection_executor_sqlite_paramstyle(): executor = db.ConnectionExecutor(None, backend="sqlite") - assert executor.paramstyle() == "pyformat" + assert executor.paramstyle() == sqlite3.paramstyle diff --git a/test/test_initdb.py b/test/test_initdb.py index eae33007..c7d29ee2 100644 --- a/test/test_initdb.py +++ b/test/test_initdb.py @@ -1,27 +1,19 @@ -import pytest - import aurweb.config import aurweb.db import aurweb.initdb from aurweb.models.account_type import AccountType -from aurweb.schema import metadata -from aurweb.testing import setup_test_db -@pytest.fixture(autouse=True) -def setup(): - setup_test_db() - - tables = metadata.tables.keys() - for table in tables: - aurweb.db.session.execute(f"DROP TABLE IF EXISTS {table}") +class Args: + use_alembic = True + verbose = True def test_run(): - class Args: - use_alembic = True - verbose = False + from aurweb.schema import metadata + aurweb.db.kill_engine() + metadata.drop_all(aurweb.db.get_engine()) aurweb.initdb.run(Args()) record = aurweb.db.query(AccountType, AccountType.AccountType == "User").first() diff --git a/test/test_package_relation.py b/test/test_package_relation.py index dd0455cd..96932f40 100644 --- a/test/test_package_relation.py +++ b/test/test_package_relation.py @@ -1,6 +1,6 @@ import pytest -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import IntegrityError, OperationalError from aurweb.db import create, query from aurweb.models.account_type import AccountType @@ -36,7 +36,7 @@ def setup(): URL="https://test.package") -def test_package_dependencies(): +def test_package_relation(): conflicts = query(RelationType, RelationType.Name == "conflicts").first() pkgrel = create(PackageRelation, Package=package, RelationType=conflicts, @@ -68,10 +68,12 @@ def test_package_dependencies(): assert pkgrel in package.package_relations -def test_package_dependencies_null_package_raises_exception(): +def test_package_relation_null_package_raises_exception(): from aurweb.db import session conflicts = query(RelationType, RelationType.Name == "conflicts").first() + assert conflicts is not None + with pytest.raises(IntegrityError): create(PackageRelation, RelationType=conflicts, @@ -79,7 +81,7 @@ def test_package_dependencies_null_package_raises_exception(): session.rollback() -def test_package_dependencies_null_dependency_type_raises_exception(): +def test_package_relation_null_relation_type_raises_exception(): from aurweb.db import session with pytest.raises(IntegrityError): @@ -89,11 +91,13 @@ def test_package_dependencies_null_dependency_type_raises_exception(): session.rollback() -def test_package_dependencies_null_depname_raises_exception(): +def test_package_relation_null_relname_raises_exception(): from aurweb.db import session - depends = query(RelationType, RelationType.Name == "depends").first() - with pytest.raises(IntegrityError): + depends = query(RelationType, RelationType.Name == "conflicts").first() + assert depends is not None + + with pytest.raises((OperationalError, IntegrityError)): create(PackageRelation, Package=package, RelationType=depends) diff --git a/test/test_session.py b/test/test_session.py index 2877ea7f..c324a739 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -25,7 +25,7 @@ def setup(): ResetKey="testReset", Passwd="testPassword", AccountType=account_type) session = create(Session, UsersID=user.ID, SessionID="testSession", - LastUpdateTS=datetime.utcnow()) + LastUpdateTS=datetime.utcnow().timestamp()) def test_session(): From 228bc8fe7c3ea7cef66f00f1608b699d00838c43 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 4 Jun 2021 23:09:38 -0700 Subject: [PATCH 144/844] fix aurweb.auth test coverage With mysqlclient, we no longer need to account for a user not existing when an ssh key is found. Signed-off-by: Kevin Morris --- aurweb/auth.py | 14 +++++++++----- test/test_auth.py | 7 ++++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index a4ff2167..401ed6ae 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -4,7 +4,8 @@ from datetime import datetime from http import HTTPStatus from fastapi.responses import RedirectResponse -from starlette.authentication import AuthCredentials, AuthenticationBackend, AuthenticationError +from sqlalchemy import and_ +from starlette.authentication import AuthCredentials, AuthenticationBackend from starlette.requests import HTTPConnection import aurweb.config @@ -42,14 +43,17 @@ class BasicAuthBackend(AuthenticationBackend): now_ts = datetime.utcnow().timestamp() record = session.query(Session).filter( - Session.SessionID == sid, Session.LastUpdateTS >= now_ts).first() + and_(Session.SessionID == sid, + Session.LastUpdateTS >= now_ts)).first() + + # If no session with sid and a LastUpdateTS now or later exists. if not record: return None, AnonymousUser() + # At this point, we cannot have an invalid user if the record + # exists, due to ForeignKey constraints in the schema upheld + # by mysqlclient. user = session.query(User).filter(User.ID == record.UsersID).first() - if not user: - raise AuthenticationError(f"Invalid User ID: {record.UsersID}") - user.authenticated = True return AuthCredentials(["authenticated"]), user diff --git a/test/test_auth.py b/test/test_auth.py index 42eac040..05dd2020 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2,7 +2,7 @@ from datetime import datetime import pytest -from starlette.authentication import AuthenticationError +from sqlalchemy.exc import IntegrityError import aurweb.config @@ -53,13 +53,13 @@ async def test_auth_backend_invalid_sid(): @pytest.mark.asyncio async def test_auth_backend_invalid_user_id(): + from aurweb.db import session + # Create a new session with a fake user id. now_ts = datetime.utcnow().timestamp() - db_backend = aurweb.config.get("database", "backend") with pytest.raises(IntegrityError): create(Session, UsersID=666, SessionID="realSession", LastUpdateTS=now_ts + 5) - session.rollback() @@ -70,6 +70,7 @@ async def test_basic_auth_backend(): now_ts = datetime.utcnow().timestamp() create(Session, UsersID=user.ID, SessionID="realSession", LastUpdateTS=now_ts + 5) + request.cookies["AURSID"] = "realSession" _, result = await backend.authenticate(request) assert result == user From 62e58b122f905b4e5462df6b0bbebd278bb56419 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 4 Jun 2021 23:13:05 -0700 Subject: [PATCH 145/844] fix test_accounts_routes test coverage Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 2 +- test/test_accounts_routes.py | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5bdf427c..8e14f77f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ cache: before_script: - pacman -Syu --noconfirm --noprogressbar --needed --cachedir .pkg-cache - base-devel git gpgme protobuf pyalpm python-mysql-connector + base-devel git gpgme protobuf pyalpm python-mysqlclient python-pygit2 python-srcinfo python-bleach python-markdown python-sqlalchemy python-alembic python-pytest python-werkzeug python-pytest-tap python-fastapi hypercorn nginx python-authlib diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index 3080a505..d5fd089e 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -30,6 +30,20 @@ client = TestClient(app) user = None +def make_ssh_pubkey(): + # Create a public key with ssh-keygen (this adds ssh-keygen as a + # dependency to passing this test). + with tempfile.TemporaryDirectory() as tmpdir: + with open("/dev/null", "w") as null: + proc = Popen(["ssh-keygen", "-f", f"{tmpdir}/test.ssh", "-N", ""], + stdout=null, stderr=null) + proc.wait() + assert proc.returncode == 0 + + # Read in the public key, then delete the temp dir we made. + return open(f"{tmpdir}/test.ssh.pub").read().rstrip() + + @pytest.fixture(autouse=True) def setup(): global user @@ -770,27 +784,13 @@ def test_post_account_edit_error_unauthorized(): def test_post_account_edit_ssh_pub_key(): - pk = str() - - # Create a public key with ssh-keygen (this adds ssh-keygen as a - # dependency to passing this test). - with tempfile.TemporaryDirectory() as tmpdir: - with open("/dev/null", "w") as null: - proc = Popen(["ssh-keygen", "-f", f"{tmpdir}/test.ssh", "-N", ""], - stdout=null, stderr=null) - proc.wait() - assert proc.returncode == 0 - - # Read in the public key, then delete the temp dir we made. - pk = open(f"{tmpdir}/test.ssh.pub").read().rstrip() - request = Request() sid = user.login(request, "testPassword") post_data = { "U": "test", "E": "test@example.org", - "PK": pk, + "PK": make_ssh_pubkey(), "passwd": "testPassword" } From 4d1faca4477bc510e81513b659f199f0d8c41bb1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 5 Jun 2021 01:07:56 -0700 Subject: [PATCH 146/844] test both mysql and sqlite in .gitlab-ci.yml Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e14f77f..e65b4343 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,9 @@ cache: # For some reason Gitlab CI only supports storing cache/artifacts in a path relative to the build directory - .pkg-cache +variables: + AUR_CONFIG: conf/config + before_script: - pacman -Syu --noconfirm --noprogressbar --needed --cachedir .pkg-cache base-devel git gpgme protobuf pyalpm python-mysqlclient @@ -15,17 +18,31 @@ before_script: python-itsdangerous python-httpx python-jinja python-pytest-cov python-requests python-aiofiles python-python-multipart python-pytest-asyncio python-coverage python-bcrypt - python-email-validator openssh python-lxml + python-email-validator openssh python-lxml mariadb - bash -c "echo '127.0.0.1' > /etc/hosts" - bash -c "echo '::1' >> /etc/hosts" + - mariadb-install-db --user=mysql --basedir=/usr --datadir=/var/lib/mysql + - (cd '/usr' && /usr/bin/mysqld_safe --datadir='/var/lib/mysql') & + - 'until : > /dev/tcp/127.0.0.1/3306; do sleep 1s; done' + - mysql -u root -e "CREATE USER 'aur'@'localhost' IDENTIFIED BY 'aur';" + - mysql -u root -e "CREATE DATABASE aurweb;" + - mysql -u root -e "GRANT ALL PRIVILEGES ON aurweb.* TO 'aur'@'localhost';" + - mysql -u root -e "FLUSH PRIVILEGES;" + - sed -r "s;YOUR_AUR_ROOT;$(pwd);g" conf/config.dev > conf/config + - cp conf/config conf/config.sqlite + - cp conf/config.defaults conf/config.sqlite.defaults + - sed -i -r 's;backend = .*;backend = sqlite;' conf/config.sqlite + - sed -i -r "s;name = .*;name = $(pwd)/aurweb.sqlite3;" conf/config.sqlite + - AUR_CONFIG=conf/config.sqlite python -m aurweb.initdb test: script: - python setup.py install - - sed -r "s;YOUR_AUR_ROOT;$(pwd);g" conf/config.dev > conf/config - - AUR_CONFIG=conf/config make -C po all install - - AUR_CONFIG=conf/config python -m aurweb.initdb - - make -C test + - make -C po all install + - python -m aurweb.initdb + - make -C test sh # sharness tests use sqlite. + - make -C test pytest # pytest with mysql. + - AUR_CONFIG=conf/config.sqlite make -C test pytest # pytest with sqlite. - coverage report --include='aurweb/*' - coverage xml --include='aurweb/*' artifacts: From 5ceeb88bee1575427008c119d2c7cd37274f0919 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 5 Jun 2021 21:19:20 -0700 Subject: [PATCH 147/844] remove unused imports, rectify isort violations Files got into the branch that violate both PEP-8 guidelines and isorts. This fixes them. Signed-off-by: Kevin Morris --- aurweb/git/update.py | 4 ++-- aurweb/models/user.py | 2 -- aurweb/routers/html.py | 3 +-- ...56e2ce8e2ffa_utf8mb4_charset_and_collation.py | 16 ++++++++-------- ...fcd6e1cd_add_sso_account_id_in_table_users.py | 1 + .../versions/f47cad5d6d03_initial_revision.py | 5 ----- test/test_auth.py | 2 -- 7 files changed, 12 insertions(+), 21 deletions(-) diff --git a/aurweb/git/update.py b/aurweb/git/update.py index 3c9c3785..2424bf6c 100755 --- a/aurweb/git/update.py +++ b/aurweb/git/update.py @@ -305,9 +305,9 @@ def main(): # noqa: C901 try: metadata_pkgbase = metadata['pkgbase'] - except KeyError as e: + except KeyError: die_commit('invalid .SRCINFO, does not contain a pkgbase (is the file empty?)', - str(commit.id)) + str(commit.id)) if not re.match(repo_regex, metadata_pkgbase): die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase), str(commit.id)) diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 6c5c6e21..1961228e 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -65,8 +65,6 @@ class User: def valid_password(self, password: str): """ Check authentication against a given password. """ - from aurweb.db import session - if password is None: return False diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index 8f89e05c..890aff88 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -2,9 +2,8 @@ decorators in some way; more complex routes should be defined in their own modules and imported here. """ from http import HTTPStatus -from urllib.parse import unquote -from fastapi import APIRouter, Form, Request, HTTPException +from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse from aurweb.templates import make_context, render_template diff --git a/migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py b/migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py index e198c34c..67f0c065 100644 --- a/migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py +++ b/migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py @@ -56,14 +56,14 @@ db_backend = aurweb.config.get("database", "backend") def rebuild_unique_indexes_with_str_cols(): for idx_name in indexes: sql = f""" -DROP INDEX IF EXISTS {idx_name} +DROP INDEX IF EXISTS {idx_name} ON {indexes.get(idx_name)[0]} """ op.execute(sql) sql = f""" -CREATE UNIQUE INDEX {idx_name} -ON {indexes.get(idx_name)[0]} -({indexes.get(idx_name)[1]}, {indexes.get(idx_name)[2]}) +CREATE UNIQUE INDEX {idx_name} +ON {indexes.get(idx_name)[0]} +({indexes.get(idx_name)[1]}, {indexes.get(idx_name)[2]}) """ op.execute(sql) @@ -77,8 +77,8 @@ def upgrade(): def op_execute(table_meta): table, charset, collate = table_meta sql = f""" -ALTER TABLE {table} -CONVERT TO CHARACTER SET {charset} +ALTER TABLE {table} +CONVERT TO CHARACTER SET {charset} COLLATE {collate} """ op.execute(sql) @@ -94,8 +94,8 @@ def downgrade(): def op_execute(table_meta): table, charset, collate = table_meta sql = f""" -ALTER TABLE {table} -CONVERT TO CHARACTER SET {src_charset} +ALTER TABLE {table} +CONVERT TO CHARACTER SET {src_charset} COLLATE {src_collate} """ op.execute(sql) diff --git a/migrations/versions/ef39fcd6e1cd_add_sso_account_id_in_table_users.py b/migrations/versions/ef39fcd6e1cd_add_sso_account_id_in_table_users.py index 2b257e9d..49bf055a 100644 --- a/migrations/versions/ef39fcd6e1cd_add_sso_account_id_in_table_users.py +++ b/migrations/versions/ef39fcd6e1cd_add_sso_account_id_in_table_users.py @@ -6,6 +6,7 @@ Create Date: 2020-06-08 10:04:13.898617 """ import sqlalchemy as sa + from alembic import op from sqlalchemy.engine.reflection import Inspector diff --git a/migrations/versions/f47cad5d6d03_initial_revision.py b/migrations/versions/f47cad5d6d03_initial_revision.py index 9e99490f..b214beea 100644 --- a/migrations/versions/f47cad5d6d03_initial_revision.py +++ b/migrations/versions/f47cad5d6d03_initial_revision.py @@ -1,14 +1,9 @@ """initial revision Revision ID: f47cad5d6d03 -Revises: Create Date: 2020-02-23 13:23:32.331396 """ -from alembic import op -import sqlalchemy as sa - - # revision identifiers, used by Alembic. revision = 'f47cad5d6d03' down_revision = None diff --git a/test/test_auth.py b/test/test_auth.py index 05dd2020..e5e1de11 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -4,8 +4,6 @@ import pytest from sqlalchemy.exc import IntegrityError -import aurweb.config - from aurweb.auth import BasicAuthBackend, has_credential from aurweb.db import create, query from aurweb.models.account_type import AccountType From e865a6347f16edba73e405a4dabc1f76d3ca6509 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 5 Jun 2021 21:28:26 -0700 Subject: [PATCH 148/844] .gitlab-ci.yml: enforce isort and flake8 compliance Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e65b4343..a9947dfe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,6 +19,7 @@ before_script: python-requests python-aiofiles python-python-multipart python-pytest-asyncio python-coverage python-bcrypt python-email-validator openssh python-lxml mariadb + python-isort flake8 - bash -c "echo '127.0.0.1' > /etc/hosts" - bash -c "echo '::1' >> /etc/hosts" - mariadb-install-db --user=mysql --basedir=/usr --datadir=/var/lib/mysql @@ -45,6 +46,12 @@ test: - AUR_CONFIG=conf/config.sqlite make -C test pytest # pytest with sqlite. - coverage report --include='aurweb/*' - coverage xml --include='aurweb/*' + - flake8 --count aurweb # Assert no flake8 violations in aurweb. + - flake8 --count test # Assert no flake8 violations in test. + - flake8 --count migrations # Assert no flake8 violations in migrations. + - isort --check-only aurweb # Assert no isort violations in aurweb. + - isort --check-only test # Assert no flake8 violations in test. + - isort --check-only migrations # Assert no flake8 violations in migrations. artifacts: reports: cobertura: coverage.xml From 1874e821f5883c2338b07c4803b67c6802621c38 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 5 Jun 2021 18:13:10 -0700 Subject: [PATCH 149/844] add case [in]sensitivity tests + add OfficialProvider model `ci` in this context means "Case Insensitive". `cs` in this context means "Case Sensitive". New models created: - OfficialProvider This was required to write a test for checking that OfficialProviders behaves as we expect, which was the starter for the original aurblup bug. New tests created: - test_official_provider Modified tests: - test_package_base: add ci test - test_package: add ci test - test_session: add cs test - test_ssh_pub_key: add cs test Signed-off-by: Kevin Morris --- aurweb/models/official_provider.py | 34 ++++++++++++++ test/test_official_provider.py | 75 ++++++++++++++++++++++++++++++ test/test_package.py | 16 +++++++ test/test_package_base.py | 21 +++++++++ test/test_session.py | 9 ++++ test/test_ssh_pub_key.py | 12 +++++ 6 files changed, 167 insertions(+) create mode 100644 aurweb/models/official_provider.py create mode 100644 test/test_official_provider.py diff --git a/aurweb/models/official_provider.py b/aurweb/models/official_provider.py new file mode 100644 index 00000000..073eb435 --- /dev/null +++ b/aurweb/models/official_provider.py @@ -0,0 +1,34 @@ +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import mapper + +from aurweb.schema import OfficialProviders + + +class OfficialProvider: + def __init__(self, + Name: str = None, + Repo: str = None, + Provides: str = None): + self.Name = Name + if not self.Name: + raise IntegrityError( + statement="Column Name cannot be null.", + orig="OfficialProviders.Name", + params=("NULL")) + + self.Repo = Repo + if not self.Repo: + raise IntegrityError( + statement="Column Repo cannot be null.", + orig="OfficialProviders.Repo", + params=("NULL")) + + self.Provides = Provides + if not self.Provides: + raise IntegrityError( + statement="Column Provides cannot be null.", + orig="OfficialProviders.Provides", + params=("NULL")) + + +mapper(OfficialProvider, OfficialProviders) diff --git a/test/test_official_provider.py b/test/test_official_provider.py new file mode 100644 index 00000000..a1d3d54a --- /dev/null +++ b/test/test_official_provider.py @@ -0,0 +1,75 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create +from aurweb.models.official_provider import OfficialProvider +from aurweb.testing import setup_test_db + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db("OfficialProviders") + + +def test_official_provider_creation(): + oprovider = create(OfficialProvider, + Name="some-name", + Repo="some-repo", + Provides="some-provides") + assert bool(oprovider.ID) + assert oprovider.Name == "some-name" + assert oprovider.Repo == "some-repo" + assert oprovider.Provides == "some-provides" + + +def test_official_provider_cs(): + """ Test case sensitivity of the database table. """ + oprovider = create(OfficialProvider, + Name="some-name", + Repo="some-repo", + Provides="some-provides") + assert bool(oprovider.ID) + + oprovider_cs = create(OfficialProvider, + Name="SOME-NAME", + Repo="SOME-REPO", + Provides="SOME-PROVIDES") + assert bool(oprovider_cs.ID) + + assert oprovider.ID != oprovider_cs.ID + + assert oprovider.Name == "some-name" + assert oprovider.Repo == "some-repo" + assert oprovider.Provides == "some-provides" + + assert oprovider_cs.Name == "SOME-NAME" + assert oprovider_cs.Repo == "SOME-REPO" + assert oprovider_cs.Provides == "SOME-PROVIDES" + + +def test_official_provider_null_name_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(OfficialProvider, + Repo="some-repo", + Provides="some-provides") + session.rollback() + + +def test_official_provider_null_repo_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(OfficialProvider, + Name="some-name", + Provides="some-provides") + session.rollback() + + +def test_official_provider_null_provides_raises_exception(): + from aurweb.db import session + with pytest.raises(IntegrityError): + create(OfficialProvider, + Name="some-name", + Repo="some-repo") + session.rollback() diff --git a/test/test_package.py b/test/test_package.py index 66d557f3..a994f096 100644 --- a/test/test_package.py +++ b/test/test_package.py @@ -3,6 +3,8 @@ import pytest from sqlalchemy import and_ from sqlalchemy.exc import IntegrityError +import aurweb.config + from aurweb.db import create, query from aurweb.models.account_type import AccountType from aurweb.models.package import Package @@ -55,6 +57,20 @@ def test_package(): assert record is not None +def test_package_ci(): + """ Test case insensitivity of the database table. """ + if aurweb.config.get("database", "backend") == "sqlite": + return None # SQLite doesn't seem handle this. + + from aurweb.db import session + + with pytest.raises(IntegrityError): + create(Package, + PackageBase=pkgbase, + Name="Beautiful-Package") + session.rollback() + + def test_package_null_pkgbase_raises_exception(): from aurweb.db import session diff --git a/test/test_package_base.py b/test/test_package_base.py index e0359f4f..7f608c2c 100644 --- a/test/test_package_base.py +++ b/test/test_package_base.py @@ -2,6 +2,8 @@ import pytest from sqlalchemy.exc import IntegrityError +import aurweb.config + from aurweb.db import create, query from aurweb.models.account_type import AccountType from aurweb.models.package_base import PackageBase @@ -35,6 +37,25 @@ def test_package_base(): assert pkgbase.ModifiedTS > 0 +def test_package_base_ci(): + """ Test case insensitivity of the database table. """ + if aurweb.config.get("database", "backend") == "sqlite": + return None # SQLite doesn't seem handle this. + + from aurweb.db import session + + pkgbase = create(PackageBase, + Name="beautiful-package", + Maintainer=user) + assert bool(pkgbase.ID) + + with pytest.raises(IntegrityError): + create(PackageBase, + Name="Beautiful-Package", + Maintainer=user) + session.rollback() + + def test_package_base_relationships(): pkgbase = create(PackageBase, Name="beautiful-package", diff --git a/test/test_session.py b/test/test_session.py index c324a739..1dd82db1 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -33,6 +33,15 @@ def test_session(): assert session.UsersID == user.ID +def test_session_cs(): + """ Test case sensitivity of the database table. """ + session_cs = create(Session, UsersID=user.ID, + SessionID="TESTSESSION", + LastUpdateTS=datetime.utcnow().timestamp()) + assert session_cs.SessionID == "TESTSESSION" + assert session.SessionID == "testSession" + + def test_session_user_association(): # Make sure that the Session user attribute is correct. assert session.User == user diff --git a/test/test_ssh_pub_key.py b/test/test_ssh_pub_key.py index 4072549e..0793199a 100644 --- a/test/test_ssh_pub_key.py +++ b/test/test_ssh_pub_key.py @@ -41,6 +41,18 @@ def test_ssh_pub_key(): assert ssh_pub_key.PubKey == "testPubKey" +def test_ssh_pub_key_cs(): + """ Test case sensitivity of the database table. """ + ssh_pub_key_cs = create(SSHPubKey, UserID=user.ID, + Fingerprint="TESTFINGERPRINT", + PubKey="TESTPUBKEY") + + assert ssh_pub_key_cs.Fingerprint == "TESTFINGERPRINT" + assert ssh_pub_key_cs.PubKey == "TESTPUBKEY" + assert ssh_pub_key.Fingerprint == "testFingerprint" + assert ssh_pub_key.PubKey == "testPubKey" + + def test_ssh_pub_key_fingerprint(): assert get_fingerprint(TEST_SSH_PUBKEY) is not None From 889d358a6daff60d5aac3df62b54a7f939b9bc8a Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Sun, 6 Jun 2021 21:49:27 +0200 Subject: [PATCH 150/844] Add missing ) for addvote.php --- web/html/addvote.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/html/addvote.php b/web/html/addvote.php index 70280cfd..3e4def44 100644 --- a/web/html/addvote.php +++ b/web/html/addvote.php @@ -69,7 +69,7 @@ if (has_credential(CRED_TU_ADD_VOTE)) { if (!empty($_POST['addVote']) && empty($error)) { // Convert $quorum to a String of maximum length "12.34" (5). - $quorum_str = substr(strval($quorum), min(5, strlen($quorum)); + $quorum_str = substr(strval($quorum), min(5, strlen($quorum))); add_tu_proposal($_POST['agenda'], $_POST['user'], $len, $quorum_str, $uid); From f9f41dc99beb9fd75d3f772bf4023168b53c9180 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 6 Jun 2021 16:30:16 -0700 Subject: [PATCH 151/844] restore TU_VoteInfo -> utf8mb4_general_ci Signed-off-by: Kevin Morris --- aurweb/schema.py | 4 +++- .../versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/aurweb/schema.py b/aurweb/schema.py index f0162045..a0bb7e09 100644 --- a/aurweb/schema.py +++ b/aurweb/schema.py @@ -389,7 +389,9 @@ TU_VoteInfo = Table( Column('No', TINYINT(3, unsigned=True), nullable=False, server_default=text("'0'")), Column('Abstain', TINYINT(3, unsigned=True), nullable=False, server_default=text("'0'")), Column('ActiveTUs', TINYINT(3, unsigned=True), nullable=False, server_default=text("'0'")), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) diff --git a/migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py b/migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py index e198c34c..03982676 100644 --- a/migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py +++ b/migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py @@ -37,7 +37,7 @@ tables = [ ('RequestTypes', 'utf8mb4', 'utf8mb4_general_ci'), ('SSHPubKeys', 'utf8mb4', 'utf8mb4_bin'), ('Sessions', 'utf8mb4', 'utf8mb4_bin'), - ('TU_VoteInfo', 'utf8mb4', 'utf8mb4_bin'), + ('TU_VoteInfo', 'utf8mb4', 'utf8mb4_general_ci'), ('Terms', 'utf8mb4', 'utf8mb4_general_ci'), ('Users', 'utf8mb4', 'utf8mb4_general_ci') ] From 4f09e939ae1d605f6568a180d8ab86033e847a0f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 6 Jun 2021 21:34:42 -0700 Subject: [PATCH 152/844] bugfix: gendummydata.py was producing invalid usernames As per our regex and policies, usernames should consist of ascii alphanumeric characters and possibly (-, _ or .). gendummydata.py was creating unicode versions of some usernames and adding them into the DB. With our newfound collations, this becomes a problem as it treats them as the same. This should have never been the case here, and so, gendummydata.py has been patched to normalize all of its usernames and package names. Signed-off-by: Kevin Morris --- schema/gendummydata.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/schema/gendummydata.py b/schema/gendummydata.py index c7b3a06d..35805d6c 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -98,11 +98,19 @@ if MAX_USERS > len(contents): MAX_USERS = len(contents) if MAX_PKGS > len(contents): MAX_PKGS = len(contents) -if len(contents) - MAX_USERS > MAX_PKGS: - need_dupes = 0 -else: + +need_dupes = 0 +if not len(contents) - MAX_USERS > MAX_PKGS: need_dupes = 1 + +def normalize(unicode_data): + """ We only accept ascii for usernames. Also use this to normalize + package names; our database utf8mb4 collations compare with Unicode + Equivalence. """ + return unicode_data.encode('ascii', 'ignore').decode('ascii') + + # select random usernames # log.debug("Generating random user names...") @@ -110,12 +118,13 @@ user_id = USER_ID while len(seen_users) < MAX_USERS: user = random.randrange(0, len(contents)) word = contents[user].replace("'", "").replace(".", "").replace(" ", "_") - word = word.strip().lower() + word = normalize(word.strip().lower()) if word not in seen_users: seen_users[word] = user_id user_id += 1 user_keys = list(seen_users.keys()) + # select random package names # log.debug("Generating random package names...") @@ -123,7 +132,7 @@ num_pkgs = PKG_ID while len(seen_pkgs) < MAX_PKGS: pkg = random.randrange(0, len(contents)) word = contents[pkg].replace("'", "").replace(".", "").replace(" ", "_") - word = word.strip().lower() + word = normalize(word.strip().lower()) if not need_dupes: if word not in seen_pkgs and word not in seen_users: seen_pkgs[word] = num_pkgs @@ -285,10 +294,10 @@ for p in seen_pkgs_keys: for i in range(num_sources): src_file = user_keys[random.randrange(0, len(user_keys))] src = "%s%s.%s/%s/%s-%s.tar.gz" % ( - RANDOM_URL[random.randrange(0, len(RANDOM_URL))], - p, RANDOM_TLDS[random.randrange(0, len(RANDOM_TLDS))], - RANDOM_LOCS[random.randrange(0, len(RANDOM_LOCS))], - src_file, genVersion()) + RANDOM_URL[random.randrange(0, len(RANDOM_URL))], + p, RANDOM_TLDS[random.randrange(0, len(RANDOM_TLDS))], + RANDOM_LOCS[random.randrange(0, len(RANDOM_LOCS))], + src_file, genVersion()) s = "INSERT INTO PackageSources(PackageID, Source) VALUES (%d, '%s');\n" s = s % (seen_pkgs[p], src) out.write(s) From 7f7a975614f3c02517b4cbb94537fc4c60b26603 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 10 Jun 2021 01:11:41 -0700 Subject: [PATCH 153/844] remove autoflush from aurweb.db.Session This causes issues with the declarative API. Signed-off-by: Kevin Morris --- aurweb/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aurweb/db.py b/aurweb/db.py index 590712e0..1f6f50d8 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -141,7 +141,7 @@ def get_engine(echo: bool = False): engine = create_engine(get_sqlalchemy_url(), connect_args=connect_args, echo=echo) - Session = sessionmaker(autocommit=False, autoflush=True, bind=engine) + Session = sessionmaker(autocommit=False, autoflush=False, bind=engine) session = Session() return engine From a625df07e294866398385548501656eb8645e610 Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Thu, 10 Jun 2021 14:46:24 -0400 Subject: [PATCH 154/844] Source valid ssh prefixes from config Signed-off-by: Eli Schwartz --- web/lib/acctfuncs.inc.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/web/lib/acctfuncs.inc.php b/web/lib/acctfuncs.inc.php index df016c6d..0d021f99 100644 --- a/web/lib/acctfuncs.inc.php +++ b/web/lib/acctfuncs.inc.php @@ -875,10 +875,7 @@ function valid_pgp_fingerprint($fingerprint) { * @return bool True if the SSH public key is valid, otherwise false */ function valid_ssh_pubkey($pubkey) { - $valid_prefixes = array( - "ssh-rsa", "ssh-dss", "ecdsa-sha2-nistp256", - "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521", "ssh-ed25519" - ); + $valid_prefixes = explode(' ', config_get('auth', 'valid-keytypes')); $has_valid_prefix = false; foreach ($valid_prefixes as $prefix) { From b32022a176ede116068a405c928cf25e23ffb691 Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Thu, 10 Jun 2021 14:35:13 -0400 Subject: [PATCH 155/844] Add FIDO/U2F ssh keytypes to default config Signed-off-by: Eli Schwartz --- conf/config.defaults | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/config.defaults b/conf/config.defaults index 98e033b7..e6961520 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -62,7 +62,7 @@ ECDSA = SHA256:L71Q91yHwmHPYYkJMDgj0xmUuw16qFOhJbBr1mzsiOI RSA = SHA256:Ju+yWiMb/2O+gKQ9RJCDqvRg7l+Q95KFAeqM5sr6l2s [auth] -valid-keytypes = ssh-rsa ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 +valid-keytypes = ssh-rsa ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 sk-ssh-ecdsa@openssh.com sk-ssh-ed25519@openssh.com username-regex = [a-zA-Z0-9]+[.\-_]?[a-zA-Z0-9]+$ git-serve-cmd = /usr/local/bin/aurweb-git-serve ssh-options = restrict From 888cf5118aa1f794f9c87413aa29b5db54adc84a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 6 Jun 2021 22:45:40 -0700 Subject: [PATCH 156/844] use declarative_base for all ORM models This rewrites the entire model base as declarative models. This allows us to more easily customize overlay fields in tables and is more common. This effort also brought some DB violations to light which this commit addresses. Signed-off-by: Kevin Morris --- aurweb/db.py | 13 ++---- aurweb/models/__init__.py | 1 + aurweb/models/accepted_term.py | 41 ++++++++++------- aurweb/models/account_type.py | 14 +++--- aurweb/models/api_rate_limit.py | 18 +++++--- aurweb/models/ban.py | 15 ++++--- aurweb/models/declarative.py | 10 +++++ aurweb/models/dependency_type.py | 15 ++++--- aurweb/models/group.py | 15 ++++--- aurweb/models/license.py | 15 ++++--- aurweb/models/official_provider.py | 15 ++++--- aurweb/models/package.py | 42 +++++++++++------- aurweb/models/package_base.py | 58 +++++++++++++++--------- aurweb/models/package_dependency.py | 55 ++++++++++++++--------- aurweb/models/package_group.py | 48 +++++++++++--------- aurweb/models/package_keyword.py | 32 +++++++------ aurweb/models/package_license.py | 52 ++++++++++++---------- aurweb/models/package_relation.py | 52 ++++++++++++---------- aurweb/models/relation_type.py | 15 ++++--- aurweb/models/session.py | 24 ++++++---- aurweb/models/ssh_pub_key.py | 26 +++++++---- aurweb/models/term.py | 15 ++++--- aurweb/models/user.py | 69 ++++++++++++----------------- test/test_accepted_term.py | 4 +- test/test_account_type.py | 3 +- test/test_package.py | 10 ++--- test/test_package_group.py | 2 +- test/test_package_license.py | 2 +- test/test_session.py | 9 ++-- 29 files changed, 398 insertions(+), 292 deletions(-) create mode 100644 aurweb/models/declarative.py diff --git a/aurweb/db.py b/aurweb/db.py index 1f6f50d8..9837c746 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -1,7 +1,5 @@ import math -from sqlalchemy.orm import backref, relationship - import aurweb.config import aurweb.util @@ -53,12 +51,6 @@ def make_random_value(table: str, column: str): return string -def make_relationship(model, foreign_key: str, backref_: str, **kwargs): - return relationship(model, foreign_keys=[foreign_key], - backref=backref(backref_, lazy="dynamic"), - **kwargs) - - def query(model, *args, **kwargs): return session.query(model).filter(*args, **kwargs) @@ -77,6 +69,10 @@ def delete(model, *args, **kwargs): session.commit() +def rollback(): + session.rollback() + + def get_sqlalchemy_url(): """ Build an SQLAlchemy for use with create_engine based on the aurweb configuration. @@ -137,7 +133,6 @@ def get_engine(echo: bool = False): # check_same_thread is for a SQLite technicality # https://fastapi.tiangolo.com/tutorial/sql-databases/#note connect_args["check_same_thread"] = False - engine = create_engine(get_sqlalchemy_url(), connect_args=connect_args, echo=echo) diff --git a/aurweb/models/__init__.py b/aurweb/models/__init__.py index e69de29b..ed0532c6 100644 --- a/aurweb/models/__init__.py +++ b/aurweb/models/__init__.py @@ -0,0 +1 @@ +# aurweb SQLAlchemy ORM model collection. diff --git a/aurweb/models/accepted_term.py b/aurweb/models/accepted_term.py index 483109f1..b46d086b 100644 --- a/aurweb/models/accepted_term.py +++ b/aurweb/models/accepted_term.py @@ -1,15 +1,33 @@ +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper +from sqlalchemy.orm import backref, relationship -from aurweb.db import make_relationship -from aurweb.models.term import Term -from aurweb.models.user import User -from aurweb.schema import AcceptedTerms +import aurweb.models.term +import aurweb.models.user + +from aurweb.models.declarative import Base -class AcceptedTerm: +class AcceptedTerm(Base): + __tablename__ = "AcceptedTerms" + + UsersID = Column(Integer, ForeignKey("Users.ID", ondelete="CASCADE"), + nullable=False) + User = relationship( + "User", backref=backref("accepted_terms", lazy="dynamic"), + foreign_keys=[UsersID]) + + TermsID = Column(Integer, ForeignKey("Terms.ID", ondelete="CASCADE"), + nullable=False) + Term = relationship( + "Term", backref=backref("accepted_terms", lazy="dynamic"), + foreign_keys=[TermsID]) + + __mapper_args__ = {"primary_key": [TermsID]} + def __init__(self, - User: User = None, Term: Term = None, + User: aurweb.models.user.User = None, + Term: aurweb.models.term.Term = None, Revision: int = None): self.User = User if not self.User: @@ -26,12 +44,3 @@ class AcceptedTerm: params=("NULL")) self.Revision = Revision - - -properties = { - "User": make_relationship(User, AcceptedTerms.c.UsersID, "accepted_terms"), - "Term": make_relationship(Term, AcceptedTerms.c.TermsID, "accepted") -} - -mapper(AcceptedTerm, AcceptedTerms, properties=properties, - primary_key=[AcceptedTerms.c.UsersID, AcceptedTerms.c.TermsID]) diff --git a/aurweb/models/account_type.py b/aurweb/models/account_type.py index 44225e35..502a86b1 100644 --- a/aurweb/models/account_type.py +++ b/aurweb/models/account_type.py @@ -1,10 +1,15 @@ -from sqlalchemy.orm import mapper +from sqlalchemy import Column, Integer -from aurweb.schema import AccountTypes +from aurweb.models.declarative import Base -class AccountType: +class AccountType(Base): """ An ORM model of a single AccountTypes record. """ + __tablename__ = "AccountTypes" + + ID = Column(Integer, primary_key=True) + + __mapper_args__ = {"primary_key": [ID]} def __init__(self, **kwargs): self.AccountType = kwargs.pop("AccountType") @@ -15,6 +20,3 @@ class AccountType: def __repr__(self): return "" % ( self.ID, str(self)) - - -mapper(AccountType, AccountTypes, confirm_deleted_rows=False) diff --git a/aurweb/models/api_rate_limit.py b/aurweb/models/api_rate_limit.py index 8b945b6a..f4590553 100644 --- a/aurweb/models/api_rate_limit.py +++ b/aurweb/models/api_rate_limit.py @@ -1,11 +1,18 @@ +from sqlalchemy import Column, String from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper -from aurweb.schema import ApiRateLimit as _ApiRateLimit +from aurweb.models.declarative import Base -class ApiRateLimit: - def __init__(self, IP: str = None, +class ApiRateLimit(Base): + __tablename__ = "ApiRateLimit" + + IP = Column(String(45), primary_key=True, unique=True, default=str()) + + __mapper_args__ = {"primary_key": [IP]} + + def __init__(self, + IP: str = None, Requests: int = None, WindowStart: int = None): self.IP = IP @@ -23,6 +30,3 @@ class ApiRateLimit: statement="Column WindowStart cannot be null.", orig="ApiRateLimit.WindowStart", params=("NULL")) - - -mapper(ApiRateLimit, _ApiRateLimit, primary_key=[_ApiRateLimit.c.IP]) diff --git a/aurweb/models/ban.py b/aurweb/models/ban.py index be030380..e10087b0 100644 --- a/aurweb/models/ban.py +++ b/aurweb/models/ban.py @@ -1,10 +1,16 @@ from fastapi import Request -from sqlalchemy.orm import mapper +from sqlalchemy import Column, String -from aurweb.schema import Bans +from aurweb.models.declarative import Base -class Ban: +class Ban(Base): + __tablename__ = "Bans" + + IPAddress = Column(String(45), primary_key=True) + + __mapper_args__ = {"primary_key": [IPAddress]} + def __init__(self, **kwargs): self.IPAddress = kwargs.get("IPAddress") self.BanTS = kwargs.get("BanTS") @@ -14,6 +20,3 @@ def is_banned(request: Request): from aurweb.db import session ip = request.client.host return session.query(Ban).filter(Ban.IPAddress == ip).first() is not None - - -mapper(Ban, Bans) diff --git a/aurweb/models/declarative.py b/aurweb/models/declarative.py new file mode 100644 index 00000000..45a629ce --- /dev/null +++ b/aurweb/models/declarative.py @@ -0,0 +1,10 @@ +from sqlalchemy.ext.declarative import declarative_base + +import aurweb.db + +Base = declarative_base() +Base.__table_args__ = { + "autoload": True, + "autoload_with": aurweb.db.get_engine(), + "extend_existing": True +} diff --git a/aurweb/models/dependency_type.py b/aurweb/models/dependency_type.py index 87b38069..71acf368 100644 --- a/aurweb/models/dependency_type.py +++ b/aurweb/models/dependency_type.py @@ -1,11 +1,14 @@ -from sqlalchemy.orm import mapper +from sqlalchemy import Column, Integer -from aurweb.schema import DependencyTypes +from aurweb.models.declarative import Base -class DependencyType: +class DependencyType(Base): + __tablename__ = "DependencyTypes" + + ID = Column(Integer, primary_key=True) + + __mapper_args__ = {"primary_key": [ID]} + def __init__(self, Name: str = None): self.Name = Name - - -mapper(DependencyType, DependencyTypes) diff --git a/aurweb/models/group.py b/aurweb/models/group.py index c5583eb4..1bd3a402 100644 --- a/aurweb/models/group.py +++ b/aurweb/models/group.py @@ -1,10 +1,16 @@ +from sqlalchemy import Column, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper -from aurweb.schema import Groups +from aurweb.models.declarative import Base -class Group: +class Group(Base): + __tablename__ = "Groups" + + ID = Column(Integer, primary_key=True) + + __mapper_args__ = {"primary_key": [ID]} + def __init__(self, Name: str = None): self.Name = Name if not self.Name: @@ -12,6 +18,3 @@ class Group: statement="Column Name cannot be null.", orig="Groups.Name", params=("NULL")) - - -mapper(Group, Groups) diff --git a/aurweb/models/license.py b/aurweb/models/license.py index bcc02713..aef6a619 100644 --- a/aurweb/models/license.py +++ b/aurweb/models/license.py @@ -1,10 +1,16 @@ +from sqlalchemy import Column, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper -from aurweb.schema import Licenses +from aurweb.models.declarative import Base -class License: +class License(Base): + __tablename__ = "Licenses" + + ID = Column(Integer, primary_key=True) + + __mapper_args__ = {"primary_key": [ID]} + def __init__(self, Name: str = None): self.Name = Name if not self.Name: @@ -12,6 +18,3 @@ class License: statement="Column Name cannot be null.", orig="Licenses.Name", params=("NULL")) - - -mapper(License, Licenses) diff --git a/aurweb/models/official_provider.py b/aurweb/models/official_provider.py index 073eb435..756be843 100644 --- a/aurweb/models/official_provider.py +++ b/aurweb/models/official_provider.py @@ -1,10 +1,16 @@ +from sqlalchemy import Column, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper -from aurweb.schema import OfficialProviders +from aurweb.models.declarative import Base -class OfficialProvider: +class OfficialProvider(Base): + __tablename__ = "OfficialProviders" + + ID = Column(Integer, primary_key=True) + + __mapper_args__ = {"primary_key": [ID]} + def __init__(self, Name: str = None, Repo: str = None, @@ -29,6 +35,3 @@ class OfficialProvider: statement="Column Provides cannot be null.", orig="OfficialProviders.Provides", params=("NULL")) - - -mapper(OfficialProvider, OfficialProviders) diff --git a/aurweb/models/package.py b/aurweb/models/package.py index 28a13791..ff518f20 100644 --- a/aurweb/models/package.py +++ b/aurweb/models/package.py @@ -1,20 +1,37 @@ +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper +from sqlalchemy.orm import backref, relationship -from aurweb.db import make_relationship -from aurweb.models.package_base import PackageBase -from aurweb.schema import Packages +import aurweb.db +import aurweb.models.package_base + +from aurweb.models.declarative import Base -class Package: +class Package(Base): + __tablename__ = "Packages" + + ID = Column(Integer, primary_key=True) + + PackageBaseID = Column( + Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), + nullable=False) + PackageBase = relationship( + "PackageBase", backref=backref("package", uselist=False), + foreign_keys=[PackageBaseID]) + + __mapper_args__ = {"primary_key": [ID]} + def __init__(self, - PackageBase: PackageBase = None, - Name: str = None, Version: str = None, - Description: str = None, URL: str = None): + PackageBase: aurweb.models.package_base.PackageBase = None, + Name: str = None, + Version: str = None, + Description: str = None, + URL: str = None): self.PackageBase = PackageBase if not self.PackageBase: raise IntegrityError( - statement="Foreign key UserID cannot be null.", + statement="Foreign key PackageBaseID cannot be null.", orig="Packages.PackageBaseID", params=("NULL")) @@ -28,10 +45,3 @@ class Package: self.Version = Version self.Description = Description self.URL = URL - - -mapper(Package, Packages, properties={ - "PackageBase": make_relationship(PackageBase, - Packages.c.PackageBaseID, - "package", uselist=False) -}) diff --git a/aurweb/models/package_base.py b/aurweb/models/package_base.py index 699559d5..261c30f3 100644 --- a/aurweb/models/package_base.py +++ b/aurweb/models/package_base.py @@ -1,17 +1,47 @@ from datetime import datetime +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper +from sqlalchemy.orm import backref, relationship -from aurweb.db import make_relationship -from aurweb.models.user import User -from aurweb.schema import PackageBases +import aurweb.models.user + +from aurweb.models.declarative import Base -class PackageBase: - def __init__(self, Name: str = None, Flagger: User = None, - Maintainer: User = None, Submitter: User = None, - Packager: User = None, **kwargs): +class PackageBase(Base): + __tablename__ = "PackageBases" + + FlaggerUID = Column(Integer, + ForeignKey("Users.ID", ondelete="SET NULL")) + Flagger = relationship( + "User", backref=backref("flagged_bases", lazy="dynamic"), + foreign_keys=[FlaggerUID]) + + SubmitterUID = Column(Integer, + ForeignKey("Users.ID", ondelete="SET NULL")) + Submitter = relationship( + "User", backref=backref("submitted_bases", lazy="dynamic"), + foreign_keys=[SubmitterUID]) + + MaintainerUID = Column(Integer, + ForeignKey("Users.ID", ondelete="SET NULL")) + Maintainer = relationship( + "User", backref=backref("maintained_bases", lazy="dynamic"), + foreign_keys=[MaintainerUID]) + + PackagerUID = Column(Integer, ForeignKey("Users.ID", ondelete="SET NULL")) + Packager = relationship( + "User", backref=backref("package_bases", lazy="dynamic"), + foreign_keys=[PackagerUID]) + + def __init__(self, Name: str = None, + Flagger: aurweb.models.user.User = None, + Maintainer: aurweb.models.user.User = None, + Submitter: aurweb.models.user.User = None, + Packager: aurweb.models.user.User = None, + **kwargs): + super().__init__(**kwargs) self.Name = Name if not self.Name: raise IntegrityError( @@ -32,15 +62,3 @@ class PackageBase: datetime.utcnow().timestamp()) self.ModifiedTS = kwargs.get("ModifiedTS", datetime.utcnow().timestamp()) - - -mapper(PackageBase, PackageBases, properties={ - "Flagger": make_relationship(User, PackageBases.c.FlaggerUID, - "flagged_bases"), - "Submitter": make_relationship(User, PackageBases.c.SubmitterUID, - "submitted_bases"), - "Maintainer": make_relationship(User, PackageBases.c.MaintainerUID, - "maintained_bases"), - "Packager": make_relationship(User, PackageBases.c.PackagerUID, - "package_bases") -}) diff --git a/aurweb/models/package_dependency.py b/aurweb/models/package_dependency.py index 21801802..0bd84073 100644 --- a/aurweb/models/package_dependency.py +++ b/aurweb/models/package_dependency.py @@ -1,17 +1,40 @@ +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper +from sqlalchemy.orm import backref, relationship -from aurweb.db import make_relationship -from aurweb.models.dependency_type import DependencyType -from aurweb.models.package import Package -from aurweb.schema import PackageDepends +import aurweb.models.package + +from aurweb.models import dependency_type +from aurweb.models.declarative import Base -class PackageDependency: - def __init__(self, Package: Package = None, - DependencyType: DependencyType = None, - DepName: str = None, DepDesc: str = None, - DepCondition: str = None, DepArch: str = None): +class PackageDependency(Base): + __tablename__ = "PackageDepends" + + PackageID = Column( + Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), + nullable=False) + Package = relationship( + "Package", backref=backref("package_dependencies", lazy="dynamic"), + foreign_keys=[PackageID], lazy="select") + + DepTypeID = Column( + Integer, ForeignKey("DependencyTypes.ID", ondelete="NO ACTION"), + nullable=False) + DependencyType = relationship( + "DependencyType", + backref=backref("package_dependencies", lazy="dynamic"), + foreign_keys=[DepTypeID], lazy="select") + + __mapper_args__ = {"primary_key": [PackageID, DepTypeID]} + + def __init__(self, + Package: aurweb.models.package.Package = None, + DependencyType: dependency_type.DependencyType = None, + DepName: str = None, + DepDesc: str = None, + DepCondition: str = None, + DepArch: str = None): self.Package = Package if not self.Package: raise IntegrityError( @@ -36,15 +59,3 @@ class PackageDependency: self.DepDesc = DepDesc self.DepCondition = DepCondition self.DepArch = DepArch - - -properties = { - "Package": make_relationship(Package, PackageDepends.c.PackageID, - "package_dependencies"), - "DependencyType": make_relationship(DependencyType, - PackageDepends.c.DepTypeID, - "package_dependencies") -} - -mapper(PackageDependency, PackageDepends, properties=properties, - primary_key=[PackageDepends.c.PackageID, PackageDepends.c.DepTypeID]) diff --git a/aurweb/models/package_group.py b/aurweb/models/package_group.py index 19a11c80..a8031e0d 100644 --- a/aurweb/models/package_group.py +++ b/aurweb/models/package_group.py @@ -1,14 +1,33 @@ +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper +from sqlalchemy.orm import backref, relationship -from aurweb.db import make_relationship -from aurweb.models.group import Group -from aurweb.models.package import Package -from aurweb.schema import PackageGroups +import aurweb.models.group +import aurweb.models.package + +from aurweb.models.declarative import Base -class PackageGroup: - def __init__(self, Package: Package = None, Group: Group = None): +class PackageGroup(Base): + __tablename__ = "PackageGroups" + + PackageID = Column(Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), + primary_key=True, nullable=True) + Package = relationship( + "Package", backref=backref("package_groups", lazy="dynamic"), + foreign_keys=[PackageID]) + + GroupID = Column(Integer, ForeignKey("Groups.ID", ondelete="CASCADE"), + primary_key=True, nullable=True) + Group = relationship( + "Group", backref=backref("package_groups", lazy="dynamic"), + foreign_keys=[GroupID]) + + __mapper_args__ = {"primary_key": [PackageID, GroupID]} + + def __init__(self, + Package: aurweb.models.package.Package = None, + Group: aurweb.models.group.Group = None): self.Package = Package if not self.Package: raise IntegrityError( @@ -22,18 +41,3 @@ class PackageGroup: statement="Primary key GroupID cannot be null.", orig="PackageGroups.GroupID", params=("NULL")) - - -properties = { - "Package": make_relationship(Package, - PackageGroups.c.PackageID, - "package_group", - uselist=False), - "Group": make_relationship(Group, - PackageGroups.c.GroupID, - "package_group", - uselist=False) -} - -mapper(PackageGroup, PackageGroups, properties=properties, - primary_key=[PackageGroups.c.PackageID, PackageGroups.c.GroupID]) diff --git a/aurweb/models/package_keyword.py b/aurweb/models/package_keyword.py index 2bae223c..2926740d 100644 --- a/aurweb/models/package_keyword.py +++ b/aurweb/models/package_keyword.py @@ -1,14 +1,27 @@ +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper +from sqlalchemy.orm import backref, relationship -from aurweb.db import make_relationship -from aurweb.models.package_base import PackageBase -from aurweb.schema import PackageKeywords +import aurweb.db +import aurweb.models.package_base + +from aurweb.models.declarative import Base -class PackageKeyword: +class PackageKeyword(Base): + __tablename__ = "PackageKeywords" + + PackageBaseID = Column( + Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), + primary_key=True, nullable=True) + PackageBase = relationship( + "PackageBase", backref=backref("keywords", lazy="dynamic"), + foreign_keys=[PackageBaseID]) + + __mapper_args__ = {"primary_key": [PackageBaseID]} + def __init__(self, - PackageBase: PackageBase = None, + PackageBase: aurweb.models.package_base.PackageBase = None, Keyword: str = None): self.PackageBase = PackageBase if not self.PackageBase: @@ -18,10 +31,3 @@ class PackageKeyword: params=("NULL")) self.Keyword = Keyword - - -mapper(PackageKeyword, PackageKeywords, properties={ - "PackageBase": make_relationship(PackageBase, - PackageKeywords.c.PackageBaseID, - "keywords") -}) diff --git a/aurweb/models/package_license.py b/aurweb/models/package_license.py index 491874a4..0689562f 100644 --- a/aurweb/models/package_license.py +++ b/aurweb/models/package_license.py @@ -1,14 +1,35 @@ +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper +from sqlalchemy.orm import backref, relationship -from aurweb.db import make_relationship -from aurweb.models.license import License -from aurweb.models.package import Package -from aurweb.schema import PackageLicenses +import aurweb.models.license +import aurweb.models.package + +from aurweb.models.declarative import Base -class PackageLicense: - def __init__(self, Package: Package = None, License: License = None): +class PackageLicense(Base): + __tablename__ = "PackageLicenses" + + PackageID = Column( + Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), + primary_key=True, nullable=True) + Package = relationship( + "Package", backref=backref("package_license", uselist=False), + foreign_keys=[PackageID]) + + LicenseID = Column( + Integer, ForeignKey("Licenses.ID", ondelete="CASCADE"), + primary_key=True, nullable=True) + License = relationship( + "License", backref=backref("package_license", uselist=False), + foreign_keys=[LicenseID]) + + __mapper_args__ = {"primary_key": [PackageID, LicenseID]} + + def __init__(self, + Package: aurweb.models.package.Package = None, + License: aurweb.models.license.License = None): self.Package = Package if not self.Package: raise IntegrityError( @@ -22,20 +43,3 @@ class PackageLicense: statement="Primary key LicenseID cannot be null.", orig="PackageLicenses.LicenseID", params=("NULL")) - - -properties = { - "Package": make_relationship(Package, - PackageLicenses.c.PackageID, - "package_license", - uselist=False), - "License": make_relationship(License, - PackageLicenses.c.LicenseID, - "package_license", - uselist=False) - - -} - -mapper(PackageLicense, PackageLicenses, properties=properties, - primary_key=[PackageLicenses.c.PackageID, PackageLicenses.c.LicenseID]) diff --git a/aurweb/models/package_relation.py b/aurweb/models/package_relation.py index d9ade727..9204af59 100644 --- a/aurweb/models/package_relation.py +++ b/aurweb/models/package_relation.py @@ -1,15 +1,36 @@ +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper +from sqlalchemy.orm import backref, relationship -from aurweb.db import make_relationship -from aurweb.models.package import Package -from aurweb.models.relation_type import RelationType -from aurweb.schema import PackageRelations +import aurweb.db +import aurweb.models.package +import aurweb.models.relation_type + +from aurweb.models.declarative import Base -class PackageRelation: - def __init__(self, Package: Package = None, - RelationType: RelationType = None, +class PackageRelation(Base): + __tablename__ = "PackageRelations" + + PackageID = Column( + Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), + nullable=False) + Package = relationship( + "Package", backref=backref("package_relations", lazy="dynamic"), + foreign_keys=[PackageID]) + + RelTypeID = Column( + Integer, ForeignKey("RelationTypes.ID", ondelete="CASCADE"), + nullable=False) + RelationType = relationship( + "RelationType", backref=backref("package_relations", lazy="dynamic"), + foreign_keys=[RelTypeID]) + + __mapper_args__ = {"primary_key": [PackageID, RelTypeID]} + + def __init__(self, + Package: aurweb.models.package.Package = None, + RelationType: aurweb.models.relation_type.RelationType = None, RelName: str = None, RelCondition: str = None, RelArch: str = None): self.Package = Package @@ -35,18 +56,3 @@ class PackageRelation: self.RelCondition = RelCondition self.RelArch = RelArch - - -properties = { - "Package": make_relationship(Package, PackageRelations.c.PackageID, - "package_relations"), - "RelationType": make_relationship(RelationType, - PackageRelations.c.RelTypeID, - "package_relations") -} - -mapper(PackageRelation, PackageRelations, properties=properties, - primary_key=[ - PackageRelations.c.PackageID, - PackageRelations.c.RelTypeID - ]) diff --git a/aurweb/models/relation_type.py b/aurweb/models/relation_type.py index b4d1efbc..319fb7f4 100644 --- a/aurweb/models/relation_type.py +++ b/aurweb/models/relation_type.py @@ -1,11 +1,14 @@ -from sqlalchemy.orm import mapper +from sqlalchemy import Column, Integer -from aurweb.schema import RelationTypes +from aurweb.models.declarative import Base -class RelationType: +class RelationType(Base): + __tablename__ = "RelationTypes" + + ID = Column(Integer, primary_key=True) + + __mapper_args__ = {"primary_key": [ID]} + def __init__(self, Name: str = None): self.Name = Name - - -mapper(RelationType, RelationTypes) diff --git a/aurweb/models/session.py b/aurweb/models/session.py index f1e0fff5..9154178e 100644 --- a/aurweb/models/session.py +++ b/aurweb/models/session.py @@ -1,12 +1,24 @@ +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import backref, mapper, relationship +from sqlalchemy.orm import backref, relationship from aurweb.db import make_random_value, query +from aurweb.models.declarative import Base from aurweb.models.user import User -from aurweb.schema import Sessions -class Session: +class Session(Base): + __tablename__ = "Sessions" + + UsersID = Column( + Integer, ForeignKey("Users.ID", ondelete="CASCADE"), + nullable=False) + User = relationship( + "User", backref=backref("session", uselist=False), + foreign_keys=[UsersID]) + + __mapper_args__ = {"primary_key": [UsersID]} + def __init__(self, **kwargs): self.UsersID = kwargs.get("UsersID") if not query(User, User.ID == self.UsersID).first(): @@ -19,11 +31,5 @@ class Session: self.LastUpdateTS = kwargs.get("LastUpdateTS") -mapper(Session, Sessions, primary_key=[Sessions.c.SessionID], properties={ - "User": relationship(User, backref=backref("session", - uselist=False)) -}) - - def generate_unique_sid(): return make_random_value(Session, Session.SessionID) diff --git a/aurweb/models/ssh_pub_key.py b/aurweb/models/ssh_pub_key.py index 01ff558e..268a585b 100644 --- a/aurweb/models/ssh_pub_key.py +++ b/aurweb/models/ssh_pub_key.py @@ -3,13 +3,26 @@ import tempfile from subprocess import PIPE, Popen -from sqlalchemy.orm import backref, mapper, relationship +from sqlalchemy import Column, ForeignKey, Integer, String +from sqlalchemy.orm import backref, relationship -from aurweb.models.user import User -from aurweb.schema import SSHPubKeys +from aurweb.models.declarative import Base -class SSHPubKey: +class SSHPubKey(Base): + __tablename__ = "SSHPubKeys" + + UserID = Column( + Integer, ForeignKey("Users.ID", ondelete="CASCADE"), + nullable=False) + User = relationship( + "User", backref=backref("ssh_pub_key", uselist=False), + foreign_keys=[UserID]) + + Fingerprint = Column(String(44), primary_key=True) + + __mapper_args__ = {"primary_key": Fingerprint} + def __init__(self, **kwargs): self.UserID = kwargs.get("UserID") self.Fingerprint = kwargs.get("Fingerprint") @@ -34,8 +47,3 @@ def get_fingerprint(pubkey): fp = parts[1].replace("SHA256:", "") return fp - - -mapper(SSHPubKey, SSHPubKeys, properties={ - "User": relationship(User, backref=backref("ssh_pub_key", uselist=False)) -}) diff --git a/aurweb/models/term.py b/aurweb/models/term.py index 1a0780df..b0da71f7 100644 --- a/aurweb/models/term.py +++ b/aurweb/models/term.py @@ -1,10 +1,16 @@ +from sqlalchemy import Column, Integer from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import mapper -from aurweb.schema import Terms +from aurweb.models.declarative import Base -class Term: +class Term(Base): + __tablename__ = "Terms" + + ID = Column(Integer, primary_key=True) + + __mapper_args__ = {"primary_key": [ID]} + def __init__(self, Description: str = None, URL: str = None, Revision: int = None): @@ -23,6 +29,3 @@ class Term: params=("NULL")) self.Revision = Revision - - -mapper(Term, Terms) diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 1961228e..83cde5f1 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -5,50 +5,44 @@ from datetime import datetime import bcrypt from fastapi import Request -from sqlalchemy.orm import backref, mapper, relationship +from sqlalchemy import Column, ForeignKey, Integer, String, text +from sqlalchemy.orm import backref, relationship import aurweb.config +import aurweb.models.account_type +import aurweb.schema -from aurweb.models.account_type import AccountType from aurweb.models.ban import is_banned -from aurweb.schema import Users +from aurweb.models.declarative import Base -class User: +class User(Base): """ An ORM model of a single Users record. """ + __tablename__ = "Users" + + ID = Column(Integer, primary_key=True) + + AccountTypeID = Column( + Integer, ForeignKey("AccountTypes.ID", ondelete="NO ACTION"), + nullable=False, server_default=text("1")) + AccountType = relationship( + "AccountType", + backref=backref("users", lazy="dynamic"), + foreign_keys=[AccountTypeID], + uselist=False) + + Passwd = Column(String(255), default=str()) + + __mapper_args__ = {"primary_key": [ID]} + + # High-level variables used to track authentication (not in DB). authenticated = False - def __init__(self, **kwargs): - # Set AccountTypeID if it was passed. - self.AccountTypeID = kwargs.get("AccountTypeID") + def __init__(self, Passwd: str = str(), **kwargs): + super().__init__(**kwargs) - account_type = kwargs.get("AccountType") - if account_type: - self.AccountType = account_type - - self.Username = kwargs.get("Username") - - self.ResetKey = kwargs.get("ResetKey") - self.Email = kwargs.get("Email") - self.BackupEmail = kwargs.get("BackupEmail") - self.RealName = kwargs.get("RealName") - self.LangPreference = kwargs.get("LangPreference") - self.Timezone = kwargs.get("Timezone") - self.Homepage = kwargs.get("Homepage") - self.IRCNick = kwargs.get("IRCNick") - self.PGPKey = kwargs.get("PGPKey") - self.RegistrationTS = datetime.utcnow() - self.CommentNotify = kwargs.get("CommentNotify") - self.UpdateNotify = kwargs.get("UpdateNotify") - self.OwnershipNotify = kwargs.get("OwnershipNotify") - self.SSOAccountID = kwargs.get("SSOAccountID") - - self.Salt = None - self.Passwd = str() - - passwd = kwargs.get("Passwd") - if passwd: - self.update_password(passwd) + if Passwd: + self.update_password(Passwd) def update_password(self, password, salt_rounds=12): self.Passwd = bcrypt.hashpw( @@ -154,10 +148,3 @@ class User: def __repr__(self): return "" % ( self.ID, str(self.AccountType), self.Username) - - -# Map schema.Users to User and give it some relationships. -mapper(User, Users, properties={ - "AccountType": relationship(AccountType, - backref=backref("users", lazy="dynamic")) -}) diff --git a/test/test_accepted_term.py b/test/test_accepted_term.py index 4ddf1fc3..8569b021 100644 --- a/test/test_accepted_term.py +++ b/test/test_accepted_term.py @@ -22,7 +22,7 @@ def setup(): AccountType.AccountType == "User").first() user = create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", - account_type=account_type) + AccountType=account_type) term = create(Term, Description="Test term", URL="https://test.term") @@ -33,7 +33,7 @@ def test_accepted_term(): # Make sure our AcceptedTerm relationships got initialized properly. assert accepted_term.User == user assert accepted_term in user.accepted_terms - assert accepted_term in term.accepted + assert accepted_term in term.accepted_terms def test_accepted_term_null_user_raises_exception(): diff --git a/test/test_account_type.py b/test/test_account_type.py index 3bd76d1e..fa4bc5ad 100644 --- a/test/test_account_type.py +++ b/test/test_account_type.py @@ -43,6 +43,7 @@ def test_user_account_type_relationship(): AccountType=account_type) assert user.AccountType == account_type - assert account_type.users.filter(User.ID == user.ID).first() + # This must be deleted here to avoid foreign key issues when + # deleting the temporary AccountType in the fixture. delete(User, User.ID == user.ID) diff --git a/test/test_package.py b/test/test_package.py index a994f096..9532823d 100644 --- a/test/test_package.py +++ b/test/test_package.py @@ -1,7 +1,7 @@ import pytest from sqlalchemy import and_ -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import IntegrityError, OperationalError import aurweb.config @@ -19,7 +19,7 @@ user = pkgbase = package = None def setup(): global user, pkgbase, package - setup_test_db("Users", "PackageBases", "Packages") + setup_test_db("Packages", "PackageBases", "Users") account_type = query(AccountType, AccountType.AccountType == "User").first() @@ -57,17 +57,17 @@ def test_package(): assert record is not None -def test_package_ci(): +def test_package_package_base_cant_change(): """ Test case insensitivity of the database table. """ if aurweb.config.get("database", "backend") == "sqlite": return None # SQLite doesn't seem handle this. from aurweb.db import session - with pytest.raises(IntegrityError): + with pytest.raises(OperationalError): create(Package, PackageBase=pkgbase, - Name="Beautiful-Package") + Name="invalidates-old-package-packagebase-relationship") session.rollback() diff --git a/test/test_package_group.py b/test/test_package_group.py index 28047a7f..0e6e41e3 100644 --- a/test/test_package_group.py +++ b/test/test_package_group.py @@ -25,7 +25,7 @@ def setup(): AccountType.AccountType == "User").first() user = create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", - account_type=account_type) + AccountType=account_type) group = create(Group, Name="Test Group") pkgbase = create(PackageBase, Name="test-package", Maintainer=user) diff --git a/test/test_package_license.py b/test/test_package_license.py index 72eb3681..f7654dee 100644 --- a/test/test_package_license.py +++ b/test/test_package_license.py @@ -25,7 +25,7 @@ def setup(): AccountType.AccountType == "User").first() user = create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", - account_type=account_type) + AccountType=account_type) license = create(License, Name="Test License") pkgbase = create(PackageBase, Name="test-package", Maintainer=user) diff --git a/test/test_session.py b/test/test_session.py index 1dd82db1..1ba11556 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -10,12 +10,12 @@ from aurweb.models.session import Session, generate_unique_sid from aurweb.models.user import User from aurweb.testing import setup_test_db -user = session = None +account_type = user = session = None @pytest.fixture(autouse=True) def setup(): - global user, session + global account_type, user, session setup_test_db("Users", "Sessions") @@ -35,7 +35,10 @@ def test_session(): def test_session_cs(): """ Test case sensitivity of the database table. """ - session_cs = create(Session, UsersID=user.ID, + user2 = create(User, Username="test2", Email="test2@example.org", + ResetKey="testReset2", Passwd="testPassword", + AccountType=account_type) + session_cs = create(Session, UsersID=user2.ID, SessionID="TESTSESSION", LastUpdateTS=datetime.utcnow().timestamp()) assert session_cs.SessionID == "TESTSESSION" From 5de7ff64df0dc33a20999ae7042ff04a77451819 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 10 Jun 2021 13:55:07 -0700 Subject: [PATCH 157/844] add PackageVote SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/package_vote.py | 53 ++++++++++++++++++++++++++++++++ test/test_package_vote.py | 58 +++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 aurweb/models/package_vote.py create mode 100644 test/test_package_vote.py diff --git a/aurweb/models/package_vote.py b/aurweb/models/package_vote.py new file mode 100644 index 00000000..55a9ecbb --- /dev/null +++ b/aurweb/models/package_vote.py @@ -0,0 +1,53 @@ +from sqlalchemy import Column, ForeignKey, Integer +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import backref, relationship + +import aurweb.models.package_base +import aurweb.models.user + +from aurweb.models.declarative import Base + + +class PackageVote(Base): + __tablename__ = "PackageVotes" + + UsersID = Column( + Integer, ForeignKey("Users.ID", ondelete="CASCADE"), + nullable=False) + User = relationship( + "User", backref=backref("package_votes", lazy="dynamic"), + foreign_keys=[UsersID]) + + PackageBaseID = Column( + Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), + nullable=False) + PackageBase = relationship( + "PackageBase", backref=backref("package_votes", lazy="dynamic"), + foreign_keys=[PackageBaseID]) + + __mapper_args__ = {"primary_key": [UsersID, PackageBaseID]} + + def __init__(self, + User: aurweb.models.user.User = None, + PackageBase: aurweb.models.package_base.PackageBase = None, + VoteTS: int = None): + self.User = User + if not self.User: + raise IntegrityError( + statement="Foreign key UsersID cannot be null.", + orig="PackageVotes.UsersID", + params=("NULL")) + + self.PackageBase = PackageBase + if not self.PackageBase: + raise IntegrityError( + statement="Foreign key PackageBaseID cannot be null.", + orig="PackageVotes.PackageBaseID", + params=("NULL")) + + self.VoteTS = VoteTS + if not self.VoteTS: + raise IntegrityError( + statement="Column VoteTS cannot be null.", + orig="PackageVotes.VoteTS", + params=("NULL")) diff --git a/test/test_package_vote.py b/test/test_package_vote.py new file mode 100644 index 00000000..b352bf11 --- /dev/null +++ b/test/test_package_vote.py @@ -0,0 +1,58 @@ +from datetime import datetime + +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query, rollback +from aurweb.models.account_type import AccountType +from aurweb.models.package_base import PackageBase +from aurweb.models.package_vote import PackageVote +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = pkgbase = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, pkgbase + + setup_test_db("Users", "PackageBases", "PackageVotes") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + print(account_type.ID) + print(account_type.AccountType) + + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword") + pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + + +def test_package_vote_creation(): + ts = int(datetime.utcnow().timestamp()) + package_vote = create(PackageVote, User=user, PackageBase=pkgbase, + VoteTS=ts) + assert bool(package_vote) + assert package_vote.User == user + assert package_vote.PackageBase == pkgbase + assert package_vote.VoteTS == ts + + +def test_package_vote_null_user_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageVote, PackageBase=pkgbase, VoteTS=1) + rollback() + + +def test_package_vote_null_pkgbase_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageVote, User=user, VoteTS=1) + rollback() + + +def test_package_vote_null_votets_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageVote, User=user, PackageBase=pkgbase) + rollback() From d18cfad63eeaab54fd21d386a769d85067153f8f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 10 Jun 2021 14:18:39 -0700 Subject: [PATCH 158/844] use djangos method of wiping sqlite3 tables Django uses a reference graph to determine the order in table deletions that occur. Do the same here. This commit also adds in the `REGEXP` sqlite function, exactly how Django uses it in its reference graphing. Signed-off-by: Kevin Morris --- aurweb/db.py | 24 +++++++++++++++++++++++- aurweb/testing/__init__.py | 36 ++++++++++++++++++++++++++++++++++++ test/test_db.py | 3 +++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/aurweb/db.py b/aurweb/db.py index 9837c746..04c8653a 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -1,4 +1,8 @@ +import functools import math +import re + +from sqlalchemy import event import aurweb.config import aurweb.util @@ -129,13 +133,31 @@ def get_engine(echo: bool = False): if engine is None: connect_args = dict() - if aurweb.config.get("database", "backend") == "sqlite": + + db_backend = aurweb.config.get("database", "backend") + if db_backend == "sqlite": # check_same_thread is for a SQLite technicality # https://fastapi.tiangolo.com/tutorial/sql-databases/#note connect_args["check_same_thread"] = False + engine = create_engine(get_sqlalchemy_url(), connect_args=connect_args, echo=echo) + + if db_backend == "sqlite": + # For SQLite, we need to add some custom functions as + # they are used in the reference graph method. + def regexp(regex, item): + return bool(re.search(regex, str(item))) + + @event.listens_for(engine, "begin") + def do_begin(conn): + create_deterministic_function = functools.partial( + conn.connection.create_function, + deterministic=True + ) + create_deterministic_function("REGEXP", 2, regexp) + Session = sessionmaker(autocommit=False, autoflush=False, bind=engine) session = Session() diff --git a/aurweb/testing/__init__.py b/aurweb/testing/__init__.py index 02c21a4c..90d46720 100644 --- a/aurweb/testing/__init__.py +++ b/aurweb/testing/__init__.py @@ -1,6 +1,28 @@ +from itertools import chain + import aurweb.db +def references_graph(table): + """ Taken from Django's sqlite3/operations.py. """ + query = """ + WITH tables AS ( + SELECT :table name + UNION + SELECT sqlite_master.name + FROM sqlite_master + JOIN tables ON (sql REGEXP :regexp_1 || tables.name || :regexp_2) + ) SELECT name FROM tables; + """ + params = { + "table": table, + "regexp_1": r'(?i)\s+references\s+("|\')?', + "regexp_2": r'("|\')?\s*\(', + } + cursor = aurweb.db.session.execute(query, params=params) + return [row[0] for row in cursor.fetchall()] + + def setup_test_db(*args): """ This function is to be used to setup a test database before using it. It takes a variable number of table strings, and for @@ -25,8 +47,22 @@ def setup_test_db(*args): aurweb.db.get_engine() tables = list(args) + + db_backend = aurweb.config.get("database", "backend") + + if db_backend != "sqlite": + aurweb.db.session.execute("SET FOREIGN_KEY_CHECKS = 0") + else: + # We're using sqlite, setup tables to be deleted without violating + # foreign key constraints by graphing references. + tables = set(chain.from_iterable( + references_graph(table) for table in tables)) + for table in tables: aurweb.db.session.execute(f"DELETE FROM {table}") + if db_backend != "sqlite": + aurweb.db.session.execute("SET FOREIGN_KEY_CHECKS = 1") + # Expunge all objects from SQLAlchemy's IdentityMap. aurweb.db.session.expunge_all() diff --git a/test/test_db.py b/test/test_db.py index 3911134f..9298c53d 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -200,6 +200,9 @@ def test_connection_execute_paramstyle_format(): aurweb.db.kill_engine() aurweb.initdb.run(Args()) + # Test SQLite route of clearing tables. + setup_test_db("Users", "Bans") + conn = db.Connection() # First, test ? to %s format replacement. From 11c4926502e0767ee2435a79dd1c8ecaee727086 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 10 Jun 2021 17:46:29 -0700 Subject: [PATCH 159/844] add PackageSource SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/package_source.py | 31 +++++++++++++++++++++++ test/test_package_source.py | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 aurweb/models/package_source.py create mode 100644 test/test_package_source.py diff --git a/aurweb/models/package_source.py b/aurweb/models/package_source.py new file mode 100644 index 00000000..4ffa23df --- /dev/null +++ b/aurweb/models/package_source.py @@ -0,0 +1,31 @@ +from sqlalchemy import Column, ForeignKey, Integer +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import backref, relationship + +import aurweb.models.package + +from aurweb.models.declarative import Base + + +class PackageSource(Base): + __tablename__ = "PackageSources" + + PackageID = Column(Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), + nullable=False) + Package = relationship( + "Package", backref=backref("package_sources", lazy="dynamic"), + foreign_keys=[PackageID]) + + __mapper_args__ = {"primary_key": [PackageID]} + + def __init__(self, + Package: aurweb.models.package.Package = None, + **kwargs): + super().__init__(**kwargs) + + self.Package = Package + if not self.Package: + raise IntegrityError( + statement="Foreign key PackageID cannot be null.", + orig="PackageSources.PackageID", + params=("NULL")) diff --git a/test/test_package_source.py b/test/test_package_source.py new file mode 100644 index 00000000..7453f756 --- /dev/null +++ b/test/test_package_source.py @@ -0,0 +1,44 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query, rollback +from aurweb.models.account_type import AccountType +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.package_source import PackageSource +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = pkgbase = package = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, pkgbase, package + + setup_test_db("PackageSources", "Packages", "PackageBases", "Users") + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + pkgbase = create(PackageBase, + Name="test-package", + Maintainer=user) + package = create(Package, PackageBase=pkgbase, Name="test-package") + + +def test_package_source(): + pkgsource = create(PackageSource, Package=package) + assert pkgsource.Package == package + # By default, PackageSources.Source assigns the string '/dev/null'. + assert pkgsource.Source == "/dev/null" + assert pkgsource.SourceArch is None + + +def test_package_source_null_package_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageSource) + rollback() From fc28c1e5fd137ab36786eae864a19617bdce90e6 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 00:35:18 -0700 Subject: [PATCH 160/844] add PackageComment SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/package_comment.py | 72 ++++++++++++++++++++++++++++++++ test/test_package_comment.py | 63 ++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 aurweb/models/package_comment.py create mode 100644 test/test_package_comment.py diff --git a/aurweb/models/package_comment.py b/aurweb/models/package_comment.py new file mode 100644 index 00000000..42a0661d --- /dev/null +++ b/aurweb/models/package_comment.py @@ -0,0 +1,72 @@ +from sqlalchemy import Column, ForeignKey, Integer +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import backref, relationship + +import aurweb.models.package_base +import aurweb.models.user + +from aurweb.models.declarative import Base + + +class PackageComment(Base): + __tablename__ = "PackageComments" + + ID = Column(Integer, primary_key=True) + + PackageBaseID = Column( + Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), + nullable=False) + PackageBase = relationship( + "PackageBase", backref=backref("comments", lazy="dynamic"), + foreign_keys=[PackageBaseID]) + + UsersID = Column(Integer, ForeignKey("Users.ID", ondelete="SET NULL")) + User = relationship( + "User", backref=backref("package_comments", lazy="dynamic"), + foreign_keys=[UsersID]) + + EditedUsersID = Column( + Integer, ForeignKey("Users.ID", ondelete="SET NULL")) + Editor = relationship( + "User", backref=backref("edited_comments", lazy="dynamic"), + foreign_keys=[EditedUsersID]) + + DelUsersID = Column( + Integer, ForeignKey("Users.ID", ondelete="SET NULL")) + Deleter = relationship( + "User", backref=backref("deleted_comments", lazy="dynamic"), + foreign_keys=[DelUsersID]) + + __mapper_args__ = {"primary_key": [ID]} + + def __init__(self, + PackageBase: aurweb.models.package_base.PackageBase = None, + User: aurweb.models.user.User = None, + **kwargs): + super().__init__(**kwargs) + + self.PackageBase = PackageBase + if not self.PackageBase: + raise IntegrityError( + statement="Foreign key PackageBaseID cannot be null.", + orig="PackageComments.PackageBaseID", + params=("NULL")) + + self.User = User + if not self.User: + raise IntegrityError( + statement="Foreign key UsersID cannot be null.", + orig="PackageComments.UsersID", + params=("NULL")) + + if self.Comments is None: + raise IntegrityError( + statement="Column Comments cannot be null.", + orig="PackageComments.Comments", + params=("NULL")) + + if self.RenderedComment is None: + raise IntegrityError( + statement="Column RenderedComment cannot be null.", + orig="PackageComments.RenderedComment", + params=("NULL")) diff --git a/test/test_package_comment.py b/test/test_package_comment.py new file mode 100644 index 00000000..fb734071 --- /dev/null +++ b/test/test_package_comment.py @@ -0,0 +1,63 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query, rollback +from aurweb.models.account_type import AccountType +from aurweb.models.package_base import PackageBase +from aurweb.models.package_comment import PackageComment +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = pkgbase = None + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db("PackageBases", "PackageComments", "Users") + + global user, pkgbase + + account_type = query(AccountType, + AccountType.AccountType == "User").first() + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + + +def test_package_comment_creation(): + package_comment = create(PackageComment, + PackageBase=pkgbase, + User=user, + Comments="Test comment.", + RenderedComment="Test rendered comment.") + assert bool(package_comment.ID) + + +def test_package_comment_null_package_base_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageComment, User=user, Comments="Test comment.", + RenderedComment="Test rendered comment.") + rollback() + + +def test_package_comment_null_user_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageComment, PackageBase=pkgbase, Comments="Test comment.", + RenderedComment="Test rendered comment.") + rollback() + + +def test_package_comment_null_comments_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageComment, PackageBase=pkgbase, User=user, + RenderedComment="Test rendered comment.") + rollback() + + +def test_package_comment_null_renderedcomment_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageComment, PackageBase=pkgbase, User=user, + Comments="Test comment.") + rollback() From ebd216edfd4f78db864f044fdc10d13cdc7b20b9 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 16:52:45 -0700 Subject: [PATCH 161/844] add PackageComaintainer SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/package_comaintainer.py | 53 +++++++++++++++++++++++++++ test/test_package_comaintainer.py | 49 +++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 aurweb/models/package_comaintainer.py create mode 100644 test/test_package_comaintainer.py diff --git a/aurweb/models/package_comaintainer.py b/aurweb/models/package_comaintainer.py new file mode 100644 index 00000000..88fd58ae --- /dev/null +++ b/aurweb/models/package_comaintainer.py @@ -0,0 +1,53 @@ +from sqlalchemy import Column, ForeignKey, Integer +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import backref, relationship + +import aurweb.models.package_base +import aurweb.models.user + +from aurweb.models.declarative import Base + + +class PackageComaintainer(Base): + __tablename__ = "PackageComaintainers" + + UsersID = Column( + Integer, ForeignKey("Users.ID", ondelete="CASCADE"), + nullable=False) + User = relationship( + "User", backref=backref("comaintained", lazy="dynamic"), + foreign_keys=[UsersID]) + + PackageBaseID = Column( + Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), + nullable=False) + PackageBase = relationship( + "PackageBase", backref=backref("comaintainers", lazy="dynamic"), + foreign_keys=[PackageBaseID]) + + __mapper_args__ = {"primary_key": [UsersID, PackageBaseID]} + + def __init__(self, + User: aurweb.models.user.User = None, + PackageBase: aurweb.models.package_base.PackageBase = None, + Priority: int = None): + self.User = User + if not self.User: + raise IntegrityError( + statement="Foreign key UsersID cannot be null.", + orig="PackageComaintainers.UsersID", + params=("NULL")) + + self.PackageBase = PackageBase + if not self.PackageBase: + raise IntegrityError( + statement="Foreign key PackageBaseID cannot be null.", + orig="PackageComaintainers.PackageBaseID", + params=("NULL")) + + self.Priority = Priority + if not self.Priority: + raise IntegrityError( + statement="Column Priority cannot be null.", + orig="PackageComaintainers.Priority", + params=("NULL")) diff --git a/test/test_package_comaintainer.py b/test/test_package_comaintainer.py new file mode 100644 index 00000000..ac94a9ba --- /dev/null +++ b/test/test_package_comaintainer.py @@ -0,0 +1,49 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, rollback +from aurweb.models.package_base import PackageBase +from aurweb.models.package_comaintainer import PackageComaintainer +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = pkgbase = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, pkgbase + + setup_test_db("Users", "PackageBases", "PackageComaintainers") + + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword") + pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + + +def test_package_comaintainer_creation(): + package_comaintainer = create(PackageComaintainer, User=user, PackageBase=pkgbase, + Priority=5) + assert bool(package_comaintainer) + assert package_comaintainer.User == user + assert package_comaintainer.PackageBase == pkgbase + assert package_comaintainer.Priority == 5 + + +def test_package_comaintainer_null_user_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageComaintainer, PackageBase=pkgbase, Priority=1) + rollback() + + +def test_package_comaintainer_null_pkgbase_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageComaintainer, User=user, Priority=1) + rollback() + + +def test_package_comaintainer_null_priority_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageComaintainer, User=user, PackageBase=pkgbase) + rollback() From 229df1adefb709959e70754845ed1bc65501064c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 16:56:15 -0700 Subject: [PATCH 162/844] test_package_vote: remove useless stuff Signed-off-by: Kevin Morris --- test/test_package_vote.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/test_package_vote.py b/test/test_package_vote.py index b352bf11..cb15e217 100644 --- a/test/test_package_vote.py +++ b/test/test_package_vote.py @@ -4,8 +4,7 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, query, rollback -from aurweb.models.account_type import AccountType +from aurweb.db import create, rollback from aurweb.models.package_base import PackageBase from aurweb.models.package_vote import PackageVote from aurweb.models.user import User @@ -20,11 +19,6 @@ def setup(): setup_test_db("Users", "PackageBases", "PackageVotes") - account_type = query(AccountType, - AccountType.AccountType == "User").first() - print(account_type.ID) - print(account_type.AccountType) - user = create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword") pkgbase = create(PackageBase, Name="test-package", Maintainer=user) From 5b856c7af2e023d24ca7ad0208f537aff45239db Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 17:14:28 -0700 Subject: [PATCH 163/844] add PackageNotification SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/package_notification.py | 47 +++++++++++++++++++++++++++ test/test_package_notification.py | 42 ++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 aurweb/models/package_notification.py create mode 100644 test/test_package_notification.py diff --git a/aurweb/models/package_notification.py b/aurweb/models/package_notification.py new file mode 100644 index 00000000..ab23a212 --- /dev/null +++ b/aurweb/models/package_notification.py @@ -0,0 +1,47 @@ +from sqlalchemy import Column, ForeignKey, Integer +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import backref, relationship + +import aurweb.models.package_base +import aurweb.models.user + +from aurweb.models.declarative import Base + + +class PackageNotification(Base): + __tablename__ = "PackageNotifications" + + UserID = Column( + Integer, ForeignKey("Users.ID", ondelete="CASCADE"), + nullable=False) + User = relationship( + "User", backref=backref("package_notifications", lazy="dynamic"), + foreign_keys=[UserID]) + + PackageBaseID = Column( + Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), + nullable=False) + PackageBase = relationship( + "PackageBase", + backref=backref("package_notifications", lazy="dynamic"), + foreign_keys=[PackageBaseID]) + + __mapper_args__ = {"primary_key": [UserID, PackageBaseID]} + + def __init__(self, + User: aurweb.models.user.User = None, + PackageBase: aurweb.models.package_base.PackageBase = None, + NotificationTS: int = None): + self.User = User + if not self.User: + raise IntegrityError( + statement="Foreign key UserID cannot be null.", + orig="PackageNotifications.UserID", + params=("NULL")) + + self.PackageBase = PackageBase + if not self.PackageBase: + raise IntegrityError( + statement="Foreign key PackageBaseID cannot be null.", + orig="PackageNotifications.PackageBaseID", + params=("NULL")) diff --git a/test/test_package_notification.py b/test/test_package_notification.py new file mode 100644 index 00000000..2898a904 --- /dev/null +++ b/test/test_package_notification.py @@ -0,0 +1,42 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, rollback +from aurweb.models.package_base import PackageBase +from aurweb.models.package_notification import PackageNotification +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = pkgbase = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, pkgbase + + setup_test_db("Users", "PackageBases", "PackageNotifications") + + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword") + pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + + +def test_package_notification_creation(): + package_notification = create(PackageNotification, User=user, + PackageBase=pkgbase) + assert bool(package_notification) + assert package_notification.User == user + assert package_notification.PackageBase == pkgbase + + +def test_package_notification_null_user_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageNotification, PackageBase=pkgbase) + rollback() + + +def test_package_notification_null_pkgbase_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageNotification, User=user) + rollback() From 163e4d738999932fb7e109aba48170c8f2526ca6 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 17:15:18 -0700 Subject: [PATCH 164/844] test_package_comaintainer: sanitize newlines Signed-off-by: Kevin Morris --- test/test_package_comaintainer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_package_comaintainer.py b/test/test_package_comaintainer.py index ac94a9ba..cba99ba0 100644 --- a/test/test_package_comaintainer.py +++ b/test/test_package_comaintainer.py @@ -23,8 +23,8 @@ def setup(): def test_package_comaintainer_creation(): - package_comaintainer = create(PackageComaintainer, User=user, PackageBase=pkgbase, - Priority=5) + package_comaintainer = create(PackageComaintainer, User=user, + PackageBase=pkgbase, Priority=5) assert bool(package_comaintainer) assert package_comaintainer.User == user assert package_comaintainer.PackageBase == pkgbase From 511f174c8ba704b3cf53ae47fcf56e55c74026f8 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 17:28:08 -0700 Subject: [PATCH 165/844] add PackageBlacklist SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/package_blacklist.py | 20 ++++++++++++++++++ test/test_package_blacklist.py | 34 ++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 aurweb/models/package_blacklist.py create mode 100644 test/test_package_blacklist.py diff --git a/aurweb/models/package_blacklist.py b/aurweb/models/package_blacklist.py new file mode 100644 index 00000000..7702c877 --- /dev/null +++ b/aurweb/models/package_blacklist.py @@ -0,0 +1,20 @@ +from sqlalchemy import Column, Integer +from sqlalchemy.exc import IntegrityError + +from aurweb.models.declarative import Base + + +class PackageBlacklist(Base): + __tablename__ = "PackageBlacklist" + + ID = Column(Integer, primary_key=True) + + __mapper_args__ = {"primary_key": [ID]} + + def __init__(self, Name: str = None): + self.Name = Name + if not self.Name: + raise IntegrityError( + statement="Column Name cannot be null.", + orig="PackageBlacklist.Name", + params=("NULL")) diff --git a/test/test_package_blacklist.py b/test/test_package_blacklist.py new file mode 100644 index 00000000..3c64cc21 --- /dev/null +++ b/test/test_package_blacklist.py @@ -0,0 +1,34 @@ +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, rollback +from aurweb.models.package_base import PackageBase +from aurweb.models.package_blacklist import PackageBlacklist +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = pkgbase = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, pkgbase + + setup_test_db("PackageBlacklist", "PackageBases", "Users") + + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword") + pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + + +def test_package_blacklist_creation(): + package_blacklist = create(PackageBlacklist, Name="evil-package") + assert bool(package_blacklist.ID) + assert package_blacklist.Name == "evil-package" + + +def test_package_blacklist_null_name_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageBlacklist) + rollback() From 3bf4b3717a6baed1326168053daf5f374a6e5003 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 17:37:51 -0700 Subject: [PATCH 166/844] add RequestType SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/request_type.py | 11 +++++++++++ test/test_request_type.py | 24 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 aurweb/models/request_type.py create mode 100644 test/test_request_type.py diff --git a/aurweb/models/request_type.py b/aurweb/models/request_type.py new file mode 100644 index 00000000..2c8276e8 --- /dev/null +++ b/aurweb/models/request_type.py @@ -0,0 +1,11 @@ +from sqlalchemy import Column, Integer + +from aurweb.models.declarative import Base + + +class RequestType(Base): + __tablename__ = "RequestTypes" + + ID = Column(Integer, primary_key=True) + + __mapper_args__ = {"primary_key": [ID]} diff --git a/test/test_request_type.py b/test/test_request_type.py new file mode 100644 index 00000000..a470a60b --- /dev/null +++ b/test/test_request_type.py @@ -0,0 +1,24 @@ +import pytest + +from aurweb.db import create, delete +from aurweb.models.request_type import RequestType +from aurweb.testing import setup_test_db + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db() + + +def test_request_type_creation(): + request_type = create(RequestType, Name="Test Request") + assert bool(request_type.ID) + assert request_type.Name == "Test Request" + delete(RequestType, RequestType.ID == request_type.ID) + + +def test_request_type_null_name_returns_empty_string(): + request_type = create(RequestType) + assert bool(request_type.ID) + assert request_type.Name == str() + delete(RequestType, RequestType.ID == request_type.ID) From 65ff0e76da9ea6cf7ba382e38414424148d7c487 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 19:57:52 -0700 Subject: [PATCH 167/844] aurweb.schema: Fix off-by-one String impls of DECIMAL Signed-off-by: Kevin Morris --- aurweb/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/schema.py b/aurweb/schema.py index 9caf6374..fb8f0dee 100644 --- a/aurweb/schema.py +++ b/aurweb/schema.py @@ -109,7 +109,7 @@ PackageBases = Table( Column('NumVotes', INTEGER(unsigned=True), nullable=False, server_default=text("0")), Column('Popularity', DECIMAL(10, 6, unsigned=True) - if db_backend == "mysql" else String(16), # Stubbed out to test. + if db_backend == "mysql" else String(17), nullable=False, server_default=text("0")), Column('OutOfDateTS', BIGINT(unsigned=True)), Column('FlaggerComment', Text, nullable=False), @@ -388,7 +388,7 @@ TU_VoteInfo = Table( Column('End', BIGINT(unsigned=True), nullable=False), Column('Quorum', DECIMAL(2, 2, unsigned=True) - if db_backend == "mysql" else String(4), + if db_backend == "mysql" else String(5), nullable=False), Column('SubmitterID', ForeignKey('Users.ID', ondelete='CASCADE'), nullable=False), Column('Yes', TINYINT(3, unsigned=True), nullable=False, server_default=text("'0'")), From 809939ab03c5192d9f06d2ffa138d4e064fd0b35 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 20:36:32 -0700 Subject: [PATCH 168/844] add TUVoteInfo SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/tu_voteinfo.py | 74 +++++++++++++++++++++++ test/test_tu_voteinfo.py | 111 +++++++++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 aurweb/models/tu_voteinfo.py create mode 100644 test/test_tu_voteinfo.py diff --git a/aurweb/models/tu_voteinfo.py b/aurweb/models/tu_voteinfo.py new file mode 100644 index 00000000..2225b4d7 --- /dev/null +++ b/aurweb/models/tu_voteinfo.py @@ -0,0 +1,74 @@ +from sqlalchemy import Column, ForeignKey, Integer +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import backref, relationship + +import aurweb.models.user + +from aurweb.models.declarative import Base + + +class TUVoteInfo(Base): + __tablename__ = "TU_VoteInfo" + + ID = Column(Integer, primary_key=True) + + SubmitterID = Column( + Integer, ForeignKey("Users.ID", ondelete="CASCADE"), + nullable=False) + Submitter = relationship( + "User", backref=backref("tu_voteinfo_set", lazy="dynamic"), + foreign_keys=[SubmitterID]) + + __mapper_args__ = {"primary_key": [ID]} + + def __init__(self, + Agenda: str = None, + User: str = None, + Submitted: int = None, + End: int = None, + Quorum: float = None, + Submitter: aurweb.models.user.User = None, + **kwargs): + super().__init__(**kwargs) + + self.Agenda = Agenda + if self.Agenda is None: + raise IntegrityError( + statement="Column Agenda cannot be null.", + orig="TU_VoteInfo.Agenda", + params=("NULL")) + + self.User = User + if self.User is None: + raise IntegrityError( + statement="Column User cannot be null.", + orig="TU_VoteInfo.User", + params=("NULL")) + + self.Submitted = Submitted + if self.Submitted is None: + raise IntegrityError( + statement="Column Submitted cannot be null.", + orig="TU_VoteInfo.Submitted", + params=("NULL")) + + self.End = End + if self.End is None: + raise IntegrityError( + statement="Column End cannot be null.", + orig="TU_VoteInfo.End", + params=("NULL")) + + if Quorum is None: + raise IntegrityError( + statement="Column Quorum cannot be null.", + orig="TU_VoteInfo.Quorum", + params=("NULL")) + self.Quorum = str(Quorum) + + self.Submitter = Submitter + if not self.Submitter: + raise IntegrityError( + statement="Foreign key SubmitterID cannot be null.", + orig="TU_VoteInfo.SubmitterID", + params=("NULL")) diff --git a/test/test_tu_voteinfo.py b/test/test_tu_voteinfo.py new file mode 100644 index 00000000..e95f174b --- /dev/null +++ b/test/test_tu_voteinfo.py @@ -0,0 +1,111 @@ +from datetime import datetime + +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query, rollback +from aurweb.models.account_type import AccountType +from aurweb.models.tu_voteinfo import TUVoteInfo +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = None + + +@pytest.fixture(autouse=True) +def setup(): + global user + + setup_test_db("Users", "PackageBases", "TU_VoteInfo") + + tu_type = query(AccountType, + AccountType.AccountType == "Trusted User").first() + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=tu_type) + + +def test_tu_voteinfo_creation(): + ts = int(datetime.utcnow().timestamp()) + tu_voteinfo = create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=ts, End=ts + 5, + Quorum=0.5, + Submitter=user) + assert bool(tu_voteinfo.ID) + assert tu_voteinfo.Agenda == "Blah blah." + assert tu_voteinfo.User == user.Username + assert tu_voteinfo.Submitted == ts + assert tu_voteinfo.End == ts + 5 + assert float(tu_voteinfo.Quorum) == 0.5 + assert tu_voteinfo.Submitter == user + assert tu_voteinfo.Yes == 0 + assert tu_voteinfo.No == 0 + assert tu_voteinfo.Abstain == 0 + assert tu_voteinfo.ActiveTUs == 0 + + assert tu_voteinfo in user.tu_voteinfo_set + + +def test_tu_voteinfo_null_submitter_raises_exception(): + with pytest.raises(IntegrityError): + create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=0, End=0, + Quorum=0.50) + rollback() + + +def test_tu_voteinfo_null_agenda_raises_exception(): + with pytest.raises(IntegrityError): + create(TUVoteInfo, + User=user.Username, + Submitted=0, End=0, + Quorum=0.50, + Submitter=user) + rollback() + + +def test_tu_voteinfo_null_user_raises_exception(): + with pytest.raises(IntegrityError): + create(TUVoteInfo, + Agenda="Blah blah.", + Submitted=0, End=0, + Quorum=0.50, + Submitter=user) + rollback() + + +def test_tu_voteinfo_null_submitted_raises_exception(): + with pytest.raises(IntegrityError): + create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + End=0, + Quorum=0.50, + Submitter=user) + rollback() + + +def test_tu_voteinfo_null_end_raises_exception(): + with pytest.raises(IntegrityError): + create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=0, + Quorum=0.50, + Submitter=user) + rollback() + + +def test_tu_voteinfo_null_quorum_raises_exception(): + with pytest.raises(IntegrityError): + create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=0, End=0, + Submitter=user) + rollback() From 541c978ac4a7bdec79e1ce7359c23d0ff42723fd Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 19:14:33 -0700 Subject: [PATCH 169/844] add PackageRequest SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/package_request.py | 93 ++++++++++++++++++++++++ test/test_package_request.py | 119 +++++++++++++++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 aurweb/models/package_request.py create mode 100644 test/test_package_request.py diff --git a/aurweb/models/package_request.py b/aurweb/models/package_request.py new file mode 100644 index 00000000..00f46ce2 --- /dev/null +++ b/aurweb/models/package_request.py @@ -0,0 +1,93 @@ +from sqlalchemy import Column, ForeignKey, Integer +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import backref, relationship + +import aurweb.models.package_base +import aurweb.models.request_type +import aurweb.models.user + +from aurweb.models.declarative import Base + + +class PackageRequest(Base): + __tablename__ = "PackageRequests" + + ID = Column(Integer, primary_key=True) + + ReqTypeID = Column( + Integer, ForeignKey("RequestTypes.ID", ondelete="NO ACTION"), + nullable=False) + RequestType = relationship( + "RequestType", backref=backref("package_requests", lazy="dynamic"), + foreign_keys=[ReqTypeID]) + + UsersID = Column(Integer, ForeignKey("Users.ID", ondelete="SET NULL")) + User = relationship( + "User", backref=backref("package_requests", lazy="dynamic"), + foreign_keys=[UsersID]) + + PackageBaseID = Column( + Integer, ForeignKey("PackageBases.ID", ondelete="SET NULL"), + nullable=False) + PackageBase = relationship( + "PackageBase", backref=backref("requests", lazy="dynamic"), + foreign_keys=[PackageBaseID]) + + ClosedUID = Column(Integer, ForeignKey("Users.ID", ondelete="SET NULL")) + Closer = relationship( + "User", backref=backref("closed_requests", lazy="dynamic"), + foreign_keys=[ClosedUID]) + + __mapper_args__ = {"primary_key": [ID]} + + def __init__(self, + RequestType: aurweb.models.request_type.RequestType = None, + PackageBase: aurweb.models.package_base.PackageBase = None, + PackageBaseName: str = None, + User: aurweb.models.user.User = None, + Comments: str = None, + ClosureComment: str = None, + **kwargs): + super().__init__(**kwargs) + + self.RequestType = RequestType + if not self.RequestType: + raise IntegrityError( + statement="Foreign key ReqTypeID cannot be null.", + orig="PackageRequests.ReqTypeID", + params=("NULL")) + + self.PackageBase = PackageBase + if not self.PackageBase: + raise IntegrityError( + statement="Foreign key PackageBaseID cannot be null.", + orig="PackageRequests.PackageBaseID", + params=("NULL")) + + self.PackageBaseName = PackageBaseName + if not self.PackageBaseName: + raise IntegrityError( + statement="Column PackageBaseName cannot be null.", + orig="PackageRequests.PackageBaseName", + params=("NULL")) + + self.User = User + if not self.User: + raise IntegrityError( + statement="Foreign key UsersID cannot be null.", + orig="PackageRequests.UsersID", + params=("NULL")) + + self.Comments = Comments + if self.Comments is None: + raise IntegrityError( + statement="Column Comments cannot be null.", + orig="PackageRequests.Comments", + params=("NULL")) + + self.ClosureComment = ClosureComment + if self.ClosureComment is None: + raise IntegrityError( + statement="Column ClosureComment cannot be null.", + orig="PackageRequests.ClosureComment", + params=("NULL")) diff --git a/test/test_package_request.py b/test/test_package_request.py new file mode 100644 index 00000000..fc839836 --- /dev/null +++ b/test/test_package_request.py @@ -0,0 +1,119 @@ +from datetime import datetime + +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query, rollback +from aurweb.models.package_base import PackageBase +from aurweb.models.package_request import PackageRequest +from aurweb.models.request_type import RequestType +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = pkgbase = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, pkgbase + + setup_test_db("PackageRequests", "PackageBases", "Users") + + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword") + pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + + +def test_package_request_creation(): + request_type = query(RequestType, RequestType.Name == "merge").first() + assert request_type.Name == "merge" + + package_request = create(PackageRequest, RequestType=request_type, + User=user, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str()) + + assert bool(package_request.ID) + assert package_request.RequestType == request_type + assert package_request.User == user + assert package_request.PackageBase == pkgbase + assert package_request.PackageBaseName == pkgbase.Name + assert package_request.Comments == str() + assert package_request.ClosureComment == str() + + # Make sure that everything is cross-referenced with relationships. + assert package_request in request_type.package_requests + assert package_request in user.package_requests + assert package_request in pkgbase.requests + + +def test_package_request_closed(): + request_type = query(RequestType, RequestType.Name == "merge").first() + assert request_type.Name == "merge" + + ts = int(datetime.utcnow().timestamp()) + package_request = create(PackageRequest, RequestType=request_type, + User=user, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Closer=user, ClosedTS=ts, + Comments=str(), ClosureComment=str()) + + assert package_request.Closer == user + assert package_request.ClosedTS == ts + + # Test relationships. + assert package_request in user.closed_requests + + +def test_package_request_null_request_type_raises_exception(): + with pytest.raises(IntegrityError): + create(PackageRequest, User=user, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str()) + rollback() + + +def test_package_request_null_user_raises_exception(): + request_type = query(RequestType, RequestType.Name == "merge").first() + with pytest.raises(IntegrityError): + create(PackageRequest, RequestType=request_type, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str()) + rollback() + + +def test_package_request_null_package_base_raises_exception(): + request_type = query(RequestType, RequestType.Name == "merge").first() + with pytest.raises(IntegrityError): + create(PackageRequest, RequestType=request_type, + User=user, PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str()) + rollback() + + +def test_package_request_null_package_base_name_raises_exception(): + request_type = query(RequestType, RequestType.Name == "merge").first() + with pytest.raises(IntegrityError): + create(PackageRequest, RequestType=request_type, + User=user, PackageBase=pkgbase, + Comments=str(), ClosureComment=str()) + rollback() + + +def test_package_request_null_comments_raises_exception(): + request_type = query(RequestType, RequestType.Name == "merge").first() + with pytest.raises(IntegrityError): + create(PackageRequest, RequestType=request_type, + User=user, PackageBase=pkgbase, PackageBaseName=pkgbase.Name, + ClosureComment=str()) + rollback() + + +def test_package_request_null_closure_comment_raises_exception(): + request_type = query(RequestType, RequestType.Name == "merge").first() + with pytest.raises(IntegrityError): + create(PackageRequest, RequestType=request_type, + User=user, PackageBase=pkgbase, PackageBaseName=pkgbase.Name, + Comments=str()) + rollback() From 8c345a04488a07b9836cff9559bb17722b5fc77e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 21:48:39 -0700 Subject: [PATCH 170/844] TUVoteInfo: generalize Quorum SQLite does not support native DECIMAL columns, and for that reason, we had to switch to using Strings that can hold the data in the case we are using sqlite. This commit sets the TUVoteInfo model up in a generic way, that it always converts to string when setting Quorum (OK for DECIMAL) and always converts to float when getting Quorum. This way, we can treat TUVoteInfo.Quorum as the same thing everywhere. Signed-off-by: Kevin Morris --- aurweb/models/tu_voteinfo.py | 15 ++++++++++++++- test/test_tu_voteinfo.py | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/aurweb/models/tu_voteinfo.py b/aurweb/models/tu_voteinfo.py index 2225b4d7..a246f132 100644 --- a/aurweb/models/tu_voteinfo.py +++ b/aurweb/models/tu_voteinfo.py @@ -1,3 +1,5 @@ +import typing + from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship @@ -64,7 +66,7 @@ class TUVoteInfo(Base): statement="Column Quorum cannot be null.", orig="TU_VoteInfo.Quorum", params=("NULL")) - self.Quorum = str(Quorum) + self.Quorum = Quorum self.Submitter = Submitter if not self.Submitter: @@ -72,3 +74,14 @@ class TUVoteInfo(Base): statement="Foreign key SubmitterID cannot be null.", orig="TU_VoteInfo.SubmitterID", params=("NULL")) + + def __setattr__(self, key: str, value: typing.Any): + """ Customize setattr to stringify any Quorum keys given. """ + if key == "Quorum": + value = str(value) + return super().__setattr__(key, value) + + def __getattribute__(self, key: str): + """ Customize getattr to floatify any fetched Quorum values. """ + attr = super().__getattribute__(key) + return float(attr) if key == "Quorum" else attr diff --git a/test/test_tu_voteinfo.py b/test/test_tu_voteinfo.py index e95f174b..37609efd 100644 --- a/test/test_tu_voteinfo.py +++ b/test/test_tu_voteinfo.py @@ -39,7 +39,7 @@ def test_tu_voteinfo_creation(): assert tu_voteinfo.User == user.Username assert tu_voteinfo.Submitted == ts assert tu_voteinfo.End == ts + 5 - assert float(tu_voteinfo.Quorum) == 0.5 + assert tu_voteinfo.Quorum == 0.5 assert tu_voteinfo.Submitter == user assert tu_voteinfo.Yes == 0 assert tu_voteinfo.No == 0 From 0c1241f8bbe53b587cb149d0daee32731bffed46 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 22:14:38 -0700 Subject: [PATCH 171/844] add TUVote SQLAlchemy ORM model Signed-off-by: Kevin Morris --- aurweb/models/tu_vote.py | 43 ++++++++++++++++++++++++++++++ test/test_tu_vote.py | 56 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 aurweb/models/tu_vote.py create mode 100644 test/test_tu_vote.py diff --git a/aurweb/models/tu_vote.py b/aurweb/models/tu_vote.py new file mode 100644 index 00000000..2b7bf2d0 --- /dev/null +++ b/aurweb/models/tu_vote.py @@ -0,0 +1,43 @@ +from sqlalchemy import Column, ForeignKey, Integer +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import backref, relationship + +import aurweb.models.tu_voteinfo +import aurweb.models.user + +from aurweb.models.declarative import Base + + +class TUVote(Base): + __tablename__ = "TU_Votes" + + VoteID = Column(Integer, ForeignKey("TU_VoteInfo.ID", ondelete="CASCADE"), + nullable=False) + VoteInfo = relationship( + "TUVoteInfo", backref=backref("tu_votes", lazy="dynamic"), + foreign_keys=[VoteID]) + + UserID = Column(Integer, ForeignKey("Users.ID", ondelete="CASCADE"), + nullable=False) + User = relationship( + "User", backref=backref("tu_votes", lazy="dynamic"), + foreign_keys=[UserID]) + + __mapper_args__ = {"primary_key": [VoteID, UserID]} + + def __init__(self, + VoteInfo: aurweb.models.tu_voteinfo.TUVoteInfo = None, + User: aurweb.models.user.User = None): + self.VoteInfo = VoteInfo + if self.VoteInfo is None: + raise IntegrityError( + statement="Foreign key VoteID cannot be null.", + orig="TU_Votes.VoteID", + params=("NULL")) + + self.User = User + if self.User is None: + raise IntegrityError( + statement="Foreign key UserID cannot be null.", + orig="TU_Votes.UserID", + params=("NULL")) diff --git a/test/test_tu_vote.py b/test/test_tu_vote.py new file mode 100644 index 00000000..9ff4a8d9 --- /dev/null +++ b/test/test_tu_vote.py @@ -0,0 +1,56 @@ +from datetime import datetime + +import pytest + +from sqlalchemy.exc import IntegrityError + +from aurweb.db import create, query, rollback +from aurweb.models.account_type import AccountType +from aurweb.models.tu_vote import TUVote +from aurweb.models.tu_voteinfo import TUVoteInfo +from aurweb.models.user import User +from aurweb.testing import setup_test_db + +user = tu_voteinfo = None + + +@pytest.fixture(autouse=True) +def setup(): + global user, tu_voteinfo + + setup_test_db("Users", "TU_VoteInfo", "TU_Votes") + + tu_type = query(AccountType, + AccountType.AccountType == "Trusted User").first() + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=tu_type) + + ts = int(datetime.utcnow().timestamp()) + tu_voteinfo = create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=ts, End=ts + 5, + Quorum=0.5, + Submitter=user) + + +def test_tu_vote_creation(): + tu_vote = create(TUVote, User=user, VoteInfo=tu_voteinfo) + assert tu_vote.VoteInfo == tu_voteinfo + assert tu_vote.User == user + + assert tu_vote in user.tu_votes + assert tu_vote in tu_voteinfo.tu_votes + + +def test_tu_vote_null_user_raises_exception(): + with pytest.raises(IntegrityError): + create(TUVote, VoteInfo=tu_voteinfo) + rollback() + + +def test_tu_vote_null_voteinfo_raises_exception(): + with pytest.raises(IntegrityError): + create(TUVote, User=user) + rollback() From 18ec8e3cc8ca3e18f87c1b892b68a0857be39597 Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Fri, 20 Nov 2020 00:19:15 +0100 Subject: [PATCH 172/844] RSS: Add ability to specify isPermaLink="false" for GUID --- web/lib/feedcreator.class.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/web/lib/feedcreator.class.php b/web/lib/feedcreator.class.php index a1fe24c9..bfc29b20 100644 --- a/web/lib/feedcreator.class.php +++ b/web/lib/feedcreator.class.php @@ -183,7 +183,7 @@ class FeedItem extends HtmlDescribable { /** * Optional attributes of an item. */ - var $author, $authorEmail, $image, $category, $comments, $guid, $source, $creator; + var $author, $authorEmail, $image, $category, $comments, $guid, $guidIsPermaLink, $source, $creator; /** * Publishing date of an item. May be in one of the following formats: @@ -995,7 +995,11 @@ class RSSCreator091 extends FeedCreator { $feed.= " ".htmlspecialchars($itemDate->rfc822())."\n"; } if ($this->items[$i]->guid!="") { - $feed.= " ".htmlspecialchars($this->items[$i]->guid)."\n"; + $feed.= " items[$i]->guidIsPermaLink == false) { + $feed.= " isPermaLink=\"false\""; + } + $feed.= ">".htmlspecialchars($this->items[$i]->guid)."\n"; } $feed.= $this->_createAdditionalElements($this->items[$i]->additionalElements, " "); $feed.= " \n"; From 2bb30f9bf53446189b6079ac349acc82909c3a49 Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Fri, 20 Nov 2020 00:19:54 +0100 Subject: [PATCH 173/844] Add RSS feed for modified packages --- web/html/modified-rss.php | 62 +++++++++++++++++++++++++++++++++++++++ web/lib/pkgfuncs.inc.php | 17 +++++++++-- web/lib/routing.inc.php | 1 + 3 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 web/html/modified-rss.php diff --git a/web/html/modified-rss.php b/web/html/modified-rss.php new file mode 100644 index 00000000..4c5c47e0 --- /dev/null +++ b/web/html/modified-rss.php @@ -0,0 +1,62 @@ +cssStyleSheet = false; +$rss->xslStyleSheet = false; + +# Use UTF-8 (fixes FS#10706). +$rss->encoding = "UTF-8"; + +#All the general RSS setup +$rss->title = "AUR Latest Modified Packages"; +$rss->description = "The latest modified packages in the AUR"; +$rss->link = "${protocol}://{$host}"; +$rss->syndicationURL = "{$protocol}://{$host}" . get_uri('/rss/'); +$image = new FeedImage(); +$image->title = "AUR Latest Modified Packages"; +$image->url = "{$protocol}://{$host}/css/archnavbar/aurlogo.png"; +$image->link = $rss->link; +$image->description = "AUR Latest Modified Packages Feed"; +$rss->image = $image; + +#Get the latest packages and add items for them +$packages = latest_modified_pkgs(100); + +foreach ($packages as $indx => $row) { + $item = new FeedItem(); + $item->title = $row["Name"]; + $item->link = "{$protocol}://{$host}" . get_pkg_uri($row["Name"]); + $item->description = $row["Description"]; + $item->date = intval($row["ModifiedTS"]); + $item->source = "{$protocol}://{$host}"; + $item->author = username_from_id($row["MaintainerUID"]); + $item->guidIsPermaLink = true; + $item->guid = $row["Name"] . "-" . $row["ModifiedTS"]; + $rss->addItem($item); +} + +#save it so that useCached() can find it +$feedContent = $rss->createFeed(); +set_cache_value($feed_key, $feedContent, 600); +echo $feedContent; +?> diff --git a/web/lib/pkgfuncs.inc.php b/web/lib/pkgfuncs.inc.php index eb3afab6..140c7ec1 100644 --- a/web/lib/pkgfuncs.inc.php +++ b/web/lib/pkgfuncs.inc.php @@ -925,13 +925,13 @@ function sanitize_ids($ids) { * * @return array $packages Package info for the specified number of recent packages */ -function latest_pkgs($numpkgs) { +function latest_pkgs($numpkgs, $orderBy='SubmittedTS') { $dbh = DB::connect(); - $q = "SELECT Packages.*, MaintainerUID, SubmittedTS "; + $q = "SELECT Packages.*, MaintainerUID, SubmittedTS, ModifiedTS "; $q.= "FROM Packages LEFT JOIN PackageBases ON "; $q.= "PackageBases.ID = Packages.PackageBaseID "; - $q.= "ORDER BY SubmittedTS DESC "; + $q.= "ORDER BY " . $orderBy . " DESC "; $q.= "LIMIT " . intval($numpkgs); $result = $dbh->query($q); @@ -944,3 +944,14 @@ function latest_pkgs($numpkgs) { return $packages; } + +/** + * Determine package information for latest modified packages + * + * @param int $numpkgs Number of packages to get information on + * + * @return array $packages Package info for the specified number of recently modified packages + */ +function latest_modified_pkgs($numpkgs) { + return latest_pkgs($numpkgs, 'ModifiedTS'); +} diff --git a/web/lib/routing.inc.php b/web/lib/routing.inc.php index 7d9750a0..73c667d2 100644 --- a/web/lib/routing.inc.php +++ b/web/lib/routing.inc.php @@ -15,6 +15,7 @@ $ROUTES = array( '/logout' => 'logout.php', '/passreset' => 'passreset.php', '/rpc' => 'rpc.php', + '/rss/modified' => 'modified-rss.php', '/rss' => 'rss.php', '/tos' => 'tos.php', '/tu' => 'tu.php', From 537349e124c158a6537b4026ed3a1394a75a7206 Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Fri, 20 Nov 2020 00:20:26 +0100 Subject: [PATCH 174/844] Add modified packages RSS feed to frontend --- web/html/css/archweb.css | 4 ++++ web/template/header.php | 1 + web/template/stats/updates_table.php | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/web/html/css/archweb.css b/web/html/css/archweb.css index f95e3843..b935d7db 100644 --- a/web/html/css/archweb.css +++ b/web/html/css/archweb.css @@ -556,6 +556,10 @@ h3 span.arrow { margin: -2em 0 0 0; } + #pkg-updates .rss-icon.latest { + margin-right: 1em; + } + #pkg-updates table { margin: 0; } diff --git a/web/template/header.php b/web/template/header.php index f7409400..afe7a9b6 100644 --- a/web/template/header.php +++ b/web/template/header.php @@ -9,6 +9,7 @@ ' /> + ' /> diff --git a/web/template/stats/updates_table.php b/web/template/stats/updates_table.php index b4c6215f..23a86288 100644 --- a/web/template/stats/updates_table.php +++ b/web/template/stats/updates_table.php @@ -1,6 +1,7 @@

    ()

    -RSS Feed +RSS Feed +RSS Feed From e7db894eb716132411bd88c094bc8df8b5f378de Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Fri, 20 Nov 2020 00:19:15 +0100 Subject: [PATCH 175/844] RSS: Add ability to specify isPermaLink="false" for GUID --- web/lib/feedcreator.class.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/web/lib/feedcreator.class.php b/web/lib/feedcreator.class.php index a1fe24c9..bfc29b20 100644 --- a/web/lib/feedcreator.class.php +++ b/web/lib/feedcreator.class.php @@ -183,7 +183,7 @@ class FeedItem extends HtmlDescribable { /** * Optional attributes of an item. */ - var $author, $authorEmail, $image, $category, $comments, $guid, $source, $creator; + var $author, $authorEmail, $image, $category, $comments, $guid, $guidIsPermaLink, $source, $creator; /** * Publishing date of an item. May be in one of the following formats: @@ -995,7 +995,11 @@ class RSSCreator091 extends FeedCreator { $feed.= " ".htmlspecialchars($itemDate->rfc822())."\n"; } if ($this->items[$i]->guid!="") { - $feed.= " ".htmlspecialchars($this->items[$i]->guid)."\n"; + $feed.= " items[$i]->guidIsPermaLink == false) { + $feed.= " isPermaLink=\"false\""; + } + $feed.= ">".htmlspecialchars($this->items[$i]->guid)."\n"; } $feed.= $this->_createAdditionalElements($this->items[$i]->additionalElements, " "); $feed.= " \n"; From 4330fe4f335a2c1b3b68743337576501dc1f6c92 Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Fri, 20 Nov 2020 00:19:54 +0100 Subject: [PATCH 176/844] Add RSS feed for modified packages --- web/html/modified-rss.php | 62 +++++++++++++++++++++++++++++++++++++++ web/lib/pkgfuncs.inc.php | 17 +++++++++-- web/lib/routing.inc.php | 1 + 3 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 web/html/modified-rss.php diff --git a/web/html/modified-rss.php b/web/html/modified-rss.php new file mode 100644 index 00000000..4c5c47e0 --- /dev/null +++ b/web/html/modified-rss.php @@ -0,0 +1,62 @@ +cssStyleSheet = false; +$rss->xslStyleSheet = false; + +# Use UTF-8 (fixes FS#10706). +$rss->encoding = "UTF-8"; + +#All the general RSS setup +$rss->title = "AUR Latest Modified Packages"; +$rss->description = "The latest modified packages in the AUR"; +$rss->link = "${protocol}://{$host}"; +$rss->syndicationURL = "{$protocol}://{$host}" . get_uri('/rss/'); +$image = new FeedImage(); +$image->title = "AUR Latest Modified Packages"; +$image->url = "{$protocol}://{$host}/css/archnavbar/aurlogo.png"; +$image->link = $rss->link; +$image->description = "AUR Latest Modified Packages Feed"; +$rss->image = $image; + +#Get the latest packages and add items for them +$packages = latest_modified_pkgs(100); + +foreach ($packages as $indx => $row) { + $item = new FeedItem(); + $item->title = $row["Name"]; + $item->link = "{$protocol}://{$host}" . get_pkg_uri($row["Name"]); + $item->description = $row["Description"]; + $item->date = intval($row["ModifiedTS"]); + $item->source = "{$protocol}://{$host}"; + $item->author = username_from_id($row["MaintainerUID"]); + $item->guidIsPermaLink = true; + $item->guid = $row["Name"] . "-" . $row["ModifiedTS"]; + $rss->addItem($item); +} + +#save it so that useCached() can find it +$feedContent = $rss->createFeed(); +set_cache_value($feed_key, $feedContent, 600); +echo $feedContent; +?> diff --git a/web/lib/pkgfuncs.inc.php b/web/lib/pkgfuncs.inc.php index eb3afab6..140c7ec1 100644 --- a/web/lib/pkgfuncs.inc.php +++ b/web/lib/pkgfuncs.inc.php @@ -925,13 +925,13 @@ function sanitize_ids($ids) { * * @return array $packages Package info for the specified number of recent packages */ -function latest_pkgs($numpkgs) { +function latest_pkgs($numpkgs, $orderBy='SubmittedTS') { $dbh = DB::connect(); - $q = "SELECT Packages.*, MaintainerUID, SubmittedTS "; + $q = "SELECT Packages.*, MaintainerUID, SubmittedTS, ModifiedTS "; $q.= "FROM Packages LEFT JOIN PackageBases ON "; $q.= "PackageBases.ID = Packages.PackageBaseID "; - $q.= "ORDER BY SubmittedTS DESC "; + $q.= "ORDER BY " . $orderBy . " DESC "; $q.= "LIMIT " . intval($numpkgs); $result = $dbh->query($q); @@ -944,3 +944,14 @@ function latest_pkgs($numpkgs) { return $packages; } + +/** + * Determine package information for latest modified packages + * + * @param int $numpkgs Number of packages to get information on + * + * @return array $packages Package info for the specified number of recently modified packages + */ +function latest_modified_pkgs($numpkgs) { + return latest_pkgs($numpkgs, 'ModifiedTS'); +} diff --git a/web/lib/routing.inc.php b/web/lib/routing.inc.php index 7d9750a0..73c667d2 100644 --- a/web/lib/routing.inc.php +++ b/web/lib/routing.inc.php @@ -15,6 +15,7 @@ $ROUTES = array( '/logout' => 'logout.php', '/passreset' => 'passreset.php', '/rpc' => 'rpc.php', + '/rss/modified' => 'modified-rss.php', '/rss' => 'rss.php', '/tos' => 'tos.php', '/tu' => 'tu.php', From 8d9f20939c864800a45fc6f8994ad9af8e8fe837 Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Fri, 20 Nov 2020 00:20:26 +0100 Subject: [PATCH 177/844] Add modified packages RSS feed to frontend --- web/html/css/archweb.css | 4 ++++ web/template/header.php | 1 + web/template/stats/updates_table.php | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/web/html/css/archweb.css b/web/html/css/archweb.css index f95e3843..b935d7db 100644 --- a/web/html/css/archweb.css +++ b/web/html/css/archweb.css @@ -556,6 +556,10 @@ h3 span.arrow { margin: -2em 0 0 0; } + #pkg-updates .rss-icon.latest { + margin-right: 1em; + } + #pkg-updates table { margin: 0; } diff --git a/web/template/header.php b/web/template/header.php index f7409400..afe7a9b6 100644 --- a/web/template/header.php +++ b/web/template/header.php @@ -9,6 +9,7 @@ ' /> + ' /> diff --git a/web/template/stats/updates_table.php b/web/template/stats/updates_table.php index b4c6215f..23a86288 100644 --- a/web/template/stats/updates_table.php +++ b/web/template/stats/updates_table.php @@ -1,6 +1,7 @@

    ()

    -RSS Feed +RSS Feed +RSS Feed
    From bd8f5280112b2cf1cf6113453977629be2ee92c5 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 13 Jun 2021 10:48:31 -0700 Subject: [PATCH 178/844] add Base.as_dict() and Base.json() Two utility functions for all of our ORM models that will allow us to easily convert them to Python structures and JSON data. Signed-off-by: Kevin Morris --- aurweb/models/declarative.py | 30 ++++++++++++++++++++++++++++++ aurweb/util.py | 8 ++++++++ test/test_user.py | 19 +++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/aurweb/models/declarative.py b/aurweb/models/declarative.py index 45a629ce..96ee1829 100644 --- a/aurweb/models/declarative.py +++ b/aurweb/models/declarative.py @@ -1,10 +1,40 @@ +import json + from sqlalchemy.ext.declarative import declarative_base import aurweb.db +from aurweb import util + + +def to_dict(model): + return { + c.name: getattr(model, c.name) + for c in model.__table__.columns + } + + +def to_json(model, indent: int = None): + return json.dumps({ + k: util.jsonify(v) + for k, v in to_dict(model).items() + }, indent=indent) + + Base = declarative_base() + +# Setup __table_args__ applicable to every table. Base.__table_args__ = { "autoload": True, "autoload_with": aurweb.db.get_engine(), "extend_existing": True } + + +# Setup Base.as_dict and Base.json. +# +# With this, declarative models can use .as_dict() or .json() +# at any time to produce a dict and json out of table columns. +# +Base.as_dict = to_dict +Base.json = to_json diff --git a/aurweb/util.py b/aurweb/util.py index 8e4b291d..ad8ac6b7 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -3,6 +3,7 @@ import random import re import string +from datetime import datetime from urllib.parse import urlparse from email_validator import EmailNotValidError, EmailUndeliverableError, validate_email @@ -94,3 +95,10 @@ def account_url(context, user): if request.url.scheme == "http" and request.url.port != 80: base += f":{request.url.port}" return f"{base}/account/{user.Username}" + + +def jsonify(obj): + """ Perform a conversion on obj if it's needed. """ + if isinstance(obj, datetime): + obj = int(obj.timestamp()) + return obj diff --git a/test/test_user.py b/test/test_user.py index 8b4da61e..06585207 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -1,4 +1,5 @@ import hashlib +import json from datetime import datetime, timedelta @@ -198,3 +199,21 @@ def test_user_credential_types(): assert aurweb.auth.trusted_user(user) assert aurweb.auth.developer(user) assert aurweb.auth.trusted_user_or_dev(user) + + +def test_user_json(): + data = json.loads(user.json()) + assert data.get("ID") == user.ID + assert data.get("Username") == user.Username + assert data.get("Email") == user.Email + # .json() converts datetime values to integer timestamps. + assert isinstance(data.get("RegistrationTS"), int) + + +def test_user_as_dict(): + data = user.as_dict() + assert data.get("ID") == user.ID + assert data.get("Username") == user.Username + assert data.get("Email") == user.Email + # .as_dict() does not convert values to json-capable types. + assert isinstance(data.get("RegistrationTS"), datetime) From 40448ccd34d32bcb4c1f5357e6196dc8f5c17dc6 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 13 Jun 2021 12:16:13 -0700 Subject: [PATCH 179/844] aurweb.db: add commit(), add() and autocommit arg With the addition of these two, some code has been swapped to use these in some of the other db wrappers with an additional autocommit kwarg in create and delete, to control batch transactions. Signed-off-by: Kevin Morris --- aurweb/db.py | 21 ++++++++++++++++----- test/test_db.py | 25 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/aurweb/db.py b/aurweb/db.py index 04c8653a..c0147720 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -59,24 +59,35 @@ def query(model, *args, **kwargs): return session.query(model).filter(*args, **kwargs) -def create(model, *args, **kwargs): +def create(model, autocommit: bool = True, *args, **kwargs): instance = model(*args, **kwargs) - session.add(instance) - session.commit() + add(instance) + if autocommit is True: + commit() return instance -def delete(model, *args, **kwargs): +def delete(model, *args, autocommit: bool = True, **kwargs): instance = session.query(model).filter(*args, **kwargs) for record in instance: session.delete(record) - session.commit() + if autocommit is True: + commit() def rollback(): session.rollback() +def add(model): + session.add(model) + return model + + +def commit(): + session.commit() + + def get_sqlalchemy_url(): """ Build an SQLAlchemy for use with create_engine based on the aurweb configuration. diff --git a/test/test_db.py b/test/test_db.py index 9298c53d..d7a91813 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -273,6 +273,31 @@ def test_create_delete(): record = db.query(AccountType, AccountType.AccountType == "test").first() assert record is None + # Create and delete a record with autocommit=False. + db.create(AccountType, AccountType="test", autocommit=False) + db.commit() + db.delete(AccountType, AccountType.AccountType == "test", autocommit=False) + db.commit() + record = db.query(AccountType, AccountType.AccountType == "test").first() + assert record is None + + +def test_add_commit(): + # Use db.add and db.commit to add a temporary record. + account_type = AccountType(AccountType="test") + db.add(account_type) + db.commit() + + # Assert it got created in the DB. + assert bool(account_type.ID) + + # Query the DB for it and compare the record with our object. + record = db.query(AccountType, AccountType.AccountType == "test").first() + assert record == account_type + + # Remove the record. + db.delete(AccountType, AccountType.ID == account_type.ID) + def test_connection_executor_mysql_paramstyle(): executor = db.ConnectionExecutor(None, backend="mysql") From 7ae95ac90813dc3f521d810ccb95adfc74a64496 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 18 Jun 2021 04:39:58 -0700 Subject: [PATCH 180/844] bugfix: removed extra space in " My Account" nav link Signed-off-by: Kevin Morris --- templates/partials/archdev-navbar.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/partials/archdev-navbar.html b/templates/partials/archdev-navbar.html index 4d54f6af..c935fd41 100644 --- a/templates/partials/archdev-navbar.html +++ b/templates/partials/archdev-navbar.html @@ -9,7 +9,7 @@ {% if request.user.is_authenticated() %}
  • - {% trans %} My Account{% endtrans %} + {% trans %}My Account{% endtrans %}
  • From b7d67bf5fcf02d172fce1a290c500f9fb432c49f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 19 Jun 2021 05:06:24 -0700 Subject: [PATCH 181/844] render_template: convert HTTPStatus objects This will automate a lot of conversion that happens around the codebase in terms of status_code. As of this commit, we should improve usage and remove int(status_code) casts wherever we can. Signed-off-by: Kevin Morris --- aurweb/templates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/templates.py b/aurweb/templates.py index c0472b2e..7474da1c 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -65,7 +65,7 @@ async def make_variable_context(request: Request, title: str, next: str = None): def render_template(request: Request, path: str, context: dict, - status_code=int(HTTPStatus.OK)): + status_code: HTTPStatus = HTTPStatus.OK): """ Render a Jinja2 multi-lingual template with some context. """ # Create a deep copy of our jinja2 environment. The environment in @@ -81,7 +81,7 @@ def render_template(request: Request, template = templates.get_template(path) rendered = template.render(context) - response = HTMLResponse(rendered, status_code=status_code) + response = HTMLResponse(rendered, status_code=int(status_code)) response.set_cookie("AURLANG", context.get("language")) response.set_cookie("AURTZ", context.get("timezone")) return response From f89d06d092aa0f20a7b267b6cb274ae175c294df Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 19 Jun 2021 05:20:23 -0700 Subject: [PATCH 182/844] setup_test_db: remove mysql-dependent coverage path Signed-off-by: Kevin Morris --- aurweb/testing/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/testing/__init__.py b/aurweb/testing/__init__.py index 90d46720..65d34253 100644 --- a/aurweb/testing/__init__.py +++ b/aurweb/testing/__init__.py @@ -50,7 +50,7 @@ def setup_test_db(*args): db_backend = aurweb.config.get("database", "backend") - if db_backend != "sqlite": + if db_backend != "sqlite": # pragma: no cover aurweb.db.session.execute("SET FOREIGN_KEY_CHECKS = 0") else: # We're using sqlite, setup tables to be deleted without violating @@ -61,7 +61,7 @@ def setup_test_db(*args): for table in tables: aurweb.db.session.execute(f"DELETE FROM {table}") - if db_backend != "sqlite": + if db_backend != "sqlite": # pragma: no cover aurweb.db.session.execute("SET FOREIGN_KEY_CHECKS = 1") # Expunge all objects from SQLAlchemy's IdentityMap. From ac67268a28f02dcdc2fb765c6bd6d76555e0056a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 18 Jun 2021 16:49:41 -0700 Subject: [PATCH 183/844] add util.timezone_to_datetime -> `dt` Jinja2 filter Signed-off-by: Kevin Morris --- aurweb/templates.py | 3 +++ aurweb/util.py | 4 ++++ test/test_util.py | 9 +++++++++ 3 files changed, 16 insertions(+) create mode 100644 test/test_util.py diff --git a/aurweb/templates.py b/aurweb/templates.py index 7474da1c..8c6f3294 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -23,6 +23,9 @@ env = jinja2.Environment(loader=loader, autoescape=True, # Add tr translation filter. env.filters["tr"] = l10n.tr +# Utility filters. +env.filters["dt"] = util.timestamp_to_datetime + # Add captcha filters. env.filters["captcha_salt"] = captcha.captcha_salt_filter env.filters["captcha_cmdline"] = captcha.captcha_cmdline_filter diff --git a/aurweb/util.py b/aurweb/util.py index ad8ac6b7..ce18853b 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -97,6 +97,10 @@ def account_url(context, user): return f"{base}/account/{user.Username}" +def timestamp_to_datetime(timestamp: int): + return datetime.utcfromtimestamp(int(timestamp)) + + def jsonify(obj): """ Perform a conversion on obj if it's needed. """ if isinstance(obj, datetime): diff --git a/test/test_util.py b/test/test_util.py new file mode 100644 index 00000000..cd7b7a57 --- /dev/null +++ b/test/test_util.py @@ -0,0 +1,9 @@ +from datetime import datetime + +from aurweb import util + + +def test_timestamp_to_datetime(): + ts = datetime.utcnow().timestamp() + dt = datetime.utcfromtimestamp(int(ts)) + assert util.timestamp_to_datetime(ts) == dt From b1baf769985949bfa496c9d8276dbcd2e101072b Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 18 Jun 2021 16:55:01 -0700 Subject: [PATCH 184/844] add util.as_timezone -> `as_timezone` Jinja2 filter Signed-off-by: Kevin Morris --- aurweb/templates.py | 1 + aurweb/util.py | 5 +++++ test/test_util.py | 7 +++++++ 3 files changed, 13 insertions(+) diff --git a/aurweb/templates.py b/aurweb/templates.py index 8c6f3294..9439f3a3 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -25,6 +25,7 @@ env.filters["tr"] = l10n.tr # Utility filters. env.filters["dt"] = util.timestamp_to_datetime +env.filters["as_timezone"] = util.as_timezone # Add captcha filters. env.filters["captcha_salt"] = captcha.captcha_salt_filter diff --git a/aurweb/util.py b/aurweb/util.py index ce18853b..1615e00a 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -5,6 +5,7 @@ import string from datetime import datetime from urllib.parse import urlparse +from zoneinfo import ZoneInfo from email_validator import EmailNotValidError, EmailUndeliverableError, validate_email from jinja2 import pass_context @@ -101,6 +102,10 @@ def timestamp_to_datetime(timestamp: int): return datetime.utcfromtimestamp(int(timestamp)) +def as_timezone(dt: datetime, timezone: str): + return dt.astimezone(tz=ZoneInfo(timezone)) + + def jsonify(obj): """ Perform a conversion on obj if it's needed. """ if isinstance(obj, datetime): diff --git a/test/test_util.py b/test/test_util.py index cd7b7a57..d58a8ae2 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,4 +1,5 @@ from datetime import datetime +from zoneinfo import ZoneInfo from aurweb import util @@ -7,3 +8,9 @@ def test_timestamp_to_datetime(): ts = datetime.utcnow().timestamp() dt = datetime.utcfromtimestamp(int(ts)) assert util.timestamp_to_datetime(ts) == dt + + +def test_as_timezone(): + ts = datetime.utcnow().timestamp() + dt = util.timestamp_to_datetime(ts) + assert util.as_timezone(dt, "UTC") == dt.astimezone(tz=ZoneInfo("UTC")) From d5e650a33930286bae0263e8b0aff12ed94a319e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 18 Jun 2021 16:57:06 -0700 Subject: [PATCH 185/844] add util.dedupe_qs -> `dedupe_qs` Jinja2 filter Signed-off-by: Kevin Morris --- aurweb/templates.py | 1 + aurweb/util.py | 26 +++++++++++++++++++++++++- test/test_util.py | 16 ++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/aurweb/templates.py b/aurweb/templates.py index 9439f3a3..1e09bf61 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -26,6 +26,7 @@ env.filters["tr"] = l10n.tr # Utility filters. env.filters["dt"] = util.timestamp_to_datetime env.filters["as_timezone"] = util.as_timezone +env.filters["dedupe_qs"] = util.dedupe_qs # Add captcha filters. env.filters["captcha_salt"] = captcha.captcha_salt_filter diff --git a/aurweb/util.py b/aurweb/util.py index 1615e00a..0aec6f45 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -3,8 +3,9 @@ import random import re import string +from collections import OrderedDict from datetime import datetime -from urllib.parse import urlparse +from urllib.parse import quote_plus, urlparse from zoneinfo import ZoneInfo from email_validator import EmailNotValidError, EmailUndeliverableError, validate_email @@ -106,6 +107,29 @@ def as_timezone(dt: datetime, timezone: str): return dt.astimezone(tz=ZoneInfo(timezone)) +def dedupe_qs(query_string: str, *additions): + """ Dedupe keys found in a query string by rewriting it without + duplicates found while iterating from the end to the beginning, + using an ordered memo to track keys found and persist locations. + + That is, query string 'a=1&b=1&a=2' will be deduped and converted + to 'b=1&a=2'. + + :param query_string: An HTTP URL query string. + :param *additions: Optional additional fields to add to the query string. + :return: Deduped query string, including *additions at the tail. + """ + for addition in list(additions): + query_string += f"&{addition}" + + qs = OrderedDict() + for item in reversed(query_string.split('&')): + key, value = item.split('=') + if key not in qs: + qs[key] = value + return '&'.join([f"{k}={quote_plus(v)}" for k, v in reversed(qs.items())]) + + def jsonify(obj): """ Perform a conversion on obj if it's needed. """ if isinstance(obj, datetime): diff --git a/test/test_util.py b/test/test_util.py index d58a8ae2..074de494 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,3 +1,4 @@ +from collections import OrderedDict from datetime import datetime from zoneinfo import ZoneInfo @@ -14,3 +15,18 @@ def test_as_timezone(): ts = datetime.utcnow().timestamp() dt = util.timestamp_to_datetime(ts) assert util.as_timezone(dt, "UTC") == dt.astimezone(tz=ZoneInfo("UTC")) + + +def test_dedupe_qs(): + items = OrderedDict() + items["key1"] = "test" + items["key2"] = "blah" + items["key3"] = 1 + + # Construct and test our query string. + query_string = '&'.join([f"{k}={v}" for k, v in items.items()]) + assert query_string == "key1=test&key2=blah&key3=1" + + # Add key1=changed and key2=changed to the query and dedupe it. + deduped = util.dedupe_qs(query_string, "key1=changed", "key3=changed") + assert deduped == "key2=blah&key1=changed&key3=changed" From d7941e6bedfed32df0299a5869df40f33b202374 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 18 Jun 2021 16:57:44 -0700 Subject: [PATCH 186/844] urllib.parse.quote_plus -> `urlencode` Jinja2 filter Signed-off-by: Kevin Morris --- aurweb/templates.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aurweb/templates.py b/aurweb/templates.py index 1e09bf61..015f8c9f 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -4,6 +4,7 @@ import zoneinfo from datetime import datetime from http import HTTPStatus +from urllib.parse import quote_plus import jinja2 @@ -27,6 +28,7 @@ env.filters["tr"] = l10n.tr env.filters["dt"] = util.timestamp_to_datetime env.filters["as_timezone"] = util.as_timezone env.filters["dedupe_qs"] = util.dedupe_qs +env.filters["urlencode"] = quote_plus # Add captcha filters. env.filters["captcha_salt"] = captcha.captcha_salt_filter From 8b6f92f9e907267288c9a2d757c7747b22c7bca8 Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Tue, 11 May 2021 00:01:13 +0200 Subject: [PATCH 187/844] Use the clipboard API for copy paste The Document.execCommand API is deprecated and no longer recommended to be used. It's replacement is the much simpler navigator.clipboard API which is supported in all browsers except internet explorer. Signed-off-by: Eli Schwartz --- web/template/pkg_details.php | 10 +++------- web/template/pkgbase_details.php | 10 +++------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/web/template/pkg_details.php b/web/template/pkg_details.php index c6bb32d8..047de9a7 100644 --- a/web/template/pkg_details.php +++ b/web/template/pkg_details.php @@ -308,14 +308,10 @@ endif; diff --git a/web/template/pkgbase_details.php b/web/template/pkgbase_details.php index a6857c4e..35ad217a 100644 --- a/web/template/pkgbase_details.php +++ b/web/template/pkgbase_details.php @@ -137,14 +137,10 @@ endif; From d7603fa4d3d31e8c50b2988730652809ae1f42b7 Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Mon, 10 May 2021 23:55:36 +0200 Subject: [PATCH 188/844] Port package details page to pure JavaScript Use a CSS animation for jQuery.Animate and replace the rest with pure vanilla JavaScript. Signed-off-by: Eli Schwartz --- web/html/css/aurweb.css | 5 +++ web/html/packages.php | 96 +++++++++++++++++++++++++---------------- 2 files changed, 65 insertions(+), 36 deletions(-) diff --git a/web/html/css/aurweb.css b/web/html/css/aurweb.css index 81bf9ab6..bb4e3ad7 100644 --- a/web/html/css/aurweb.css +++ b/web/html/css/aurweb.css @@ -199,3 +199,8 @@ label.confirmation, .error { color: red; } + +.article-content > div { + overflow: hidden; + transition: height 1s; +} diff --git a/web/html/packages.php b/web/html/packages.php index a989428e..559a8f45 100644 --- a/web/html/packages.php +++ b/web/html/packages.php @@ -46,70 +46,94 @@ if (isset($pkgname)) { html_header($title, $details); ?> - From 06fa8ab5f32061ae1d06abbe1cc502883f3884da Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Mon, 14 Jun 2021 22:13:07 +0200 Subject: [PATCH 189/844] Convert comment editing to vanilla JavaScript Signed-off-by: Eli Schwartz --- web/template/pkg_comments.php | 90 ++++++++++++++++++++++++----------- 1 file changed, 62 insertions(+), 28 deletions(-) diff --git a/web/template/pkg_comments.php b/web/template/pkg_comments.php index 3bcf1a38..ffa9e137 100644 --- a/web/template/pkg_comments.php +++ b/web/template/pkg_comments.php @@ -169,37 +169,71 @@ if ($comment_section == "package") { From af76e660d0f03712901d2d1ddda07d383fafcb08 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 21 Jun 2021 21:35:05 -0700 Subject: [PATCH 190/844] auth_required: allow formattable template tuples See docstring for updates. template= has been modified. status_code= has been added as an optional template status_code. Signed-off-by: Kevin Morris --- aurweb/auth.py | 67 +++++++++++++++++++++++++++++++----- aurweb/routers/accounts.py | 16 ++++++--- test/test_accounts_routes.py | 1 - 3 files changed, 70 insertions(+), 14 deletions(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index 401ed6ae..f57e18bf 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -10,9 +10,10 @@ from starlette.requests import HTTPConnection import aurweb.config +from aurweb import l10n from aurweb.models.session import Session from aurweb.models.user import User -from aurweb.templates import make_context, render_template +from aurweb.templates import make_variable_context, render_template class AnonymousUser: @@ -60,7 +61,8 @@ class BasicAuthBackend(AuthenticationBackend): def auth_required(is_required: bool = True, redirect: str = "/", - template: tuple = None): + template: tuple = None, + status_code: HTTPStatus = HTTPStatus.UNAUTHORIZED): """ Authentication route decorator. If redirect is given, the user will be redirected if the auth state @@ -69,26 +71,73 @@ def auth_required(is_required: bool = True, If template is given, it will be rendered with Unauthorized if is_required does not match and take priority over redirect. + A precondition of this function is that, if template is provided, + it **must** match the following format: + + template=("template.html", ["Some Template For", "{}"], ["username"]) + + Where `username` is a FastAPI request path parameter, fitting + a route like: `/some_route/{username}`. + + If you wish to supply a non-formatted template, just omit any Python + format strings (with the '{}' substring). The third tuple element + will not be used, and so anything can be supplied. + + template=("template.html", ["Some Page"], None) + + All title shards and format parameters will be translated before + applying any format operations. + :param is_required: A boolean indicating whether the function requires auth :param redirect: Path to redirect to if is_required isn't True - :param template: A template tuple: ("template.html", "Template Page") + :param template: A three-element template tuple: + (path, title_iterable, variable_iterable) + :param status_code: An optional status_code for template render. + Redirects are always SEE_OTHER. """ def decorator(func): @functools.wraps(func) async def wrapper(request, *args, **kwargs): if request.user.is_authenticated() != is_required: - status_code = int(HTTPStatus.UNAUTHORIZED) url = "/" if redirect: - status_code = int(HTTPStatus.SEE_OTHER) url = redirect if template: - path, title = template - context = make_context(request, title) + # template=("template.html", + # ["Some Title", "someFormatted {}"], + # ["variable"]) + # => render template.html with title: + # "Some Title someFormatted variables" + path, title_parts, variables = template + _ = l10n.get_translator_for_request(request) + + # Step through title_parts; for each part which contains + # a '{}' in it, apply .format(var) where var = the current + # iteration of variables. + # + # This implies that len(variables) is equal to + # len([part for part in title_parts if '{}' in part]) + # and this must always be true. + # + sanitized = [] + _variables = iter(variables) + for part in title_parts: + if "{}" in part: # If this part is formattable. + key = next(_variables) + var = request.path_params.get(key) + sanitized.append(_(part.format(var))) + else: # Otherwise, just add the translated part. + sanitized.append(_(part)) + + # Glue all title parts together, separated by spaces. + title = " ".join(sanitized) + + context = await make_variable_context(request, title) return render_template(request, path, context, - status_code=int(HTTPStatus.UNAUTHORIZED)) - return RedirectResponse(url=url, status_code=status_code) + status_code=status_code) + return RedirectResponse(url, + status_code=int(HTTPStatus.SEE_OTHER)) return await func(request, *args, **kwargs) return wrapper diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index c7c96003..966f8409 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -555,13 +555,21 @@ async def account_edit_post(request: Request, return util.migrate_cookies(request, response) +account_template = ( + "account/show.html", + ["Account", "{}"], + ["username"] # Query parameters to replace in the title string. +) + + @router.get("/account/{username}") -@auth_required(True, template=("account/show.html", "Accounts")) +@auth_required(True, template=account_template, + status_code=HTTPStatus.UNAUTHORIZED) async def account(request: Request, username: str): + _ = l10n.get_translator_for_request(request) + context = await make_variable_context(request, _("Account") + username) + user = db.query(User, User.Username == username).first() - - context = await make_variable_context(request, "Accounts") - if not user: raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index d5fd089e..bd0d9d4b 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -915,7 +915,6 @@ def test_get_account_not_found(): def test_get_account_unauthenticated(): with client as request: response = request.get("/account/test", allow_redirects=False) - assert response.status_code == int(HTTPStatus.UNAUTHORIZED) content = response.content.decode() From 959e535126bdb6863f62ccf1e4a32482793b1386 Mon Sep 17 00:00:00 2001 From: Kristian Klausen Date: Wed, 23 Jun 2021 03:09:37 +0200 Subject: [PATCH 191/844] Use the real ml email address instead of alias All the arch-x@archlinux.org -> arch-x@lists.archlinux.org aliases will be dropped soon[1]. [1] https://lists.archlinux.org/pipermail/arch-dev-public/2021-June/030462.html --- conf/config.defaults | 2 +- test/setup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/config.defaults b/conf/config.defaults index e6961520..b7bc0368 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -22,7 +22,7 @@ git_clone_uri_anon = https://aur.archlinux.org/%s.git git_clone_uri_priv = ssh://aur@aur.archlinux.org/%s.git max_rpc_results = 5000 max_depends = 1000 -aur_request_ml = aur-requests@archlinux.org +aur_request_ml = aur-requests@lists.archlinux.org request_idle_time = 1209600 request_archive_time = 15552000 auto_orphan_age = 15552000 diff --git a/test/setup.sh b/test/setup.sh index 4a6eb3b1..589c8c3f 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -26,7 +26,7 @@ name = aur.db [options] aur_location = https://aur.archlinux.org -aur_request_ml = aur-requests@archlinux.org +aur_request_ml = aur-requests@lists.archlinux.org enable-maintenance = 0 maintenance-exceptions = 127.0.0.1 commit_uri = https://aur.archlinux.org/cgit/aur.git/log/?h=%s&id=%s From ec632a7091df6df3940f62cfee3d9a09641dd4b5 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 11 Jun 2021 23:09:34 -0700 Subject: [PATCH 192/844] use secure=True when options.disable_http_login is enabled We'll piggyback off of the current existing configuration item, `disable_http_login`, to decide how we should submit cookies to an HTTP response. Previously, in `sso.py`, the http schema was used to make this decision. There is an issue with that, however: We cannot actually test properly if we depend on the https schema. This change allows us to toggle `disable_http_login` to modify the behavior of cookies sent with an http response to be secure. We test this behavior in test/test_auth_routes.py#L81: `test_secure_login(mock)`. Signed-off-by: Kevin Morris --- aurweb/routers/auth.py | 5 +++- aurweb/routers/html.py | 6 ++++- aurweb/routers/sso.py | 8 ++++--- aurweb/templates.py | 9 +++++--- aurweb/util.py | 3 ++- test/test_auth_routes.py | 50 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 72 insertions(+), 9 deletions(-) diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index e4864424..4aca9304 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -59,7 +59,10 @@ async def login_post(request: Request, response = RedirectResponse(url=next, status_code=int(HTTPStatus.SEE_OTHER)) - response.set_cookie("AURSID", sid, expires=expires_at) + + secure_cookies = aurweb.config.getboolean("options", "disable_http_login") + response.set_cookie("AURSID", sid, expires=expires_at, + secure=secure_cookies, httponly=True) return response diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index 890aff88..ed0c039b 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -6,6 +6,8 @@ from http import HTTPStatus from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse +import aurweb.config + from aurweb.templates import make_context, render_template router = APIRouter() @@ -45,7 +47,9 @@ async def language(request: Request, # In any case, set the response's AURLANG cookie that never expires. response = RedirectResponse(url=f"{next}{query_string}", status_code=int(HTTPStatus.SEE_OTHER)) - response.set_cookie("AURLANG", set_lang) + secure_cookies = aurweb.config.getboolean("options", "disable_http_login") + response.set_cookie("AURLANG", set_lang, + secure=secure_cookies, httponly=True) return response diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index 4b12b932..093807fe 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -131,13 +131,15 @@ async def authenticate(request: Request, redirect: str = None, conn=Depends(aurw elif len(aur_accounts) == 1: sid = open_session(request, conn, aur_accounts[0][Users.c.ID]) response = RedirectResponse(redirect if redirect and is_aur_url(redirect) else "/") + secure_cookies = aurweb.config.getboolean("options", "disable_http_login") response.set_cookie(key="AURSID", value=sid, httponly=True, - secure=request.url.scheme == "https") + secure=secure_cookies) if "id_token" in token: # We save the id_token for the SSO logout. It’s not too important # though, so if we can’t find it, we can live without it. - response.set_cookie(key="SSO_ID_TOKEN", value=token["id_token"], path="/sso/", - httponly=True, secure=request.url.scheme == "https") + response.set_cookie(key="SSO_ID_TOKEN", value=token["id_token"], + path="/sso/", httponly=True, + secure=secure_cookies) return response else: # We’ve got a severe integrity violation. diff --git a/aurweb/templates.py b/aurweb/templates.py index 015f8c9f..640b9447 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -88,7 +88,10 @@ def render_template(request: Request, template = templates.get_template(path) rendered = template.render(context) - response = HTMLResponse(rendered, status_code=int(status_code)) - response.set_cookie("AURLANG", context.get("language")) - response.set_cookie("AURTZ", context.get("timezone")) + response = HTMLResponse(rendered, status_code=status_code) + secure_cookies = aurweb.config.getboolean("options", "disable_http_login") + response.set_cookie("AURLANG", context.get("language"), + secure=secure_cookies, httponly=True) + response.set_cookie("AURTZ", context.get("timezone"), + secure=secure_cookies, httponly=True) return response diff --git a/aurweb/util.py b/aurweb/util.py index 0aec6f45..1da85606 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -85,8 +85,9 @@ def valid_ssh_pubkey(pk): def migrate_cookies(request, response): + secure_cookies = aurweb.config.getboolean("options", "disable_http_login") for k, v in request.cookies.items(): - response.set_cookie(k, v) + response.set_cookie(k, v, secure=secure_cookies, httponly=True) return response diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index 360b48cc..a443be72 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -1,5 +1,6 @@ from datetime import datetime from http import HTTPStatus +from unittest import mock import pytest @@ -70,6 +71,55 @@ def test_login_logout(): assert "AURSID" not in response.cookies +def mock_getboolean(a, b): + if a == "options" and b == "disable_http_login": + return True + return bool(aurweb.config.get(a, b)) + + +@mock.patch("aurweb.config.getboolean", side_effect=mock_getboolean) +def test_secure_login(mock): + """ In this test, we check to verify the course of action taken + by starlette when providing secure=True to a response cookie. + This is achieved by mocking aurweb.config.getboolean to return + True (or 1) when looking for `options.disable_http_login`. + When we receive a response with `disable_http_login` enabled, + we check the fields in cookies received for the secure and + httponly fields, in addition to the rest of the fields given + on such a request. """ + + # Create a local TestClient here since we mocked configuration. + client = TestClient(app) + + # Data used for our upcoming http post request. + post_data = { + "user": user.Username, + "passwd": "testPassword", + "next": "/" + } + + # Perform a login request with the data matching our user. + with client as request: + response = request.post("/login", data=post_data, + allow_redirects=False) + + # Make sure we got the expected status out of it. + assert response.status_code == int(HTTPStatus.SEE_OTHER) + + # Let's check what we got in terms of cookies for AURSID. + # Make sure that a secure cookie got passed to us. + cookie = next(c for c in response.cookies if c.name == "AURSID") + assert cookie.secure is True + assert cookie.has_nonstandard_attr("HttpOnly") is True + assert cookie.value is not None and len(cookie.value) > 0 + + # Let's make sure we actually have a session relationship + # with the AURSID we ended up with. + record = query(Session, Session.SessionID == cookie.value).first() + assert record is not None and record.User == user + assert user.session == record + + def test_authenticated_login_forbidden(): post_data = { "user": "test", From 91dc3efc75700fd9c559a908028ff39eeb7d2bfe Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 12 Jun 2021 03:23:58 -0700 Subject: [PATCH 193/844] add util.add_samesite_fields(response, value) This function adds f"SameSite={value}" to each cookie's header stored in response. This is needed because starlette does not currently support the `samesite` argument in Response.set_cookie. It is merged, however, and waiting for next release. Signed-off-by: Kevin Morris --- aurweb/routers/auth.py | 3 ++- aurweb/routers/html.py | 3 ++- aurweb/routers/sso.py | 3 ++- aurweb/templates.py | 2 +- aurweb/util.py | 15 ++++++++++++++- test/test_auth_routes.py | 2 ++ 6 files changed, 23 insertions(+), 5 deletions(-) diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index 4aca9304..2b05784b 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -6,6 +6,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse import aurweb.config +from aurweb import util from aurweb.auth import auth_required from aurweb.models.user import User from aurweb.templates import make_context, render_template @@ -63,7 +64,7 @@ async def login_post(request: Request, secure_cookies = aurweb.config.getboolean("options", "disable_http_login") response.set_cookie("AURSID", sid, expires=expires_at, secure=secure_cookies, httponly=True) - return response + return util.add_samesite_fields(response, "strict") @router.get("/logout") diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index ed0c039b..580ee0d4 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -8,6 +8,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse import aurweb.config +from aurweb import util from aurweb.templates import make_context, render_template router = APIRouter() @@ -50,7 +51,7 @@ async def language(request: Request, secure_cookies = aurweb.config.getboolean("options", "disable_http_login") response.set_cookie("AURLANG", set_lang, secure=secure_cookies, httponly=True) - return response + return util.add_samesite_fields(response, "strict") @router.get("/", response_class=HTMLResponse) diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index 093807fe..edeb7c6b 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -14,6 +14,7 @@ from starlette.requests import Request import aurweb.config import aurweb.db +from aurweb import util from aurweb.l10n import get_translator_for_request from aurweb.schema import Bans, Sessions, Users @@ -140,7 +141,7 @@ async def authenticate(request: Request, redirect: str = None, conn=Depends(aurw response.set_cookie(key="SSO_ID_TOKEN", value=token["id_token"], path="/sso/", httponly=True, secure=secure_cookies) - return response + return util.add_samesite_fields(response, "strict") else: # We’ve got a severe integrity violation. raise Exception("Multiple accounts found for SSO account " + sub) diff --git a/aurweb/templates.py b/aurweb/templates.py index 640b9447..bb4047f4 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -94,4 +94,4 @@ def render_template(request: Request, secure=secure_cookies, httponly=True) response.set_cookie("AURTZ", context.get("timezone"), secure=secure_cookies, httponly=True) - return response + return util.add_samesite_fields(response, "strict") diff --git a/aurweb/util.py b/aurweb/util.py index 1da85606..b34226a2 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -9,6 +9,7 @@ from urllib.parse import quote_plus, urlparse from zoneinfo import ZoneInfo from email_validator import EmailNotValidError, EmailUndeliverableError, validate_email +from fastapi.responses import Response from jinja2 import pass_context import aurweb.config @@ -88,7 +89,7 @@ def migrate_cookies(request, response): secure_cookies = aurweb.config.getboolean("options", "disable_http_login") for k, v in request.cookies.items(): response.set_cookie(k, v, secure=secure_cookies, httponly=True) - return response + return add_samesite_fields(response, "strict") @pass_context @@ -136,3 +137,15 @@ def jsonify(obj): if isinstance(obj, datetime): obj = int(obj.timestamp()) return obj + + +def add_samesite_fields(response: Response, value: str): + """ Set the SameSite field on all cookie headers found. + Taken from https://github.com/tiangolo/fastapi/issues/1099. """ + for idx, header in enumerate(response.raw_headers): + if header[0].decode() == "set-cookie": + cookie = header[1].decode() + if f"SameSite={value}" not in cookie: + cookie += f"; SameSite={value}" + response.raw_headers[idx] = (header[0], cookie.encode()) + return response diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index a443be72..b0dd5648 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -111,6 +111,8 @@ def test_secure_login(mock): cookie = next(c for c in response.cookies if c.name == "AURSID") assert cookie.secure is True assert cookie.has_nonstandard_attr("HttpOnly") is True + assert cookie.has_nonstandard_attr("SameSite") is True + assert cookie.get_nonstandard_attr("SameSite") == "strict" assert cookie.value is not None and len(cookie.value) > 0 # Let's make sure we actually have a session relationship From 13456fea1e6eab4971e4ec9f38456ccaeccda352 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 12 Jun 2021 03:26:05 -0700 Subject: [PATCH 194/844] set AURLANG + AURTZ on login Signed-off-by: Kevin Morris --- aurweb/routers/auth.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index 2b05784b..8f37fe27 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -64,6 +64,10 @@ async def login_post(request: Request, secure_cookies = aurweb.config.getboolean("options", "disable_http_login") response.set_cookie("AURSID", sid, expires=expires_at, secure=secure_cookies, httponly=True) + response.set_cookie("AURTZ", user.Timezone, + secure=secure_cookies, httponly=True) + response.set_cookie("AURLANG", user.LangPreference, + secure=secure_cookies, httponly=True) return util.add_samesite_fields(response, "strict") From 865c41450466c8392605f7c1aabc5607b77cb5a1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 12 Jun 2021 03:54:41 -0700 Subject: [PATCH 195/844] aurweb.asgi: add security headers middleware This commit introduces a middleware function which adds the following security headers to each response: - Content-Security-Policy - This includes a new `nonce`, which is tied to a user via authentication middleware. Both an anonymous user and an authenticated user recieve their own random nonces. - X-Content-Type-Options - Referrer-Policy - X-Frame-Options They are then tested for existence in test/test_routes.py. Note: The overcomplicated-looking asyncio behavior in the middleware function is used to avoid a warning about the old coroutine awaits being deprecated. See https://docs.python.org/3/library/asyncio-task.html#asyncio.wait for more detail. Signed-off-by: Kevin Morris --- aurweb/asgi.py | 43 ++++++++++++++++++++++++++++++- aurweb/auth.py | 10 ++++++- aurweb/models/user.py | 1 + aurweb/util.py | 11 ++++++++ templates/partials/typeahead.html | 2 +- test/test_routes.py | 42 ++++++++++++++++++++++++++++++ 6 files changed, 106 insertions(+), 3 deletions(-) diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 861f6056..6c4d457d 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -1,6 +1,8 @@ +import asyncio import http +import typing -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, HTTPException, Request from fastapi.responses import HTMLResponse from fastapi.staticfiles import StaticFiles from starlette.middleware.authentication import AuthenticationMiddleware @@ -55,3 +57,42 @@ async def http_exception_handler(request, exc): phrase = http.HTTPStatus(exc.status_code).phrase return HTMLResponse(f"

    {exc.status_code} {phrase}

    {exc.detail}

    ", status_code=exc.status_code) + + +@app.middleware("http") +async def add_security_headers(request: Request, call_next: typing.Callable): + """ This middleware adds the CSP, XCTO, XFO and RP security + headers to the HTTP response associated with request. + + CSP: Content-Security-Policy + XCTO: X-Content-Type-Options + RP: Referrer-Policy + XFO: X-Frame-Options + """ + response = asyncio.create_task(call_next(request)) + await asyncio.wait({response}, return_when=asyncio.FIRST_COMPLETED) + response = response.result() + + # Add CSP header. + nonce = request.user.nonce + csp = "default-src 'self'; " + script_hosts = [ + "ajax.googleapis.com", + "cdn.jsdelivr.net" + ] + csp += f"script-src 'self' 'nonce-{nonce}' " + ' '.join(script_hosts) + response.headers["Content-Security-Policy"] = csp + + # Add XTCO header. + xcto = "nosniff" + response.headers["X-Content-Type-Options"] = xcto + + # Add Referrer Policy header. + rp = "same-origin" + response.headers["Referrer-Policy"] = rp + + # Add X-Frame-Options header. + xfo = "SAMEORIGIN" + response.headers["X-Frame-Options"] = xfo + + return response diff --git a/aurweb/auth.py b/aurweb/auth.py index f57e18bf..ba5f0fea 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -10,7 +10,7 @@ from starlette.requests import HTTPConnection import aurweb.config -from aurweb import l10n +from aurweb import l10n, util from aurweb.models.session import Session from aurweb.models.user import User from aurweb.templates import make_variable_context, render_template @@ -25,6 +25,12 @@ class AnonymousUser: # A stub ssh_pub_key relationship. ssh_pub_key = None + # A nonce attribute, needed for all browser sessions; set in __init__. + nonce = None + + def __init__(self): + self.nonce = util.make_nonce() + @staticmethod def is_authenticated(): return False @@ -55,7 +61,9 @@ class BasicAuthBackend(AuthenticationBackend): # exists, due to ForeignKey constraints in the schema upheld # by mysqlclient. user = session.query(User).filter(User.ID == record.UsersID).first() + user.nonce = util.make_nonce() user.authenticated = True + return AuthCredentials(["authenticated"]), user diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 83cde5f1..9db9add0 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -37,6 +37,7 @@ class User(Base): # High-level variables used to track authentication (not in DB). authenticated = False + nonce = None def __init__(self, Passwd: str = str(), **kwargs): super().__init__(**kwargs) diff --git a/aurweb/util.py b/aurweb/util.py index b34226a2..e5f510ce 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -1,6 +1,8 @@ import base64 +import math import random import re +import secrets import string from collections import OrderedDict @@ -20,6 +22,15 @@ def make_random_string(length): string.digits, k=length)) +def make_nonce(length: int = 8): + """ Generate a single random nonce. Here, token_hex generates a hex + string of 2 hex characters per byte, where the length give is + nbytes. This means that to get our proper string length, we need to + cut it in half and truncate off any remaining (in the case that + length was uneven). """ + return secrets.token_hex(math.ceil(length / 2))[:length] + + def valid_username(username): min_len = aurweb.config.getint("options", "username_min_len") max_len = aurweb.config.getint("options", "username_max_len") diff --git a/templates/partials/typeahead.html b/templates/partials/typeahead.html index d943dbc4..c218b8d1 100644 --- a/templates/partials/typeahead.html +++ b/templates/partials/typeahead.html @@ -1,6 +1,6 @@ - - - + "+t+""})},render:function(t){var n=this;return t=e(t).map(function(t,r){return t=e(n.options.item).attr("data-value",r),t.find("a").html(n.highlighter(r)),t[0]}),this.$menu.html(t),this},next:function(t){var n=this.$menu.find(".active").removeClass("active"),r=n.next();r.length||(r=e(this.$menu.find("li")[0])),r.addClass("active")},prev:function(e){var t=this.$menu.find(".active").removeClass("active"),n=t.prev();n.length||(n=this.$menu.find("li").last()),n.addClass("active")},listen:function(){this.$element.on("blur",e.proxy(this.blur,this)).on("keypress",e.proxy(this.keypress,this)).on("keyup",e.proxy(this.keyup,this)),(e.browser.chrome||e.browser.webkit||e.browser.msie)&&this.$element.on("keydown",e.proxy(this.keydown,this)),this.$menu.on("click",e.proxy(this.click,this)).on("mouseenter","li",e.proxy(this.mouseenter,this))},move:function(e){if(!this.shown)return;switch(e.keyCode){case 9:case 13:case 27:e.preventDefault();break;case 38:e.preventDefault(),this.prev();break;case 40:e.preventDefault(),this.next()}e.stopPropagation()},keydown:function(t){this.suppressKeyPressRepeat=!~e.inArray(t.keyCode,[40,38,9,13,27]),this.move(t)},keypress:function(e){if(this.suppressKeyPressRepeat)return;this.move(e)},keyup:function(e){switch(e.keyCode){case 40:case 38:break;case 9:case 13:if(!this.shown)return;this.select();break;case 27:if(!this.shown)return;this.hide();break;default:this.lookup()}e.stopPropagation(),e.preventDefault()},blur:function(e){var t=this;setTimeout(function(){t.hide()},150)},click:function(e){e.stopPropagation(),e.preventDefault(),this.select()},mouseenter:function(t){this.$menu.find(".active").removeClass("active"),e(t.currentTarget).addClass("active")}},e.fn.typeahead=function(n){return this.each(function(){var r=e(this),i=r.data("typeahead"),s=typeof n=="object"&&n;i||r.data("typeahead",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.typeahead.defaults={source:[],items:8,menu:'',item:'
  • ',minLength:1},e.fn.typeahead.Constructor=t,e(function(){e("body").on("focus.typeahead.data-api",'[data-provide="typeahead"]',function(t){var n=e(this);if(n.data("typeahead"))return;t.preventDefault(),n.typeahead(n.data())})})}(window.jQuery) \ No newline at end of file diff --git a/web/html/js/typeahead.js b/web/html/js/typeahead.js new file mode 100644 index 00000000..1b7252d7 --- /dev/null +++ b/web/html/js/typeahead.js @@ -0,0 +1,151 @@ +"use strict"; + +const typeahead = (function() { + var input; + var form; + var suggest_type; + var list; + var submit = true; + + function resetResults() { + if (!list) return; + list.style.display = "none"; + list.innerHTML = ""; + } + + function getCompleteList() { + if (!list) { + list = document.createElement("UL"); + list.setAttribute("class", "pkgsearch-typeahead"); + form.appendChild(list); + setListLocation(); + } + return list; + } + + function onListClick(e) { + let target = e.target; + while (!target.getAttribute('data-value')) { + target = target.parentNode; + } + input.value = target.getAttribute('data-value'); + if (submit) { + form.submit(); + } + } + + function setListLocation() { + if (!list) return; + const rects = input.getClientRects()[0]; + list.style.top = (rects.top + rects.height) + "px"; + list.style.left = rects.left + "px"; + } + + function loadData(letter, data) { + const pkgs = data.slice(0, 10); // Show maximum of 10 results + + resetResults(); + + if (pkgs.length === 0) { + return; + } + + const ul = getCompleteList(); + ul.style.display = "block"; + const fragment = document.createDocumentFragment(); + + for (let i = 0; i < pkgs.length; i++) { + const item = document.createElement("li"); + const text = pkgs[i].replace(letter, '' + letter + ''); + item.innerHTML = '' + text + ''; + item.setAttribute('data-value', pkgs[i]); + fragment.appendChild(item); + } + + ul.appendChild(fragment); + ul.addEventListener('click', onListClick); + } + + function fetchData(letter) { + const url = '/rpc?type=' + suggest_type + '&arg=' + letter; + fetch(url).then(function(response) { + return response.json(); + }).then(function(data) { + loadData(letter, data); + }); + } + + function onInputClick() { + if (input.value === "") { + resetResults(); + return; + } + fetchData(input.value); + } + + function onKeyDown(e) { + if (!list) return; + + const elem = document.querySelector(".pkgsearch-typeahead li.active"); + switch(e.keyCode) { + case 13: // enter + if (!submit) { + return; + } + if (elem) { + input.value = elem.getAttribute('data-value'); + form.submit(); + } else { + form.submit(); + } + e.preventDefault(); + break; + case 38: // up + if (elem && elem.previousElementSibling) { + elem.className = ""; + elem.previousElementSibling.className = "active"; + } + e.preventDefault(); + break; + case 40: // down + if (elem && elem.nextElementSibling) { + elem.className = ""; + elem.nextElementSibling.className = "active"; + } else if (!elem && list.childElementCount !== 0) { + list.children[0].className = "active"; + } + e.preventDefault(); + break; + } + } + + // debounce https://davidwalsh.name/javascript-debounce-function + function debounce(func, wait, immediate) { + var timeout; + return function() { + var context = this, args = arguments; + var later = function() { + timeout = null; + if (!immediate) func.apply(context, args); + }; + var callNow = immediate && !timeout; + clearTimeout(timeout); + timeout = setTimeout(later, wait); + if (callNow) func.apply(context, args); + }; + } + + return { + init: function(type, inputfield, formfield, submitdata = true) { + suggest_type = type; + input = inputfield; + form = formfield; + submit = submitdata; + + input.addEventListener("input", onInputClick); + input.addEventListener("keydown", onKeyDown); + window.addEventListener('resize', debounce(setListLocation, 150)); + document.addEventListener("click", resetResults); + } + } +}()); diff --git a/web/html/pkgmerge.php b/web/html/pkgmerge.php index d583c239..d96562a7 100644 --- a/web/html/pkgmerge.php +++ b/web/html/pkgmerge.php @@ -25,7 +25,7 @@ if (has_credential(CRED_PKGBASE_DELETE)): ?>

    -
    +
    @@ -33,25 +33,17 @@ if (has_credential(CRED_PKGBASE_DELETE)): ?> - - +

    -

    +

    " />

    diff --git a/web/template/pkgreq_form.php b/web/template/pkgreq_form.php index d80a422c..9d74093e 100644 --- a/web/template/pkgreq_form.php +++ b/web/template/pkgreq_form.php @@ -9,7 +9,7 @@
  • - +
    @@ -24,44 +24,41 @@

    - - +

    - +

    From c8d88464b1f62154a58d8ee403f3eefb3168aa0f Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Fri, 25 Jun 2021 17:25:24 +0200 Subject: [PATCH 210/844] Update mailing list address https://lists.archlinux.org/pipermail/arch-dev-public/2021-June/030462.html --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7b9ff466..9c8d747e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing -Patches should be sent to the [aur-dev@archlinux.org][1] mailing list. +Patches should be sent to the [aur-dev@lists.archlinux.org][1] mailing list. Before sending patches, you are recommended to run `flake8` and `isort`. From d95e4ec4431ff4e08e409eb1a670f1b0c90035cc Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 25 Jun 2021 17:09:21 -0700 Subject: [PATCH 211/844] Docker: create missing 'aurweb' DB if needed Signed-off-by: Kevin Morris --- docker/scripts/run-mariadb.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/scripts/run-mariadb.sh b/docker/scripts/run-mariadb.sh index 7e908129..d27d8124 100755 --- a/docker/scripts/run-mariadb.sh +++ b/docker/scripts/run-mariadb.sh @@ -8,9 +8,16 @@ done # Create test database. mysql -u root -e "CREATE USER 'aur'@'%' IDENTIFIED BY 'aur'" \ 2>/dev/null || /bin/true + +# Create a brand new 'aurweb_test' DB. mysql -u root -e "DROP DATABASE aurweb_test" 2>/dev/null || /bin/true mysql -u root -e "CREATE DATABASE aurweb_test" mysql -u root -e "GRANT ALL PRIVILEGES ON aurweb_test.* TO 'aur'@'%'" + +# Create the 'aurweb' DB if it does not yet exist. +mysql -u root -e "CREATE DATABASE aurweb" 2>/dev/null || /bin/true +mysql -u root -e "GRANT ALL PRIVILEGES ON aurweb.* TO 'aur'@'%'" + mysql -u root -e "FLUSH PRIVILEGES" # Shutdown mariadb. From 201a04ffb9dddadbd7be2fc587057017426ace2e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 25 Jun 2021 16:17:38 -0700 Subject: [PATCH 212/844] gendummydata: employ a salted hash for users As of Python updates, we are no longer considering rows with empty salts to be legacy hashes. Update gendummydata.py to generate salts for the legacy passwords it uses with salt rounds = 4. Signed-off-by: Kevin Morris --- schema/gendummydata.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/schema/gendummydata.py b/schema/gendummydata.py index 35805d6c..11f2838a 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -16,6 +16,8 @@ import random import sys import time +import bcrypt + LOG_LEVEL = logging.DEBUG # logging level. set to logging.INFO to reduce output SEED_FILE = "/usr/share/dict/words" USER_ID = 5 # Users.ID of first bogus user @@ -182,11 +184,17 @@ for u in user_keys: # pass + # For dummy data, we just use 4 salt rounds. + salt = bcrypt.gensalt(rounds=4).decode() + + # "{salt}{username}" + to_hash = f"{salt}{u}" + h = hashlib.new('md5') - h.update(u.encode()) - s = ("INSERT INTO Users (ID, AccountTypeID, Username, Email, Passwd)" - " VALUES (%d, %d, '%s', '%s@example.com', '%s');\n") - s = s % (seen_users[u], account_type, u, u, h.hexdigest()) + h.update(to_hash.encode()) + s = ("INSERT INTO Users (ID, AccountTypeID, Username, Email, Passwd, Salt)" + " VALUES (%d, %d, '%s', '%s@example.com', '%s', '%s');\n") + s = s % (seen_users[u], account_type, u, u, h.hexdigest(), salt) out.write(s) log.debug("Number of developers: %d" % len(developers)) From eb56305091f13c44716e45746d23fe850c22803c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 25 Jun 2021 17:27:43 -0700 Subject: [PATCH 213/844] gendummydata: lower record counts This commit halves MAX_USERS and MAX_PKGS, in addition to setting OPEN_PROPOSALS to 15 and CLOSE_PROPOSALS to 50. A few counts are now configurable via environment variable: - MAX_USERS, default: 38000 - MAX_PKGS, default: 32000 - OPEN_PROPOSALS, default: 15 - CLOSE_PROPOSALS, default: 15 Signed-off-by: Kevin Morris --- schema/gendummydata.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/schema/gendummydata.py b/schema/gendummydata.py index 11f2838a..9224b051 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -22,18 +22,22 @@ LOG_LEVEL = logging.DEBUG # logging level. set to logging.INFO to reduce output SEED_FILE = "/usr/share/dict/words" USER_ID = 5 # Users.ID of first bogus user PKG_ID = 1 # Packages.ID of first package -MAX_USERS = 76000 # how many users to 'register' +# how many users to 'register' +MAX_USERS = int(os.environ.get("MAX_USERS", 38000)) MAX_DEVS = .1 # what percentage of MAX_USERS are Developers MAX_TUS = .2 # what percentage of MAX_USERS are Trusted Users -MAX_PKGS = 64000 # how many packages to load +# how many packages to load +MAX_PKGS = int(os.environ.get("MAX_PKGS", 32000)) PKG_DEPS = (1, 15) # min/max depends a package has PKG_RELS = (1, 5) # min/max relations a package has PKG_SRC = (1, 3) # min/max sources a package has PKG_CMNTS = (1, 5) # min/max number of comments a package has CATEGORIES_COUNT = 17 # the number of categories from aur-schema VOTING = (0, .001) # percentage range for package voting -OPEN_PROPOSALS = 5 # number of open trusted user proposals -CLOSE_PROPOSALS = 15 # number of closed trusted user proposals +# number of open trusted user proposals +OPEN_PROPOSALS = int(os.environ.get("OPEN_PROPOSALS", 15)) +# number of closed trusted user proposals +CLOSE_PROPOSALS = int(os.environ.get("CLOSE_PROPOSALS", 50)) RANDOM_TLDS = ("edu", "com", "org", "net", "tw", "ru", "pl", "de", "es") RANDOM_URL = ("http://www.", "ftp://ftp.", "http://", "ftp://") RANDOM_LOCS = ("pub", "release", "files", "downloads", "src") From d8556b0d868ef4d3696bbe5a3fe1d5d400537ee4 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 25 Jun 2021 21:22:54 -0700 Subject: [PATCH 214/844] config: add options.salt_rounds During development, the lower this value is (must be >= 4) equals faster User generation. This is particularly useful for running tests. In production, a higher value (like 12 which is used by various popular frameworks) should be used. Signed-off-by: Kevin Morris --- conf/config.defaults | 1 + conf/config.dev | 2 ++ 2 files changed, 3 insertions(+) diff --git a/conf/config.defaults b/conf/config.defaults index 6da4d754..ebc21e51 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -40,6 +40,7 @@ localedir = /srv/http/aurweb/aur.git/web/locale/ cache = none cache_pkginfo_ttl = 86400 memcache_servers = 127.0.0.1:11211 +salt_rounds = 12 [ratelimit] request_limit = 4000 diff --git a/conf/config.dev b/conf/config.dev index 6ef3bb79..fc3bde91 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -26,6 +26,8 @@ aur_location = http://127.0.0.1:8080 disable_http_login = 0 enable-maintenance = 0 localedir = YOUR_AUR_ROOT/web/locale +; In production, salt_rounds should be higher; suggested: 12. +salt_rounds = 4 [notifications] ; For development/testing, use /usr/bin/sendmail From cec07c76b63460865de326e60ab4be8c148b6bc0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 25 Jun 2021 21:24:33 -0700 Subject: [PATCH 215/844] User: use aurweb.config options.salt_rounds Signed-off-by: Kevin Morris --- aurweb/config.py | 4 ++-- aurweb/models/user.py | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/aurweb/config.py b/aurweb/config.py index 2a6cfc3e..73db58dc 100644 --- a/aurweb/config.py +++ b/aurweb/config.py @@ -44,5 +44,5 @@ def getboolean(section, option): return _get_parser().getboolean(section, option) -def getint(section, option): - return _get_parser().getint(section, option) +def getint(section, option, fallback=None): + return _get_parser().getint(section, option, fallback=fallback) diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 9db9add0..bcb47754 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -15,6 +15,8 @@ import aurweb.schema from aurweb.models.ban import is_banned from aurweb.models.declarative import Base +SALT_ROUNDS_DEFAULT = 12 + class User(Base): """ An ORM model of a single Users record. """ @@ -39,16 +41,24 @@ class User(Base): authenticated = False nonce = None + # Make this static to the class just in case SQLAlchemy ever + # does something to bypass our constructor. + salt_rounds = aurweb.config.getint("options", "salt_rounds", + SALT_ROUNDS_DEFAULT) + def __init__(self, Passwd: str = str(), **kwargs): super().__init__(**kwargs) + # Run this again in the constructor in case we rehashed config. + self.salt_rounds = aurweb.config.getint("options", "salt_rounds", + SALT_ROUNDS_DEFAULT) if Passwd: self.update_password(Passwd) - def update_password(self, password, salt_rounds=12): + def update_password(self, password): self.Passwd = bcrypt.hashpw( password.encode(), - bcrypt.gensalt(rounds=salt_rounds)).decode() + bcrypt.gensalt(rounds=self.salt_rounds)).decode() @staticmethod def minimum_passwd_length(): From ff3519ae113dd0f8b19051e9f08ddffefb7adf56 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 25 Jun 2021 22:12:01 -0700 Subject: [PATCH 216/844] [alembic] Log db name being used in a migration Signed-off-by: Kevin Morris --- migrations/env.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/migrations/env.py b/migrations/env.py index dfe14804..7130d141 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -1,3 +1,4 @@ +import logging import logging.config import sqlalchemy @@ -19,12 +20,14 @@ target_metadata = aurweb.schema.metadata # my_important_option = config.get_main_option("my_important_option") # ... etc. - # If configure_logger is either True or not specified, # configure the logger via fileConfig. if config.attributes.get("configure_logger", True): logging.config.fileConfig(config.config_file_name) +# This grabs the root logger in env.py. +logger = logging.getLogger(__name__) + def run_migrations_offline(): """Run migrations in 'offline' mode. @@ -38,6 +41,8 @@ def run_migrations_offline(): script output. """ + db_name = aurweb.config.get("database", "name") + logging.info(f"Performing offline migration on database '{db_name}'.") context.configure( url=aurweb.db.get_sqlalchemy_url(), target_metadata=target_metadata, @@ -56,6 +61,8 @@ def run_migrations_online(): and associate a connection with the context. """ + db_name = aurweb.config.get("database", "name") + logging.info(f"Performing online migration on database '{db_name}'.") connectable = sqlalchemy.create_engine( aurweb.db.get_sqlalchemy_url(), poolclass=sqlalchemy.pool.NullPool, From 9ee7be4a1c8299010df384e91c8e56ec25cc925a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 26 Jun 2021 00:33:32 -0700 Subject: [PATCH 217/844] Docker: remove web/locale from volume mounts This caused a bug where generated locale would not be used. Also, removed appending to /etc/hosts which was bugging out on Mac OS X. archlinux:base-devel seems to come with a valid /etc/hosts. Additionally, remove AUR_CONFIG from Dockerfile. We don't set it up; just use the defaults during installation. Signed-off-by: Kevin Morris --- Dockerfile | 3 --- docker-compose.yml | 28 +++++++++++++++++++++------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5fd3ec07..da9c8d3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,6 @@ FROM archlinux:base-devel # Setup some default system stuff. -RUN bash -c 'echo "127.0.0.1 localhost" >> /etc/hosts' -RUN bash -c 'echo "::1 localhost" >> /etc/hosts' RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime RUN mkdir -p .pkg-cache @@ -28,6 +26,5 @@ WORKDIR /aurweb COPY . . ENV PYTHONPATH=/aurweb -ENV AUR_CONFIG=conf/config RUN make -C po all install diff --git a/docker-compose.yml b/docker-compose.yml index c2c948f5..795236c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -116,7 +116,9 @@ services: - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations - ./test:/aurweb/test - - ./web:/aurweb/web + - ./web/html:/aurweb/web/html + - ./web/template:/aurweb/web/template + - ./web/lib:/aurweb/web/lib - ./templates:/aurweb/templates fastapi: @@ -148,7 +150,9 @@ services: - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations - ./test:/aurweb/test - - ./web:/aurweb/web + - ./web/html:/aurweb/web/html + - ./web/template:/aurweb/web/template + - ./web/lib:/aurweb/web/lib - ./templates:/aurweb/templates nginx: @@ -180,7 +184,9 @@ services: - git_data:/aurweb/aur.git - ./cache:/cache - ./logs:/var/log/nginx - - ./web:/aurweb/web + - ./web/html:/aurweb/web/html + - ./web/template:/aurweb/web/template + - ./web/lib:/aurweb/web/lib sharness: image: aurweb:latest @@ -202,7 +208,9 @@ services: - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations - ./test:/aurweb/test - - ./web:/aurweb/web + - ./web/html:/aurweb/web/html + - ./web/template:/aurweb/web/template + - ./web/lib:/aurweb/web/lib - ./templates:/aurweb/templates pytest-mysql: @@ -229,7 +237,9 @@ services: - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations - ./test:/aurweb/test - - ./web:/aurweb/web + - ./web/html:/aurweb/web/html + - ./web/template:/aurweb/web/template + - ./web/lib:/aurweb/web/lib - ./templates:/aurweb/templates pytest-sqlite: @@ -248,7 +258,9 @@ services: - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations - ./test:/aurweb/test - - ./web:/aurweb/web + - ./web/html:/aurweb/web/html + - ./web/template:/aurweb/web/template + - ./web/lib:/aurweb/web/lib - ./templates:/aurweb/templates depends_on: git: @@ -277,7 +289,9 @@ services: - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations - ./test:/aurweb/test - - ./web:/aurweb/web + - ./web/html:/aurweb/web/html + - ./web/template:/aurweb/web/template + - ./web/lib:/aurweb/web/lib - ./templates:/aurweb/templates volumes: From 07c4be0afbfb467d7c09620a6f01c0761dc0ba6e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 26 Jun 2021 00:42:20 -0700 Subject: [PATCH 218/844] Docker: add .dockerignore Currently, this ignores compiled translation files. Signed-off-by: Kevin Morris --- .dockerignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..30747517 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +*/*.mo From 4927a61378a15cdd8ab0fe277c4acdc8cfd973d9 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 18 Jun 2021 04:40:54 -0700 Subject: [PATCH 219/844] add TUVoteInfo.is_running() method Signed-off-by: Kevin Morris --- aurweb/models/tu_voteinfo.py | 5 +++++ test/test_tu_voteinfo.py | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/aurweb/models/tu_voteinfo.py b/aurweb/models/tu_voteinfo.py index a246f132..fd0031a7 100644 --- a/aurweb/models/tu_voteinfo.py +++ b/aurweb/models/tu_voteinfo.py @@ -1,5 +1,7 @@ import typing +from datetime import datetime + from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship @@ -85,3 +87,6 @@ class TUVoteInfo(Base): """ Customize getattr to floatify any fetched Quorum values. """ attr = super().__getattribute__(key) return float(attr) if key == "Quorum" else attr + + def is_running(self): + return self.End > int(datetime.utcnow().timestamp()) diff --git a/test/test_tu_voteinfo.py b/test/test_tu_voteinfo.py index 37609efd..bd5709fb 100644 --- a/test/test_tu_voteinfo.py +++ b/test/test_tu_voteinfo.py @@ -4,7 +4,7 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, query, rollback +from aurweb.db import commit, create, query, rollback from aurweb.models.account_type import AccountType from aurweb.models.tu_voteinfo import TUVoteInfo from aurweb.models.user import User @@ -49,6 +49,21 @@ def test_tu_voteinfo_creation(): assert tu_voteinfo in user.tu_voteinfo_set +def test_tu_voteinfo_is_running(): + ts = int(datetime.utcnow().timestamp()) + tu_voteinfo = create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=ts, End=ts + 1000, + Quorum=0.5, + Submitter=user) + assert tu_voteinfo.is_running() is True + + tu_voteinfo.End = ts - 5 + commit() + assert tu_voteinfo.is_running() is False + + def test_tu_voteinfo_null_submitter_raises_exception(): with pytest.raises(IntegrityError): create(TUVoteInfo, From ef4a7308ee16e0e42b6adff170dccc259807cade Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 24 Jun 2021 19:28:36 -0700 Subject: [PATCH 220/844] add AccountType constants New constants (in aurweb.models.account_type): - USER: "User" - USER_ID: USER's ID - TRUSTED_USER: "Trusted User" - TRUSTED_USER_ID: TRUSTED_USER's ID - DEVELOPER: "Developer" - DEVELOPER_ID: DEVELOPER's ID - TRUSTED_USER_AND_DEV: "TRUSTED_USER_AND_DEV" - TRUSTED_USER_AND_DEV_ID: TRUSTED_USER_AND_DEV's ID Signed-off-by: Kevin Morris --- aurweb/models/account_type.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/aurweb/models/account_type.py b/aurweb/models/account_type.py index 502a86b1..ca302e5b 100644 --- a/aurweb/models/account_type.py +++ b/aurweb/models/account_type.py @@ -1,5 +1,6 @@ from sqlalchemy import Column, Integer +from aurweb import db from aurweb.models.declarative import Base @@ -20,3 +21,30 @@ class AccountType(Base): def __repr__(self): return "" % ( self.ID, str(self)) + + +# Define some AccountType.AccountType constants. +USER = "User" +TRUSTED_USER = "Trusted User" +DEVELOPER = "Developer" +TRUSTED_USER_AND_DEV = "Trusted User & Developer" + +# Fetch account type IDs from the database for constants. +_account_types = db.query(AccountType) +USER_ID = _account_types.filter( + AccountType.AccountType == USER).first().ID +TRUSTED_USER_ID = _account_types.filter( + AccountType.AccountType == TRUSTED_USER).first().ID +DEVELOPER_ID = _account_types.filter( + AccountType.AccountType == DEVELOPER).first().ID +TRUSTED_USER_AND_DEV_ID = _account_types.filter( + AccountType.AccountType == TRUSTED_USER_AND_DEV).first().ID +_account_types = None # Get rid of the query handle. + +# Map string constants to integer constants. +ACCOUNT_TYPE_ID = { + USER: USER_ID, + TRUSTED_USER: TRUSTED_USER_ID, + DEVELOPER: DEVELOPER_ID, + TRUSTED_USER_AND_DEV: TRUSTED_USER_AND_DEV_ID +} From d606ebc0f1c5805d92a2a1dae86d097dde38ab30 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 18 Jun 2021 04:41:28 -0700 Subject: [PATCH 221/844] add User.is_trusted_user() and User.is_developer() Signed-off-by: Kevin Morris --- aurweb/models/user.py | 12 ++++++++++++ test/test_user.py | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/aurweb/models/user.py b/aurweb/models/user.py index bcb47754..1762f004 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -156,6 +156,18 @@ class User(Base): session.delete(self.session) session.commit() + def is_trusted_user(self): + return self.AccountType.ID in { + aurweb.models.account_type.TRUSTED_USER_ID, + aurweb.models.account_type.TRUSTED_USER_AND_DEV_ID + } + + def is_developer(self): + return self.AccountType.ID in { + aurweb.models.account_type.DEVELOPER_ID, + aurweb.models.account_type.TRUSTED_USER_AND_DEV_ID + } + def __repr__(self): return "" % ( self.ID, str(self.AccountType), self.Username) diff --git a/test/test_user.py b/test/test_user.py index 06585207..9ab40801 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -9,7 +9,7 @@ import pytest import aurweb.auth import aurweb.config -from aurweb.db import create, query +from aurweb.db import commit, create, query from aurweb.models.account_type import AccountType from aurweb.models.ban import Ban from aurweb.models.session import Session @@ -217,3 +217,35 @@ def test_user_as_dict(): assert data.get("Email") == user.Email # .as_dict() does not convert values to json-capable types. assert isinstance(data.get("RegistrationTS"), datetime) + + +def test_user_is_trusted_user(): + tu_type = query(AccountType, + AccountType.AccountType == "Trusted User").first() + user.AccountType = tu_type + commit() + assert user.is_trusted_user() is True + + # Do it again with the combined role. + tu_type = query( + AccountType, + AccountType.AccountType == "Trusted User & Developer").first() + user.AccountType = tu_type + commit() + assert user.is_trusted_user() is True + + +def test_user_is_developer(): + dev_type = query(AccountType, + AccountType.AccountType == "Developer").first() + user.AccountType = dev_type + commit() + assert user.is_developer() is True + + # Do it again with the combined role. + dev_type = query( + AccountType, + AccountType.AccountType == "Trusted User & Developer").first() + user.AccountType = dev_type + commit() + assert user.is_developer() is True From a6bba601a98aef9b19a4a2b5114557b21706c1d5 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 19 Jun 2021 10:46:44 -0700 Subject: [PATCH 222/844] add util.get_vote -> `get_vote` Jinja2 filter This filter gets a vote of a request's user toward a voteinfo. Example: {% set vote = (voteinfo | get_vote(request)) %} Signed-off-by: Kevin Morris --- aurweb/templates.py | 1 + aurweb/util.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/aurweb/templates.py b/aurweb/templates.py index bb4047f4..b8853593 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -29,6 +29,7 @@ env.filters["dt"] = util.timestamp_to_datetime env.filters["as_timezone"] = util.as_timezone env.filters["dedupe_qs"] = util.dedupe_qs env.filters["urlencode"] = quote_plus +env.filters["get_vote"] = util.get_vote # Add captcha filters. env.filters["captcha_salt"] = captcha.captcha_salt_filter diff --git a/aurweb/util.py b/aurweb/util.py index e5f510ce..adbff755 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -10,6 +10,8 @@ from datetime import datetime from urllib.parse import quote_plus, urlparse from zoneinfo import ZoneInfo +import fastapi + from email_validator import EmailNotValidError, EmailUndeliverableError, validate_email from fastapi.responses import Response from jinja2 import pass_context @@ -143,6 +145,11 @@ def dedupe_qs(query_string: str, *additions): return '&'.join([f"{k}={quote_plus(v)}" for k, v in reversed(qs.items())]) +def get_vote(voteinfo, request: fastapi.Request): + from aurweb.models.tu_vote import TUVote + return voteinfo.tu_votes.filter(TUVote.User == request.user).first() + + def jsonify(obj): """ Perform a conversion on obj if it's needed. """ if isinstance(obj, datetime): From d674aaf736383a82d0bc900a5b8bcfc7e41537b9 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 18 Jun 2021 04:33:48 -0700 Subject: [PATCH 223/844] add /tu/ (get) index This commit implements the '/tu' Trusted User index page. In addition to this functionality, this commit introduces the following jinja2 filters: - dt: util.timestamp_to_datetime - as_timezone: util.as_timezone - dedupe_qs: util.dedupe_qs - urlencode: urllib.parse.quote_plus There's also a new decorator that can be used to enforce permissions: `account_type_required`. If a user does not meet account type requirements, they are redirected to '/'. ``` @auth_required(True) @account_type_required({"Trusted User"}) async def some_route(request: fastapi.Request): return Response("You are a Trusted User!") ``` Routes added: - `GET /tu`: aurweb.routers.trusted_user.trusted_user Signed-off-by: Kevin Morris --- aurweb/asgi.py | 3 +- aurweb/auth.py | 39 +++ aurweb/models/account_type.py | 5 + aurweb/routers/trusted_user.py | 97 ++++++ templates/partials/archdev-navbar.html | 18 + templates/partials/tu/last_votes.html | 33 ++ templates/partials/tu/proposals.html | 120 +++++++ templates/tu/index.html | 35 ++ test/test_auth.py | 18 +- test/test_trusted_user_routes.py | 443 +++++++++++++++++++++++++ 10 files changed, 808 insertions(+), 3 deletions(-) create mode 100644 aurweb/routers/trusted_user.py create mode 100644 templates/partials/tu/last_votes.html create mode 100644 templates/partials/tu/proposals.html create mode 100644 templates/tu/index.html create mode 100644 test/test_trusted_user_routes.py diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 65318907..a674fec6 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -16,7 +16,7 @@ from aurweb.auth import BasicAuthBackend from aurweb.db import get_engine, query from aurweb.models.accepted_term import AcceptedTerm from aurweb.models.term import Term -from aurweb.routers import accounts, auth, errors, html, sso +from aurweb.routers import accounts, auth, errors, html, sso, trusted_user # Setup the FastAPI app. app = FastAPI(exception_handlers=errors.exceptions) @@ -47,6 +47,7 @@ async def app_startup(): app.include_router(html.router) app.include_router(auth.router) app.include_router(accounts.router) + app.include_router(trusted_user.router) # Initialize the database engine and ORM. get_engine() diff --git a/aurweb/auth.py b/aurweb/auth.py index ba5f0fea..316e7293 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -3,6 +3,8 @@ import functools from datetime import datetime from http import HTTPStatus +import fastapi + from fastapi.responses import RedirectResponse from sqlalchemy import and_ from starlette.authentication import AuthCredentials, AuthenticationBackend @@ -11,6 +13,7 @@ from starlette.requests import HTTPConnection import aurweb.config from aurweb import l10n, util +from aurweb.models.account_type import ACCOUNT_TYPE_ID from aurweb.models.session import Session from aurweb.models.user import User from aurweb.templates import make_variable_context, render_template @@ -152,6 +155,42 @@ def auth_required(is_required: bool = True, return decorator +def account_type_required(one_of: set): + """ A decorator that can be used on FastAPI routes to dictate + that a user belongs to one of the types defined in one_of. + + This decorator should be run after an @auth_required(True) is + dictated. + + - Example code: + + @router.get('/some_route') + @auth_required(True) + @account_type_required({"Trusted User", "Trusted User & Developer"}) + async def some_route(request: fastapi.Request): + return Response() + + :param one_of: A set consisting of strings to match against AccountType. + :return: Return the FastAPI function this decorator wraps. + """ + # Convert any account type string constants to their integer IDs. + one_of = { + ACCOUNT_TYPE_ID[atype] + for atype in one_of + if isinstance(atype, str) + } + + def decorator(func): + @functools.wraps(func) + async def wrapper(request: fastapi.Request, *args, **kwargs): + if request.user.AccountType.ID not in one_of: + return RedirectResponse("/", + status_code=int(HTTPStatus.SEE_OTHER)) + return await func(request, *args, **kwargs) + return wrapper + return decorator + + CRED_ACCOUNT_CHANGE_TYPE = 1 CRED_ACCOUNT_EDIT = 2 CRED_ACCOUNT_EDIT_DEV = 3 diff --git a/aurweb/models/account_type.py b/aurweb/models/account_type.py index ca302e5b..0db37ced 100644 --- a/aurweb/models/account_type.py +++ b/aurweb/models/account_type.py @@ -3,6 +3,11 @@ from sqlalchemy import Column, Integer from aurweb import db from aurweb.models.declarative import Base +USER = "User" +TRUSTED_USER = "Trusted User" +DEVELOPER = "Developer" +TRUSTED_USER_AND_DEV = "Trusted User & Developer" + class AccountType(Base): """ An ORM model of a single AccountTypes record. """ diff --git a/aurweb/routers/trusted_user.py b/aurweb/routers/trusted_user.py new file mode 100644 index 00000000..c027f67d --- /dev/null +++ b/aurweb/routers/trusted_user.py @@ -0,0 +1,97 @@ +from datetime import datetime +from urllib.parse import quote_plus + +from fastapi import APIRouter, Request +from sqlalchemy import and_, or_ + +from aurweb import db +from aurweb.auth import account_type_required, auth_required +from aurweb.models.account_type import DEVELOPER, TRUSTED_USER, TRUSTED_USER_AND_DEV +from aurweb.models.tu_vote import TUVote +from aurweb.models.tu_voteinfo import TUVoteInfo +from aurweb.models.user import User +from aurweb.templates import make_context, render_template + +router = APIRouter() + +# Some TU route specific constants. +ITEMS_PER_PAGE = 10 # Paged table size. +MAX_AGENDA_LENGTH = 75 # Agenda table column length. + +# A set of account types that will approve a user for TU actions. +REQUIRED_TYPES = { + TRUSTED_USER, + DEVELOPER, + TRUSTED_USER_AND_DEV +} + + +@router.get("/tu") +@auth_required(True, redirect="/") +@account_type_required(REQUIRED_TYPES) +async def trusted_user(request: Request, + coff: int = 0, # current offset + cby: str = "desc", # current by + poff: int = 0, # past offset + pby: str = "desc"): # past by + context = make_context(request, "Trusted User") + + current_by, past_by = cby, pby + current_off, past_off = coff, poff + + context["pp"] = pp = ITEMS_PER_PAGE + context["prev_len"] = MAX_AGENDA_LENGTH + + ts = int(datetime.utcnow().timestamp()) + + if current_by not in {"asc", "desc"}: + # If a malicious by was given, default to desc. + current_by = "desc" + context["current_by"] = current_by + + if past_by not in {"asc", "desc"}: + # If a malicious by was given, default to desc. + past_by = "desc" + context["past_by"] = past_by + + current_votes = db.query(TUVoteInfo, TUVoteInfo.End > ts).order_by( + TUVoteInfo.Submitted.desc()) + context["current_votes_count"] = current_votes.count() + current_votes = current_votes.limit(pp).offset(current_off) + context["current_votes"] = reversed(current_votes.all()) \ + if current_by == "asc" else current_votes.all() + context["current_off"] = current_off + + past_votes = db.query(TUVoteInfo, TUVoteInfo.End <= ts).order_by( + TUVoteInfo.Submitted.desc()) + context["past_votes_count"] = past_votes.count() + past_votes = past_votes.limit(pp).offset(past_off) + context["past_votes"] = reversed(past_votes.all()) \ + if past_by == "asc" else past_votes.all() + context["past_off"] = past_off + + # TODO + # We order last votes by TUVote.VoteID and User.Username. + # This is really bad. We should add a Created column to + # TUVote of type Timestamp and order by that instead. + last_votes_by_tu = db.query(TUVote).filter( + and_(TUVote.VoteID == TUVoteInfo.ID, + TUVoteInfo.End <= ts, + TUVote.UserID == User.ID, + or_(User.AccountTypeID == 2, + User.AccountTypeID == 4)) + ).group_by(User.ID).order_by( + TUVote.VoteID.desc(), User.Username.asc()) + context["last_votes_by_tu"] = last_votes_by_tu.all() + + context["current_by_next"] = "asc" if current_by == "desc" else "desc" + context["past_by_next"] = "asc" if past_by == "desc" else "desc" + + context["q"] = '&'.join([ + f"coff={current_off}", + f"cby={quote_plus(current_by)}", + f"poff={past_off}", + f"pby={quote_plus(past_by)}" + ]) + + return render_template(request, "tu/index.html", context) diff --git a/templates/partials/archdev-navbar.html b/templates/partials/archdev-navbar.html index c935fd41..c6cd3f19 100644 --- a/templates/partials/archdev-navbar.html +++ b/templates/partials/archdev-navbar.html @@ -7,11 +7,29 @@ {% endif %}

  • {% trans %}Packages{% endtrans %}
  • {% if request.user.is_authenticated() %} + + {% if request.user.is_trusted_user() or request.user.is_developer() %} +
  • + {% trans %}Requests{% endtrans %} +
  • + +
  • + {% trans %}Accounts{% endtrans %} +
  • + {% endif %} +
  • {% trans %}My Account{% endtrans %}
  • + + {% if request.user.is_trusted_user() %} +
  • + {% trans %}Trusted User{% endtrans %} +
  • + {% endif %} +
  • {% trans %}Logout{% endtrans %} diff --git a/templates/partials/tu/last_votes.html b/templates/partials/tu/last_votes.html new file mode 100644 index 00000000..94b9c1e8 --- /dev/null +++ b/templates/partials/tu/last_votes.html @@ -0,0 +1,33 @@ +
    +

    {% trans %}{{ title }}{% endtrans %}

    + +
  • + + + + + + + {% if not votes %} + + + + + {% else %} + {% for vote in votes %} + + + + + {% endfor %} + {% endif %} + +
    {{ "User" | tr }}{{ "Last vote" | tr }}
    + {{ "No results found." | tr }} +
    {{ vote.User.Username }} + + {{ vote.VoteID }} + +
    + +
    diff --git a/templates/partials/tu/proposals.html b/templates/partials/tu/proposals.html new file mode 100644 index 00000000..13e705fc --- /dev/null +++ b/templates/partials/tu/proposals.html @@ -0,0 +1,120 @@ +
    +

    {% trans %}{{ title }}{% endtrans %}

    + + {% if title == "Current Votes" %} +
    + {% endif %} + + {% if not results %} +

    + {% trans %}No results found.{% endtrans %} +

    + {% else %} + + + + + + + + + {% if title != "Current Votes" %} + + + {% endif %} + + + + + + {% for result in results %} + + + + + {% set submitted = result.Submitted | dt | as_timezone(timezone) %} + + + + {% set end = result.End | dt | as_timezone(timezone) %} + + + + + {% if title != "Current Votes" %} + + + {% endif %} + + {% set vote = (result | get_vote(request)) %} + + + {% endfor %} + +
    {{ "Proposal" | tr }} + {% set off_qs = "%s=%d" | format(off_param, off) %} + {% set by_qs = "%s=%s" | format(by_param, by_next | urlencode) %} + + {{ "Start" | tr }} + + {{ "End" | tr }}{{ "User" | tr }}{{ "Yes" | tr }}{{ "No" | tr }}{{ "Voted" | tr }}
    + + {% set agenda = result.Agenda[:prev_len] %} + {{ agenda }} + {{ submitted.strftime("%Y-%m-%d") }}{{ end.strftime("%Y-%m-%d") }} + {% if not result.User %} + N/A + {% else %} + + {{ result.User }} + + {% endif %} + {{ result.Yes }}{{ result.No }} + {% if vote %} + + {{ "Yes" | tr }} + + {% else %} + + {{ "No" | tr }} + + {% endif %} +
    + +
    +

    + {% if total_votes > pp %} + + {% if off > 0 %} + {% set off_qs = "%s=%d" | format(off_param, off - 10) %} + {% set by_qs = "%s=%s" | format(by_param, by | urlencode) %} + + ‹ Back + + {% endif %} + + {% if off < total_votes - pp %} + {% set off_qs = "%s=%d" | format(off_param, off + 10) %} + {% set by_qs = "%s=%s" | format(by_param, by | urlencode) %} + + Next › + + {% endif %} + + {% endif %} +

    +
    + + {% endif %} + +
    diff --git a/templates/tu/index.html b/templates/tu/index.html new file mode 100644 index 00000000..5060e1f7 --- /dev/null +++ b/templates/tu/index.html @@ -0,0 +1,35 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} + {% + with table_class = "current-votes", + total_votes = current_votes_count, + results = current_votes, + off_param = "coff", + by_param = "cby", + by_next = current_by_next, + title = "Current Votes", + off = current_off, + by = current_by + %} + {% include "partials/tu/proposals.html" %} + {% endwith %} + + {% + with table_class = "past-votes", + total_votes = past_votes_count, + results = past_votes, + off_param = "poff", + by_param = "pby", + by_next = past_by_next, + title = "Past Votes", + off = past_off, + by = past_by + %} + {% include "partials/tu/proposals.html" %} + {% endwith %} + + {% with title = "Last Votes by TU", votes = last_votes_by_tu %} + {% include "partials/tu/last_votes.html" %} + {% endwith %} +{% endblock %} diff --git a/test/test_auth.py b/test/test_auth.py index e5e1de11..b386bea1 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -4,9 +4,9 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.auth import BasicAuthBackend, has_credential +from aurweb.auth import BasicAuthBackend, account_type_required, has_credential from aurweb.db import create, query -from aurweb.models.account_type import AccountType +from aurweb.models.account_type import USER, USER_ID, AccountType from aurweb.models.session import Session from aurweb.models.user import User from aurweb.testing import setup_test_db @@ -76,3 +76,17 @@ async def test_basic_auth_backend(): def test_has_fake_credential_fails(): # Fake credential 666 does not exist. assert not has_credential(user, 666) + + +def test_account_type_required(): + """ This test merely asserts that a few different paths + do not raise exceptions. """ + # This one shouldn't raise. + account_type_required({USER}) + + # This one also shouldn't raise. + account_type_required({USER_ID}) + + # But this one should! We have no "FAKE" key. + with pytest.raises(KeyError): + account_type_required({'FAKE'}) diff --git a/test/test_trusted_user_routes.py b/test/test_trusted_user_routes.py new file mode 100644 index 00000000..a6527e6f --- /dev/null +++ b/test/test_trusted_user_routes.py @@ -0,0 +1,443 @@ +import re + +from datetime import datetime +from http import HTTPStatus +from io import StringIO + +import lxml.etree +import pytest + +from fastapi.testclient import TestClient + +from aurweb import db +from aurweb.models.account_type import AccountType +from aurweb.models.tu_vote import TUVote +from aurweb.models.tu_voteinfo import TUVoteInfo +from aurweb.models.user import User +from aurweb.testing import setup_test_db +from aurweb.testing.requests import Request + +DATETIME_REGEX = r'^[0-9]{4}-[0-9]{2}-[0-9]{2}$' + + +def parse_root(html): + parser = lxml.etree.HTMLParser(recover=True) + tree = lxml.etree.parse(StringIO(html), parser) + return tree.getroot() + + +def get_table(root, class_name): + table = root.xpath(f'//table[contains(@class, "{class_name}")]')[0] + return table + + +def get_table_rows(table): + tbody = table.xpath("./tbody")[0] + return tbody.xpath("./tr") + + +def get_pkglist_directions(table): + stats = table.getparent().xpath("./div[@class='pkglist-stats']")[0] + nav = stats.xpath("./p[@class='pkglist-nav']")[0] + return nav.xpath("./a") + + +def get_a(node): + return node.xpath('./a')[0].text.strip() + + +def get_span(node): + return node.xpath('./span')[0].text.strip() + + +def assert_current_vote_html(row, expected): + columns = row.xpath("./td") + proposal, start, end, user, voted = columns + p, s, e, u, v = expected # Column expectations. + assert re.match(p, get_a(proposal)) is not None + assert re.match(s, start.text) is not None + assert re.match(e, end.text) is not None + assert re.match(u, get_a(user)) is not None + assert re.match(v, get_span(voted)) is not None + + +def assert_past_vote_html(row, expected): + columns = row.xpath("./td") + proposal, start, end, user, yes, no, voted = columns # Real columns. + p, s, e, u, y, n, v = expected # Column expectations. + assert re.match(p, get_a(proposal)) is not None + assert re.match(s, start.text) is not None + assert re.match(e, end.text) is not None + assert re.match(u, get_a(user)) is not None + assert re.match(y, yes.text) is not None + assert re.match(n, no.text) is not None + assert re.match(v, get_span(voted)) is not None + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db("TU_Votes", "TU_VoteInfo", "Users") + + +@pytest.fixture +def client(): + from aurweb.asgi import app + yield TestClient(app=app) + + +@pytest.fixture +def tu_user(): + tu_type = db.query(AccountType, + AccountType.AccountType == "Trusted User").first() + yield db.create(User, Username="test_tu", Email="test_tu@example.org", + RealName="Test TU", Passwd="testPassword", + AccountType=tu_type) + + +@pytest.fixture +def user(): + user_type = db.query(AccountType, + AccountType.AccountType == "User").first() + yield db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=user_type) + + +def test_tu_index_guest(client): + with client as request: + response = request.get("/tu", allow_redirects=False) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.headers.get("location") == "/" + + +def test_tu_index_unauthorized(client, user): + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + # Login as a normal user, not a TU. + response = request.get("/tu", cookies=cookies, allow_redirects=False) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.headers.get("location") == "/" + + +def test_tu_empty_index(client, tu_user): + """ Check an empty index when we don't create any records. """ + + # Make a default get request to /tu. + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + response = request.get("/tu", cookies=cookies, allow_redirects=False) + assert response.status_code == int(HTTPStatus.OK) + + # Parse lxml root. + root = parse_root(response.text) + + # Check that .current-votes does not exist. + tables = root.xpath('//table[contains(@class, "current-votes")]') + assert len(tables) == 0 + + # Check that .past-votes has does not exist. + tables = root.xpath('//table[contains(@class, "current-votes")]') + assert len(tables) == 0 + + +def test_tu_index(client, tu_user): + ts = int(datetime.utcnow().timestamp()) + + # Create some test votes: (Agenda, Start, End). + votes = [ + ("Test agenda 1", ts - 5, ts + 1000), # Still running. + ("Test agenda 2", ts - 1000, ts - 5) # Not running anymore. + ] + vote_records = [] + for vote in votes: + agenda, start, end = vote + vote_records.append( + db.create(TUVoteInfo, Agenda=agenda, + User=tu_user.Username, + Submitted=start, End=end, + Quorum=0.0, + Submitter=tu_user)) + + # Vote on an ended proposal. + vote_record = vote_records[1] + vote_record.Yes += 1 + vote_record.ActiveTUs += 1 + db.create(TUVote, VoteInfo=vote_record, User=tu_user) + + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + # Pass an invalid cby and pby; let them default to "desc". + response = request.get("/tu", cookies=cookies, params={ + "cby": "BAD!", + "pby": "blah" + }, allow_redirects=False) + + assert response.status_code == int(HTTPStatus.OK) + + # Rows we expect to exist in HTML produced by /tu for current votes. + expected_rows = [ + ( + r'Test agenda 1', + DATETIME_REGEX, + DATETIME_REGEX, + tu_user.Username, + r'^(Yes|No)$' + ) + ] + + # Assert that we are matching the number of current votes. + current_votes = [c for c in votes if c[2] > ts] + assert len(current_votes) == len(expected_rows) + + # Parse lxml.etree root. + root = parse_root(response.text) + + table = get_table(root, "current-votes") + rows = get_table_rows(table) + for i, row in enumerate(rows): + assert_current_vote_html(row, expected_rows[i]) + + # Assert that we are matching the number of past votes. + past_votes = [c for c in votes if c[2] <= ts] + assert len(past_votes) == len(expected_rows) + + # Rows we expect to exist in HTML produced by /tu for past votes. + expected_rows = [ + ( + r'Test agenda 2', + DATETIME_REGEX, + DATETIME_REGEX, + tu_user.Username, + r'^\d+$', + r'^\d+$', + r'^(Yes|No)$' + ) + ] + + table = get_table(root, "past-votes") + rows = get_table_rows(table) + for i, row in enumerate(rows): + assert_past_vote_html(row, expected_rows[i]) + + # Get the .last-votes table and check that our vote shows up. + table = get_table(root, "last-votes") + rows = get_table_rows(table) + assert len(rows) == 1 + + # Check to see the rows match up to our user and related vote. + username, vote_id = rows[0] + vote_id = vote_id.xpath("./a")[0] + assert username.text.strip() == tu_user.Username + assert int(vote_id.text.strip()) == vote_records[1].ID + + +def test_tu_index_table_paging(client, tu_user): + ts = int(datetime.utcnow().timestamp()) + + for i in range(25): + # Create 25 current votes. + db.create(TUVoteInfo, Agenda=f"Agenda #{i}", + User=tu_user.Username, + Submitted=(ts - 5), End=(ts + 1000), + Quorum=0.0, + Submitter=tu_user, autocommit=False) + + for i in range(25): + # Create 25 past votes. + db.create(TUVoteInfo, Agenda=f"Agenda #{25 + i}", + User=tu_user.Username, + Submitted=(ts - 1000), End=(ts - 5), + Quorum=0.0, + Submitter=tu_user, autocommit=False) + db.commit() + + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + response = request.get("/tu", cookies=cookies, allow_redirects=False) + assert response.status_code == int(HTTPStatus.OK) + + # Parse lxml.etree root. + root = parse_root(response.text) + + table = get_table(root, "current-votes") + rows = get_table_rows(table) + assert len(rows) == 10 + + def make_expectation(offset, i): + return [ + f"Agenda #{offset + i}", + DATETIME_REGEX, + DATETIME_REGEX, + tu_user.Username, + r'^(Yes|No)$' + ] + + for i, row in enumerate(rows): + assert_current_vote_html(row, make_expectation(0, i)) + + # Parse out Back/Next buttons. + directions = get_pkglist_directions(table) + assert len(directions) == 1 + assert "Next" in directions[0].text + + # Now, get the next page of current votes. + offset = 10 # Specify coff=10 + with client as request: + response = request.get("/tu", cookies=cookies, params={ + "coff": offset + }, allow_redirects=False) + assert response.status_code == int(HTTPStatus.OK) + + old_rows = rows + root = parse_root(response.text) + + table = get_table(root, "current-votes") + rows = get_table_rows(table) + assert rows != old_rows + + for i, row in enumerate(rows): + assert_current_vote_html(row, make_expectation(offset, i)) + + # Parse out Back/Next buttons. + directions = get_pkglist_directions(table) + assert len(directions) == 2 + assert "Back" in directions[0].text + assert "Next" in directions[1].text + + # Make sure past-votes' Back/Next were not affected. + past_votes = get_table(root, "past-votes") + past_directions = get_pkglist_directions(past_votes) + assert len(past_directions) == 1 + assert "Next" in past_directions[0].text + + offset = 20 # Specify coff=10 + with client as request: + response = request.get("/tu", cookies=cookies, params={ + "coff": offset + }, allow_redirects=False) + assert response.status_code == int(HTTPStatus.OK) + + # Do it again, we only have five left. + old_rows = rows + root = parse_root(response.text) + + table = get_table(root, "current-votes") + rows = get_table_rows(table) + assert rows != old_rows + for i, row in enumerate(rows): + assert_current_vote_html(row, make_expectation(offset, i)) + + # Parse out Back/Next buttons. + directions = get_pkglist_directions(table) + assert len(directions) == 1 + assert "Back" in directions[0].text + + # Make sure past-votes' Back/Next were not affected. + past_votes = get_table(root, "past-votes") + past_directions = get_pkglist_directions(past_votes) + assert len(past_directions) == 1 + assert "Next" in past_directions[0].text + + +def test_tu_index_sorting(client, tu_user): + ts = int(datetime.utcnow().timestamp()) + + for i in range(2): + # Create 'Agenda #1' and 'Agenda #2'. + db.create(TUVoteInfo, Agenda=f"Agenda #{i + 1}", + User=tu_user.Username, + Submitted=(ts + 5), End=(ts + 1000), + Quorum=0.0, + Submitter=tu_user, autocommit=False) + + # Let's order each vote one day after the other. + # This will allow us to test the sorting nature + # of the tables. + ts += 86405 + + # Make a default request to /tu. + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + response = request.get("/tu", cookies=cookies, allow_redirects=False) + assert response.status_code == int(HTTPStatus.OK) + + # Get lxml handles of the document. + root = parse_root(response.text) + table = get_table(root, "current-votes") + rows = get_table_rows(table) + + # The latest Agenda is at the top by default. + expected = [ + "Agenda #2", + "Agenda #1" + ] + + assert len(rows) == len(expected) + for i, row in enumerate(rows): + assert_current_vote_html(row, [ + expected[i], + DATETIME_REGEX, + DATETIME_REGEX, + tu_user.Username, + r'^(Yes|No)$' + ]) + + # Make another request; one that sorts the current votes + # in ascending order instead of the default descending order. + with client as request: + response = request.get("/tu", cookies=cookies, params={ + "cby": "asc" + }, allow_redirects=False) + assert response.status_code == int(HTTPStatus.OK) + + # Get lxml handles of the document. + root = parse_root(response.text) + table = get_table(root, "current-votes") + rows = get_table_rows(table) + + # Reverse our expectations and assert that the proposals got flipped. + rev_expected = list(reversed(expected)) + assert len(rows) == len(rev_expected) + for i, row in enumerate(rows): + assert_current_vote_html(row, [ + rev_expected[i], + DATETIME_REGEX, + DATETIME_REGEX, + tu_user.Username, + r'^(Yes|No)$' + ]) + + +def test_tu_index_last_votes(client, tu_user, user): + ts = int(datetime.utcnow().timestamp()) + + # Create a proposal which has ended. + voteinfo = db.create(TUVoteInfo, Agenda="Test agenda", + User=user.Username, + Submitted=(ts - 1000), + End=(ts - 5), + Yes=1, + ActiveTUs=1, + Quorum=0.0, + Submitter=tu_user) + + # Create a vote on it from tu_user. + db.create(TUVote, VoteInfo=voteinfo, User=tu_user) + + # Now, check that tu_user got populated in the .last-votes table. + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + response = request.get("/tu", cookies=cookies) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + table = get_table(root, "last-votes") + rows = get_table_rows(table) + assert len(rows) == 1 + + last_vote = rows[0] + user, vote_id = last_vote.xpath("./td") + vote_id = vote_id.xpath("./a")[0] + + assert user.text.strip() == tu_user.Username + assert int(vote_id.text.strip()) == voteinfo.ID From e534704a98ed7d421c04279e0c4f57aa8ea837b1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 26 Jun 2021 01:10:20 -0700 Subject: [PATCH 224/844] [FastAPI] remove unused Requests navbar item Signed-off-by: Kevin Morris --- templates/partials/archdev-navbar.html | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/templates/partials/archdev-navbar.html b/templates/partials/archdev-navbar.html index c6cd3f19..13459e1a 100644 --- a/templates/partials/archdev-navbar.html +++ b/templates/partials/archdev-navbar.html @@ -7,17 +7,13 @@ {% endif %}
  • {% trans %}Packages{% endtrans %}
  • {% if request.user.is_authenticated() %} - {% if request.user.is_trusted_user() or request.user.is_developer() %}
  • - {% trans %}Requests{% endtrans %} -
  • - -
  • - {% trans %}Accounts{% endtrans %} + + {% trans %}Accounts{% endtrans %} +
  • {% endif %} -
  • {% trans %}My Account{% endtrans %} From dc4cc9b604a9085f631c2649909f6767e6f2ce3e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 19 Jun 2021 01:10:53 -0700 Subject: [PATCH 225/844] add aurweb.asgi.id_redirect_middleware A new middleware which redirects requests going to '/route?id=some_id' to '/route/some_id'. In the FastAPI application, we'll prefer using restful layouts where possible where resource-based ids are parameters of the request uri: '/route/{resource_id}'. Signed-off-by: Kevin Morris --- aurweb/asgi.py | 22 ++++++++++++++++++++++ test/test_routes.py | 10 ++++++++++ 2 files changed, 32 insertions(+) diff --git a/aurweb/asgi.py b/aurweb/asgi.py index a674fec6..35166c73 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -2,6 +2,8 @@ import asyncio import http import typing +from urllib.parse import quote_plus + from fastapi import FastAPI, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse from fastapi.staticfiles import StaticFiles @@ -120,3 +122,23 @@ async def check_terms_of_service(request: Request, call_next: typing.Callable): task = asyncio.create_task(call_next(request)) await asyncio.wait({task}, return_when=asyncio.FIRST_COMPLETED) return task.result() + + +@app.middleware("http") +async def id_redirect_middleware(request: Request, call_next: typing.Callable): + id = request.query_params.get("id") + + if id is not None: + # Preserve query string. + qs = [] + for k, v in request.query_params.items(): + if k != "id": + qs.append(f"{k}={quote_plus(str(v))}") + qs = str() if not qs else '?' + '&'.join(qs) + + path = request.url.path.rstrip('/') + return RedirectResponse(f"{path}/{id}{qs}") + + task = asyncio.create_task(call_next(request)) + await asyncio.wait({task}, return_when=asyncio.FIRST_COMPLETED) + return task.result() diff --git a/test/test_routes.py b/test/test_routes.py index d67f4a48..a2d1786e 100644 --- a/test/test_routes.py +++ b/test/test_routes.py @@ -148,3 +148,13 @@ def test_nonce_csp(): if not (nonce_verified := (script.get("nonce") == nonce)): break assert nonce_verified is True + + +def test_id_redirect(): + with client as request: + response = request.get("/", params={ + "id": "test", # This param will be rewritten into Location. + "key": "value", # Test that this param persists. + "key2": "value2" # And this one. + }, allow_redirects=False) + assert response.headers.get("location") == "/test?key=value&key2=value2" From ac1779b705d9b0ad87deaa1856e9b5e05d9ae944 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 19 Jun 2021 01:14:10 -0700 Subject: [PATCH 226/844] add util.number_format -> `number_format` Jinja2 filter Implement a `number_format` equivalent to PHP's version. Signed-off-by: Kevin Morris --- aurweb/templates.py | 1 + aurweb/util.py | 5 +++++ test/test_util.py | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/aurweb/templates.py b/aurweb/templates.py index b8853593..8b507425 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -30,6 +30,7 @@ env.filters["as_timezone"] = util.as_timezone env.filters["dedupe_qs"] = util.dedupe_qs env.filters["urlencode"] = quote_plus env.filters["get_vote"] = util.get_vote +env.filters["number_format"] = util.number_format # Add captcha filters. env.filters["captcha_salt"] = captcha.captcha_salt_filter diff --git a/aurweb/util.py b/aurweb/util.py index adbff755..539af40e 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -150,6 +150,11 @@ def get_vote(voteinfo, request: fastapi.Request): return voteinfo.tu_votes.filter(TUVote.User == request.user).first() +def number_format(value: float, places: int): + """ A converter function similar to PHP's number_format. """ + return f"{value:.{places}f}" + + def jsonify(obj): """ Perform a conversion on obj if it's needed. """ if isinstance(obj, datetime): diff --git a/test/test_util.py b/test/test_util.py index 074de494..f54a98a0 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -30,3 +30,8 @@ def test_dedupe_qs(): # Add key1=changed and key2=changed to the query and dedupe it. deduped = util.dedupe_qs(query_string, "key1=changed", "key3=changed") assert deduped == "key2=blah&key1=changed&key3=changed" + + +def test_number_format(): + assert util.number_format(0.222, 2) == "0.22" + assert util.number_format(0.226, 2) == "0.23" From 83c038a42ac50a087bff82490b21acc7e55d65b9 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 19 Jun 2021 01:18:40 -0700 Subject: [PATCH 227/844] add TUVoteInfo.total_votes() Returns the sum of TUVoteInfo.Yes, TUVoteInfo.No and TUVoteInfo.Abstain. Signed-off-by: Kevin Morris --- aurweb/models/tu_voteinfo.py | 3 +++ test/test_tu_voteinfo.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/aurweb/models/tu_voteinfo.py b/aurweb/models/tu_voteinfo.py index fd0031a7..b80073f4 100644 --- a/aurweb/models/tu_voteinfo.py +++ b/aurweb/models/tu_voteinfo.py @@ -90,3 +90,6 @@ class TUVoteInfo(Base): def is_running(self): return self.End > int(datetime.utcnow().timestamp()) + + def total_votes(self): + return self.Yes + self.No + self.Abstain diff --git a/test/test_tu_voteinfo.py b/test/test_tu_voteinfo.py index bd5709fb..494300c5 100644 --- a/test/test_tu_voteinfo.py +++ b/test/test_tu_voteinfo.py @@ -64,6 +64,24 @@ def test_tu_voteinfo_is_running(): assert tu_voteinfo.is_running() is False +def test_tu_voteinfo_total_votes(): + ts = int(datetime.utcnow().timestamp()) + tu_voteinfo = create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=ts, End=ts + 1000, + Quorum=0.5, + Submitter=user) + + tu_voteinfo.Yes = 1 + tu_voteinfo.No = 3 + tu_voteinfo.Abstain = 5 + commit() + + # total_votes() should be the sum of Yes, No and Abstain: 1 + 3 + 5 = 9. + assert tu_voteinfo.total_votes() == 9 + + def test_tu_voteinfo_null_submitter_raises_exception(): with pytest.raises(IntegrityError): create(TUVoteInfo, From 85ba4a33a865ab78cd9e3e1b4e9bd6e410ea8816 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 19 Jun 2021 05:08:25 -0700 Subject: [PATCH 228/844] add /tu/{proposal_id} (get, post) routes This commit ports the `/tu/?id={proposal_id}` PHP routes to FastAPI into two individual GET and POST routes. With this port of the single proposal view and POST logic, several things have changed. - The only parameter used is now `decision`, which must contain `Yes`, `No`, or `Abstain` as a string. When an invalid value is given, a BAD_REQUEST response is returned in plaintext: Invalid 'decision' value. - The `doVote` parameter has been removed. - The details section has been rearranged into a set of divs with specific classes that can be used for testing. CSS has been added to persist the layout with the element changes. - Several errors that can be discovered in the POST path now trigger their own non-200 HTTPStatus codes. Signed-off-by: Kevin Morris --- aurweb/routers/trusted_user.py | 127 ++++++++- templates/partials/tu/proposal/details.html | 106 +++++++ templates/partials/tu/proposal/form.html | 14 + templates/partials/tu/proposal/voters.html | 10 + templates/tu/show.html | 20 ++ test/test_trusted_user_routes.py | 288 ++++++++++++++++++++ web/html/css/aurweb.css | 8 + 7 files changed, 571 insertions(+), 2 deletions(-) create mode 100644 templates/partials/tu/proposal/details.html create mode 100644 templates/partials/tu/proposal/form.html create mode 100644 templates/partials/tu/proposal/voters.html create mode 100644 templates/tu/show.html diff --git a/aurweb/routers/trusted_user.py b/aurweb/routers/trusted_user.py index c027f67d..efdcfc73 100644 --- a/aurweb/routers/trusted_user.py +++ b/aurweb/routers/trusted_user.py @@ -1,7 +1,11 @@ +import typing + from datetime import datetime +from http import HTTPStatus from urllib.parse import quote_plus -from fastapi import APIRouter, Request +from fastapi import APIRouter, Form, HTTPException, Request +from fastapi.responses import Response from sqlalchemy import and_, or_ from aurweb import db @@ -10,7 +14,7 @@ from aurweb.models.account_type import DEVELOPER, TRUSTED_USER, TRUSTED_USER_AND from aurweb.models.tu_vote import TUVote from aurweb.models.tu_voteinfo import TUVoteInfo from aurweb.models.user import User -from aurweb.templates import make_context, render_template +from aurweb.templates import make_context, make_variable_context, render_template router = APIRouter() @@ -95,3 +99,122 @@ async def trusted_user(request: Request, ]) return render_template(request, "tu/index.html", context) + + +def render_proposal(request: Request, + context: dict, + proposal: int, + voteinfo: TUVoteInfo, + voters: typing.Iterable[User], + vote: TUVote, + status_code: HTTPStatus = HTTPStatus.OK): + """ Render a single TU proposal. """ + context["proposal"] = proposal + context["voteinfo"] = voteinfo + context["voters"] = voters + + participation = voteinfo.ActiveTUs / voteinfo.total_votes() \ + if voteinfo.total_votes() else 0 + context["participation"] = participation + + accepted = (voteinfo.Yes > voteinfo.ActiveTUs / 2) or \ + (participation > voteinfo.Quorum and voteinfo.Yes > voteinfo.No) + context["accepted"] = accepted + + can_vote = voters.filter(TUVote.User == request.user).first() is None + context["can_vote"] = can_vote + + if not voteinfo.is_running(): + context["error"] = "Voting is closed for this proposal." + + context["vote"] = vote + context["has_voted"] = vote is not None + + return render_template(request, "tu/show.html", context, + status_code=status_code) + + +@router.get("/tu/{proposal}") +@auth_required(True, redirect="/") +@account_type_required(REQUIRED_TYPES) +async def trusted_user_proposal(request: Request, proposal: int): + context = await make_variable_context(request, "Trusted User") + proposal = int(proposal) + + voteinfo = db.query(TUVoteInfo, TUVoteInfo.ID == proposal).first() + if not voteinfo: + raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) + + voters = db.query(User).join(TUVote).filter(TUVote.VoteID == voteinfo.ID) + vote = db.query(TUVote, and_(TUVote.UserID == request.user.ID, + TUVote.VoteID == voteinfo.ID)).first() + + if not request.user.is_trusted_user(): + context["error"] = "Only Trusted Users are allowed to vote." + elif voteinfo.User == request.user.Username: + context["error"] = "You cannot vote in an proposal about you." + elif vote is not None: + context["error"] = "You've already voted for this proposal." + + context["vote"] = vote + return render_proposal(request, context, proposal, voteinfo, voters, vote) + + +@router.post("/tu/{proposal}") +@auth_required(True, redirect="/") +@account_type_required(REQUIRED_TYPES) +async def trusted_user_proposal_post(request: Request, + proposal: int, + decision: str = Form(...)): + context = await make_variable_context(request, "Trusted User") + proposal = int(proposal) # Make sure it's an int. + + voteinfo = db.query(TUVoteInfo, TUVoteInfo.ID == proposal).first() + if not voteinfo: + raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) + + voters = db.query(User).join(TUVote).filter(TUVote.VoteID == voteinfo.ID) + + # status_code we'll use for responses later. + status_code = HTTPStatus.OK + + if not request.user.is_trusted_user(): + # Test: Create a proposal and view it as a "Developer". It + # should give us this error. + context["error"] = "Only Trusted Users are allowed to vote." + status_code = HTTPStatus.UNAUTHORIZED + elif voteinfo.User == request.user.Username: + context["error"] = "You cannot vote in an proposal about you." + status_code = HTTPStatus.BAD_REQUEST + + vote = db.query(TUVote, and_(TUVote.UserID == request.user.ID, + TUVote.VoteID == voteinfo.ID)).first() + + if status_code != HTTPStatus.OK: + return render_proposal(request, context, proposal, + voteinfo, voters, vote, + status_code=status_code) + + if vote is not None: + context["error"] = "You've already voted for this proposal." + status_code = HTTPStatus.BAD_REQUEST + + if status_code != HTTPStatus.OK: + return render_proposal(request, context, proposal, + voteinfo, voters, vote, + status_code=status_code) + + if decision in {"Yes", "No", "Abstain"}: + # Increment whichever decision was given to us. + setattr(voteinfo, decision, getattr(voteinfo, decision) + 1) + else: + return Response("Invalid 'decision' value.", + status_code=int(HTTPStatus.BAD_REQUEST)) + + vote = db.create(TUVote, User=request.user, VoteInfo=voteinfo, + autocommit=False) + voteinfo.ActiveTUs += 1 + db.commit() + + context["error"] = "You've already voted for this proposal." + return render_proposal(request, context, proposal, voteinfo, voters, vote) diff --git a/templates/partials/tu/proposal/details.html b/templates/partials/tu/proposal/details.html new file mode 100644 index 00000000..3f15a6eb --- /dev/null +++ b/templates/partials/tu/proposal/details.html @@ -0,0 +1,106 @@ +

    {% trans %}Proposal Details{% endtrans %}

    + +{% if voteinfo.is_running() %} +

    + {% trans %}This vote is still running.{% endtrans %} +

    +{% endif %} + + +
    + + + {% set submitted = voteinfo.Submitted | dt | as_timezone(timezone) %} + {% set end = voteinfo.End | dt | as_timezone(timezone) %} + + +
    + {{ "End" | tr }}: + + {{ end.strftime("%Y-%m-%d %H:%M") }} + +
    + + {% if not voteinfo.is_running() %} +
    + {{ "Result" | tr }}: + {% if not voteinfo.ActiveTUs %} + {{ "unknown" | tr }} + {% elif accepted %} + + {{ "Accepted" | tr }} + + {% else %} + + {{ "Rejected" | tr }} + + {% endif %} +
    + {% endif %} +
    + +
    +

    + + {{ voteinfo.Agenda | replace("\n", "
    \n") | safe | e }} +

    +
    + + + + {% if not voteinfo.is_running() %} + + + + {% endif %} + + + + + + + + {% if not voteinfo.is_running() %} + + + + {% endif %} + + + + + +
    {{ "Yes" | tr }}{{ "No" | tr }}{{ "Abstain" | tr }}{{ "Total" | tr }}{{ "Voted" | tr }}{{ "Participation" | tr }}
    {{ voteinfo.Yes }}{{ voteinfo.No }}{{ voteinfo.Abstain }}{{ voteinfo.total_votes() }} + {% if not has_voted %} + + {{ "No" | tr }} + + {% else %} + + {{ "Yes" | tr }} + + {% endif %} + + {% if voteinfo.ActiveTUs %} + {{ (participation * 100) | number_format(2) }}% + {% else %} + {{ "unknown" | tr }} + {% endif %} +
    diff --git a/templates/partials/tu/proposal/form.html b/templates/partials/tu/proposal/form.html new file mode 100644 index 00000000..d783a622 --- /dev/null +++ b/templates/partials/tu/proposal/form.html @@ -0,0 +1,14 @@ + + +
    + + + +
    + diff --git a/templates/partials/tu/proposal/voters.html b/templates/partials/tu/proposal/voters.html new file mode 100644 index 00000000..2fd42bdf --- /dev/null +++ b/templates/partials/tu/proposal/voters.html @@ -0,0 +1,10 @@ +

    {{ "Voters" | tr }}

    + diff --git a/templates/tu/show.html b/templates/tu/show.html new file mode 100644 index 00000000..ca5cbe63 --- /dev/null +++ b/templates/tu/show.html @@ -0,0 +1,20 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} +
    + {% include "partials/tu/proposal/details.html" %} +
    + +
    + {% include "partials/tu/proposal/voters.html" %} +
    + +
    + {% if error %} + {{ error | tr }} + {% else %} + {% include "partials/tu/proposal/form.html" %} + {% endif %} +
    + +{% endblock %} diff --git a/test/test_trusted_user_routes.py b/test/test_trusted_user_routes.py index a6527e6f..73cea9bf 100644 --- a/test/test_trusted_user_routes.py +++ b/test/test_trusted_user_routes.py @@ -18,6 +18,7 @@ from aurweb.testing import setup_test_db from aurweb.testing.requests import Request DATETIME_REGEX = r'^[0-9]{4}-[0-9]{2}-[0-9]{2}$' +PARTICIPATION_REGEX = r'^1?[0-9]{2}[%]$' # 0% - 100% def parse_root(html): @@ -103,6 +104,26 @@ def user(): AccountType=user_type) +@pytest.fixture +def proposal(tu_user): + ts = int(datetime.utcnow().timestamp()) + agenda = "Test proposal." + start = ts - 5 + end = ts + 1000 + + user_type = db.query(AccountType, + AccountType.AccountType == "User").first() + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=user_type) + + voteinfo = db.create(TUVoteInfo, + Agenda=agenda, Quorum=0.0, + User=user.Username, Submitter=tu_user, + Submitted=start, End=end) + yield (tu_user, user, voteinfo) + + def test_tu_index_guest(client): with client as request: response = request.get("/tu", allow_redirects=False) @@ -441,3 +462,270 @@ def test_tu_index_last_votes(client, tu_user, user): assert user.text.strip() == tu_user.Username assert int(vote_id.text.strip()) == voteinfo.ID + + +def test_tu_proposal_not_found(client, tu_user): + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + response = request.get("/tu", params={"id": 1}, cookies=cookies) + assert response.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_tu_running_proposal(client, proposal): + tu_user, user, voteinfo = proposal + + # Initiate an authenticated GET request to /tu/{proposal_id}. + proposal_id = voteinfo.ID + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + response = request.get(f"/tu/{proposal_id}", cookies=cookies) + assert response.status_code == int(HTTPStatus.OK) + + # Alright, now let's continue on to verifying some markup. + # First, let's verify that the proposal details match. + root = parse_root(response.text) + details = root.xpath('//div[@class="proposal details"]')[0] + + vote_running = root.xpath('//p[contains(@class, "vote-running")]')[0] + assert vote_running.text.strip() == "This vote is still running." + + # Verify User field. + username = details.xpath( + './div[contains(@class, "user")]/strong/a/text()')[0] + assert username.strip() == user.Username + + submitted = details.xpath( + './div[contains(@class, "submitted")]/text()')[0] + assert re.match(r'^Submitted: \d{4}-\d{2}-\d{2} \d{2}:\d{2} by .+$', + submitted.strip()) is not None + + end = details.xpath('./div[contains(@class, "end")]')[0] + end_label = end.xpath("./text()")[0] + assert end_label.strip() == "End:" + + end_datetime = end.xpath("./strong/text()")[0] + assert re.match(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}$', + end_datetime.strip()) is not None + + # We have not voted yet. Assert that our voting form is shown. + form = root.xpath('//form[contains(@class, "action-form")]')[0] + fields = form.xpath("./fieldset")[0] + buttons = fields.xpath('./button[@name="decision"]') + assert len(buttons) == 3 + + # Check the button names and values. + yes, no, abstain = buttons + + # Yes + assert yes.attrib["name"] == "decision" + assert yes.attrib["value"] == "Yes" + + # No + assert no.attrib["name"] == "decision" + assert no.attrib["value"] == "No" + + # Abstain + assert abstain.attrib["name"] == "decision" + assert abstain.attrib["value"] == "Abstain" + + # Create a vote. + db.create(TUVote, VoteInfo=voteinfo, User=tu_user) + voteinfo.ActiveTUs += 1 + voteinfo.Yes += 1 + db.commit() + + # Make another request now that we've voted. + with client as request: + response = request.get( + "/tu", params={"id": voteinfo.ID}, cookies=cookies) + assert response.status_code == int(HTTPStatus.OK) + + # Parse our new root. + root = parse_root(response.text) + + # Check that we no longer have a voting form. + form = root.xpath('//form[contains(@class, "action-form")]') + assert not form + + # Check that we're told we've voted. + status = root.xpath('//span[contains(@class, "status")]/text()')[0] + assert status == "You've already voted for this proposal." + + +def test_tu_ended_proposal(client, proposal): + tu_user, user, voteinfo = proposal + + ts = int(datetime.utcnow().timestamp()) + voteinfo.End = ts - 5 # 5 seconds ago. + db.commit() + + # Initiate an authenticated GET request to /tu/{proposal_id}. + proposal_id = voteinfo.ID + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + response = request.get(f"/tu/{proposal_id}", cookies=cookies) + assert response.status_code == int(HTTPStatus.OK) + + # Alright, now let's continue on to verifying some markup. + # First, let's verify that the proposal details match. + root = parse_root(response.text) + details = root.xpath('//div[@class="proposal details"]')[0] + + vote_running = root.xpath('//p[contains(@class, "vote-running")]') + assert not vote_running + + result_node = details.xpath('./div[contains(@class, "result")]')[0] + result_label = result_node.xpath("./text()")[0] + assert result_label.strip() == "Result:" + + result = result_node.xpath("./span/text()")[0] + assert result.strip() == "unknown" + + # Check that voting has ended. + form = root.xpath('//form[contains(@class, "action-form")]') + assert not form + + # We should see a status about it. + status = root.xpath('//span[contains(@class, "status")]/text()')[0] + assert status == "Voting is closed for this proposal." + + +def test_tu_proposal_vote_not_found(client, tu_user): + """ Test POST request to a missing vote. """ + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + data = {"decision": "Yes"} + response = request.post("/tu/1", cookies=cookies, + data=data, allow_redirects=False) + assert response.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_tu_proposal_vote(client, proposal): + tu_user, user, voteinfo = proposal + + # Store the current related values. + yes = voteinfo.Yes + active_tus = voteinfo.ActiveTUs + + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + data = {"decision": "Yes"} + response = request.post(f"/tu/{voteinfo.ID}", cookies=cookies, + data=data) + assert response.status_code == int(HTTPStatus.OK) + + # Check that the proposal record got updated. + assert voteinfo.Yes == yes + 1 + assert voteinfo.ActiveTUs == active_tus + 1 + + # Check that the new TUVote exists. + vote = db.query(TUVote, TUVote.VoteInfo == voteinfo, + TUVote.User == tu_user).first() + assert vote is not None + + root = parse_root(response.text) + + # Check that we're told we've voted. + status = root.xpath('//span[contains(@class, "status")]/text()')[0] + assert status == "You've already voted for this proposal." + + +def test_tu_proposal_vote_unauthorized(client, proposal): + tu_user, user, voteinfo = proposal + + dev_type = db.query(AccountType, + AccountType.AccountType == "Developer").first() + tu_user.AccountType = dev_type + db.commit() + + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + data = {"decision": "Yes"} + response = request.post(f"/tu/{voteinfo.ID}", cookies=cookies, + data=data, allow_redirects=False) + assert response.status_code == int(HTTPStatus.UNAUTHORIZED) + + root = parse_root(response.text) + status = root.xpath('//span[contains(@class, "status")]/text()')[0] + assert status == "Only Trusted Users are allowed to vote." + + with client as request: + data = {"decision": "Yes"} + response = request.get(f"/tu/{voteinfo.ID}", cookies=cookies, + data=data, allow_redirects=False) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + status = root.xpath('//span[contains(@class, "status")]/text()')[0] + assert status == "Only Trusted Users are allowed to vote." + + +def test_tu_proposal_vote_cant_self_vote(client, proposal): + tu_user, user, voteinfo = proposal + + # Update voteinfo.User. + voteinfo.User = tu_user.Username + db.commit() + + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + data = {"decision": "Yes"} + response = request.post(f"/tu/{voteinfo.ID}", cookies=cookies, + data=data, allow_redirects=False) + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + root = parse_root(response.text) + status = root.xpath('//span[contains(@class, "status")]/text()')[0] + assert status == "You cannot vote in an proposal about you." + + with client as request: + data = {"decision": "Yes"} + response = request.get(f"/tu/{voteinfo.ID}", cookies=cookies, + data=data, allow_redirects=False) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + status = root.xpath('//span[contains(@class, "status")]/text()')[0] + assert status == "You cannot vote in an proposal about you." + + +def test_tu_proposal_vote_already_voted(client, proposal): + tu_user, user, voteinfo = proposal + + db.create(TUVote, VoteInfo=voteinfo, User=tu_user) + voteinfo.Yes += 1 + voteinfo.ActiveTUs += 1 + db.commit() + + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + data = {"decision": "Yes"} + response = request.post(f"/tu/{voteinfo.ID}", cookies=cookies, + data=data, allow_redirects=False) + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + + root = parse_root(response.text) + status = root.xpath('//span[contains(@class, "status")]/text()')[0] + assert status == "You've already voted for this proposal." + + with client as request: + data = {"decision": "Yes"} + response = request.get(f"/tu/{voteinfo.ID}", cookies=cookies, + data=data, allow_redirects=False) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + status = root.xpath('//span[contains(@class, "status")]/text()')[0] + assert status == "You've already voted for this proposal." + + +def test_tu_proposal_vote_invalid_decision(client, proposal): + tu_user, user, voteinfo = proposal + + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + data = {"decision": "EVIL"} + response = request.post(f"/tu/{voteinfo.ID}", cookies=cookies, + data=data) + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + assert response.text == "Invalid 'decision' value." diff --git a/web/html/css/aurweb.css b/web/html/css/aurweb.css index bb4e3ad7..2748462f 100644 --- a/web/html/css/aurweb.css +++ b/web/html/css/aurweb.css @@ -204,3 +204,11 @@ label.confirmation, overflow: hidden; transition: height 1s; } + +.proposal.details { + margin: .33em 0 1em; +} + +button[type="submit"] { + padding: 0 0.6em; +} From bfffdd4d912eb012f947a81ff4c51489015fa2df Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 26 Jun 2021 04:13:28 -0700 Subject: [PATCH 229/844] aurweb.asgi: Allow unsafe-inline style-src in CSP Signed-off-by: Kevin Morris --- aurweb/asgi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 35166c73..26893232 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -88,6 +88,8 @@ async def add_security_headers(request: Request, call_next: typing.Callable): "cdn.jsdelivr.net" ] csp += f"script-src 'self' 'nonce-{nonce}' " + ' '.join(script_hosts) + # It's fine if css is inlined. + csp += f"; style-src 'self' 'unsafe-inline'" response.headers["Content-Security-Policy"] = csp # Add XTCO header. From 04ab98907aa6b7e432bbc3b0bd71462bf9ecd513 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 26 Jun 2021 04:20:55 -0700 Subject: [PATCH 230/844] aurweb.asgi: patch invalid f-string Signed-off-by: Kevin Morris --- aurweb/asgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 26893232..228b9a65 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -89,7 +89,7 @@ async def add_security_headers(request: Request, call_next: typing.Callable): ] csp += f"script-src 'self' 'nonce-{nonce}' " + ' '.join(script_hosts) # It's fine if css is inlined. - csp += f"; style-src 'self' 'unsafe-inline'" + csp += "; style-src 'self' 'unsafe-inline'" response.headers["Content-Security-Policy"] = csp # Add XTCO header. From 97c1247b577adb13fac793577d411f0f2be9274a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 26 Jun 2021 04:43:00 -0700 Subject: [PATCH 231/844] /tu/{proposal_id}: Do not show voters if there are none This was different than PHP. Signed-off-by: Kevin Morris --- aurweb/routers/trusted_user.py | 2 +- templates/tu/show.html | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/aurweb/routers/trusted_user.py b/aurweb/routers/trusted_user.py index efdcfc73..55f7b7e1 100644 --- a/aurweb/routers/trusted_user.py +++ b/aurweb/routers/trusted_user.py @@ -111,7 +111,7 @@ def render_proposal(request: Request, """ Render a single TU proposal. """ context["proposal"] = proposal context["voteinfo"] = voteinfo - context["voters"] = voters + context["voters"] = voters.all() participation = voteinfo.ActiveTUs / voteinfo.total_votes() \ if voteinfo.total_votes() else 0 diff --git a/templates/tu/show.html b/templates/tu/show.html index ca5cbe63..ff2d4bb6 100644 --- a/templates/tu/show.html +++ b/templates/tu/show.html @@ -4,10 +4,12 @@
    {% include "partials/tu/proposal/details.html" %}
    - -
    - {% include "partials/tu/proposal/voters.html" %} -
    + + {% if voters %} +
    + {% include "partials/tu/proposal/voters.html" %} +
    + {% endif %}
    {% if error %} From 83f93c8dbb1bc823fa8bcc3966d572255dc742e2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 27 Jun 2021 03:54:13 -0700 Subject: [PATCH 232/844] aurweb.routers.accounts: strip host out of ssh pubkeys We must store the paired key, otherwise aurweb-git-auth will fail. Signed-off-by: Kevin Morris --- aurweb/routers/accounts.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 3e3469ca..36871595 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -1,4 +1,5 @@ import copy +import logging import typing from datetime import datetime @@ -24,6 +25,7 @@ from aurweb.scripts.notify import ResetKeyNotification from aurweb.templates import make_variable_context, render_template router = APIRouter() +logger = logging.getLogger(__name__) @router.get("/passreset", response_class=HTMLResponse) @@ -402,6 +404,10 @@ async def account_register_post(request: Request, if PK: # Get the second element in the PK, which is the actual key. pubkey = PK.strip().rstrip() + parts = pubkey.split(" ") + if len(parts) == 3: + # Remove the host part. + pubkey = parts[0] + " " + parts[1] fingerprint = get_fingerprint(pubkey) user.ssh_pub_key = SSHPubKey(UserID=user.ID, PubKey=pubkey, @@ -522,15 +528,19 @@ async def account_edit_post(request: Request, if PK: # Get the second token in the public key, which is the actual key. pubkey = PK.strip().rstrip() + parts = pubkey.split(" ") + if len(parts) == 3: + # Remove the host part. + pubkey = parts[0] + " " + parts[1] fingerprint = get_fingerprint(pubkey) if not user.ssh_pub_key: # No public key exists, create one. user.ssh_pub_key = SSHPubKey(UserID=user.ID, - PubKey=PK, + PubKey=pubkey, Fingerprint=fingerprint) - elif user.ssh_pub_key.Fingerprint != fingerprint: + elif user.ssh_pub_key.PubKey != pubkey: # A public key already exists, update it. - user.ssh_pub_key.PubKey = PK + user.ssh_pub_key.PubKey = pubkey user.ssh_pub_key.Fingerprint = fingerprint elif user.ssh_pub_key: # Else, if the user has a public key already, delete it. From 0a3aa40f209e755f4a345ff86015d7a42acdb1f4 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 27 Jun 2021 05:16:12 -0700 Subject: [PATCH 233/844] Docker: Fix `git` sshd This was completely bugged out. This commit fixes git, provides two separate cgit servers for the different URL bases and also supplies a smartgit service for $AURWEB_URL/repo.git interaction. Docker image needs to be rebuilt with this change: $ docker build -t aurweb:latest . Signed-off-by: Kevin Morris --- Dockerfile | 11 +++-- docker-compose.yml | 72 +++++++++++++++++++++++++----- docker/cgit-entrypoint.sh | 7 ++- docker/fastapi-entrypoint.sh | 3 ++ docker/git-entrypoint.sh | 81 +++++++++++++++++++++++++++------- docker/health/cgit.sh | 2 +- docker/health/smartgit.sh | 2 + docker/health/sshd.sh | 5 ++- docker/nginx-entrypoint.sh | 38 ++++++++++++++-- docker/php-entrypoint.sh | 3 ++ docker/scripts/run-cgit.sh | 4 ++ docker/scripts/run-smartgit.sh | 9 ++++ docker/scripts/run-sshd.sh | 2 +- docker/smartgit-entrypoint.sh | 4 ++ 14 files changed, 202 insertions(+), 41 deletions(-) create mode 100755 docker/health/smartgit.sh create mode 100755 docker/scripts/run-cgit.sh create mode 100755 docker/scripts/run-smartgit.sh create mode 100755 docker/smartgit-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index da9c8d3b..4141a4c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ FROM archlinux:base-devel +ENV PYTHONPATH=/aurweb +ENV AUR_CONFIG=conf/config # Setup some default system stuff. RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime @@ -16,7 +18,7 @@ RUN pacman -Syu --noconfirm --noprogressbar \ python-pytest-asyncio python-coverage hypercorn python-bcrypt \ python-email-validator openssh python-lxml mariadb mariadb-libs \ python-isort flake8 cgit uwsgi uwsgi-plugin-cgi php php-fpm \ - python-asgiref uvicorn + python-asgiref uvicorn python-pip python-wheel RUN useradd -U -d /aurweb -c 'AUR User' aur @@ -25,6 +27,9 @@ COPY docker /docker WORKDIR /aurweb COPY . . -ENV PYTHONPATH=/aurweb - RUN make -C po all install +RUN pip3 install -t /aurweb/app --upgrade -I . + +# Set permissions on directories and binaries. +RUN bash -c 'find /aurweb/app -type d -exec chmod 755 {} \;' +RUN chmod 755 /aurweb/app/bin/* diff --git a/docker-compose.yml b/docker-compose.yml index 795236c7..6bf36166 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,33 +50,77 @@ services: image: aurweb:latest init: true environment: - - AUR_CONFIG=conf/config + - AUR_CONFIG=/aurweb/conf/config entrypoint: /docker/git-entrypoint.sh command: /docker/scripts/run-sshd.sh ports: - - "2222:22" + - "2222:2222" healthcheck: test: "bash /docker/health/sshd.sh" interval: 2s timeout: 30s + depends_on: + mariadb: + condition: service_healthy + links: + - mariadb volumes: - mariadb_run:/var/run/mysqld - mariadb_data:/var/lib/mysql - git_data:/aurweb/aur.git - ./cache:/cache - cgit: + smartgit: + image: aurweb:latest + init: true + environment: + - AUR_CONFIG=/aurweb/conf/config + entrypoint: /docker/smartgit-entrypoint.sh + command: /docker/scripts/run-smartgit.sh + healthcheck: + test: "bash /docker/health/smartgit.sh" + interval: 2s + timeout: 30s + depends_on: + mariadb: + condition: service_healthy + links: + - mariadb + volumes: + - mariadb_run:/var/run/mysqld + - mariadb_data:/var/lib/mysql + - git_data:/aurweb/aur.git + - ./cache:/cache + - smartgit_run:/var/run/smartgit + + cgit-php: image: aurweb:latest init: true environment: - AUR_CONFIG=/aurweb/conf/config entrypoint: /docker/cgit-entrypoint.sh - command: >- - uwsgi --socket 0.0.0.0:3000 - --plugins cgi - --cgi /usr/share/webapps/cgit/cgit.cgi + command: /docker/scripts/run-cgit.sh 3000 "https://localhost:8443/cgit" healthcheck: - test: "bash /docker/health/cgit.sh" + test: "bash /docker/health/cgit.sh 3000" + interval: 2s + timeout: 30s + depends_on: + git: + condition: service_healthy + links: + - git + volumes: + - git_data:/aurweb/aur.git + + cgit-fastapi: + image: aurweb:latest + init: true + environment: + - AUR_CONFIG=/aurweb/conf/config + entrypoint: /docker/cgit-entrypoint.sh + command: /docker/scripts/run-cgit.sh 3000 "https://localhost:8444/cgit" + healthcheck: + test: "bash /docker/health/cgit.sh 3000" interval: 2s timeout: 30s depends_on: @@ -170,14 +214,20 @@ services: interval: 2s timeout: 30s depends_on: - cgit: + cgit-php: + condition: service_healthy + cgit-fastapi: + condition: service_healthy + smartgit: condition: service_healthy fastapi: condition: service_healthy php-fpm: condition: service_healthy links: - - cgit + - cgit-php + - cgit-fastapi + - smartgit - fastapi - php-fpm volumes: @@ -187,6 +237,7 @@ services: - ./web/html:/aurweb/web/html - ./web/template:/aurweb/web/template - ./web/lib:/aurweb/web/lib + - smartgit_run:/var/run/smartgit sharness: image: aurweb:latest @@ -298,3 +349,4 @@ volumes: mariadb_run: {} # Share /var/run/mysqld/mysqld.sock mariadb_data: {} # Share /var/lib/mysql git_data: {} # Share aurweb/aur.git + smartgit_run: {} diff --git a/docker/cgit-entrypoint.sh b/docker/cgit-entrypoint.sh index e05e1b7a..9abc5091 100755 --- a/docker/cgit-entrypoint.sh +++ b/docker/cgit-entrypoint.sh @@ -1,13 +1,12 @@ #!/bin/bash set -eou pipefail -cp -vf conf/cgitrc.proto /etc/cgitrc +mkdir -p /var/cache/cgit -sed -ri 's|clone-prefix=.*|clone-prefix=https://localhost:8443|' /etc/cgitrc +cp -vf conf/cgitrc.proto /etc/cgitrc +sed -ri "s|clone-prefix=.*|clone-prefix=${2}|" /etc/cgitrc sed -ri 's|header=.*|header=/aurweb/web/template/cgit/header.html|' /etc/cgitrc sed -ri 's|footer=.*|footer=/aurweb/web/template/cgit/footer.html|' /etc/cgitrc sed -ri 's|repo\.path=.*|repo.path=/aurweb/aur.git|' /etc/cgitrc -mkdir -p /var/cache/cgit - exec "$@" diff --git a/docker/fastapi-entrypoint.sh b/docker/fastapi-entrypoint.sh index 2f04c29f..11b8ac5a 100755 --- a/docker/fastapi-entrypoint.sh +++ b/docker/fastapi-entrypoint.sh @@ -7,4 +7,7 @@ bash $dir/test-mysql-entrypoint.sh sed -ri "s;^(aur_location) = .+;\1 = https://localhost:8444;" conf/config sed -ri 's/^(name) = .+/\1 = aurweb/' conf/config +sed -ri "s|^(git_clone_uri_anon) = .+|\1 = https://localhost:8444/cgit/aur.git -b %s|" conf/config.defaults +sed -ri "s|^(git_clone_uri_priv) = .+|\1 = ssh://aur@localhost:2222/%s.git|" conf/config.defaults + exec "$@" diff --git a/docker/git-entrypoint.sh b/docker/git-entrypoint.sh index d17ceeaf..e6d3ad97 100755 --- a/docker/git-entrypoint.sh +++ b/docker/git-entrypoint.sh @@ -2,44 +2,91 @@ set -eou pipefail SSHD_CONFIG=/etc/ssh/sshd_config +AUTH_SCRIPT=/aurweb/app/git-auth.sh -GIT_REPO=aur.git -GIT_KEY=/cache/git.key +GIT_REPO=/aurweb/aur.git +GIT_BRANCH=master # 'Master' branch. -# Setup SSH Keys. -ssh-keygen -A +if ! grep -q 'PYTHONPATH' /etc/environment; then + echo "PYTHONPATH='/aurweb:/aurweb/app'" >> /etc/environment +else + sed -ri "s|^(PYTHONPATH)=.*$|\1='/aurweb:/aurweb/app'|" /etc/environment +fi + +if ! grep -q 'AUR_CONFIG' /etc/environment; then + echo "AUR_CONFIG='/aurweb/conf/config'" >> /etc/environment +else + sed -ri "s|^(AUR_CONFIG)=.*$|\1='/aurweb/conf/config'|" /etc/environment +fi + +if ! grep -q '/aurweb/app/bin' /etc/environment; then + echo "PATH='/aurweb/app/bin:\${PATH}'" >> /etc/environment +fi # Add AUR SSH config. cat >> $SSHD_CONFIG << EOF Match User aur PasswordAuthentication no - AuthorizedKeysCommand /usr/local/bin/aurweb-git-auth "%t" "%k" + AuthorizedKeysCommand $AUTH_SCRIPT "%t" "%k" AuthorizedKeysCommandUser aur AcceptEnv AUR_OVERWRITE - SetEnv AUR_CONFIG=/aurweb/config/config EOF +cat >> $AUTH_SCRIPT << EOF +#!/usr/bin/env bash +export PYTHONPATH="$PYTHONPATH" +export AUR_CONFIG="$AUR_CONFIG" +export PATH="/aurweb/app/bin:\${PATH}" + +exec /aurweb/app/bin/aurweb-git-auth "\$@" +EOF +chmod 755 $AUTH_SCRIPT + +DB_NAME="aurweb" +DB_HOST="mariadb" +DB_USER="aur" +DB_PASS="aur" + +# Setup a config for our mysql db. +cp -vf conf/config.dev $AUR_CONFIG +sed -i "s;YOUR_AUR_ROOT;$(pwd);g" $AUR_CONFIG +sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" $AUR_CONFIG +sed -ri "s/^(host) = .+/\1 = ${DB_HOST}/" $AUR_CONFIG +sed -ri "s/^(user) = .+/\1 = ${DB_USER}/" $AUR_CONFIG +sed -ri "s/^;?(password) = .+/\1 = ${DB_PASS}/" $AUR_CONFIG +sed -i "s|/usr/local/bin|/aurweb/app/bin|g" $AUR_CONFIG + +AUR_CONFIG_DEFAULTS="${AUR_CONFIG}.defaults" + +if [[ "$AUR_CONFIG_DEFAULTS" != "/aurweb/conf/config.defaults" ]]; then + cp -vf conf/config.defaults $AUR_CONFIG_DEFAULTS +fi + +# Set some defaults needed for pathing and ssh uris. +sed -i "s|/usr/local/bin|/aurweb/app/bin|g" $AUR_CONFIG_DEFAULTS +sed -ri "s|^(repo-path) = .+|\1 = /aurweb/aur.git/|" $AUR_CONFIG_DEFAULTS + +ssh_cmdline='ssh ssh://aur@localhost:2222' +sed -ri "s|^(ssh-cmdline) = .+|\1 = $ssh_cmdline|" $AUR_CONFIG_DEFAULTS + +# Setup SSH Keys. +ssh-keygen -A + # Taken from INSTALL. mkdir -pv $GIT_REPO # Initialize git repository. if [ ! -f $GIT_REPO/config ]; then + curdir="$(pwd)" cd $GIT_REPO + git config --global init.defaultBranch $GIT_BRANCH git init --bare git config --local transfer.hideRefs '^refs/' git config --local --add transfer.hideRefs '!refs/' git config --local --add transfer.hideRefs '!HEAD' - ln -sf /usr/local/bin/aurweb-git-update hooks/update - chown -R aur . - cd .. + ln -sf /aurweb/app/bin/aurweb-git-update hooks/update + cd $curdir + chown -R aur:aur $GIT_REPO fi -if [ ! -f $GIT_KEY ]; then - # Create a DSA ssh private/pubkey at /cache/git.key{.pub,}. - ssh-keygen -f $GIT_KEY -t dsa -N '' -C 'AUR Git Key' -fi - -# Users should modify these permissions on their local machines. -chmod 666 ${GIT_KEY}{.pub,} - exec "$@" diff --git a/docker/health/cgit.sh b/docker/health/cgit.sh index add33031..2f0cfeb1 100755 --- a/docker/health/cgit.sh +++ b/docker/health/cgit.sh @@ -1,2 +1,2 @@ #!/bin/bash -exec printf "" >>/dev/tcp/127.0.0.1/3000 +exec printf "" >>/dev/tcp/127.0.0.1/${1} diff --git a/docker/health/smartgit.sh b/docker/health/smartgit.sh new file mode 100755 index 00000000..b4e7ebd4 --- /dev/null +++ b/docker/health/smartgit.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec pgrep uwsgi diff --git a/docker/health/sshd.sh b/docker/health/sshd.sh index 6befdfb5..d9da9ea1 100755 --- a/docker/health/sshd.sh +++ b/docker/health/sshd.sh @@ -1,2 +1,5 @@ #!/bin/bash -exec printf "" >>/dev/tcp/127.0.0.1/22 +# Opt to just pgrep sshd instead of connecting here. This health +# script is used on a regular interval and it ends up spamming +# the git service's logs with accesses. +exec pgrep sshd diff --git a/docker/nginx-entrypoint.sh b/docker/nginx-entrypoint.sh index 1e442ef7..238cd167 100755 --- a/docker/nginx-entrypoint.sh +++ b/docker/nginx-entrypoint.sh @@ -45,8 +45,16 @@ http { server fastapi:8000; } - upstream cgit { - server cgit:3000; + upstream cgit-php { + server cgit-php:3000; + } + + upstream cgit-fastapi { + server cgit-fastapi:3000; + } + + upstream smartgit { + server unix:/var/run/smartgit/smartgit.sock; } server { @@ -59,12 +67,23 @@ http { root /aurweb/web/html; index index.php; + location ~ "^/([a-z0-9][a-z0-9.+_-]*?)(\.git)?/(git-(receive|upload)-pack|HEAD|info/refs|objects/(info/(http-)?alternates|packs)|[0-9a-f]{2}/[0-9a-f]{38}|pack/pack-[0-9a-f]{40}\.(pack|idx))$" { + include uwsgi_params; + uwsgi_pass smartgit; + uwsgi_modifier1 9; + uwsgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend; + uwsgi_param PATH_INFO /aur.git/\$3; + uwsgi_param GIT_HTTP_EXPORT_ALL ""; + uwsgi_param GIT_NAMESPACE \$1; + uwsgi_param GIT_PROJECT_ROOT /aurweb; + } + location ~ ^/cgit { include uwsgi_params; rewrite ^/cgit/([^?/]+/[^?]*)?(?:\?(.*))?$ /cgit.cgi?url=\$1&\$2 last; uwsgi_modifier1 9; uwsgi_param CGIT_CONFIG /etc/cgitrc; - uwsgi_pass uwsgi://cgit; + uwsgi_pass uwsgi://cgit-php; } location ~ ^/[^/]+\.php($|/) { @@ -95,12 +114,23 @@ http { try_files \$uri @proxy_to_app; } + location ~ "^/([a-z0-9][a-z0-9.+_-]*?)(\.git)?/(git-(receive|upload)-pack|HEAD|info/refs|objects/(info/(http-)?alternates|packs)|[0-9a-f]{2}/[0-9a-f]{38}|pack/pack-[0-9a-f]{40}\.(pack|idx))$" { + include uwsgi_params; + uwsgi_pass smartgit; + uwsgi_modifier1 9; + uwsgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend; + uwsgi_param PATH_INFO /aur.git/\$3; + uwsgi_param GIT_HTTP_EXPORT_ALL ""; + uwsgi_param GIT_NAMESPACE \$1; + uwsgi_param GIT_PROJECT_ROOT /aurweb; + } + location ~ ^/cgit { include uwsgi_params; rewrite ^/cgit/([^?/]+/[^?]*)?(?:\?(.*))?$ /cgit.cgi?url=\$1&\$2 last; uwsgi_modifier1 9; uwsgi_param CGIT_CONFIG /etc/cgitrc; - uwsgi_pass uwsgi://cgit; + uwsgi_pass uwsgi://cgit-fastapi; } location @proxy_to_app { diff --git a/docker/php-entrypoint.sh b/docker/php-entrypoint.sh index 19c6d059..4d49ef17 100755 --- a/docker/php-entrypoint.sh +++ b/docker/php-entrypoint.sh @@ -7,6 +7,9 @@ bash $dir/test-mysql-entrypoint.sh sed -ri "s;^(aur_location) = .+;\1 = https://localhost:8443;" conf/config sed -ri 's/^(name) = .+/\1 = aurweb/' conf/config +sed -ri "s|^(git_clone_uri_anon) = .+|\1 = https://localhost:8443/cgit/aur.git -b %s|" conf/config.defaults +sed -ri "s|^(git_clone_uri_priv) = .+|\1 = ssh://aur@localhost:2222/%s.git|" conf/config.defaults + sed -ri 's/^(listen).*/\1 = 0.0.0.0:9000/' /etc/php/php-fpm.d/www.conf sed -ri 's/^;?(clear_env).*/\1 = no/' /etc/php/php-fpm.d/www.conf diff --git a/docker/scripts/run-cgit.sh b/docker/scripts/run-cgit.sh new file mode 100755 index 00000000..67bdc079 --- /dev/null +++ b/docker/scripts/run-cgit.sh @@ -0,0 +1,4 @@ +#!/bin/bash +exec uwsgi --socket 0.0.0.0:${1} \ + --plugins cgi \ + --cgi /usr/share/webapps/cgit/cgit.cgi diff --git a/docker/scripts/run-smartgit.sh b/docker/scripts/run-smartgit.sh new file mode 100755 index 00000000..b6869a6c --- /dev/null +++ b/docker/scripts/run-smartgit.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +exec uwsgi \ + --socket /var/run/smartgit/smartgit.sock \ + --uid root \ + --gid http \ + --chmod-socket=666 \ + --plugins cgi \ + --cgi /usr/lib/git-core/git-http-backend diff --git a/docker/scripts/run-sshd.sh b/docker/scripts/run-sshd.sh index a69af7e2..d488e80d 100755 --- a/docker/scripts/run-sshd.sh +++ b/docker/scripts/run-sshd.sh @@ -1,2 +1,2 @@ #!/bin/bash -exec /usr/sbin/sshd -D +exec /usr/sbin/sshd -e -p 2222 -D diff --git a/docker/smartgit-entrypoint.sh b/docker/smartgit-entrypoint.sh new file mode 100755 index 00000000..daa9edeb --- /dev/null +++ b/docker/smartgit-entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -eou pipefail + +exec "$@" From 12911a101e8a5adb4097ed503c3923bedaf98fa9 Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Sun, 27 Jun 2021 13:55:51 +0200 Subject: [PATCH 234/844] Port homepage intro to fastapi Port the main home page content to fastapi. --- aurweb/config.py | 6 +++ aurweb/routers/html.py | 2 + aurweb/util.py | 10 +++++ templates/index.html | 92 ++++++++++++++++++++++++++++++++++++++++++ test/test_homepage.py | 36 +++++++++++++++++ 5 files changed, 146 insertions(+) create mode 100644 test/test_homepage.py diff --git a/aurweb/config.py b/aurweb/config.py index 73db58dc..52fadda2 100644 --- a/aurweb/config.py +++ b/aurweb/config.py @@ -46,3 +46,9 @@ def getboolean(section, option): def getint(section, option, fallback=None): return _get_parser().getint(section, option, fallback=fallback) + + +def get_section(section_name): + for section in _get_parser().sections(): + if section == section_name: + return _get_parser()[section] diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index 580ee0d4..f6f1a54e 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -58,6 +58,8 @@ async def language(request: Request, async def index(request: Request): """ Homepage route. """ context = make_context(request, "Home") + context['ssh_fingerprints'] = util.get_ssh_fingerprints() + return render_template(request, "index.html", context) diff --git a/aurweb/util.py b/aurweb/util.py index 539af40e..d4a0b221 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -172,3 +172,13 @@ def add_samesite_fields(response: Response, value: str): cookie += f"; SameSite={value}" response.raw_headers[idx] = (header[0], cookie.encode()) return response + + +def get_ssh_fingerprints(): + fingerprints = {} + fingerprint_section = aurweb.config.get_section("fingerprints") + + if fingerprint_section: + fingerprints = {key: fingerprint_section[key] for key in fingerprint_section.keys()} + + return fingerprints diff --git a/templates/index.html b/templates/index.html index 27d3375d..8cd1cc78 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,4 +1,96 @@ {% extends 'partials/layout.html' %} {% block pageContent %} +
    +

    AUR {% trans %}Home{% endtrans %}

    +

    + {{ "Welcome to the AUR! Please read the %sAUR User Guidelines%s and %sAUR TU Guidelines%s for more information." + | tr + | format('', "", + '', "") + | safe + }} + {{ "Contributed PKGBUILDs %smust%s conform to the %sArch Packaging Standards%s otherwise they will be deleted!" + | tr + | format("", "", + '', + "") + | safe + }} + {% trans %}Remember to vote for your favourite packages!{% endtrans %} + {% trans %}Some packages may be provided as binaries in [community].{% endtrans %} +

    + {% trans %}DISCLAIMER{% endtrans %}: + {% trans %}AUR packages are user produced content. Any use of the provided files is at your own risk.{% endtrans %} +

    +

    {% trans %}Learn more...{% endtrans %}

    +

    +
    +
    +

    {% trans %}Support{% endtrans %}

    +

    {% trans %}Package Requests{% endtrans %}

    +
    +

    + {{ "There are three types of requests that can be filed in the %sPackage Actions%s box on the package details page:" + | tr + | format("", "") + | safe + }} +

    +
      +
    • {% trans %}Orphan Request{% endtrans %}: {% trans %}Request a package to be disowned, e.g. when the maintainer is inactive and the package has been flagged out-of-date for a long time.{% endtrans %}
    • +
    • {% trans %}Deletion Request{% endtrans %}: {%trans %}Request a package to be removed from the Arch User Repository. Please do not use this if a package is broken and can be fixed easily. Instead, contact the package maintainer and file orphan request if necessary.{% endtrans %}
    • +
    • {% trans %}Merge Request{% endtrans %}: {% trans %}Request a package to be merged into another one. Can be used when a package needs to be renamed or replaced by a split package.{% endtrans %}
    • +
    +

    + {{ "If you want to discuss a request, you can use the %saur-requests%s mailing list. However, please do not use that list to file requests." + | tr + | format('', "") + | safe + }} +

    +
    +

    {% trans %}Submitting Packages{% endtrans %}

    +
    +

    + {{ "Git over SSH is now used to submit packages to the AUR. See the %sSubmitting packages%s section of the Arch User Repository ArchWiki page for more details." + | tr + | format('', "") + | safe + }} +

    + {% if ssh_fingerprints %} +

    + {% trans %}The following SSH fingerprints are used for the AUR:{% endtrans %} +

    +

      + {% for keytype in ssh_fingerprints %} +
    • {{ keytype }}: {{ ssh_fingerprints[keytype] }} + {% endfor %} +
    + {% endif %} +
    +

    {% trans %}Discussion{% endtrans %}

    +
    +

    + {{ "General discussion regarding the Arch User Repository (AUR) and Trusted User structure takes place on %saur-general%s. For discussion relating to the development of the AUR web interface, use the %saur-dev%s mailing list." + | tr + | format('', "", + '', "") + | safe + }} +

    +

    +

    {% trans %}Bug Reporting{% endtrans %}

    +
    +

    + {{ "If you find a bug in the AUR web interface, please fill out a bug report on our %sbug tracker%s. Use the tracker to report bugs in the AUR web interface %sonly%s. To report packaging bugs contact the package maintainer or leave a comment on the appropriate package page." + | tr + | format('', "", + "", "") + | safe + }} +

    +
    +
    {% endblock %} diff --git a/test/test_homepage.py b/test/test_homepage.py new file mode 100644 index 00000000..23d7185f --- /dev/null +++ b/test/test_homepage.py @@ -0,0 +1,36 @@ +from http import HTTPStatus +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from aurweb.asgi import app + +client = TestClient(app) + + +def test_homepage(): + with client as request: + response = request.get("/") + assert response.status_code == int(HTTPStatus.OK) + + +@patch('aurweb.util.get_ssh_fingerprints') +def test_homepage_ssh_fingerprints(get_ssh_fingerprints_mock): + fingerprints = {'Ed25519': "SHA256:RFzBCUItH9LZS0cKB5UE6ceAYhBD5C8GeOBip8Z11+4"} + get_ssh_fingerprints_mock.return_value = fingerprints + + with client as request: + response = request.get("/") + + assert list(fingerprints.values())[0] in response.content.decode() + assert 'The following SSH fingerprints are used for the AUR' in response.content.decode() + + +@patch('aurweb.util.get_ssh_fingerprints') +def test_homepage_no_ssh_fingerprints(get_ssh_fingerprints_mock): + get_ssh_fingerprints_mock.return_value = {} + + with client as request: + response = request.get("/") + + assert 'The following SSH fingerprints are used for the AUR' not in response.content.decode() From acc100eb5220169b257d33e40818ab9361ecb5e5 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 27 Jun 2021 06:26:18 -0700 Subject: [PATCH 235/844] Docker: Fix installation, remove pip, simplify sshd Signed-off-by: Kevin Morris --- Dockerfile | 8 ++------ docker/git-entrypoint.sh | 28 +++++++++++----------------- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4141a4c9..cec36158 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN pacman -Syu --noconfirm --noprogressbar \ python-pytest-asyncio python-coverage hypercorn python-bcrypt \ python-email-validator openssh python-lxml mariadb mariadb-libs \ python-isort flake8 cgit uwsgi uwsgi-plugin-cgi php php-fpm \ - python-asgiref uvicorn python-pip python-wheel + python-asgiref uvicorn RUN useradd -U -d /aurweb -c 'AUR User' aur @@ -28,8 +28,4 @@ WORKDIR /aurweb COPY . . RUN make -C po all install -RUN pip3 install -t /aurweb/app --upgrade -I . - -# Set permissions on directories and binaries. -RUN bash -c 'find /aurweb/app -type d -exec chmod 755 {} \;' -RUN chmod 755 /aurweb/app/bin/* +RUN python3 setup.py install --install-scripts=/usr/local/bin diff --git a/docker/git-entrypoint.sh b/docker/git-entrypoint.sh index e6d3ad97..89537853 100755 --- a/docker/git-entrypoint.sh +++ b/docker/git-entrypoint.sh @@ -2,7 +2,7 @@ set -eou pipefail SSHD_CONFIG=/etc/ssh/sshd_config -AUTH_SCRIPT=/aurweb/app/git-auth.sh +AUTH_SCRIPT=/app/git-auth.sh GIT_REPO=/aurweb/aur.git GIT_BRANCH=master # 'Master' branch. @@ -10,7 +10,7 @@ GIT_BRANCH=master # 'Master' branch. if ! grep -q 'PYTHONPATH' /etc/environment; then echo "PYTHONPATH='/aurweb:/aurweb/app'" >> /etc/environment else - sed -ri "s|^(PYTHONPATH)=.*$|\1='/aurweb:/aurweb/app'|" /etc/environment + sed -ri "s|^(PYTHONPATH)=.*$|\1='/aurweb'|" /etc/environment fi if ! grep -q 'AUR_CONFIG' /etc/environment; then @@ -19,9 +19,15 @@ else sed -ri "s|^(AUR_CONFIG)=.*$|\1='/aurweb/conf/config'|" /etc/environment fi -if ! grep -q '/aurweb/app/bin' /etc/environment; then - echo "PATH='/aurweb/app/bin:\${PATH}'" >> /etc/environment -fi +mkdir -p /app +chmod 755 /app + +cat >> $AUTH_SCRIPT << EOF +#!/usr/bin/env bash +export AUR_CONFIG="$AUR_CONFIG" +exec /usr/local/bin/aurweb-git-auth "\$@" +EOF +chmod 755 $AUTH_SCRIPT # Add AUR SSH config. cat >> $SSHD_CONFIG << EOF @@ -32,16 +38,6 @@ Match User aur AcceptEnv AUR_OVERWRITE EOF -cat >> $AUTH_SCRIPT << EOF -#!/usr/bin/env bash -export PYTHONPATH="$PYTHONPATH" -export AUR_CONFIG="$AUR_CONFIG" -export PATH="/aurweb/app/bin:\${PATH}" - -exec /aurweb/app/bin/aurweb-git-auth "\$@" -EOF -chmod 755 $AUTH_SCRIPT - DB_NAME="aurweb" DB_HOST="mariadb" DB_USER="aur" @@ -54,7 +50,6 @@ sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" $AUR_CONFIG sed -ri "s/^(host) = .+/\1 = ${DB_HOST}/" $AUR_CONFIG sed -ri "s/^(user) = .+/\1 = ${DB_USER}/" $AUR_CONFIG sed -ri "s/^;?(password) = .+/\1 = ${DB_PASS}/" $AUR_CONFIG -sed -i "s|/usr/local/bin|/aurweb/app/bin|g" $AUR_CONFIG AUR_CONFIG_DEFAULTS="${AUR_CONFIG}.defaults" @@ -63,7 +58,6 @@ if [[ "$AUR_CONFIG_DEFAULTS" != "/aurweb/conf/config.defaults" ]]; then fi # Set some defaults needed for pathing and ssh uris. -sed -i "s|/usr/local/bin|/aurweb/app/bin|g" $AUR_CONFIG_DEFAULTS sed -ri "s|^(repo-path) = .+|\1 = /aurweb/aur.git/|" $AUR_CONFIG_DEFAULTS ssh_cmdline='ssh ssh://aur@localhost:2222' From b2491ddc07fefe9612a14fb8ac9ed4bac9da8f79 Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Sun, 27 Jun 2021 17:25:46 +0200 Subject: [PATCH 236/844] Use type=email for email fields Setting the input type gives the use a hint that the field should be an email and also shows an error when a non-email is filled into the email field. --- templates/partials/account_form.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/partials/account_form.html b/templates/partials/account_form.html index 5ae18131..6455c351 100644 --- a/templates/partials/account_form.html +++ b/templates/partials/account_form.html @@ -89,7 +89,7 @@ {% trans %}Email Address{% endtrans %}: - ({% trans %}required{% endtrans %})

    @@ -119,7 +119,7 @@ {% trans %}Backup Email Address{% endtrans %}: -

    From 222d995e95ebedab00169be9e44e161d36efc292 Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Sun, 27 Jun 2021 17:27:44 +0200 Subject: [PATCH 237/844] Use backup_email field for backup email The context gives backup_email and not backup for the backup email field. Fixes: #91 --- templates/partials/account_form.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/partials/account_form.html b/templates/partials/account_form.html index 6455c351..05009594 100644 --- a/templates/partials/account_form.html +++ b/templates/partials/account_form.html @@ -120,7 +120,7 @@ + maxlength="254" name="BE" value="{{ backup_email }}">

    From a26e70334333c99ade535d92c5957fd19f7d796e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 28 Jun 2021 04:04:52 -0700 Subject: [PATCH 238/844] bugfix: use empty string if backup_email is None Signed-off-by: Kevin Morris --- templates/partials/account_form.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/partials/account_form.html b/templates/partials/account_form.html index 05009594..6374fd5e 100644 --- a/templates/partials/account_form.html +++ b/templates/partials/account_form.html @@ -90,7 +90,7 @@ + size="30" maxlength="254" name="E" value="{{ email or '' }}"> ({% trans %}required{% endtrans %})

    @@ -120,7 +120,7 @@ + maxlength="254" name="BE" value="{{ backup_email or '' }}">

    From 28300ee889f74eaf29fec5dc0bb2f4492375b0d2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 28 Jun 2021 04:12:29 -0700 Subject: [PATCH 239/844] bugfix: populate context on invalid password (account edit) Signed-off-by: Kevin Morris --- aurweb/routers/accounts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 36871595..5d798ae8 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -483,15 +483,15 @@ async def account_edit_post(request: Request, context = await make_variable_context(request, "Accounts") context["user"] = user + args = dict(await request.form()) + context = make_account_form_context(context, request, user, args) + ok, errors = process_account_form(request, user, args) + if not passwd: context["errors"] = ["Invalid password."] return render_template(request, "account/edit.html", context, status_code=int(HTTPStatus.BAD_REQUEST)) - args = dict(await request.form()) - context = make_account_form_context(context, request, user, args) - ok, errors = process_account_form(request, user, args) - if not ok: context["errors"] = errors return render_template(request, "account/edit.html", context, From 3c6b2203e92e8359f421aed5f421a8d6424fe8de Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 28 Jun 2021 05:36:12 -0700 Subject: [PATCH 240/844] Docker: bugfix: /usr/local/bin instead of /aurweb/app/bin Signed-off-by: Kevin Morris --- docker/git-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/git-entrypoint.sh b/docker/git-entrypoint.sh index 89537853..57752ac5 100755 --- a/docker/git-entrypoint.sh +++ b/docker/git-entrypoint.sh @@ -78,7 +78,7 @@ if [ ! -f $GIT_REPO/config ]; then git config --local transfer.hideRefs '^refs/' git config --local --add transfer.hideRefs '!refs/' git config --local --add transfer.hideRefs '!HEAD' - ln -sf /aurweb/app/bin/aurweb-git-update hooks/update + ln -sf /usr/local/bin/aurweb-git-update hooks/update cd $curdir chown -R aur:aur $GIT_REPO fi From f8d2d4c82a5fd442e10fa205811548cffea59da7 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 28 Jun 2021 08:22:34 -0700 Subject: [PATCH 241/844] PackageBase.package -> PackageBase.packages A PackageBase can have more than one package associated with it. Signed-off-by: Kevin Morris --- aurweb/models/package.py | 2 +- test/test_package.py | 18 +----------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/aurweb/models/package.py b/aurweb/models/package.py index ff518f20..e8159d85 100644 --- a/aurweb/models/package.py +++ b/aurweb/models/package.py @@ -17,7 +17,7 @@ class Package(Base): Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), nullable=False) PackageBase = relationship( - "PackageBase", backref=backref("package", uselist=False), + "PackageBase", backref=backref("packages", lazy="dynamic"), foreign_keys=[PackageBaseID]) __mapper_args__ = {"primary_key": [ID]} diff --git a/test/test_package.py b/test/test_package.py index 9532823d..1e940164 100644 --- a/test/test_package.py +++ b/test/test_package.py @@ -1,9 +1,7 @@ import pytest from sqlalchemy import and_ -from sqlalchemy.exc import IntegrityError, OperationalError - -import aurweb.config +from sqlalchemy.exc import IntegrityError from aurweb.db import create, query from aurweb.models.account_type import AccountType @@ -57,20 +55,6 @@ def test_package(): assert record is not None -def test_package_package_base_cant_change(): - """ Test case insensitivity of the database table. """ - if aurweb.config.get("database", "backend") == "sqlite": - return None # SQLite doesn't seem handle this. - - from aurweb.db import session - - with pytest.raises(OperationalError): - create(Package, - PackageBase=pkgbase, - Name="invalidates-old-package-packagebase-relationship") - session.rollback() - - def test_package_null_pkgbase_raises_exception(): from aurweb.db import session From 7d695f0c6af4b35688aa53c3602aee09e05eff68 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 28 Jun 2021 08:32:46 -0700 Subject: [PATCH 242/844] Update .gitignore Signed-off-by: Kevin Morris --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5d1a4de7..27ff977a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ fastapi_aw/ .idea /cache/* /logs/* +/build/ +/dist/ +/aurweb.egg-info/ From dbbafc15fae9567deba5fd02b7a4dfdc5969d1ad Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 28 Jun 2021 12:44:55 -0700 Subject: [PATCH 243/844] bugfix: PackageKeyword should have two PKs Signed-off-by: Kevin Morris --- aurweb/models/package_keyword.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/aurweb/models/package_keyword.py b/aurweb/models/package_keyword.py index 2926740d..803e6bca 100644 --- a/aurweb/models/package_keyword.py +++ b/aurweb/models/package_keyword.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, ForeignKey, Integer +from sqlalchemy import Column, ForeignKey, Integer, String, text from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship @@ -18,7 +18,11 @@ class PackageKeyword(Base): "PackageBase", backref=backref("keywords", lazy="dynamic"), foreign_keys=[PackageBaseID]) - __mapper_args__ = {"primary_key": [PackageBaseID]} + Keyword = Column( + String(255), primary_key=True, nullable=False, + server_default=text("''")) + + __mapper_args__ = {"primary_key": [PackageBaseID, Keyword]} def __init__(self, PackageBase: aurweb.models.package_base.PackageBase = None, From 2f5d9c63c479211aec63a5c29ef5a7dc8c464225 Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Mon, 28 Jun 2021 22:32:56 +0100 Subject: [PATCH 244/844] [php] Support DB mysql backend with port instead of socket Signed-off-by: Leonidas Spyropoulos --- web/lib/DB.class.php | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/web/lib/DB.class.php b/web/lib/DB.class.php index dfdbbf96..c7b3c745 100644 --- a/web/lib/DB.class.php +++ b/web/lib/DB.class.php @@ -20,15 +20,23 @@ class DB { $backend = config_get('database', 'backend'); $host = config_get('database', 'host'); $socket = config_get('database', 'socket'); + $port = config_get('database', 'port'); $name = config_get('database', 'name'); $user = config_get('database', 'user'); $password = config_get('database', 'password'); if ($backend == "mysql") { - $dsn = $backend . - ':host=' . $host . - ';unix_socket=' . $socket . - ';dbname=' . $name; + if ($port != '') { + $dsn = $backend . + ':host=' . $host . + ';port=' . $port . + ';dbname=' . $name; + } else { + $dsn = $backend . + ':host=' . $host . + ';unix_socket=' . $socket . + ';dbname=' . $name; + } self::$dbh = new PDO($dsn, $user, $password); self::$dbh->exec("SET NAMES 'utf8' COLLATE 'utf8_general_ci';"); From c3a29171cde3b7fa16de82c1f60aa9e3329393bc Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Mon, 28 Jun 2021 22:33:51 +0100 Subject: [PATCH 245/844] [php] aurweb.spawn avoid permission denied when running as user Signed-off-by: Leonidas Spyropoulos --- aurweb/spawn.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aurweb/spawn.py b/aurweb/spawn.py index f7c07dd7..6d553dde 100644 --- a/aurweb/spawn.py +++ b/aurweb/spawn.py @@ -58,6 +58,11 @@ def generate_nginx_config(): pid {os.path.join(temporary_dir, "nginx.pid")}; http {{ access_log /dev/stdout; + client_body_temp_path {os.path.join(temporary_dir, "client_body")}; + proxy_temp_path {os.path.join(temporary_dir, "proxy")}; + fastcgi_temp_path {os.path.join(temporary_dir, "fastcgi")}1 2; + uwsgi_temp_path {os.path.join(temporary_dir, "uwsgi")}; + scgi_temp_path {os.path.join(temporary_dir, "scgi")}; server {{ listen {aur_location_parts.netloc}; location / {{ From af96be7d0928172a42f92d9a4a264ca6eb2c4710 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 29 Jun 2021 10:27:45 -0700 Subject: [PATCH 246/844] Docker: move nginx config to its own file Signed-off-by: Kevin Morris --- docker/config/nginx.conf | 133 ++++++++++++++++++++++++++++++++++++ docker/nginx-entrypoint.sh | 135 +------------------------------------ 2 files changed, 134 insertions(+), 134 deletions(-) create mode 100644 docker/config/nginx.conf diff --git a/docker/config/nginx.conf b/docker/config/nginx.conf new file mode 100644 index 00000000..c1957d71 --- /dev/null +++ b/docker/config/nginx.conf @@ -0,0 +1,133 @@ +daemon off; +user root; +worker_processes auto; +pid /var/run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; + +events { + worker_connections 256; +} + +http { + sendfile on; + tcp_nopush on; + types_hash_max_size 4096; + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers on; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + gzip on; + + upstream hypercorn { + server fastapi:8000; + } + + upstream cgit-php { + server cgit-php:3000; + } + + upstream cgit-fastapi { + server cgit-fastapi:3000; + } + + upstream smartgit { + server unix:/var/run/smartgit/smartgit.sock; + } + + server { + listen 8443 ssl http2; + server_name localhost default_server; + + ssl_certificate /etc/ssl/certs/localhost.cert.pem; + ssl_certificate_key /etc/ssl/private/localhost.key.pem; + + root /aurweb/web/html; + index index.php; + + location ~ "^/([a-z0-9][a-z0-9.+_-]*?)(\.git)?/(git-(receive|upload)-pack|HEAD|info/refs|objects/(info/(http-)?alternates|packs)|[0-9a-f]{2}/[0-9a-f]{38}|pack/pack-[0-9a-f]{40}\.(pack|idx))$" { + include uwsgi_params; + uwsgi_pass smartgit; + uwsgi_modifier1 9; + uwsgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend; + uwsgi_param PATH_INFO /aur.git/$3; + uwsgi_param GIT_HTTP_EXPORT_ALL ""; + uwsgi_param GIT_NAMESPACE $1; + uwsgi_param GIT_PROJECT_ROOT /aurweb; + } + + location ~ ^/cgit { + include uwsgi_params; + rewrite ^/cgit/([^?/]+/[^?]*)?(?:\?(.*))?$ /cgit.cgi?url=$1&$2 last; + uwsgi_modifier1 9; + uwsgi_param CGIT_CONFIG /etc/cgitrc; + uwsgi_pass uwsgi://cgit-php; + } + + location ~ ^/[^/]+\.php($|/) { + fastcgi_pass php-fpm:9000; + fastcgi_index index.php; + fastcgi_split_path_info ^(/[^/]+\.php)(/.*)$; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_param PATH_INFO $fastcgi_path_info; + include fastcgi_params; + } + + location ~ .* { + rewrite ^/(.*)$ /index.php/$1 last; + } + + } + + server { + listen 8444 ssl http2; + server_name localhost default_server; + + ssl_certificate /etc/ssl/certs/localhost.cert.pem; + ssl_certificate_key /etc/ssl/private/localhost.key.pem; + + root /aurweb/web/html; + + location / { + try_files $uri @proxy_to_app; + } + + location ~ "^/([a-z0-9][a-z0-9.+_-]*?)(\.git)?/(git-(receive|upload)-pack|HEAD|info/refs|objects/(info/(http-)?alternates|packs)|[0-9a-f]{2}/[0-9a-f]{38}|pack/pack-[0-9a-f]{40}\.(pack|idx))$" { + include uwsgi_params; + uwsgi_pass smartgit; + uwsgi_modifier1 9; + uwsgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend; + uwsgi_param PATH_INFO /aur.git/$3; + uwsgi_param GIT_HTTP_EXPORT_ALL ""; + uwsgi_param GIT_NAMESPACE $1; + uwsgi_param GIT_PROJECT_ROOT /aurweb; + } + + location ~ ^/cgit { + include uwsgi_params; + rewrite ^/cgit/([^?/]+/[^?]*)?(?:\?(.*))?$ /cgit.cgi?url=$1&$2 last; + uwsgi_modifier1 9; + uwsgi_param CGIT_CONFIG /etc/cgitrc; + uwsgi_pass uwsgi://cgit-fastapi; + } + + location @proxy_to_app { + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + proxy_buffering off; + proxy_pass https://hypercorn; + } + } + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } +} + diff --git a/docker/nginx-entrypoint.sh b/docker/nginx-entrypoint.sh index 238cd167..347af50f 100755 --- a/docker/nginx-entrypoint.sh +++ b/docker/nginx-entrypoint.sh @@ -15,139 +15,6 @@ sed -ri 's/^(disable_http_login) = .+/\1 = 1/' conf/config cp -vf /cache/localhost.cert.pem /etc/ssl/certs/localhost.cert.pem cp -vf /cache/localhost.key.pem /etc/ssl/private/localhost.key.pem -cat > /etc/nginx/nginx.conf << EOF -daemon off; -user root; -worker_processes auto; -pid /var/run/nginx.pid; -include /etc/nginx/modules-enabled/*.conf; - -events { - worker_connections 256; -} - -http { - sendfile on; - tcp_nopush on; - types_hash_max_size 4096; - include /etc/nginx/mime.types; - default_type application/octet-stream; - - ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; - ssl_prefer_server_ciphers on; - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - gzip on; - - upstream hypercorn { - server fastapi:8000; - } - - upstream cgit-php { - server cgit-php:3000; - } - - upstream cgit-fastapi { - server cgit-fastapi:3000; - } - - upstream smartgit { - server unix:/var/run/smartgit/smartgit.sock; - } - - server { - listen 8443 ssl http2; - server_name localhost default_server; - - ssl_certificate /etc/ssl/certs/localhost.cert.pem; - ssl_certificate_key /etc/ssl/private/localhost.key.pem; - - root /aurweb/web/html; - index index.php; - - location ~ "^/([a-z0-9][a-z0-9.+_-]*?)(\.git)?/(git-(receive|upload)-pack|HEAD|info/refs|objects/(info/(http-)?alternates|packs)|[0-9a-f]{2}/[0-9a-f]{38}|pack/pack-[0-9a-f]{40}\.(pack|idx))$" { - include uwsgi_params; - uwsgi_pass smartgit; - uwsgi_modifier1 9; - uwsgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend; - uwsgi_param PATH_INFO /aur.git/\$3; - uwsgi_param GIT_HTTP_EXPORT_ALL ""; - uwsgi_param GIT_NAMESPACE \$1; - uwsgi_param GIT_PROJECT_ROOT /aurweb; - } - - location ~ ^/cgit { - include uwsgi_params; - rewrite ^/cgit/([^?/]+/[^?]*)?(?:\?(.*))?$ /cgit.cgi?url=\$1&\$2 last; - uwsgi_modifier1 9; - uwsgi_param CGIT_CONFIG /etc/cgitrc; - uwsgi_pass uwsgi://cgit-php; - } - - location ~ ^/[^/]+\.php($|/) { - fastcgi_pass php-fpm:9000; - fastcgi_index index.php; - fastcgi_split_path_info ^(/[^/]+\.php)(/.*)\$; - fastcgi_param SCRIPT_FILENAME \$document_root\$fastcgi_script_name; - fastcgi_param PATH_INFO \$fastcgi_path_info; - include fastcgi_params; - } - - location ~ .* { - rewrite ^/(.*)$ /index.php/\$1 last; - } - - } - - server { - listen 8444 ssl http2; - server_name localhost default_server; - - ssl_certificate /etc/ssl/certs/localhost.cert.pem; - ssl_certificate_key /etc/ssl/private/localhost.key.pem; - - root /aurweb/web/html; - - location / { - try_files \$uri @proxy_to_app; - } - - location ~ "^/([a-z0-9][a-z0-9.+_-]*?)(\.git)?/(git-(receive|upload)-pack|HEAD|info/refs|objects/(info/(http-)?alternates|packs)|[0-9a-f]{2}/[0-9a-f]{38}|pack/pack-[0-9a-f]{40}\.(pack|idx))$" { - include uwsgi_params; - uwsgi_pass smartgit; - uwsgi_modifier1 9; - uwsgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend; - uwsgi_param PATH_INFO /aur.git/\$3; - uwsgi_param GIT_HTTP_EXPORT_ALL ""; - uwsgi_param GIT_NAMESPACE \$1; - uwsgi_param GIT_PROJECT_ROOT /aurweb; - } - - location ~ ^/cgit { - include uwsgi_params; - rewrite ^/cgit/([^?/]+/[^?]*)?(?:\?(.*))?$ /cgit.cgi?url=\$1&\$2 last; - uwsgi_modifier1 9; - uwsgi_param CGIT_CONFIG /etc/cgitrc; - uwsgi_pass uwsgi://cgit-fastapi; - } - - location @proxy_to_app { - proxy_set_header Host \$http_host; - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto \$scheme; - proxy_redirect off; - proxy_buffering off; - proxy_pass https://hypercorn; - } - } - - map \$http_upgrade \$connection_upgrade { - default upgrade; - '' close; - } -} -EOF +cp -vf /docker/config/nginx.conf /etc/nginx/nginx.conf exec "$@" From 3bacfe6cd97049926893a37595bbb40158be7a48 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 29 Jun 2021 10:29:24 -0700 Subject: [PATCH 247/844] Docker: increase nginx and php-fpm logging Log toward stdout/stderr which is accessible via `docker-compose logs `. Examples: - `docker-compose logs nginx` - `docker-compose logs php-fpm` Signed-off-by: Kevin Morris --- docker/config/nginx.conf | 4 ++-- docker/php-entrypoint.sh | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/config/nginx.conf b/docker/config/nginx.conf index c1957d71..d7c0196a 100644 --- a/docker/config/nginx.conf +++ b/docker/config/nginx.conf @@ -18,8 +18,8 @@ http { ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; ssl_prefer_server_ciphers on; - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; + access_log /dev/stdout; + error_log /dev/stderr; gzip on; diff --git a/docker/php-entrypoint.sh b/docker/php-entrypoint.sh index 4d49ef17..350871d6 100755 --- a/docker/php-entrypoint.sh +++ b/docker/php-entrypoint.sh @@ -13,6 +13,11 @@ sed -ri "s|^(git_clone_uri_priv) = .+|\1 = ssh://aur@localhost:2222/%s.git|" con sed -ri 's/^(listen).*/\1 = 0.0.0.0:9000/' /etc/php/php-fpm.d/www.conf sed -ri 's/^;?(clear_env).*/\1 = no/' /etc/php/php-fpm.d/www.conf +# Log to stderr. View logs via `docker-compose logs php-fpm`. +sed -ri 's|^(error_log) = .*$|\1 = /proc/self/fd/2|g' /etc/php/php-fpm.conf +sed -ri 's|^;?(access\.log) = .*$|\1 = /proc/self/fd/2|g' \ + /etc/php/php-fpm.d/www.conf + sed -ri 's/^;?(extension=pdo_mysql)/\1/' /etc/php/php.ini sed -ri 's/^;?(open_basedir).*$/\1 = \//' /etc/php/php.ini From a120af5a005450b348dd260ac4c9b92e979d95e7 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 29 Jun 2021 10:30:26 -0700 Subject: [PATCH 248/844] Docker: remove asset forward to index.php This makes logging look a little better for development purposes. Now, `docker-compose logs php-fpm` will only show details about PHP accesses, while `docker-compose logs nginx` will show accesses regarding PHP assets. Signed-off-by: Kevin Morris --- docker/config/nginx.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/config/nginx.conf b/docker/config/nginx.conf index d7c0196a..3a8de801 100644 --- a/docker/config/nginx.conf +++ b/docker/config/nginx.conf @@ -77,6 +77,10 @@ http { include fastcgi_params; } + location ~ .+\.(css|js?|jpe?g|png|svg|ico)/?$ { + try_files $uri =404; + } + location ~ .* { rewrite ^/(.*)$ /index.php/$1 last; } From 4442ba6703de42b1cba568f582ea4f668629ac14 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 29 Jun 2021 10:41:54 -0700 Subject: [PATCH 249/844] bugfix: return null if config key doesn't exist This was previously causing a PHP warning due to returning a missing key. Signed-off-by: Kevin Morris --- web/lib/confparser.inc.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/lib/confparser.inc.php b/web/lib/confparser.inc.php index 1152e132..fdd2b78e 100644 --- a/web/lib/confparser.inc.php +++ b/web/lib/confparser.inc.php @@ -30,7 +30,9 @@ function config_get($section, $key) { global $AUR_CONFIG; config_load(); - return $AUR_CONFIG[$section][$key]; + return isset($AUR_CONFIG[$section][$key]) + ? $AUR_CONFIG[$section][$key] + : null; } function config_get_int($section, $key) { From 6c7bb04b93161580946c2ee96d0002f3bd7858d1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 29 Jun 2021 21:33:47 -0700 Subject: [PATCH 250/844] Docker: Improve mariadb init Signed-off-by: Kevin Morris --- docker-compose.yml | 8 +++++++- docker/mariadb-entrypoint.sh | 29 ++++++++++++++++++++++++++++- docker/scripts/run-mariadb.sh | 26 -------------------------- docker/scripts/run-pytests.sh | 3 ++- docker/test-mysql-entrypoint.sh | 3 ++- 5 files changed, 39 insertions(+), 30 deletions(-) delete mode 100755 docker/scripts/run-mariadb.sh diff --git a/docker-compose.yml b/docker-compose.yml index 6bf36166..40d9bc5b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,8 +32,10 @@ services: mariadb: image: aurweb:latest init: true + environment: + - DB_HOST="%" entrypoint: /docker/mariadb-entrypoint.sh - command: /docker/scripts/run-mariadb.sh mysqld_safe --datadir=/var/lib/mysql + command: /usr/bin/mysqld_safe --datadir=/var/lib/mysql ports: # This will expose mariadbd on 127.0.0.1:13306 in the host. # Ex: `mysql -uaur -paur -h 127.0.0.1 -P 13306 aurweb` @@ -136,6 +138,7 @@ services: init: true environment: - AUR_CONFIG=/aurweb/conf/config + - DB_HOST=mariadb entrypoint: /docker/php-entrypoint.sh command: /docker/scripts/run-php.sh healthcheck: @@ -170,6 +173,7 @@ services: init: true environment: - AUR_CONFIG=conf/config + - DB_HOST=mariadb entrypoint: /docker/fastapi-entrypoint.sh command: /docker/scripts/run-fastapi.sh "${FASTAPI_BACKEND}" healthcheck: @@ -269,6 +273,7 @@ services: init: true environment: - AUR_CONFIG=conf/config + - DB_HOST=mariadb entrypoint: /docker/test-mysql-entrypoint.sh command: /docker/scripts/run-pytests.sh clean stdin_open: true @@ -324,6 +329,7 @@ services: init: true environment: - AUR_CONFIG=conf/config + - DB_HOST=mariadb entrypoint: /docker/tests-entrypoint.sh command: /docker/scripts/run-tests.sh stdin_open: true diff --git a/docker/mariadb-entrypoint.sh b/docker/mariadb-entrypoint.sh index e33c61c7..48e87045 100755 --- a/docker/mariadb-entrypoint.sh +++ b/docker/mariadb-entrypoint.sh @@ -1,6 +1,33 @@ #!/bin/bash set -eou pipefail -mariadb-install-db --user=mysql --basedir=/usr --datadir=/var/lib/mysql +MYSQL_DATA=/var/lib/mysql +DB_HOST="localhost" + +mariadb-install-db --user=mysql --basedir=/usr --datadir=$MYSQL_DATA + +# Start it up. +mysqld_safe --datadir=$MYSQL_DATA --skip-networking & +while ! mysqladmin ping 2>/dev/null; do + sleep 1s +done + +# Configure databases. +DATABASE="aurweb" # Persistent database for fastapi/php-fpm. +TEST_DB="aurweb_test" # Test database (ephemereal). + +echo "Taking care of primary database '${DATABASE}'..." +mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'$DB_HOST' IDENTIFIED BY 'aur';" +mysql -u root -e "CREATE DATABASE IF NOT EXISTS $DATABASE;" +mysql -u root -e "GRANT ALL ON ${DATABASE}.* TO 'aur'@'$DB_HOST';" + +# Drop and create our test database. +echo "Dropping test database '$TEST_DB'..." +mysql -u root -e "DROP DATABASE IF EXISTS $TEST_DB;" +mysql -u root -e "CREATE DATABASE $TEST_DB;" +mysql -u root -e "GRANT ALL ON ${TEST_DB}.* TO 'aur'@'$DB_HOST';" +echo "Created new '$TEST_DB'!" + +mysqladmin -uroot shutdown exec "$@" diff --git a/docker/scripts/run-mariadb.sh b/docker/scripts/run-mariadb.sh deleted file mode 100755 index d27d8124..00000000 --- a/docker/scripts/run-mariadb.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -mysqld_safe --datadir=/var/lib/mysql --skip-networking & -until mysqladmin ping --silent; do - sleep 1s -done - -# Create test database. -mysql -u root -e "CREATE USER 'aur'@'%' IDENTIFIED BY 'aur'" \ - 2>/dev/null || /bin/true - -# Create a brand new 'aurweb_test' DB. -mysql -u root -e "DROP DATABASE aurweb_test" 2>/dev/null || /bin/true -mysql -u root -e "CREATE DATABASE aurweb_test" -mysql -u root -e "GRANT ALL PRIVILEGES ON aurweb_test.* TO 'aur'@'%'" - -# Create the 'aurweb' DB if it does not yet exist. -mysql -u root -e "CREATE DATABASE aurweb" 2>/dev/null || /bin/true -mysql -u root -e "GRANT ALL PRIVILEGES ON aurweb.* TO 'aur'@'%'" - -mysql -u root -e "FLUSH PRIVILEGES" - -# Shutdown mariadb. -mysqladmin -uroot shutdown - -exec "$@" diff --git a/docker/scripts/run-pytests.sh b/docker/scripts/run-pytests.sh index 021603b1..c6baa939 100755 --- a/docker/scripts/run-pytests.sh +++ b/docker/scripts/run-pytests.sh @@ -23,7 +23,8 @@ while [ $# -ne 0 ]; do done # Initialize the new database; ignore errors. -python -m aurweb.initdb 2>/dev/null || /bin/true +python -m aurweb.initdb 2>/dev/null || \ + (echo "Error: aurweb.initdb failed; already initialized?" && /bin/true) # Run pytest with optional targets in front of it. make -C test "${PARAMS[@]}" pytest diff --git a/docker/test-mysql-entrypoint.sh b/docker/test-mysql-entrypoint.sh index ea4df868..9594318f 100755 --- a/docker/test-mysql-entrypoint.sh +++ b/docker/test-mysql-entrypoint.sh @@ -1,8 +1,9 @@ #!/bin/bash set -eou pipefail +[[ -z "$DB_HOST" ]] && echo 'Error: $DB_HOST required but missing.' && exit 1 + DB_NAME="aurweb_test" -DB_HOST="mariadb" DB_USER="aur" DB_PASS="aur" From f4406ccf5cc27806843d7bb8a216c5aa6ad1a214 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 29 Jun 2021 21:28:12 -0700 Subject: [PATCH 251/844] Docker: Centralize repo dependencies Now, we have `docker/scripts/install-deps.sh`, a script used by both Docker and .gitlab-ci.yml. We can now focus on changing deps in this script along as well as documentation going forward. Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 47 +++++++++++----------------------- Dockerfile | 37 +++++++++++++------------- docker/scripts/install-deps.sh | 19 ++++++++++++++ 3 files changed, 52 insertions(+), 51 deletions(-) create mode 100755 docker/scripts/install-deps.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a9d80cb..7b8da2ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: archlinux +image: archlinux:base-devel cache: key: system-v1 @@ -7,45 +7,28 @@ cache: - .pkg-cache variables: - AUR_CONFIG: conf/config + AUR_CONFIG: conf/config # Default MySQL config setup in before_script. + DB_HOST: localhost before_script: - - pacman -Syu --noconfirm --noprogressbar --needed --cachedir .pkg-cache - base-devel git gpgme protobuf pyalpm python-mysqlclient - python-pygit2 python-srcinfo python-bleach python-markdown - python-sqlalchemy python-alembic python-pytest python-werkzeug - python-pytest-tap python-fastapi hypercorn nginx python-authlib - python-itsdangerous python-httpx python-jinja python-pytest-cov - python-requests python-aiofiles python-python-multipart - python-pytest-asyncio python-coverage python-bcrypt - python-email-validator openssh python-lxml mariadb - python-isort flake8 - - bash -c "echo '127.0.0.1 localhost' > /etc/hosts" - - bash -c "echo '::1 localhost' >> /etc/hosts" - - mariadb-install-db --user=mysql --basedir=/usr --datadir=/var/lib/mysql + - ./docker/scripts/install-deps.sh + - useradd -U -d /aurweb -c 'AUR User' aur + - ./docker/mariadb-entrypoint.sh - (cd '/usr' && /usr/bin/mysqld_safe --datadir='/var/lib/mysql') & - 'until : > /dev/tcp/127.0.0.1/3306; do sleep 1s; done' - - mysql -u root -e "CREATE USER 'aur'@'localhost' IDENTIFIED BY 'aur';" - - mysql -u root -e "CREATE DATABASE aurweb_test;" - - mysql -u root -e "GRANT ALL ON aurweb_test.* TO 'aur'@'localhost';" - - mysql -u root -e "FLUSH PRIVILEGES;" - - sed -r "s;YOUR_AUR_ROOT;$(pwd);g" conf/config.dev > conf/config - - cp conf/config conf/config.sqlite - - cp conf/config.defaults conf/config.sqlite.defaults - - sed -i -r 's;backend = .*;backend = sqlite;' conf/config.sqlite - - sed -i -r "s;name = .*;name = $(pwd)/aurweb.sqlite3;" conf/config.sqlite + - ./docker/test-mysql-entrypoint.sh # Create mysql AUR_CONFIG. + - ./docker/test-sqlite-entrypoint.sh # Create sqlite AUR_CONFIG. + - make -C po all install + - python setup.py install --install-scripts=/usr/local/bin + - python -m aurweb.initdb # Initialize MySQL tables. - AUR_CONFIG=conf/config.sqlite python -m aurweb.initdb + - make -C test clean test: script: - - python setup.py install - - make -C po all install - - python -m aurweb.initdb - - make -C test sh # sharness tests use sqlite. - - make -C test pytest # pytest with mysql. - - AUR_CONFIG=conf/config.sqlite make -C test pytest # pytest with sqlite. - - coverage report --include='aurweb/*' - - coverage xml --include='aurweb/*' + - make -C test sh pytest # sharness tests use sqlite & pytest w/ mysql. + - AUR_CONFIG=conf/config.sqlite make -C test pytest + - make -C test coverage # Produce coverage reports. - flake8 --count aurweb # Assert no flake8 violations in aurweb. - flake8 --count test # Assert no flake8 violations in test. - flake8 --count migrations # Assert no flake8 violations in migrations. diff --git a/Dockerfile b/Dockerfile index cec36158..2843fa1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,30 @@ FROM archlinux:base-devel + ENV PYTHONPATH=/aurweb ENV AUR_CONFIG=conf/config +# Copy our single bootstrap script. +COPY docker/scripts/install-deps.sh /install-deps.sh +RUN /install-deps.sh + +# Add our aur user. +RUN useradd -U -d /aurweb -c 'AUR User' aur + # Setup some default system stuff. RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime -RUN mkdir -p .pkg-cache +# Copy the rest of docker. +COPY ./docker /docker +COPY ./docker/scripts/*.sh /usr/local/bin/ -# Install dependencies. -RUN pacman -Syu --noconfirm --noprogressbar \ - --cachedir .pkg-cache git gpgme protobuf pyalpm \ - python-mysqlclient python-pygit2 python-srcinfo python-bleach \ - python-markdown python-sqlalchemy python-alembic python-pytest \ - python-werkzeug python-pytest-tap python-fastapi nginx python-authlib \ - python-itsdangerous python-httpx python-jinja python-pytest-cov \ - python-requests python-aiofiles python-python-multipart \ - python-pytest-asyncio python-coverage hypercorn python-bcrypt \ - python-email-validator openssh python-lxml mariadb mariadb-libs \ - python-isort flake8 cgit uwsgi uwsgi-plugin-cgi php php-fpm \ - python-asgiref uvicorn - -RUN useradd -U -d /aurweb -c 'AUR User' aur - -COPY docker /docker +# Copy from host to container. +COPY . /aurweb +# Working directory is aurweb root @ /aurweb. WORKDIR /aurweb -COPY . . +# Install translations. RUN make -C po all install -RUN python3 setup.py install --install-scripts=/usr/local/bin + +# Install package and scripts. +RUN python setup.py install --install-scripts=/usr/local/bin diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh new file mode 100755 index 00000000..fc15313f --- /dev/null +++ b/docker/scripts/install-deps.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Install Arch Linux dependencies. This is centralized here +# for CI and Docker usage and should always reflect the most +# robust development ecosystem. +set -eou pipefail + +pacman -Syu --noconfirm --noprogressbar \ + --cachedir .pkg-cache git gpgme protobuf pyalpm \ + python-mysqlclient python-pygit2 python-srcinfo python-bleach \ + python-markdown python-sqlalchemy python-alembic python-pytest \ + python-werkzeug python-pytest-tap python-fastapi nginx python-authlib \ + python-itsdangerous python-httpx python-jinja python-pytest-cov \ + python-requests python-aiofiles python-python-multipart \ + python-pytest-asyncio python-coverage hypercorn python-bcrypt \ + python-email-validator openssh python-lxml mariadb mariadb-libs \ + python-isort flake8 cgit uwsgi uwsgi-plugin-cgi php php-fpm \ + python-asgiref uvicorn + +exec "$@" From 3f60f5048e210f5248bb2b56fcfbe346e5ebac2a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 29 Jun 2021 21:35:00 -0700 Subject: [PATCH 252/844] Docker: add scripts/setup-sqlite.sh This script purely removes any existing sqlite and is used before tests are run. This causes the test flow to run `aurweb.initdb` again (if ever). Signed-off-by: Kevin Morris --- docker-compose.yml | 4 ++-- docker/scripts/setup-sqlite.sh | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100755 docker/scripts/setup-sqlite.sh diff --git a/docker-compose.yml b/docker-compose.yml index 40d9bc5b..22495a95 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -304,7 +304,7 @@ services: environment: - AUR_CONFIG=conf/config.sqlite entrypoint: /docker/test-sqlite-entrypoint.sh - command: /docker/scripts/run-pytests.sh clean + command: setup-sqlite.sh run-pytests.sh clean stdin_open: true tty: true volumes: @@ -331,7 +331,7 @@ services: - AUR_CONFIG=conf/config - DB_HOST=mariadb entrypoint: /docker/tests-entrypoint.sh - command: /docker/scripts/run-tests.sh + command: setup-sqlite.sh run-tests.sh stdin_open: true tty: true depends_on: diff --git a/docker/scripts/setup-sqlite.sh b/docker/scripts/setup-sqlite.sh new file mode 100755 index 00000000..e0b8de50 --- /dev/null +++ b/docker/scripts/setup-sqlite.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Run an sqlite test. This script really just prepares sqlite +# tests by deleting any existing databases so the test can +# initialize cleanly. +DB_NAME="$(grep 'name =' conf/config.sqlite | sed -r 's/^name = (.+)$/\1/')" +rm -vf $DB_NAME +exec "$@" From 427a30ef8adb9ff92c9ee6745c3d2985929f3a7a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 29 Jun 2021 21:37:45 -0700 Subject: [PATCH 253/844] Docker: Remove deprecated `links` In addition, remove some unneeded dependencies on tests. Though, in the future we _should_ craft tests that use these. Signed-off-by: Kevin Morris --- docker-compose.yml | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 22495a95..ab8d7c41 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,8 +64,6 @@ services: depends_on: mariadb: condition: service_healthy - links: - - mariadb volumes: - mariadb_run:/var/run/mysqld - mariadb_data:/var/lib/mysql @@ -86,8 +84,6 @@ services: depends_on: mariadb: condition: service_healthy - links: - - mariadb volumes: - mariadb_run:/var/run/mysqld - mariadb_data:/var/lib/mysql @@ -109,8 +105,6 @@ services: depends_on: git: condition: service_healthy - links: - - git volumes: - git_data:/aurweb/aur.git @@ -128,8 +122,6 @@ services: depends_on: git: condition: service_healthy - links: - - git volumes: - git_data:/aurweb/aur.git @@ -152,10 +144,6 @@ services: condition: service_healthy mariadb: condition: service_healthy - links: - - ca - - git - - mariadb volumes: - mariadb_run:/var/run/mysqld # Bind socket in this volume. - mariadb_data:/var/lib/mysql @@ -187,10 +175,6 @@ services: condition: service_healthy mariadb: condition: service_healthy - links: - - ca - - git - - mariadb volumes: - mariadb_run:/var/run/mysqld # Bind socket in this volume. - mariadb_data:/var/lib/mysql @@ -228,12 +212,6 @@ services: condition: service_healthy php-fpm: condition: service_healthy - links: - - cgit-php - - cgit-fastapi - - smartgit - - fastapi - - php-fpm volumes: - git_data:/aurweb/aur.git - ./cache:/cache @@ -255,8 +233,6 @@ services: depends_on: git: condition: service_healthy - links: - - git volumes: - git_data:/aurweb/aur.git - ./cache:/cache @@ -281,11 +257,6 @@ services: depends_on: mariadb: condition: service_healthy - git: - condition: service_healthy - links: - - mariadb - - git volumes: - mariadb_run:/var/run/mysqld - git_data:/aurweb/aur.git @@ -318,11 +289,6 @@ services: - ./web/template:/aurweb/web/template - ./web/lib:/aurweb/web/lib - ./templates:/aurweb/templates - depends_on: - git: - condition: service_healthy - links: - - git test: image: aurweb:latest @@ -337,8 +303,6 @@ services: depends_on: mariadb: condition: service_healthy - links: - - mariadb volumes: - mariadb_run:/var/run/mysqld - git_data:/aurweb/aur.git From 3a74f76ff9835dd03b4709a585195611f6a20e38 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 29 Jun 2021 22:44:41 -0700 Subject: [PATCH 254/844] FastAPI: use internal typeahead and remove jquery Awesome! Signed-off-by: Kevin Morris --- aurweb/asgi.py | 5 +---- templates/index.html | 10 ++++++++++ templates/partials/head.html | 3 +++ templates/partials/layout.html | 1 - templates/partials/typeahead.html | 30 ------------------------------ web/html/js/typeahead-home.js | 6 ++++++ 6 files changed, 20 insertions(+), 35 deletions(-) delete mode 100644 templates/partials/typeahead.html create mode 100644 web/html/js/typeahead-home.js diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 228b9a65..5f0ad01d 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -83,10 +83,7 @@ async def add_security_headers(request: Request, call_next: typing.Callable): # Add CSP header. nonce = request.user.nonce csp = "default-src 'self'; " - script_hosts = [ - "ajax.googleapis.com", - "cdn.jsdelivr.net" - ] + script_hosts = [] csp += f"script-src 'self' 'nonce-{nonce}' " + ' '.join(script_hosts) # It's fine if css is inlined. csp += "; style-src 'self' 'unsafe-inline'" diff --git a/templates/index.html b/templates/index.html index 8cd1cc78..f8745f33 100644 --- a/templates/index.html +++ b/templates/index.html @@ -93,4 +93,14 @@

  • + + + + + + + {% endblock %} diff --git a/templates/partials/head.html b/templates/partials/head.html index 0351fd6e..9b438255 100644 --- a/templates/partials/head.html +++ b/templates/partials/head.html @@ -12,5 +12,8 @@ + + + AUR ({{ language }}) - {{ title | tr }} diff --git a/templates/partials/layout.html b/templates/partials/layout.html index 019ebff7..68637ed7 100644 --- a/templates/partials/layout.html +++ b/templates/partials/layout.html @@ -6,6 +6,5 @@ {% include 'partials/navbar.html' %} {% extends 'partials/body.html' %} - {% include 'partials/typeahead.html' %} diff --git a/templates/partials/typeahead.html b/templates/partials/typeahead.html deleted file mode 100644 index c218b8d1..00000000 --- a/templates/partials/typeahead.html +++ /dev/null @@ -1,30 +0,0 @@ - - - diff --git a/web/html/js/typeahead-home.js b/web/html/js/typeahead-home.js new file mode 100644 index 00000000..5af51c53 --- /dev/null +++ b/web/html/js/typeahead-home.js @@ -0,0 +1,6 @@ +document.addEventListener('DOMContentLoaded', function() { + const input = document.getElementById('pkgsearch-field'); + const form = document.getElementById('pkgsearch-form'); + const type = 'suggest'; + typeahead.init(type, input, form); +}); From 450469e3d66059d2002a68c4ad8d435793a7bfe4 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 19 Jun 2021 08:54:22 -0700 Subject: [PATCH 255/844] add /addvote/ (get, post) routes Another part of the "Trusted User" collection of routes. This allows a Trusted User to create a proposal. New Routes: - get `/addvote/` - post `/addvote/` Signed-off-by: Kevin Morris --- aurweb/routers/trusted_user.py | 119 ++++++++++++++++++++++++++----- templates/addvote.html | 68 ++++++++++++++++++ test/test_trusted_user_routes.py | 93 ++++++++++++++++++++++++ 3 files changed, 264 insertions(+), 16 deletions(-) create mode 100644 templates/addvote.html diff --git a/aurweb/routers/trusted_user.py b/aurweb/routers/trusted_user.py index 55f7b7e1..fd5ebb04 100644 --- a/aurweb/routers/trusted_user.py +++ b/aurweb/routers/trusted_user.py @@ -1,3 +1,6 @@ +import html +import logging +import re import typing from datetime import datetime @@ -5,10 +8,10 @@ from http import HTTPStatus from urllib.parse import quote_plus from fastapi import APIRouter, Form, HTTPException, Request -from fastapi.responses import Response +from fastapi.responses import RedirectResponse, Response from sqlalchemy import and_, or_ -from aurweb import db +from aurweb import db, l10n from aurweb.auth import account_type_required, auth_required from aurweb.models.account_type import DEVELOPER, TRUSTED_USER, TRUSTED_USER_AND_DEV from aurweb.models.tu_vote import TUVote @@ -17,6 +20,7 @@ from aurweb.models.user import User from aurweb.templates import make_context, make_variable_context, render_template router = APIRouter() +logger = logging.getLogger(__name__) # Some TU route specific constants. ITEMS_PER_PAGE = 10 # Paged table size. @@ -29,6 +33,17 @@ REQUIRED_TYPES = { TRUSTED_USER_AND_DEV } +ADDVOTE_SPECIFICS = { + # This dict stores a vote duration and quorum for a proposal. + # When a proposal is added, duration is added to the current + # timestamp. + # "addvote_type": (duration, quorum) + "add_tu": (7 * 24 * 60 * 60, 0.66), + "remove_tu": (7 * 24 * 60 * 60, 0.75), + "remove_inactive_tu": (5 * 24 * 60 * 60, 0.66), + "bylaws": (7 * 24 * 60 * 60, 0.75) +} + @router.get("/tu") @auth_required(True, redirect="/") @@ -174,28 +189,17 @@ async def trusted_user_proposal_post(request: Request, raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) voters = db.query(User).join(TUVote).filter(TUVote.VoteID == voteinfo.ID) + vote = db.query(TUVote, and_(TUVote.UserID == request.user.ID, + TUVote.VoteID == voteinfo.ID)).first() - # status_code we'll use for responses later. status_code = HTTPStatus.OK - if not request.user.is_trusted_user(): - # Test: Create a proposal and view it as a "Developer". It - # should give us this error. context["error"] = "Only Trusted Users are allowed to vote." status_code = HTTPStatus.UNAUTHORIZED elif voteinfo.User == request.user.Username: context["error"] = "You cannot vote in an proposal about you." status_code = HTTPStatus.BAD_REQUEST - - vote = db.query(TUVote, and_(TUVote.UserID == request.user.ID, - TUVote.VoteID == voteinfo.ID)).first() - - if status_code != HTTPStatus.OK: - return render_proposal(request, context, proposal, - voteinfo, voters, vote, - status_code=status_code) - - if vote is not None: + elif vote is not None: context["error"] = "You've already voted for this proposal." status_code = HTTPStatus.BAD_REQUEST @@ -218,3 +222,86 @@ async def trusted_user_proposal_post(request: Request, context["error"] = "You've already voted for this proposal." return render_proposal(request, context, proposal, voteinfo, voters, vote) + + +@router.get("/addvote") +@auth_required(True) +@account_type_required({"Trusted User", "Trusted User & Developer"}) +async def trusted_user_addvote(request: Request, + user: str = str(), + type: str = "add_tu", + agenda: str = str()): + context = await make_variable_context(request, "Add Proposal") + + if type not in ADDVOTE_SPECIFICS: + context["error"] = "Invalid type." + type = "add_tu" # Default it. + + context["user"] = user + context["type"] = type + context["agenda"] = agenda + + return render_template(request, "addvote.html", context) + + +@router.post("/addvote") +@auth_required(True) +@account_type_required({TRUSTED_USER, TRUSTED_USER_AND_DEV}) +async def trusted_user_addvote_post(request: Request, + user: str = Form(default=str()), + type: str = Form(default=str()), + agenda: str = Form(default=str())): + # Build a context. + context = await make_variable_context(request, "Add Proposal") + + context["type"] = type + context["user"] = user + context["agenda"] = agenda + + def render_addvote(context, status_code): + """ Simplify render_template a bit for this test. """ + return render_template(request, "addvote.html", context, status_code) + + # Alright, get some database records, if we can. + if type != "bylaws": + user_record = db.query(User, User.Username == user).first() + if user_record is None: + context["error"] = "Username does not exist." + return render_addvote(context, HTTPStatus.NOT_FOUND) + + voteinfo = db.query(TUVoteInfo, TUVoteInfo.User == user).count() + if voteinfo: + _ = l10n.get_translator_for_request(request) + context["error"] = _( + "%s already has proposal running for them.") % ( + html.escape(user),) + return render_addvote(context, HTTPStatus.BAD_REQUEST) + + if type not in ADDVOTE_SPECIFICS: + context["error"] = "Invalid type." + context["type"] = type = "add_tu" # Default for rendering. + return render_addvote(context, HTTPStatus.BAD_REQUEST) + + if not agenda: + context["error"] = "Proposal cannot be empty." + return render_addvote(context, HTTPStatus.BAD_REQUEST) + + # Gather some mapped constants and the current timestamp. + duration, quorum = ADDVOTE_SPECIFICS.get(type) + timestamp = int(datetime.utcnow().timestamp()) + + # Remove diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html new file mode 100644 index 00000000..a25e9c9e --- /dev/null +++ b/templates/partials/packages/details.html @@ -0,0 +1,145 @@ + + + + + + {% if show_package_details | default(False) %} + + + + + + + + + + + + + {% endif %} + {% if pkgbase.keywords.count() %} + + + {% if is_maintainer %} + + {% else %} + + {% endif %} + + {% endif %} + {% if licenses and licenses.count() and show_package_details | default(False) %} + + + + + {% endif %} + {% if show_package_details | default(False) %} + + + + + {% endif %} + + + + + + + + + + + + + + + {% if not is_maintainer %} + + {% else %} + + {% endif %} + + + + + + + {% set submitted = pkgbase.SubmittedTS | dt | as_timezone(timezone) %} + + + + + + {% set updated = pkgbase.ModifiedTS | dt | as_timezone(timezone) %} + + +
    {{ "Git Clone URL" | tr }}: + {{ git_clone_uri_anon | format(pkgbase.Name) }} ({{ "read-only" | tr }}, {{ "click to copy" | tr }}) + {% if is_maintainer %} +
    {{ git_clone_uri_priv | format(pkgbase.Name) }} ({{ "click to copy" | tr }}) + {% endif %} +
    {{ "Package Base" | tr }}: + + {{ pkgbase.Name }} + +
    {{ "Description" | tr }}:{{ pkgbase.packages.first().Description }}
    {{ "Upstream URL" | tr }}: + {% set pkg = pkgbase.packages.first() %} + {% if pkg.URL %} + {{ pkg.URL }} + {% else %} + {{ "None" | tr }} + {% endif %} +
    {{ "Keywords" | tr }}: +
    +
    + + +
    +
    +
    + {% for keyword in pkgbase.keywords %} + + {{ keyword.Keyword }} + + {% endfor %} +
    {{ "Licenses" | tr }}:{{ licenses | join(', ', attribute='Name') | default('None' | tr) }}
    {{ "Conflicts" | tr }}: + {{ conflicts | join(', ', attribute='RelName') }} +
    {{ "Submitter" | tr }}: + {% if request.user.is_authenticated() %} + + {{ pkgbase.Submitter.Username | default("None" | tr) }} + + {% else %} + {{ pkgbase.Submitter.Username | default("None" | tr) }} + {% endif %} +
    {{ "Maintainer" | tr }}: + {% if request.user.is_authenticated() %} + + {{ pkgbase.Maintainer.Username | default("None" | tr) }} + + {% else %} + {{ pkgbase.Maintainer.Username | default("None" | tr) }} + {% endif %} +
    {{ "Last Packager" | tr }}: + {% if request.user.is_authenticated() %} + + {{ pkgbase.Packager.Username | default("None" | tr) }} + + {% else %} + {{ pkgbase.Packager.Username | default("None" | tr) }} + {% endif %} +
    {{ "Votes" | tr }}:{{ pkgbase.package_votes.count() }} + + {{ pkgbase.package_votes.count() }} + +
    {{ "Popularity" | tr }}:{{ pkgbase.Popularity | number_format(6 if pkgbase.Popularity <= 0.2 else 2) }}
    {{ "First Submitted" | tr }}:{{ "%s" | format(submitted.strftime("%Y-%m-%d %H:%M")) }}
    {{ "Last Updated" | tr }}:{{ "%s" | format(updated.strftime("%Y-%m-%d %H:%M")) }}
    + + + diff --git a/templates/partials/packages/package_actions.html b/templates/partials/packages/package_actions.html deleted file mode 100644 index 4e7da882..00000000 --- a/templates/partials/packages/package_actions.html +++ /dev/null @@ -1,87 +0,0 @@ - - diff --git a/templates/partials/packages/package_metadata.html b/templates/partials/packages/package_metadata.html new file mode 100644 index 00000000..767e25a9 --- /dev/null +++ b/templates/partials/packages/package_metadata.html @@ -0,0 +1,54 @@ +
    +

    Dependencies ({{ dependencies.count() }})

    +
      + {% for dep in dependencies.all() %} +
    • + {% set broken = not dep.is_package() %} + {% if broken %} + + {% else %} + + {% endif %} + {{ dep.DepName }} + {% if broken %} + + {% else %} + + {% endif %} + {{ dep.Package | provides_list(dep.DepName) | safe }} + {% set extra = dep | dep_extra %} + {% if extra %} + {{ dep | dep_extra_desc }} + {% endif %} +
    • + {% endfor %} +
    +
    + +
    +

    Required by ({{ required_by.count() }})

    + +
    + +
    +

    Sources ({{ sources.count() }})

    +
    + +
    + +
    diff --git a/templates/pkgbase.html b/templates/pkgbase.html deleted file mode 100644 index d608fa2e..00000000 --- a/templates/pkgbase.html +++ /dev/null @@ -1,95 +0,0 @@ -{% extends "partials/layout.html" %} - -{% block pageContent %} - {% include "partials/packages/search.html" %} -
    -

    Package Details: {{ pkgbase.Name }}

    - - {% set result = pkgbase %} - {% set pkgname = "result.Name" %} - {% include "partials/packages/package_actions.html" %} - - - - - - - - - {% if is_maintainer %} - - {% else %} - - {% endif %} - - - - - - - - - - - - - - - - - - - - - - - {% set submitted = pkgbase.SubmittedTS | dt | as_timezone(timezone) %} - - - - - - {% set updated = pkgbase.ModifiedTS | dt | as_timezone(timezone) %} - - -
    {{ "Git Clone URL" | tr }}: - {{ git_clone_uri_anon | format(pkgbase.Name) }} ({{ "read-only" | tr }}, {{ "click to copy" | tr }}) - {% if is_maintainer %} -
    {{ git_clone_uri_priv | format(pkgbase.Name) }} ({{ "click to copy" | tr }}) - {% endif %} -
    {{ "Keywords" | tr }}: -
    -
    - - - -
    -
    -
    - {% for item in pkgbase.keywords %} - {{ item.Keyword }} - {% endfor %} -
    {{ "Submitter" | tr }}:{{ pkgbase.Submitter.Username | default("None") }}
    {{ "Maintainer" | tr }}:{{ pkgbase.Maintainer.Username | default("None") }}
    {{ "Last Packager" | tr }}:{{ pkgbase.Packager.Username | default("None") }}
    {{ "Votes" | tr }}:{{ pkgbase.NumVotes }}
    {{ "Popularity" | tr }}:{{ '%0.2f' % pkgbase.Popularity | float }}
    {{ "First Submitted" | tr }}:{{ "%s" | tr | format(submitted.strftime("%Y-%m-%d %H:%M")) }}
    {{ "Last Updated" | tr }}:{{ "%s" | tr | format(updated.strftime("%Y-%m-%d %H:%M")) }}
    - -
    -
    - -

    Packages ({{ packages_count }})

    - -
    -
    -
    - {% set pkgname = result.Name %} - {% set pkgbase_id = result.ID %} - {% set comments = comments %} - {% include "partials/packages/comments.html" %} -{% endblock %} diff --git a/test/test_package_dependency.py b/test/test_package_dependency.py index d39091aa..e28f1781 100644 --- a/test/test_package_dependency.py +++ b/test/test_package_dependency.py @@ -77,6 +77,13 @@ def test_package_dependencies(): assert pkgdep in optdepends.package_dependencies assert pkgdep in package.package_dependencies + assert not pkgdep.is_package() + + base = create(PackageBase, Name=pkgdep.DepName, Maintainer=user) + create(Package, PackageBase=base, Name=pkgdep.DepName) + + assert pkgdep.is_package() + def test_package_dependencies_null_package_raises_exception(): from aurweb.db import session diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py new file mode 100644 index 00000000..f9592238 --- /dev/null +++ b/test/test_packages_routes.py @@ -0,0 +1,283 @@ +from datetime import datetime +from http import HTTPStatus + +import pytest + +from fastapi.testclient import TestClient + +from aurweb import asgi, db +from aurweb.models.account_type import USER_ID, AccountType +from aurweb.models.dependency_type import DependencyType +from aurweb.models.official_provider import OfficialProvider +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.package_comment import PackageComment +from aurweb.models.package_dependency import PackageDependency +from aurweb.models.package_keyword import PackageKeyword +from aurweb.models.package_relation import PackageRelation +from aurweb.models.relation_type import PROVIDES_ID, RelationType +from aurweb.models.user import User +from aurweb.testing import setup_test_db +from aurweb.testing.html import parse_root +from aurweb.testing.requests import Request + + +def package_endpoint(package: Package) -> str: + return f"/packages/{package.Name}" + + +def create_package(pkgname: str, maintainer: User, + autocommit: bool = True) -> Package: + pkgbase = db.create(PackageBase, + Name=pkgname, + Maintainer=maintainer, + autocommit=False) + return db.create(Package, Name=pkgbase.Name, PackageBase=pkgbase, + autocommit=autocommit) + + +def create_package_dep(package: Package, depname: str, + dep_type_name: str = "depends", + autocommit: bool = True) -> PackageDependency: + dep_type = db.query(DependencyType, + DependencyType.Name == dep_type_name).first() + return db.create(PackageDependency, + DependencyType=dep_type, + Package=package, + DepName=depname, + autocommit=autocommit) + + +def create_package_rel(package: Package, + relname: str, + autocommit: bool = True) -> PackageRelation: + rel_type = db.query(RelationType, + RelationType.ID == PROVIDES_ID).first() + return db.create(PackageRelation, + RelationType=rel_type, + Package=package, + RelName=relname) + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db( + User.__tablename__, + Package.__tablename__, + PackageBase.__tablename__, + PackageDependency.__tablename__, + PackageRelation.__tablename__, + PackageKeyword.__tablename__, + OfficialProvider.__tablename__ + ) + + +@pytest.fixture +def client() -> TestClient: + """ Yield a FastAPI TestClient. """ + yield TestClient(app=asgi.app) + + +@pytest.fixture +def user() -> User: + """ Yield a user. """ + account_type = db.query(AccountType, AccountType.ID == USER_ID).first() + yield db.create(User, Username="test", + Email="test@example.org", + Passwd="testPassword", + AccountType=account_type) + + +@pytest.fixture +def maintainer() -> User: + """ Yield a specific User used to maintain packages. """ + account_type = db.query(AccountType, AccountType.ID == USER_ID).first() + yield db.create(User, Username="test_maintainer", + Email="test_maintainer@example.org", + Passwd="testPassword", + AccountType=account_type) + + +@pytest.fixture +def package(maintainer: User) -> Package: + """ Yield a Package created by user. """ + pkgbase = db.create(PackageBase, + Name="test-package", + Maintainer=maintainer) + yield db.create(Package, + PackageBase=pkgbase, + Name=pkgbase.Name) + + +def test_package_not_found(client: TestClient): + with client as request: + resp = request.get("/packages/not_found") + assert resp.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_package_official_not_found(client: TestClient, package: Package): + """ When a Package has a matching OfficialProvider record, it is not + hosted on AUR, but in the official repositories. Getting a package + with this kind of record should return a status code 404. """ + db.create(OfficialProvider, + Name=package.Name, + Repo="core", + Provides=package.Name) + + with client as request: + resp = request.get(package_endpoint(package)) + assert resp.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_package(client: TestClient, package: Package): + """ Test a single /packages/{name} route. """ + with client as request: + + resp = request.get(package_endpoint(package)) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + h2 = root.find('.//div[@id="pkgdetails"]/h2') + + sections = h2.text.split(":") + assert sections[0] == "Package Details" + + name, version = sections[1].lstrip().split(" ") + assert name == package.Name + version == package.Version + + rows = root.findall('.//table[@id="pkginfo"]//tr') + row = rows[1] # Second row is our target. + + pkgbase = row.find("./td/a") + assert pkgbase.text.strip() == package.PackageBase.Name + + +def test_package_comments(client: TestClient, user: User, package: Package): + now = (datetime.utcnow().timestamp()) + comment = db.create(PackageComment, PackageBase=package.PackageBase, + User=user, Comments="Test comment", CommentTS=now) + + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.get(package_endpoint(package), cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + expected = [ + comment.Comments + ] + comments = root.xpath('.//div[contains(@class, "package-comments")]' + '/div[@class="article-content"]/div/text()') + for i, row in enumerate(expected): + assert comments[i].strip() == row + + +def test_package_authenticated(client: TestClient, user: User, + package: Package): + """ We get the same here for either authenticated or not + authenticated. Form inputs are presented to maintainers. + This process also occurs when pkgbase.html is rendered. """ + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.get(package_endpoint(package), cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + expected = [ + "View PKGBUILD", + "View Changes", + "Download snapshot", + "Search wiki", + "Flag package out-of-date", + "Vote for this package", + "Enable notifications", + "Submit Request" + ] + for expected_text in expected: + assert expected_text in resp.text + + +def test_package_authenticated_maintainer(client: TestClient, + maintainer: User, + package: Package): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + with client as request: + resp = request.get(package_endpoint(package), cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + expected = [ + "View PKGBUILD", + "View Changes", + "Download snapshot", + "Search wiki", + "Flag package out-of-date", + "Vote for this package", + "Enable notifications", + "Manage Co-Maintainers", + "Submit Request", + "Delete Package", + "Merge Package", + "Disown Package" + ] + for expected_text in expected: + assert expected_text in resp.text + + +def test_package_dependencies(client: TestClient, maintainer: User, + package: Package): + # Create a normal dependency of type depends. + dep_pkg = create_package("test-dep-1", maintainer, autocommit=False) + dep = create_package_dep(package, dep_pkg.Name, autocommit=False) + + # Also, create a makedepends. + make_dep_pkg = create_package("test-dep-2", maintainer, autocommit=False) + make_dep = create_package_dep(package, make_dep_pkg.Name, + dep_type_name="makedepends", + autocommit=False) + + # And... a checkdepends! + check_dep_pkg = create_package("test-dep-3", maintainer, autocommit=False) + check_dep = create_package_dep(package, check_dep_pkg.Name, + dep_type_name="checkdepends", + autocommit=False) + + # Geez. Just stop. This is optdepends. + opt_dep_pkg = create_package("test-dep-4", maintainer, autocommit=False) + opt_dep = create_package_dep(package, opt_dep_pkg.Name, + dep_type_name="optdepends", + autocommit=False) + + broken_dep = create_package_dep(package, "test-dep-5", + dep_type_name="depends", + autocommit=False) + + # Create an official provider record. + db.create(OfficialProvider, Name="test-dep-99", + Repo="core", Provides="test-dep-99", + autocommit=False) + official_dep = create_package_dep(package, "test-dep-99", + autocommit=False) + + # Also, create a provider who provides our test-dep-99. + provider = create_package("test-provider", maintainer, autocommit=False) + create_package_rel(provider, dep.DepName) + + with client as request: + resp = request.get(package_endpoint(package)) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + + expected = [ + dep.DepName, + make_dep.DepName, + check_dep.DepName, + opt_dep.DepName, + official_dep.DepName + ] + pkgdeps = root.findall('.//ul[@id="pkgdepslist"]/li/a') + for i, expectation in enumerate(expected): + assert pkgdeps[i].text.strip() == expectation + + broken_node = root.find('.//ul[@id="pkgdepslist"]/li/span') + assert broken_node.text.strip() == broken_dep.DepName diff --git a/test/test_packages_util.py b/test/test_packages_util.py new file mode 100644 index 00000000..17978490 --- /dev/null +++ b/test/test_packages_util.py @@ -0,0 +1,51 @@ +import pytest + +from fastapi.testclient import TestClient + +from aurweb import asgi, db +from aurweb.models.account_type import USER_ID, AccountType +from aurweb.models.official_provider import OFFICIAL_BASE, OfficialProvider +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.user import User +from aurweb.packages import util +from aurweb.testing import setup_test_db + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db( + User.__tablename__, + Package.__tablename__, + PackageBase.__tablename__, + OfficialProvider.__tablename__ + ) + + +@pytest.fixture +def maintainer() -> User: + account_type = db.query(AccountType, AccountType.ID == USER_ID).first() + yield db.create(User, Username="test_maintainer", + Email="test_maintainer@examepl.org", + Passwd="testPassword", + AccountType=account_type) + + +@pytest.fixture +def package(maintainer: User) -> Package: + pkgbase = db.create(PackageBase, Name="test-pkg", Maintainer=maintainer) + yield db.create(Package, Name=pkgbase.Name, PackageBase=pkgbase) + + +@pytest.fixture +def client() -> TestClient: + yield TestClient(app=asgi.app) + + +def test_package_link(client: TestClient, maintainer: User, package: Package): + db.create(OfficialProvider, + Name=package.Name, + Repo="core", + Provides=package.Name) + expected = f"{OFFICIAL_BASE}/packages/?q={package.Name}" + assert util.package_link(package) == expected diff --git a/test/test_templates.py b/test/test_templates.py new file mode 100644 index 00000000..8e3017b4 --- /dev/null +++ b/test/test_templates.py @@ -0,0 +1,15 @@ +import pytest + +from aurweb.templates import register_filter + + +@register_filter("func") +def func(): pass + + +def test_register_filter_exists_key_error(): + """ Most instances of register_filter are tested through module + imports or template renders, so we only test failures here. """ + with pytest.raises(KeyError): + @register_filter("func") + def some_func(): pass diff --git a/web/html/js/copy.js b/web/html/js/copy.js new file mode 100644 index 00000000..f46299b3 --- /dev/null +++ b/web/html/js/copy.js @@ -0,0 +1,6 @@ +document.addEventListener('DOMContentLoaded', function() { + document.querySelector('.copy').addEventListener('click', function(e) { + e.preventDefault(); + navigator.clipboard.writeText(event.target.text); + }); +}); From 88569b6d0987dd58e11198964e7bf597ed75d311 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 15 Jul 2021 22:54:41 -0700 Subject: [PATCH 278/844] add /pkgbase/{name} route Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 17 +++++++++ .../partials/packages/pkgbase_metadata.html | 13 +++++++ templates/pkgbase.html | 21 +++++++++++ test/test_packages_routes.py | 37 +++++++++++++++++++ 4 files changed, 88 insertions(+) create mode 100644 templates/partials/packages/pkgbase_metadata.html create mode 100644 templates/pkgbase.html diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 9650df85..cb7f4a18 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -96,3 +96,20 @@ async def package(request: Request, name: str) -> Response: context["conflicts"] = conflicts return render_template(request, "packages/show.html", context) + + +@router.get("/pkgbase/{name}") +async def package_base(request: Request, name: str) -> Response: + # Get the PackageBase. + pkgbase = get_pkgbase(name) + + # If this is not a split package, redirect to /packages/{name}. + if pkgbase.packages.count() == 1: + return RedirectResponse(f"/packages/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) + + # Add our base information. + context = await make_single_context(request, pkgbase) + context["packages"] = pkgbase.packages.all() # Doesn't need to be here. + + return render_template(request, "pkgbase.html", context) diff --git a/templates/partials/packages/pkgbase_metadata.html b/templates/partials/packages/pkgbase_metadata.html new file mode 100644 index 00000000..ba27fda5 --- /dev/null +++ b/templates/partials/packages/pkgbase_metadata.html @@ -0,0 +1,13 @@ +
    +

    Packages ({{ packages_count }})

    + +
    diff --git a/templates/pkgbase.html b/templates/pkgbase.html new file mode 100644 index 00000000..315cdf67 --- /dev/null +++ b/templates/pkgbase.html @@ -0,0 +1,21 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} + {% include "partials/packages/search.html" %} +
    +

    {{ 'Package Base Details' | tr }}: {{ pkgbase.Name }}

    + + {% set result = pkgbase %} + {% include "partials/packages/actions.html" %} + {% include "partials/packages/details.html" %} + +
    + {% include "partials/packages/pkgbase_metadata.html" %} +
    +
    + + {% set pkgname = result.Name %} + {% set pkgbase_id = result.ID %} + {% set comments = comments %} + {% include "partials/packages/comments.html" %} +{% endblock %} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index f9592238..44ef7fcd 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -281,3 +281,40 @@ def test_package_dependencies(client: TestClient, maintainer: User, broken_node = root.find('.//ul[@id="pkgdepslist"]/li/span') assert broken_node.text.strip() == broken_dep.DepName + + +def test_pkgbase_not_found(client: TestClient): + with client as request: + resp = request.get("/pkgbase/not_found") + assert resp.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_pkgbase_redirect(client: TestClient, package: Package): + with client as request: + resp = request.get(f"/pkgbase/{package.Name}", + allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert resp.headers.get("location") == f"/packages/{package.Name}" + + +def test_pkgbase(client: TestClient, package: Package): + second = db.create(Package, Name="second-pkg", + PackageBase=package.PackageBase) + + expected = [package.Name, second.Name] + with client as request: + resp = request.get(f"/pkgbase/{package.Name}", + allow_redirects=False) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + + # Check the details box title. + title = root.find('.//div[@id="pkgdetails"]/h2') + title, pkgname = title.text.split(": ") + assert title == "Package Base Details" + assert pkgname == package.Name + + pkgs = root.findall('.//div[@id="pkgs"]/ul/li/a') + for i, name in enumerate(expected): + assert pkgs[i].text.strip() == name From 04d1c81d3dc3af59749ca3555a9fd45f4a9fcb78 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 27 Jul 2021 22:03:38 -0700 Subject: [PATCH 279/844] bugfix: fix extra dependency annotations These were being displayed regardless of the dep type and state of DepDesc. This is fixed with this commit. Signed-off-by: Kevin Morris --- aurweb/packages/util.py | 2 ++ templates/partials/packages/package_metadata.html | 6 ++++-- test/test_packages_routes.py | 11 ++++++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index 6681d479..698ae1af 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -46,6 +46,8 @@ def dep_extra(dep: PackageDependency) -> str: @register_filter("dep_extra_desc") def dep_extra_desc(dep: PackageDependency) -> str: extra = dep_extra(dep) + if not dep.DepDesc: + return extra return extra + f" – {dep.DepDesc}" diff --git a/templates/partials/packages/package_metadata.html b/templates/partials/packages/package_metadata.html index 767e25a9..7ec95699 100644 --- a/templates/partials/packages/package_metadata.html +++ b/templates/partials/packages/package_metadata.html @@ -16,9 +16,11 @@ {% endif %} {{ dep.Package | provides_list(dep.DepName) | safe }} - {% set extra = dep | dep_extra %} - {% if extra %} + + {% if dep.DepTypeID == 4 %} {{ dep | dep_extra_desc }} + {% else %} + {{ dep | dep_extra }} {% endif %} {% endfor %} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 44ef7fcd..82fbba40 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -247,7 +247,15 @@ def test_package_dependencies(client: TestClient, maintainer: User, dep_type_name="optdepends", autocommit=False) - broken_dep = create_package_dep(package, "test-dep-5", + # Heh. Another optdepends to test one with a description. + opt_desc_dep_pkg = create_package("test-dep-5", maintainer, + autocommit=False) + opt_desc_dep = create_package_dep(package, opt_desc_dep_pkg.Name, + dep_type_name="optdepends", + autocommit=False) + opt_desc_dep.DepDesc = "Test description." + + broken_dep = create_package_dep(package, "test-dep-6", dep_type_name="depends", autocommit=False) @@ -273,6 +281,7 @@ def test_package_dependencies(client: TestClient, maintainer: User, make_dep.DepName, check_dep.DepName, opt_dep.DepName, + opt_desc_dep.DepName, official_dep.DepName ] pkgdeps = root.findall('.//ul[@id="pkgdepslist"]/li/a') From bace345da4c6ff4f89e6cda0f916d8789ea5dba8 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 8 Aug 2021 18:32:35 -0700 Subject: [PATCH 280/844] Docker: support both '%' and 'localhost' in mariadb This is needed to be able to reach the mysql service from other hosts or through localhost. Handling both cases here means that we can support both localhost access and host access. Signed-off-by: Kevin Morris --- docker/mariadb-entrypoint.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docker/mariadb-entrypoint.sh b/docker/mariadb-entrypoint.sh index e38900c8..945a4b82 100755 --- a/docker/mariadb-entrypoint.sh +++ b/docker/mariadb-entrypoint.sh @@ -18,15 +18,19 @@ DATABASE="aurweb" # Persistent database for fastapi/php-fpm. TEST_DB="aurweb_test" # Test database (ephemereal). echo "Taking care of primary database '${DATABASE}'..." -mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'$DB_HOST' IDENTIFIED BY 'aur';" +mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'localhost' IDENTIFIED BY 'aur';" +mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'%' IDENTIFIED BY 'aur';" mysql -u root -e "CREATE DATABASE IF NOT EXISTS $DATABASE;" -mysql -u root -e "GRANT ALL ON ${DATABASE}.* TO 'aur'@'$DB_HOST';" +mysql -u root -e "GRANT ALL ON ${DATABASE}.* TO 'aur'@'localhost';" +mysql -u root -e "GRANT ALL ON ${DATABASE}.* TO 'aur'@'%';" # Drop and create our test database. echo "Dropping test database '$TEST_DB'..." mysql -u root -e "DROP DATABASE IF EXISTS $TEST_DB;" mysql -u root -e "CREATE DATABASE $TEST_DB;" -mysql -u root -e "GRANT ALL ON ${TEST_DB}.* TO 'aur'@'$DB_HOST';" +mysql -u root -e "GRANT ALL ON ${TEST_DB}.* TO 'aur'@'localhost';" +mysql -u root -e "GRANT ALL ON ${TEST_DB}.* TO 'aur'@'%';" + echo "Created new '$TEST_DB'!" mysqladmin -uroot shutdown From 4ade8b053992663242df361477ca26282cadfc20 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 6 Aug 2021 00:59:38 -0700 Subject: [PATCH 281/844] routers.packages: Simplify some existence checks Signed-off-by: Kevin Morris --- aurweb/auth.py | 15 ++++++++ aurweb/models/package_dependency.py | 2 +- aurweb/packages/util.py | 18 ++++------ aurweb/routers/packages.py | 44 +++++++++++------------ templates/partials/packages/comments.html | 2 +- templates/partials/packages/details.html | 10 +++--- 6 files changed, 48 insertions(+), 43 deletions(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index 316e7293..26e4073d 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -19,6 +19,17 @@ from aurweb.models.user import User from aurweb.templates import make_variable_context, render_template +class StubQuery: + """ Acts as a stubbed version of an orm.Query. Typically used + to masquerade fake records for an AnonymousUser. """ + + def filter(self, *args): + return StubQuery() + + def scalar(self): + return 0 + + class AnonymousUser: # Stub attributes used to mimic a real user. ID = 0 @@ -28,6 +39,10 @@ class AnonymousUser: # A stub ssh_pub_key relationship. ssh_pub_key = None + # Add stubbed relationship backrefs. + package_notifications = StubQuery() + package_votes = StubQuery() + # A nonce attribute, needed for all browser sessions; set in __init__. nonce = None diff --git a/aurweb/models/package_dependency.py b/aurweb/models/package_dependency.py index 0e5b028b..9ce0b019 100644 --- a/aurweb/models/package_dependency.py +++ b/aurweb/models/package_dependency.py @@ -69,4 +69,4 @@ class PackageDependency(Base): pkg = db.query(Package, Package.Name == self.DepName) official = db.query(OfficialProvider, OfficialProvider.Name == self.DepName) - return pkg.count() > 0 or official.count() > 0 + return pkg.scalar() or official.scalar() diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index 698ae1af..60db2962 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -54,10 +54,9 @@ def dep_extra_desc(dep: PackageDependency) -> str: @register_filter("pkgname_link") def pkgname_link(pkgname: str) -> str: base = "/".join([OFFICIAL_BASE, "packages"]) - pkg = db.query(Package).filter(Package.Name == pkgname) official = db.query(OfficialProvider).filter( OfficialProvider.Name == pkgname) - if not pkg.count() or official.count(): + if official.scalar(): return f"{base}/?q={pkgname}" return f"/packages/{pkgname}" @@ -67,7 +66,7 @@ def package_link(package: Package) -> str: base = "/".join([OFFICIAL_BASE, "packages"]) official = db.query(OfficialProvider).filter( OfficialProvider.Name == package.Name) - if official.count(): + if official.scalar(): return f"{base}/?q={package.Name}" return f"/packages/{package.Name}" @@ -82,19 +81,14 @@ def provides_list(package: Package, depname: str) -> list: ) ) - string = str() - has_providers = providers.count() > 0 - - if has_providers: - string += "(" - - string += ", ".join([ + string = ", ".join([ f'{pkg.Name}' for pkg in providers ]) - if has_providers: - string += ")" + if string: + # If we actually constructed a string, wrap it. + string = f"({string})" return string diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index cb7f4a18..0ffcbfb9 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -19,9 +19,9 @@ from aurweb.models.package_notification import PackageNotification from aurweb.models.package_relation import PackageRelation from aurweb.models.package_source import PackageSource from aurweb.models.package_vote import PackageVote -from aurweb.models.relation_type import CONFLICTS_ID, RelationType +from aurweb.models.relation_type import CONFLICTS_ID from aurweb.packages.util import get_pkgbase -from aurweb.templates import make_variable_context, render_template +from aurweb.templates import make_context, render_template router = APIRouter() @@ -34,7 +34,7 @@ async def make_single_context(request: Request, :param pkgbase: PackageBase instance :return: A pkgbase context without specific differences """ - context = await make_variable_context(request, pkgbase.Name) + context = make_context(request, pkgbase.Name) context["git_clone_uri_anon"] = aurweb.config.get("options", "git_clone_uri_anon") context["git_clone_uri_priv"] = aurweb.config.get("options", @@ -44,20 +44,15 @@ async def make_single_context(request: Request, context["keywords"] = pkgbase.keywords context["comments"] = pkgbase.comments context["is_maintainer"] = (request.user.is_authenticated() - and request.user == pkgbase.Maintainer) - context["notified"] = db.query( - PackageNotification).join(PackageBase).filter( - and_(PackageBase.ID == pkgbase.ID, - PackageNotification.UserID == request.user.ID)).count() > 0 + and request.user.ID == pkgbase.MaintainerUID) + context["notified"] = request.user.package_notifications.filter( + PackageNotification.PackageBaseID == pkgbase.ID + ).scalar() context["out_of_date"] = bool(pkgbase.OutOfDateTS) - context["voted"] = pkgbase.package_votes.filter( - PackageVote.UsersID == request.user.ID).count() > 0 - - context["notifications_enabled"] = db.query( - PackageNotification).join(PackageBase).filter( - PackageBase.ID == pkgbase.ID).count() > 0 + context["voted"] = request.user.package_votes.filter( + PackageVote.PackageBaseID == pkgbase.ID).scalar() return context @@ -71,13 +66,12 @@ async def package(request: Request, name: str) -> Response: context = await make_single_context(request, pkgbase) # Package sources. - sources = db.query(PackageSource).join(Package).filter( - Package.PackageBaseID == pkgbase.ID) - context["sources"] = sources + context["sources"] = db.query(PackageSource).join(Package).join( + PackageBase).filter(PackageBase.ID == pkgbase.ID) # Package dependencies. - dependencies = db.query(PackageDependency).join(Package).filter( - Package.PackageBaseID == pkgbase.ID) + dependencies = db.query(PackageDependency).join(Package).join( + PackageBase).filter(PackageBase.ID == pkgbase.ID) context["dependencies"] = dependencies # Package requirements (other packages depend on this one). @@ -86,13 +80,15 @@ async def package(request: Request, name: str) -> Response: Package.Name.asc()) context["required_by"] = required_by - licenses = db.query(License).join(PackageLicense).join(Package).filter( - PackageLicense.PackageID == pkgbase.packages.first().ID) + licenses = db.query(License).join(PackageLicense).join(Package).join( + PackageBase).filter(PackageBase.ID == pkgbase.ID) context["licenses"] = licenses - conflicts = db.query(PackageRelation).join(RelationType).join(Package).join(PackageBase).filter( - and_(RelationType.ID == CONFLICTS_ID, - PackageBase.ID == pkgbase.ID)) + conflicts = db.query(PackageRelation).join(Package).join( + PackageBase).filter( + and_(PackageRelation.RelTypeID == CONFLICTS_ID, + PackageBase.ID == pkgbase.ID) + ) context["conflicts"] = conflicts return render_template(request, "packages/show.html", context) diff --git a/templates/partials/packages/comments.html b/templates/partials/packages/comments.html index f1bc020d..051849b0 100644 --- a/templates/partials/packages/comments.html +++ b/templates/partials/packages/comments.html @@ -49,7 +49,7 @@ {% endif %} -{% if comments.count() %} +{% if comments.scalar() %}

    diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index a25e9c9e..83c6d53b 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -1,3 +1,4 @@ +{% set pkg = pkgbase.packages.first() %} @@ -19,12 +20,11 @@ - + {% endif %} - {% if pkgbase.keywords.count() %} + {% if pkgbase.keywords.scalar() %} {% if is_maintainer %} @@ -63,13 +63,13 @@ {% endif %} {% endif %} - {% if licenses and licenses.count() and show_package_details | default(False) %} + {% if licenses and licenses.scalar() and show_package_details %} {% endif %} - {% if show_package_details | default(False) %} + {% if show_package_details %} - - + {% if request.user.is_authenticated() %} + + + {% endif %} @@ -21,33 +23,33 @@ {{ pkg.Name }} - {% if flagged %} - - {% else %} - - {% endif %} + {{ pkg.Version }} - - - + + {% if request.user.is_authenticated() %} + + + {% endif %} {% endfor %} From f147ef34767a91c0171b6d69ca2a2749ea0f4994 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 18 Aug 2021 22:14:35 -0700 Subject: [PATCH 297/844] models.account_type: remove duplicated constants Clearly made in mistake, removing to keep things organized. Signed-off-by: Kevin Morris --- aurweb/models/account_type.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/aurweb/models/account_type.py b/aurweb/models/account_type.py index 0db37ced..2e3dde06 100644 --- a/aurweb/models/account_type.py +++ b/aurweb/models/account_type.py @@ -28,12 +28,6 @@ class AccountType(Base): self.ID, str(self)) -# Define some AccountType.AccountType constants. -USER = "User" -TRUSTED_USER = "Trusted User" -DEVELOPER = "Developer" -TRUSTED_USER_AND_DEV = "Trusted User & Developer" - # Fetch account type IDs from the database for constants. _account_types = db.query(AccountType) USER_ID = _account_types.filter( From fb908189b61cce6241d8b4afdb4010ea279dfeea Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Sat, 28 Aug 2021 17:18:32 -0500 Subject: [PATCH 298/844] Began port of dependencies to pip Adds Python dependencies to requirements list to allow installation via pip --- requirements.txt | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..1c760057 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,40 @@ +# Arch Linux +pyalpm==0.10.6 +srcinfo==0.0.8 + +# Generic +authlib==0.15.2 +aiofiles==0.7.0 +asgiref==3.4.1 +bcrypt==3.2.0 +bleach==3.3.1 +coverage==5.5 +email-validator==1.1.3 +fakeredis==1.6.0 +fastapi==0.66.0 +feedgen==0.9.0 +flake8==3.9.2 +httpx==0.18.2 +hypercorn==0.11.2 +isort==5.9.3 +itsdangerous==2.0.1 +jinja2==3.0.1 +lxml==4.6.3 +markdown==3.3.4 +orjson==3.6.3 +protobuf==3.17.3 +pygit2==1.6.1 +pytest==6.2.4 +pytest-asyncio==0.15.1 +pytest-cov==2.12.1 +pytest-tap==3.2 +python-multipart==0.0.5 +redis==3.5.3 +requests==2.26.0 +uvicorn==0.15.0 +werkzeug==2.0.1 + +# SQL +alembic==1.6.5 +sqlalchemy==1.3.23 +mysqlclient==2.0.3 From b88fa8386ae9359fbf0d7cabf6efd008bd4d760b Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Sat, 28 Aug 2021 19:25:51 -0500 Subject: [PATCH 299/844] Removed pyalpm and srcinfo from pip requirements; Changed section title Changed 'Generic' to 'General' --- requirements.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1c760057..37a12f61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,4 @@ -# Arch Linux -pyalpm==0.10.6 -srcinfo==0.0.8 - -# Generic +# General authlib==0.15.2 aiofiles==0.7.0 asgiref==3.4.1 From 0075ba3c33c18831e277ddfceac53d8653a9dcc6 Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Sat, 28 Aug 2021 19:27:36 -0500 Subject: [PATCH 300/844] Added .python-version from Pyenv --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4fdfa790..885d9c2f 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ doc/rpc.html # Ignore any user-configured .envrc files at the root. /.envrc + +# Ignore .python-version file from Pyenv +.python-version From e69004bc4a135551fc89581147947fb8c8f0e68c Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Sat, 28 Aug 2021 19:29:44 -0500 Subject: [PATCH 301/844] Alphabetized .gitignore file so it looks prettier --- .gitignore | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 885d9c2f..f7ec5a95 100644 --- a/.gitignore +++ b/.gitignore @@ -1,23 +1,5 @@ -dummy-data.sql* -po/*.mo -po/*.po~ -po/POTFILES -web/locale/*/ -aur.git/ __pycache__/ *.py[cod] -test/test-results/ -test/trash directory* -schema/aur-schema-sqlite.sql -data.sql -aurweb.sqlite3 -conf/config -conf/config.sqlite -conf/config.sqlite.defaults -conf/docker -conf/docker.defaults -htmlcov/ -fastapi_aw/ .vim/ .pylintrc .coverage @@ -28,6 +10,24 @@ fastapi_aw/ /dist/ /aurweb.egg-info/ /pyrightconfig.json +aur.git/ +aurweb.sqlite3 +conf/config +conf/config.sqlite +conf/config.sqlite.defaults +conf/docker +conf/docker.defaults +data.sql +dummy-data.sql* +fastapi_aw/ +htmlcov/ +po/*.mo +po/*.po~ +po/POTFILES +schema/aur-schema-sqlite.sql +test/test-results/ +test/trash directory* +web/locale/*/ # Do not stage compiled asciidoc: make -C doc doc/rpc.html From e61050adcf41141d43a2018265b1c38b0c4031e4 Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Sat, 28 Aug 2021 19:31:11 -0500 Subject: [PATCH 302/844] Added env/ to .gitignore Folder will be used under virtualenv for pip dependencies --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f7ec5a95..581d5c17 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ conf/docker conf/docker.defaults data.sql dummy-data.sql* +env/ fastapi_aw/ htmlcov/ po/*.mo From 85b1a05d0138888e2f06c8fc5b474314abd090e9 Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Sat, 28 Aug 2021 19:51:05 -0500 Subject: [PATCH 303/844] Removed pip dependencies from docker/scripts/install-deps.sh --- docker/scripts/install-deps.sh | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh index a532a6b2..d22fd460 100755 --- a/docker/scripts/install-deps.sh +++ b/docker/scripts/install-deps.sh @@ -5,16 +5,12 @@ set -eou pipefail pacman -Syu --noconfirm --noprogressbar \ - --cachedir .pkg-cache git gpgme protobuf pyalpm \ - python-mysqlclient python-pygit2 python-srcinfo python-bleach \ - python-markdown python-sqlalchemy python-alembic python-pytest \ - python-werkzeug python-pytest-tap python-fastapi nginx python-authlib \ - python-itsdangerous python-httpx python-jinja python-pytest-cov \ - python-requests python-aiofiles python-python-multipart \ - python-pytest-asyncio python-coverage hypercorn python-bcrypt \ - python-email-validator openssh python-lxml mariadb mariadb-libs \ - python-isort flake8 cgit uwsgi uwsgi-plugin-cgi php php-fpm \ - python-asgiref uvicorn python-feedgen memcached php-memcached \ - python-redis redis python-fakeredis python-orjson + --cachedir .pkg-cache git gpgme \ + nginx redis openssh \ + mariadb mariadb-libs \ + cgit uwsgi uwsgi-plugin-cgi \ + php php-fpm \ + memcached php-memcached \ + pyalpm python-srcinfo exec "$@" From eff7d478ab29d541c7a486cccd27c23d0e803e5d Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Sat, 28 Aug 2021 20:12:35 -0500 Subject: [PATCH 304/844] Updated CI tests for pip dependencies; Changed styling in install-deps.sh --- .gitlab-ci.yml | 1 + Dockerfile | 26 +++++++++++++------------- docker/scripts/install-deps.sh | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7b8da2ae..d360d483 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,7 @@ variables: before_script: - ./docker/scripts/install-deps.sh + - pip install -r requirements.txt - useradd -U -d /aurweb -c 'AUR User' aur - ./docker/mariadb-entrypoint.sh - (cd '/usr' && /usr/bin/mysqld_safe --datadir='/var/lib/mysql') & diff --git a/Dockerfile b/Dockerfile index 2843fa1b..b610b8c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,19 @@ FROM archlinux:base-devel ENV PYTHONPATH=/aurweb ENV AUR_CONFIG=conf/config -# Copy our single bootstrap script. -COPY docker/scripts/install-deps.sh /install-deps.sh -RUN /install-deps.sh +# Copy Docker scripts +COPY ./docker /docker +COPY ./docker/scripts/*.sh /usr/local/bin/ + +# Copy over all aurweb files. +COPY . /aurweb + +# Working directory is aurweb root @ /aurweb. +WORKDIR /aurweb + +# Install dependencies +RUN docker/scripts/install-deps.sh +RUN pip install -r requirements.txt # Add our aur user. RUN useradd -U -d /aurweb -c 'AUR User' aur @@ -13,16 +23,6 @@ RUN useradd -U -d /aurweb -c 'AUR User' aur # Setup some default system stuff. RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime -# Copy the rest of docker. -COPY ./docker /docker -COPY ./docker/scripts/*.sh /usr/local/bin/ - -# Copy from host to container. -COPY . /aurweb - -# Working directory is aurweb root @ /aurweb. -WORKDIR /aurweb - # Install translations. RUN make -C po all install diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh index d22fd460..4985fe85 100755 --- a/docker/scripts/install-deps.sh +++ b/docker/scripts/install-deps.sh @@ -11,6 +11,6 @@ pacman -Syu --noconfirm --noprogressbar \ cgit uwsgi uwsgi-plugin-cgi \ php php-fpm \ memcached php-memcached \ - pyalpm python-srcinfo + python-pip pyalpm python-srcinfo exec "$@" From a0be0185475e38adcb628b776f440efc5685c23d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 28 Aug 2021 21:30:36 -0700 Subject: [PATCH 305/844] Docker: Reorder dependency installation for cache purposes Signed-off-by: Kevin Morris --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b610b8c1..6539bd94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,14 +7,16 @@ ENV AUR_CONFIG=conf/config COPY ./docker /docker COPY ./docker/scripts/*.sh /usr/local/bin/ +# Install system-wide dependencies. +RUN /docker/scripts/install-deps.sh + # Copy over all aurweb files. COPY . /aurweb # Working directory is aurweb root @ /aurweb. WORKDIR /aurweb -# Install dependencies -RUN docker/scripts/install-deps.sh +# Install pip directories now that we have access to /aurweb. RUN pip install -r requirements.txt # Add our aur user. From 1c26ce52a5e83abe99d5bea055aac8231b97428f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 30 Aug 2021 18:17:14 -0700 Subject: [PATCH 306/844] [FastAPI] include DepArch in dependency list Signed-off-by: Kevin Morris --- templates/partials/packages/package_metadata.html | 3 +++ test/test_packages_routes.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/templates/partials/packages/package_metadata.html b/templates/partials/packages/package_metadata.html index 7ec95699..e7b1aefb 100644 --- a/templates/partials/packages/package_metadata.html +++ b/templates/partials/packages/package_metadata.html @@ -16,6 +16,9 @@ {% endif %} {{ dep.Package | provides_list(dep.DepName) | safe }} + {% if dep.DepArch %} + ({{ dep.DepArch }}) + {% endif %} {% if dep.DepTypeID == 4 %} {{ dep | dep_extra_desc }} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 82fbba40..0c9d80e8 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -228,6 +228,7 @@ def test_package_dependencies(client: TestClient, maintainer: User, # Create a normal dependency of type depends. dep_pkg = create_package("test-dep-1", maintainer, autocommit=False) dep = create_package_dep(package, dep_pkg.Name, autocommit=False) + dep.DepArch = "x86_64" # Also, create a makedepends. make_dep_pkg = create_package("test-dep-2", maintainer, autocommit=False) @@ -288,6 +289,11 @@ def test_package_dependencies(client: TestClient, maintainer: User, for i, expectation in enumerate(expected): assert pkgdeps[i].text.strip() == expectation + # Let's make sure the DepArch was displayed for our first dep. + arch = root.findall('.//ul[@id="pkgdepslist"]/li')[0] + arch = arch.xpath('./em')[1] + assert arch.text.strip() == "(x86_64)" + broken_node = root.find('.//ul[@id="pkgdepslist"]/li/span') assert broken_node.text.strip() == broken_dep.DepName From 45fbf214b46fd5e3f90157ee2aaf94cdf62e62f4 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 20 Aug 2021 16:29:04 -0700 Subject: [PATCH 307/844] jinja2: add 'tn' filter, a numerical translation The possibly plural version of `tr`, `tn` provides a way to translate strings into singular or plural form based on a given integer being 1 or not 1. Example use: ``` {{ 1 | tn("%d package found.", "%d packages found.") | format(1) }} ``` Signed-off-by: Kevin Morris --- aurweb/l10n.py | 18 ++++++++++++++++++ aurweb/templates.py | 3 ++- test/test_l10n.py | 12 ++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/aurweb/l10n.py b/aurweb/l10n.py index 9270f3ce..c4938d64 100644 --- a/aurweb/l10n.py +++ b/aurweb/l10n.py @@ -93,3 +93,21 @@ def tr(context: typing.Any, value: str): """ A translation filter; example: {{ "Hello" | tr("de") }}. """ _ = get_translator_for_request(context.get("request")) return _(value) + + +@pass_context +def tn(context: typing.Dict[str, typing.Any], count: int, + singular: str, plural: str) -> str: + """ A singular and plural translation filter. + + Example: + {{ some_integer | tn("singular %d", "plural %d") }} + + :param context: Response context + :param count: The number used to decide singular or plural state + :param singular: The singular translation + :param plural: The plural translation + :return: Translated string + """ + gettext = get_raw_translator_for_request(context.get("request")) + return gettext.ngettext(singular, plural, count) diff --git a/aurweb/templates.py b/aurweb/templates.py index fa7aa039..7530472f 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -23,8 +23,9 @@ loader = jinja2.FileSystemLoader(os.path.join( env = jinja2.Environment(loader=loader, autoescape=True, extensions=["jinja2.ext.i18n"]) -# Add tr translation filter. +# Add t{r,n} translation filters. env.filters["tr"] = l10n.tr +env.filters["tn"] = l10n.tn # Utility filters. env.filters["dt"] = util.timestamp_to_datetime diff --git a/test/test_l10n.py b/test/test_l10n.py index e833cd44..1c2ae95a 100644 --- a/test/test_l10n.py +++ b/test/test_l10n.py @@ -36,3 +36,15 @@ def test_get_translator_for_request(): translate = l10n.get_translator_for_request(request) assert translate("Home") == "Startseite" + + +def test_tn_filter(): + request = Request() + request.cookies["AURLANG"] = "en" + context = {"language": "en", "request": request} + + translated = l10n.tn(context, 1, "%d package found.", "%d packages found.") + assert translated == "%d package found." + + translated = l10n.tn(context, 2, "%d package found.", "%d packages found.") + assert translated == "%d packages found." From 55c29c4519c200f32ad2463a63f5a46f7a850f2c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 30 Aug 2021 18:27:17 -0700 Subject: [PATCH 308/844] partials/packages/details.html: Add package request count This was missed during the original implementation merge. Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 5 +++ templates/partials/packages/actions.html | 8 ++++- test/test_packages_routes.py | 45 ++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 0ffcbfb9..0873bd9f 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -17,6 +17,7 @@ from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_license import PackageLicense from aurweb.models.package_notification import PackageNotification from aurweb.models.package_relation import PackageRelation +from aurweb.models.package_request import PackageRequest from aurweb.models.package_source import PackageSource from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import CONFLICTS_ID @@ -54,6 +55,10 @@ async def make_single_context(request: Request, context["voted"] = request.user.package_votes.filter( PackageVote.PackageBaseID == pkgbase.ID).scalar() + context["requests"] = pkgbase.requests.filter( + PackageRequest.ClosedTS.is_(None) + ).count() + return context diff --git a/templates/partials/packages/actions.html b/templates/partials/packages/actions.html index 87db3a3f..346537be 100644 --- a/templates/partials/packages/actions.html +++ b/templates/partials/packages/actions.html @@ -124,7 +124,13 @@ {% endif %} -
  • + {% if requests %} +
  • + + {{ requests | tn("%d pending request", "%d pending requests") | format(requests) }} + +
  • + {% endif %}
  • {% if not request.user.is_authenticated() %} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 0c9d80e8..ad07ec17 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -15,7 +15,9 @@ from aurweb.models.package_comment import PackageComment from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_keyword import PackageKeyword from aurweb.models.package_relation import PackageRelation +from aurweb.models.package_request import PackageRequest from aurweb.models.relation_type import PROVIDES_ID, RelationType +from aurweb.models.request_type import DELETION_ID, RequestType from aurweb.models.user import User from aurweb.testing import setup_test_db from aurweb.testing.html import parse_root @@ -173,6 +175,43 @@ def test_package_comments(client: TestClient, user: User, package: Package): assert comments[i].strip() == row +def test_package_requests_display(client: TestClient, user: User, + package: Package): + type_ = db.query(RequestType, RequestType.ID == DELETION_ID).first() + db.create(PackageRequest, PackageBase=package.PackageBase, + PackageBaseName=package.PackageBase.Name, + User=user, RequestType=type_, + Comments="Test comment.", + ClosureComment=str()) + + # Test that a single request displays "1 pending request". + with client as request: + resp = request.get(package_endpoint(package)) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + selector = '//div[@id="actionlist"]/ul/li/span[@class="flagged"]' + target = root.xpath(selector)[0] + assert target.text.strip() == "1 pending request" + + type_ = db.query(RequestType, RequestType.ID == DELETION_ID).first() + db.create(PackageRequest, PackageBase=package.PackageBase, + PackageBaseName=package.PackageBase.Name, + User=user, RequestType=type_, + Comments="Test comment2.", + ClosureComment=str()) + + # Test that a two requests display "2 pending requests". + with client as request: + resp = request.get(package_endpoint(package)) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + selector = '//div[@id="actionlist"]/ul/li/span[@class="flagged"]' + target = root.xpath(selector)[0] + assert target.text.strip() == "2 pending requests" + + def test_package_authenticated(client: TestClient, user: User, package: Package): """ We get the same here for either authenticated or not @@ -196,6 +235,12 @@ def test_package_authenticated(client: TestClient, user: User, for expected_text in expected: assert expected_text in resp.text + # When no requests are up, make sure we don't see the display for them. + root = parse_root(resp.text) + selector = '//div[@id="actionlist"]/ul/li/span[@class="flagged"]' + target = root.xpath(selector) + assert len(target) == 0 + def test_package_authenticated_maintainer(client: TestClient, maintainer: User, From 718ae1acba880d6ecf7488e88c3ac90bd357d493 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 30 Aug 2021 22:14:31 -0700 Subject: [PATCH 309/844] aurweb.templates: loader -> _loader, env -> _env These are module local globals and we don't want to expose global functionality to users, so privatize them with a leading `_` prefix. These things should **really** not be accessible by users. --- aurweb/templates.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/aurweb/templates.py b/aurweb/templates.py index 7530472f..48391b4a 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -18,29 +18,29 @@ import aurweb.config from aurweb import captcha, l10n, time, util # Prepare jinja2 objects. -loader = jinja2.FileSystemLoader(os.path.join( +_loader = jinja2.FileSystemLoader(os.path.join( aurweb.config.get("options", "aurwebdir"), "templates")) -env = jinja2.Environment(loader=loader, autoescape=True, - extensions=["jinja2.ext.i18n"]) +_env = jinja2.Environment(loader=_loader, autoescape=True, + extensions=["jinja2.ext.i18n"]) # Add t{r,n} translation filters. -env.filters["tr"] = l10n.tr -env.filters["tn"] = l10n.tn +_env.filters["tr"] = l10n.tr +_env.filters["tn"] = l10n.tn # Utility filters. -env.filters["dt"] = util.timestamp_to_datetime -env.filters["as_timezone"] = util.as_timezone -env.filters["dedupe_qs"] = util.dedupe_qs -env.filters["urlencode"] = quote_plus -env.filters["get_vote"] = util.get_vote -env.filters["number_format"] = util.number_format +_env.filters["dt"] = util.timestamp_to_datetime +_env.filters["as_timezone"] = util.as_timezone +_env.filters["dedupe_qs"] = util.dedupe_qs +_env.filters["urlencode"] = quote_plus +_env.filters["get_vote"] = util.get_vote +_env.filters["number_format"] = util.number_format # Add captcha filters. -env.filters["captcha_salt"] = captcha.captcha_salt_filter -env.filters["captcha_cmdline"] = captcha.captcha_cmdline_filter +_env.filters["captcha_salt"] = captcha.captcha_salt_filter +_env.filters["captcha_cmdline"] = captcha.captcha_cmdline_filter # Add account utility filters. -env.filters["account_url"] = util.account_url +_env.filters["account_url"] = util.account_url def register_filter(name: str) -> Callable: @@ -61,9 +61,9 @@ def register_filter(name: str) -> Callable: @functools.wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) - if name in env.filters: + if name in _env.filters: raise KeyError(f"Jinja already has a filter named '{name}'") - env.filters[name] = wrapper + _env.filters[name] = wrapper return wrapper return decorator @@ -105,12 +105,12 @@ def render_template(request: Request, status_code: HTTPStatus = HTTPStatus.OK): """ Render a Jinja2 multi-lingual template with some context. """ - # Create a deep copy of our jinja2 environment. The environment in + # Create a deep copy of our jinja2 _environment. The _environment in # total by itself is 48 bytes large (according to sys.getsizeof). # This is done so we can install gettext translations on the template - # environment being rendered without installing them into a global + # _environment being rendered without installing them into a global # which is reused in this function. - templates = copy.copy(env) + templates = copy.copy(_env) translator = l10n.get_raw_translator_for_request(context.get("request")) templates.install_gettext_translations(translator) From e15a18e9fb05afac2ddc1fdc12965b80eafa5435 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 30 Aug 2021 23:04:55 -0700 Subject: [PATCH 310/844] remove unneeded comment Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 0873bd9f..a20c97b1 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -111,6 +111,6 @@ async def package_base(request: Request, name: str) -> Response: # Add our base information. context = await make_single_context(request, pkgbase) - context["packages"] = pkgbase.packages.all() # Doesn't need to be here. + context["packages"] = pkgbase.packages.all() return render_template(request, "pkgbase.html", context) From 49cc12f99dbe2bc69edd09db39692ad2135473af Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 20 Aug 2021 14:44:36 -0700 Subject: [PATCH 311/844] jinja2: rename filter 'urlencode' to 'quote_plus' urlencode does more than just a quote_plus. Using urlencode was not sensible, so this commit addresses that. Signed-off-by: Kevin Morris --- aurweb/templates.py | 2 +- aurweb/util.py | 3 +++ templates/partials/packages/actions.html | 6 +++--- templates/partials/tu/proposal/voters.html | 2 +- templates/partials/tu/proposals.html | 6 +++--- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/aurweb/templates.py b/aurweb/templates.py index 48391b4a..a648d5a1 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -31,7 +31,7 @@ _env.filters["tn"] = l10n.tn _env.filters["dt"] = util.timestamp_to_datetime _env.filters["as_timezone"] = util.as_timezone _env.filters["dedupe_qs"] = util.dedupe_qs -_env.filters["urlencode"] = quote_plus +_env.filters["quote_plus"] = quote_plus _env.filters["get_vote"] = util.get_vote _env.filters["number_format"] = util.number_format diff --git a/aurweb/util.py b/aurweb/util.py index 860bdd12..494a988d 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -1,4 +1,5 @@ import base64 +import logging import math import random import re @@ -18,6 +19,8 @@ from jinja2 import pass_context import aurweb.config +logger = logging.getLogger(__name__) + def make_random_string(length): return ''.join(random.choices(string.ascii_lowercase diff --git a/templates/partials/packages/actions.html b/templates/partials/packages/actions.html index 346537be..d552f2dd 100644 --- a/templates/partials/packages/actions.html +++ b/templates/partials/packages/actions.html @@ -42,12 +42,12 @@
  • {% endif %}
  • - + {{ "Vote for this package" | tr }}
  • - + {{ "Enable notifications" | tr }}
  • @@ -133,7 +133,7 @@ {% endif %}
  • {% if not request.user.is_authenticated() %} - + {{ "Submit Request" | tr }} {% else %} diff --git a/templates/partials/tu/proposal/voters.html b/templates/partials/tu/proposal/voters.html index 2fd42bdf..6069f97d 100644 --- a/templates/partials/tu/proposal/voters.html +++ b/templates/partials/tu/proposal/voters.html @@ -2,7 +2,7 @@
      {% for voter in voters %}
    • - + {{ voter.Username | e }}
    • diff --git a/templates/partials/tu/proposals.html b/templates/partials/tu/proposals.html index 13e705fc..ab90444e 100644 --- a/templates/partials/tu/proposals.html +++ b/templates/partials/tu/proposals.html @@ -23,7 +23,7 @@
  • @@ -97,7 +97,7 @@ {% set off_qs = "%s=%d" | format(off_param, off - 10) %} {% set by_qs = "%s=%s" | format(by_param, by | quote_plus) %} + href="?{{ q | extend_query([off_param, ([off - 10, 0] | max)], [by_param, by]) | urlencode }}"> ‹ Back {% endif %} @@ -106,7 +106,7 @@ {% set off_qs = "%s=%d" | format(off_param, off + 10) %} {% set by_qs = "%s=%s" | format(by_param, by | quote_plus) %} + href="?{{ q | extend_query([off_param, off + pp], [by_param, by]) | urlencode }}"> Next › {% endif %} diff --git a/test/test_util.py b/test/test_util.py index 06fc08d3..0cc45409 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,4 +1,3 @@ -from collections import OrderedDict from datetime import datetime from zoneinfo import ZoneInfo @@ -17,21 +16,6 @@ def test_as_timezone(): assert util.as_timezone(dt, "UTC") == dt.astimezone(tz=ZoneInfo("UTC")) -def test_dedupe_qs(): - items = OrderedDict() - items["key1"] = "test" - items["key2"] = "blah" - items["key3"] = 1 - - # Construct and test our query string. - query_string = '&'.join([f"{k}={v}" for k, v in items.items()]) - assert query_string == "key1=test&key2=blah&key3=1" - - # Add key1=changed and key2=changed to the query and dedupe it. - deduped = util.dedupe_qs(query_string, "key1=changed", "key3=changed") - assert deduped == "key2=blah&key1=changed&key3=changed" - - def test_number_format(): assert util.number_format(0.222, 2) == "0.22" assert util.number_format(0.226, 2) == "0.23" From b52059d43758e56729e475e01039b4ed19a64fab Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 31 Aug 2021 17:13:10 -0700 Subject: [PATCH 315/844] RPC: add deprecation warning for v1-v4 usage With FastAPI starting to come closer to a close, we've got to advertise this deprecation so that users have some time to adjust before making the changes. We have not specified a specific time here, but we'd like this message to reach users of the RPC API for at least a month before any modifications are made to the interface. Signed-off-by: Kevin Morris --- web/lib/aurjson.class.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/web/lib/aurjson.class.php b/web/lib/aurjson.class.php index 86eae22b..e7bc7f97 100644 --- a/web/lib/aurjson.class.php +++ b/web/lib/aurjson.class.php @@ -272,6 +272,15 @@ class AurJSON { 'results' => $data ); + if ($this->version != 5) { + $json_array['warning'] = 'The use of versions lower than 5 is ' + . 'now deprecated and will soon be unsupported. To ensure ' + . 'your API client supports the change without issue, it ' + . 'should use version 5 and adjust for any changes in the ' + . 'API interface. See https://aur.archlinux.org/rpc for ' + . 'documentation related to v5.'; + } + if ($error) { $json_array['error'] = $error; } From cfa95ef80ad758f1804896f582c73a8f907d9686 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 31 Aug 2021 17:13:10 -0700 Subject: [PATCH 316/844] RPC: add deprecation warning for v1-v4 usage With FastAPI starting to come closer to a close, we've got to advertise this deprecation so that users have some time to adjust before making the changes. We have not specified a specific time here, but we'd like this message to reach users of the RPC API for at least a month before any modifications are made to the interface. Signed-off-by: Kevin Morris --- web/lib/aurjson.class.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/web/lib/aurjson.class.php b/web/lib/aurjson.class.php index 86eae22b..e7bc7f97 100644 --- a/web/lib/aurjson.class.php +++ b/web/lib/aurjson.class.php @@ -272,6 +272,15 @@ class AurJSON { 'results' => $data ); + if ($this->version != 5) { + $json_array['warning'] = 'The use of versions lower than 5 is ' + . 'now deprecated and will soon be unsupported. To ensure ' + . 'your API client supports the change without issue, it ' + . 'should use version 5 and adjust for any changes in the ' + . 'API interface. See https://aur.archlinux.org/rpc for ' + . 'documentation related to v5.'; + } + if ($error) { $json_array['error'] = $error; } From a5943bf2add0231925d7836e2e0b587a4f5c7f05 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 2 Sep 2021 16:26:48 -0700 Subject: [PATCH 317/844] [FastAPI] Refactor db modifications For SQLAlchemy to automatically understand updates from the external world, it must use an `autocommit=True` in its session. This change breaks how we were using commit previously, as `autocommit=True` causes SQLAlchemy to commit when a SessionTransaction context hits __exit__. So, a refactoring was required of our tests: All usage of any `db.{create,delete}` must be called **within** a SessionTransaction context, created via new `db.begin()`. From this point forward, we're going to require: ``` with db.begin(): db.create(...) db.delete(...) db.session.delete(object) ``` With this, we now get external DB modifications automatically without reloading or restarting the FastAPI server, which we absolutely need for production. Signed-off-by: Kevin Morris --- aurweb/db.py | 44 +++++--- aurweb/models/user.py | 42 ++++---- aurweb/routers/accounts.py | 145 +++++++++++++-------------- aurweb/routers/html.py | 6 +- aurweb/routers/trusted_user.py | 20 ++-- test/test_account_type.py | 18 ++-- test/test_accounts_routes.py | 166 ++++++++++++++++--------------- test/test_api_rate_limit.py | 19 ++-- test/test_auth.py | 22 ++-- test/test_auth_routes.py | 9 +- test/test_ban.py | 10 +- test/test_db.py | 22 ++-- test/test_dependency_type.py | 14 ++- test/test_group.py | 11 +- test/test_homepage.py | 49 ++++----- test/test_license.py | 11 +- test/test_official_provider.py | 59 +++++------ test/test_package.py | 68 ++++++------- test/test_package_base.py | 61 ++++++------ test/test_package_blacklist.py | 16 +-- test/test_package_comment.py | 47 +++++---- test/test_package_dependency.py | 84 ++++++++-------- test/test_package_relation.py | 75 +++++++------- test/test_package_request.py | 99 ++++++++++-------- test/test_package_source.py | 23 +++-- test/test_packages_routes.py | 160 +++++++++++++++-------------- test/test_packages_util.py | 27 +++-- test/test_relation_type.py | 15 +-- test/test_request_type.py | 24 +++-- test/test_routes.py | 14 +-- test/test_rss.py | 15 ++- test/test_session.py | 34 ++++--- test/test_ssh_pub_key.py | 32 +++--- test/test_term.py | 19 ++-- test/test_trusted_user_routes.py | 166 ++++++++++++++++--------------- test/test_tu_voteinfo.py | 130 +++++++++++++----------- test/test_user.py | 124 ++++++++++++----------- 37 files changed, 998 insertions(+), 902 deletions(-) diff --git a/aurweb/db.py b/aurweb/db.py index c0147720..ea6b6918 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -59,20 +59,15 @@ def query(model, *args, **kwargs): return session.query(model).filter(*args, **kwargs) -def create(model, autocommit: bool = True, *args, **kwargs): +def create(model, *args, **kwargs): instance = model(*args, **kwargs) - add(instance) - if autocommit is True: - commit() - return instance + return add(instance) -def delete(model, *args, autocommit: bool = True, **kwargs): +def delete(model, *args, **kwargs): instance = session.query(model).filter(*args, **kwargs) for record in instance: session.delete(record) - if autocommit is True: - commit() def rollback(): @@ -84,8 +79,25 @@ def add(model): return model -def commit(): - session.commit() +def begin(): + """ Begin an SQLAlchemy SessionTransaction. + + This context is **required** to perform an modifications to the + database. + + Example: + + with db.begin(): + object = db.create(...) + # On __exit__, db.commit() is run. + + with db.begin(): + object = db.delete(...) + # On __exit__, db.commit() is run. + + :return: A new SessionTransaction based on session + """ + return session.begin() def get_sqlalchemy_url(): @@ -155,23 +167,23 @@ def get_engine(echo: bool = False): connect_args=connect_args, echo=echo) + Session = sessionmaker(autocommit=True, autoflush=False, bind=engine) + session = Session() + if db_backend == "sqlite": # For SQLite, we need to add some custom functions as # they are used in the reference graph method. def regexp(regex, item): return bool(re.search(regex, str(item))) - @event.listens_for(engine, "begin") - def do_begin(conn): + @event.listens_for(engine, "connect") + def do_begin(conn, record): create_deterministic_function = functools.partial( - conn.connection.create_function, + conn.create_function, deterministic=True ) create_deterministic_function("REGEXP", 2, regexp) - Session = sessionmaker(autocommit=False, autoflush=False, bind=engine) - session = Session() - return engine diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 0ccf7329..70d15f88 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -102,7 +102,7 @@ class User(Base): def login(self, request: Request, password: str, session_time=0): """ Login and authenticate a request. """ - from aurweb.db import session + from aurweb import db from aurweb.models.session import Session, generate_unique_sid if not self._login_approved(request): @@ -112,10 +112,7 @@ class User(Base): if not self.authenticated: return None - self.LastLogin = now_ts = datetime.utcnow().timestamp() - self.LastLoginIPAddress = request.client.host - session.commit() - + now_ts = datetime.utcnow().timestamp() session_ts = now_ts + ( session_time if session_time else aurweb.config.getint("options", "login_timeout") @@ -123,22 +120,23 @@ class User(Base): sid = None - if not self.session: - sid = generate_unique_sid() - self.session = Session(UsersID=self.ID, SessionID=sid, - LastUpdateTS=session_ts) - session.add(self.session) - else: - last_updated = self.session.LastUpdateTS - if last_updated and last_updated < now_ts: - self.session.SessionID = sid = generate_unique_sid() + with db.begin(): + self.LastLogin = now_ts + self.LastLoginIPAddress = request.client.host + if not self.session: + sid = generate_unique_sid() + self.session = Session(UsersID=self.ID, SessionID=sid, + LastUpdateTS=session_ts) + db.add(self.session) else: - # Session is still valid; retrieve the current SID. - sid = self.session.SessionID + last_updated = self.session.LastUpdateTS + if last_updated and last_updated < now_ts: + self.session.SessionID = sid = generate_unique_sid() + else: + # Session is still valid; retrieve the current SID. + sid = self.session.SessionID - self.session.LastUpdateTS = session_ts - - session.commit() + self.session.LastUpdateTS = session_ts request.cookies["AURSID"] = self.session.SessionID return self.session.SessionID @@ -149,13 +147,11 @@ class User(Base): return aurweb.auth.has_credential(self, cred, approved) def logout(self, request): - from aurweb.db import session - del request.cookies["AURSID"] self.authenticated = False if self.session: - session.delete(self.session) - session.commit() + with db.begin(): + db.session.delete(self.session) def is_trusted_user(self): return self.AccountType.ID in { diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 466d129d..ef4b99af 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -43,8 +43,6 @@ async def passreset_post(request: Request, resetkey: str = Form(default=None), password: str = Form(default=None), confirm: str = Form(default=None)): - from aurweb.db import session - context = await make_variable_context(request, "Password Reset") # The user parameter being required, we can match against @@ -86,12 +84,11 @@ async def passreset_post(request: Request, # We got to this point; everything matched up. Update the password # and remove the ResetKey. - user.ResetKey = str() - user.update_password(password) - - if user.session: - session.delete(user.session) - session.commit() + with db.begin(): + user.ResetKey = str() + if user.session: + db.session.delete(user.session) + user.update_password(password) # Render ?step=complete. return RedirectResponse(url="/passreset?step=complete", @@ -99,8 +96,8 @@ async def passreset_post(request: Request, # If we got here, we continue with issuing a resetkey for the user. resetkey = db.make_random_value(User, User.ResetKey) - user.ResetKey = resetkey - session.commit() + with db.begin(): + user.ResetKey = resetkey executor = db.ConnectionExecutor(db.get_engine().raw_connection()) ResetKeyNotification(executor, user.ID).send() @@ -364,8 +361,6 @@ async def account_register_post(request: Request, ON: bool = Form(default=False), captcha: str = Form(default=None), captcha_salt: str = Form(...)): - from aurweb.db import session - context = await make_variable_context(request, "Register") args = dict(await request.form()) @@ -394,11 +389,13 @@ async def account_register_post(request: Request, AccountType.AccountType == "User").first() # Create a user given all parameters available. - user = db.create(User, Username=U, Email=E, HideEmail=H, BackupEmail=BE, - RealName=R, Homepage=HP, IRCNick=I, PGPKey=K, - LangPreference=L, Timezone=TZ, CommentNotify=CN, - UpdateNotify=UN, OwnershipNotify=ON, ResetKey=resetkey, - AccountType=account_type) + with db.begin(): + user = db.create(User, Username=U, + Email=E, HideEmail=H, BackupEmail=BE, + RealName=R, Homepage=HP, IRCNick=I, PGPKey=K, + LangPreference=L, Timezone=TZ, CommentNotify=CN, + UpdateNotify=UN, OwnershipNotify=ON, + ResetKey=resetkey, AccountType=account_type) # If a PK was given and either one does not exist or the given # PK mismatches the existing user's SSHPubKey.PubKey. @@ -410,10 +407,10 @@ async def account_register_post(request: Request, # Remove the host part. pubkey = parts[0] + " " + parts[1] fingerprint = get_fingerprint(pubkey) - user.ssh_pub_key = SSHPubKey(UserID=user.ID, - PubKey=pubkey, - Fingerprint=fingerprint) - session.commit() + with db.begin(): + user.ssh_pub_key = SSHPubKey(UserID=user.ID, + PubKey=pubkey, + Fingerprint=fingerprint) # Send a reset key notification to the new user. executor = db.ConnectionExecutor(db.get_engine().raw_connection()) @@ -499,63 +496,67 @@ async def account_edit_post(request: Request, status_code=int(HTTPStatus.BAD_REQUEST)) # Set all updated fields as needed. - user.Username = U or user.Username - user.Email = E or user.Email - user.HideEmail = bool(H) - user.BackupEmail = BE or user.BackupEmail - user.RealName = R or user.RealName - user.Homepage = HP or user.Homepage - user.IRCNick = I or user.IRCNick - user.PGPKey = K or user.PGPKey - user.InactivityTS = datetime.utcnow().timestamp() if J else 0 + with db.begin(): + user.Username = U or user.Username + user.Email = E or user.Email + user.HideEmail = bool(H) + user.BackupEmail = BE or user.BackupEmail + user.RealName = R or user.RealName + user.Homepage = HP or user.Homepage + user.IRCNick = I or user.IRCNick + user.PGPKey = K or user.PGPKey + user.InactivityTS = datetime.utcnow().timestamp() if J else 0 # If we update the language, update the cookie as well. if L and L != user.LangPreference: request.cookies["AURLANG"] = L - user.LangPreference = L + with db.begin(): + user.LangPreference = L context["language"] = L # If we update the timezone, also update the cookie. if TZ and TZ != user.Timezone: - user.Timezone = TZ + with db.begin(): + user.Timezone = TZ request.cookies["AURTZ"] = TZ context["timezone"] = TZ - user.CommentNotify = bool(CN) - user.UpdateNotify = bool(UN) - user.OwnershipNotify = bool(ON) + with db.begin(): + user.CommentNotify = bool(CN) + user.UpdateNotify = bool(UN) + user.OwnershipNotify = bool(ON) # If a PK is given, compare it against the target user's PK. - if PK: - # Get the second token in the public key, which is the actual key. - pubkey = PK.strip().rstrip() - parts = pubkey.split(" ") - if len(parts) == 3: - # Remove the host part. - pubkey = parts[0] + " " + parts[1] - fingerprint = get_fingerprint(pubkey) - if not user.ssh_pub_key: - # No public key exists, create one. - user.ssh_pub_key = SSHPubKey(UserID=user.ID, - PubKey=pubkey, - Fingerprint=fingerprint) - elif user.ssh_pub_key.PubKey != pubkey: - # A public key already exists, update it. - user.ssh_pub_key.PubKey = pubkey - user.ssh_pub_key.Fingerprint = fingerprint - elif user.ssh_pub_key: - # Else, if the user has a public key already, delete it. - session.delete(user.ssh_pub_key) - - # Commit changes, if any. - session.commit() + with db.begin(): + if PK: + # Get the second token in the public key, which is the actual key. + pubkey = PK.strip().rstrip() + parts = pubkey.split(" ") + if len(parts) == 3: + # Remove the host part. + pubkey = parts[0] + " " + parts[1] + fingerprint = get_fingerprint(pubkey) + if not user.ssh_pub_key: + # No public key exists, create one. + user.ssh_pub_key = SSHPubKey(UserID=user.ID, + PubKey=pubkey, + Fingerprint=fingerprint) + elif user.ssh_pub_key.PubKey != pubkey: + # A public key already exists, update it. + user.ssh_pub_key.PubKey = pubkey + user.ssh_pub_key.Fingerprint = fingerprint + elif user.ssh_pub_key: + # Else, if the user has a public key already, delete it. + session.delete(user.ssh_pub_key) if P and not user.valid_password(P): # Remove the fields we consumed for passwords. context["P"] = context["C"] = str() # If a password was given and it doesn't match the user's, update it. - user.update_password(P) + with db.begin(): + user.update_password(P) + if user == request.user: # If the target user is the request user, login with # the updated password and update AURSID. @@ -731,21 +732,17 @@ async def terms_of_service_post(request: Request, accept_needed = sorted(unaccepted + diffs) return render_terms_of_service(request, context, accept_needed) - # For each term we found, query for the matching accepted term - # and update its Revision to the term's current Revision. - for term in diffs: - accepted_term = request.user.accepted_terms.filter( - AcceptedTerm.TermsID == term.ID).first() - accepted_term.Revision = term.Revision + with db.begin(): + # For each term we found, query for the matching accepted term + # and update its Revision to the term's current Revision. + for term in diffs: + accepted_term = request.user.accepted_terms.filter( + AcceptedTerm.TermsID == term.ID).first() + accepted_term.Revision = term.Revision - # For each term that was never accepted, accept it! - for term in unaccepted: - db.create(AcceptedTerm, User=request.user, - Term=term, Revision=term.Revision, - autocommit=False) - - if diffs or unaccepted: - # If we had any terms to update, commit the changes. - db.commit() + # For each term that was never accepted, accept it! + for term in unaccepted: + db.create(AcceptedTerm, User=request.user, + Term=term, Revision=term.Revision) return RedirectResponse("/", status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index c2375f69..c3fd3db1 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -44,8 +44,6 @@ async def language(request: Request, setting the language on any page, we want to preserve query parameters across the redirect. """ - from aurweb.db import session - if next[0] != '/': return HTMLResponse(b"Invalid 'next' parameter.", status_code=400) @@ -53,8 +51,8 @@ async def language(request: Request, # If the user is authenticated, update the user's LangPreference. if request.user.is_authenticated(): - request.user.LangPreference = set_lang - session.commit() + with db.begin(): + request.user.LangPreference = set_lang # In any case, set the response's AURLANG cookie that never expires. response = RedirectResponse(url=f"{next}{query_string}", diff --git a/aurweb/routers/trusted_user.py b/aurweb/routers/trusted_user.py index 61cfec6c..a977b31a 100644 --- a/aurweb/routers/trusted_user.py +++ b/aurweb/routers/trusted_user.py @@ -214,10 +214,9 @@ async def trusted_user_proposal_post(request: Request, return Response("Invalid 'decision' value.", status_code=int(HTTPStatus.BAD_REQUEST)) - vote = db.create(TUVote, User=request.user, VoteInfo=voteinfo, - autocommit=False) - voteinfo.ActiveTUs += 1 - db.commit() + with db.begin(): + vote = db.create(TUVote, User=request.user, VoteInfo=voteinfo) + voteinfo.ActiveTUs += 1 context["error"] = "You've already voted for this proposal." return render_proposal(request, context, proposal, voteinfo, voters, vote) @@ -294,12 +293,13 @@ async def trusted_user_addvote_post(request: Request, agenda = re.sub(r'<[/]?style.*>', '', agenda) # Create a new TUVoteInfo (proposal)! - voteinfo = db.create(TUVoteInfo, - User=user, - Agenda=agenda, - Submitted=timestamp, End=timestamp + duration, - Quorum=quorum, - Submitter=request.user) + with db.begin(): + voteinfo = db.create(TUVoteInfo, + User=user, + Agenda=agenda, + Submitted=timestamp, End=timestamp + duration, + Quorum=quorum, + Submitter=request.user) # Redirect to the new proposal. return RedirectResponse(f"/tu/{voteinfo.ID}", diff --git a/test/test_account_type.py b/test/test_account_type.py index fa4bc5ad..86e68253 100644 --- a/test/test_account_type.py +++ b/test/test_account_type.py @@ -1,6 +1,6 @@ import pytest -from aurweb.db import create, delete, query +from aurweb.db import begin, create, delete, query from aurweb.models.account_type import AccountType from aurweb.models.user import User from aurweb.testing import setup_test_db @@ -14,11 +14,13 @@ def setup(): global account_type - account_type = create(AccountType, AccountType="TestUser") + with begin(): + account_type = create(AccountType, AccountType="TestUser") yield account_type - delete(AccountType, AccountType.ID == account_type.ID) + with begin(): + delete(AccountType, AccountType.ID == account_type.ID) def test_account_type(): @@ -38,12 +40,14 @@ def test_account_type(): def test_user_account_type_relationship(): - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + with begin(): + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) assert user.AccountType == account_type # This must be deleted here to avoid foreign key issues when # deleting the temporary AccountType in the fixture. - delete(User, User.ID == user.ID) + with begin(): + delete(User, User.ID == user.ID) diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index 567b3426..9120f23f 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -11,9 +11,9 @@ import pytest from fastapi.testclient import TestClient -from aurweb import captcha +from aurweb import captcha, db from aurweb.asgi import app -from aurweb.db import commit, create, query +from aurweb.db import create, query from aurweb.models.accepted_term import AcceptedTerm from aurweb.models.account_type import DEVELOPER_ID, TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID, AccountType from aurweb.models.ban import Ban @@ -57,9 +57,11 @@ def setup(): account_type = query(AccountType, AccountType.AccountType == "User").first() - user = create(User, Username=TEST_USERNAME, Email=TEST_EMAIL, - RealName="Test UserZ", Passwd="testPassword", - IRCNick="testZ", AccountType=account_type) + + with db.begin(): + user = create(User, Username=TEST_USERNAME, Email=TEST_EMAIL, + RealName="Test UserZ", Passwd="testPassword", + IRCNick="testZ", AccountType=account_type) yield user @@ -70,9 +72,10 @@ def setup(): @pytest.fixture def tu_user(): - user.AccountType = query(AccountType, - AccountType.ID == TRUSTED_USER_AND_DEV_ID).first() - commit() + with db.begin(): + user.AccountType = query(AccountType).filter( + AccountType.ID == TRUSTED_USER_AND_DEV_ID + ).first() yield user @@ -149,11 +152,9 @@ def test_post_passreset_user(): def test_post_passreset_resetkey(): - from aurweb.db import session - - user.session = Session(UsersID=user.ID, SessionID="blah", - LastUpdateTS=datetime.utcnow().timestamp()) - session.commit() + with db.begin(): + user.session = Session(UsersID=user.ID, SessionID="blah", + LastUpdateTS=datetime.utcnow().timestamp()) # Prepare a password reset. with client as request: @@ -357,7 +358,8 @@ def test_post_register_error_invalid_captcha(): def test_post_register_error_ip_banned(): # 'testclient' is used as request.client.host via FastAPI TestClient. - create(Ban, IPAddress="testclient", BanTS=datetime.utcnow()) + with db.begin(): + create(Ban, IPAddress="testclient", BanTS=datetime.utcnow()) with client as request: response = post_register(request) @@ -576,7 +578,8 @@ def test_post_register_error_ssh_pubkey_taken(): # Take the sha256 fingerprint of the ssh public key, create it. fp = get_fingerprint(pk) - create(SSHPubKey, UserID=user.ID, PubKey=pk, Fingerprint=fp) + with db.begin(): + create(SSHPubKey, UserID=user.ID, PubKey=pk, Fingerprint=fp) with client as request: response = post_register(request, PK=pk) @@ -660,13 +663,11 @@ def test_post_account_edit(): def test_post_account_edit_dev(): - from aurweb.db import session - # Modify our user to be a "Trusted User & Developer" name = "Trusted User & Developer" tu_or_dev = query(AccountType, AccountType.AccountType == name).first() - user.AccountType = tu_or_dev - session.commit() + with db.begin(): + user.AccountType = tu_or_dev request = Request() sid = user.login(request, "testPassword") @@ -1001,21 +1002,19 @@ def get_rows(html): def test_post_accounts(tu_user): # Set a PGPKey. - user.PGPKey = "5F18B20346188419750745D7335F2CB41F253D30" + with db.begin(): + user.PGPKey = "5F18B20346188419750745D7335F2CB41F253D30" # Create a few more users. users = [user] - for i in range(10): - _user = create(User, Username=f"test_{i}", - Email=f"test_{i}@example.org", - RealName=f"Test #{i}", - Passwd="testPassword", - IRCNick=f"test_#{i}", - autocommit=False) - users.append(_user) - - # Commit everything to the database. - commit() + with db.begin(): + for i in range(10): + _user = create(User, Username=f"test_{i}", + Email=f"test_{i}@example.org", + RealName=f"Test #{i}", + Passwd="testPassword", + IRCNick=f"test_#{i}") + users.append(_user) sid = user.login(Request(), "testPassword") cookies = {"AURSID": sid} @@ -1085,11 +1084,12 @@ def test_post_accounts_account_type(tu_user): # test the `u` parameter. account_type = query(AccountType, AccountType.AccountType == "User").first() - create(User, Username="test_2", - Email="test_2@example.org", - RealName="Test User 2", - Passwd="testPassword", - AccountType=account_type) + with db.begin(): + create(User, Username="test_2", + Email="test_2@example.org", + RealName="Test User 2", + Passwd="testPassword", + AccountType=account_type) # Expect no entries; we marked our only user as a User type. with client as request: @@ -1113,9 +1113,10 @@ def test_post_accounts_account_type(tu_user): assert type.text.strip() == "User" # Set our only user to a Trusted User. - user.AccountType = query(AccountType, - AccountType.ID == TRUSTED_USER_ID).first() - commit() + with db.begin(): + user.AccountType = query(AccountType).filter( + AccountType.ID == TRUSTED_USER_ID + ).first() with client as request: response = request.post("/accounts/", cookies=cookies, @@ -1130,9 +1131,10 @@ def test_post_accounts_account_type(tu_user): assert type.text.strip() == "Trusted User" - user.AccountType = query(AccountType, - AccountType.ID == DEVELOPER_ID).first() - commit() + with db.begin(): + user.AccountType = query(AccountType).filter( + AccountType.ID == DEVELOPER_ID + ).first() with client as request: response = request.post("/accounts/", cookies=cookies, @@ -1147,10 +1149,10 @@ def test_post_accounts_account_type(tu_user): assert type.text.strip() == "Developer" - user.AccountType = query(AccountType, - AccountType.ID == TRUSTED_USER_AND_DEV_ID - ).first() - commit() + with db.begin(): + user.AccountType = query(AccountType).filter( + AccountType.ID == TRUSTED_USER_AND_DEV_ID + ).first() with client as request: response = request.post("/accounts/", cookies=cookies, @@ -1182,8 +1184,8 @@ def test_post_accounts_status(tu_user): username, type, status, realname, irc, pgp_key, edit = row assert status.text.strip() == "Active" - user.Suspended = True - commit() + with db.begin(): + user.Suspended = True with client as request: response = request.post("/accounts/", cookies=cookies, @@ -1244,12 +1246,13 @@ def test_post_accounts_sortby(tu_user): # Create a second user so we can compare sorts. account_type = query(AccountType, AccountType.ID == DEVELOPER_ID).first() - create(User, Username="test2", - Email="test2@example.org", - RealName="Test User 2", - Passwd="testPassword", - IRCNick="test2", - AccountType=account_type) + with db.begin(): + create(User, Username="test2", + Email="test2@example.org", + RealName="Test User 2", + Passwd="testPassword", + IRCNick="test2", + AccountType=account_type) sid = user.login(Request(), "testPassword") cookies = {"AURSID": sid} @@ -1297,9 +1300,10 @@ def test_post_accounts_sortby(tu_user): # Test the rows are reversed when ordering by RealName. assert compare_text_values(4, first_rows, reversed(rows)) is True - user.AccountType = query(AccountType, - AccountType.ID == TRUSTED_USER_AND_DEV_ID).first() - commit() + with db.begin(): + user.AccountType = query(AccountType).filter( + AccountType.ID == TRUSTED_USER_AND_DEV_ID + ).first() # Fetch first_rows again with our new AccountType ordering. with client as request: @@ -1322,8 +1326,8 @@ def test_post_accounts_sortby(tu_user): def test_post_accounts_pgp_key(tu_user): - user.PGPKey = "5F18B20346188419750745D7335F2CB41F253D30" - commit() + with db.begin(): + user.PGPKey = "5F18B20346188419750745D7335F2CB41F253D30" sid = user.login(Request(), "testPassword") cookies = {"AURSID": sid} @@ -1343,15 +1347,14 @@ def test_post_accounts_paged(tu_user): users = [user] account_type = query(AccountType, AccountType.AccountType == "User").first() - for i in range(150): - _user = create(User, Username=f"test_#{i}", - Email=f"test_#{i}@example.org", - RealName=f"Test User #{i}", - Passwd="testPassword", - AccountType=account_type, - autocommit=False) - users.append(_user) - commit() + with db.begin(): + for i in range(150): + _user = create(User, Username=f"test_#{i}", + Email=f"test_#{i}@example.org", + RealName=f"Test User #{i}", + Passwd="testPassword", + AccountType=account_type) + users.append(_user) sid = user.login(Request(), "testPassword") cookies = {"AURSID": sid} @@ -1414,8 +1417,9 @@ def test_post_accounts_paged(tu_user): def test_get_terms_of_service(): - term = create(Term, Description="Test term.", - URL="http://localhost", Revision=1) + with db.begin(): + term = create(Term, Description="Test term.", + URL="http://localhost", Revision=1) with client as request: response = request.get("/tos", allow_redirects=False) @@ -1436,8 +1440,9 @@ def test_get_terms_of_service(): response = request.get("/tos", cookies=cookies, allow_redirects=False) assert response.status_code == int(HTTPStatus.OK) - accepted_term = create(AcceptedTerm, User=user, - Term=term, Revision=term.Revision) + with db.begin(): + accepted_term = create(AcceptedTerm, User=user, + Term=term, Revision=term.Revision) with client as request: response = request.get("/tos", cookies=cookies, allow_redirects=False) @@ -1445,8 +1450,8 @@ def test_get_terms_of_service(): assert response.status_code == int(HTTPStatus.SEE_OTHER) # Bump the term's revision. - term.Revision = 2 - commit() + with db.begin(): + term.Revision = 2 with client as request: response = request.get("/tos", cookies=cookies, allow_redirects=False) @@ -1454,8 +1459,8 @@ def test_get_terms_of_service(): # yet been agreed to via AcceptedTerm update. assert response.status_code == int(HTTPStatus.OK) - accepted_term.Revision = term.Revision - commit() + with db.begin(): + accepted_term.Revision = term.Revision with client as request: response = request.get("/tos", cookies=cookies, allow_redirects=False) @@ -1471,8 +1476,9 @@ def test_post_terms_of_service(): cookies = {"AURSID": sid} # Auth cookie. # Create a fresh Term. - term = create(Term, Description="Test term.", - URL="http://localhost", Revision=1) + with db.begin(): + term = create(Term, Description="Test term.", + URL="http://localhost", Revision=1) # Test that the term we just created is listed. with client as request: @@ -1497,8 +1503,8 @@ def test_post_terms_of_service(): assert accepted_term.Term == term # Update the term to revision 2. - term.Revision = 2 - commit() + with db.begin(): + term.Revision = 2 # A GET request gives us the new revision to accept. with client as request: diff --git a/test/test_api_rate_limit.py b/test/test_api_rate_limit.py index 536e3841..25cb3e0f 100644 --- a/test/test_api_rate_limit.py +++ b/test/test_api_rate_limit.py @@ -2,6 +2,7 @@ import pytest from sqlalchemy.exc import IntegrityError +from aurweb import db from aurweb.db import create from aurweb.models.api_rate_limit import ApiRateLimit from aurweb.testing import setup_test_db @@ -13,26 +14,28 @@ def setup(): def test_api_rate_key_creation(): - rate = create(ApiRateLimit, IP="127.0.0.1", Requests=10, WindowStart=1) + with db.begin(): + rate = create(ApiRateLimit, IP="127.0.0.1", Requests=10, WindowStart=1) assert rate.IP == "127.0.0.1" assert rate.Requests == 10 assert rate.WindowStart == 1 def test_api_rate_key_ip_default(): - api_rate_limit = create(ApiRateLimit, Requests=10, WindowStart=1) + with db.begin(): + api_rate_limit = create(ApiRateLimit, Requests=10, WindowStart=1) assert api_rate_limit.IP == str() def test_api_rate_key_null_requests_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(ApiRateLimit, IP="127.0.0.1", WindowStart=1) - session.rollback() + with db.begin(): + create(ApiRateLimit, IP="127.0.0.1", WindowStart=1) + db.rollback() def test_api_rate_key_null_window_start_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(ApiRateLimit, IP="127.0.0.1", Requests=1) - session.rollback() + with db.begin(): + create(ApiRateLimit, IP="127.0.0.1", Requests=1) + db.rollback() diff --git a/test/test_auth.py b/test/test_auth.py index b386bea1..caa39468 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -4,6 +4,7 @@ import pytest from sqlalchemy.exc import IntegrityError +from aurweb import db from aurweb.auth import BasicAuthBackend, account_type_required, has_credential from aurweb.db import create, query from aurweb.models.account_type import USER, USER_ID, AccountType @@ -23,9 +24,10 @@ def setup(): account_type = query(AccountType, AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.com", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + with db.begin(): + user = create(User, Username="test", Email="test@example.com", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) backend = BasicAuthBackend() request = Request() @@ -51,14 +53,13 @@ async def test_auth_backend_invalid_sid(): @pytest.mark.asyncio async def test_auth_backend_invalid_user_id(): - from aurweb.db import session - # Create a new session with a fake user id. now_ts = datetime.utcnow().timestamp() with pytest.raises(IntegrityError): - create(Session, UsersID=666, SessionID="realSession", - LastUpdateTS=now_ts + 5) - session.rollback() + with db.begin(): + create(Session, UsersID=666, SessionID="realSession", + LastUpdateTS=now_ts + 5) + db.rollback() @pytest.mark.asyncio @@ -66,8 +67,9 @@ async def test_basic_auth_backend(): # This time, everything matches up. We expect the user to # equal the real_user. now_ts = datetime.utcnow().timestamp() - create(Session, UsersID=user.ID, SessionID="realSession", - LastUpdateTS=now_ts + 5) + with db.begin(): + create(Session, UsersID=user.ID, SessionID="realSession", + LastUpdateTS=now_ts + 5) request.cookies["AURSID"] = "realSession" _, result = await backend.authenticate(request) assert result == user diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index b0dd5648..1d8f9cbe 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -9,7 +9,7 @@ from fastapi.testclient import TestClient import aurweb.config from aurweb.asgi import app -from aurweb.db import create, query +from aurweb.db import begin, create, query from aurweb.models.account_type import AccountType from aurweb.models.session import Session from aurweb.models.user import User @@ -32,9 +32,10 @@ def setup(): account_type = query(AccountType, AccountType.AccountType == "User").first() - user = create(User, Username=TEST_USERNAME, Email=TEST_EMAIL, - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + with begin(): + user = create(User, Username=TEST_USERNAME, Email=TEST_EMAIL, + RealName="Test User", Passwd="testPassword", + AccountType=account_type) client = TestClient(app) diff --git a/test/test_ban.py b/test/test_ban.py index b728644b..f96e9d14 100644 --- a/test/test_ban.py +++ b/test/test_ban.py @@ -6,6 +6,7 @@ import pytest from sqlalchemy import exc as sa_exc +from aurweb import db from aurweb.db import create from aurweb.models.ban import Ban, is_banned from aurweb.testing import setup_test_db @@ -21,7 +22,8 @@ def setup(): setup_test_db("Bans") ts = datetime.utcnow() + timedelta(seconds=30) - ban = create(Ban, IPAddress="127.0.0.1", BanTS=ts) + with db.begin(): + ban = create(Ban, IPAddress="127.0.0.1", BanTS=ts) request = Request() @@ -35,17 +37,17 @@ def test_invalid_ban(): with pytest.raises(sa_exc.IntegrityError): bad_ban = Ban(BanTS=datetime.utcnow()) - session.add(bad_ban) # We're adding a ban with no primary key; this causes an # SQLAlchemy warnings when committing to the DB. # Ignore them. with warnings.catch_warnings(): warnings.simplefilter("ignore", sa_exc.SAWarning) - session.commit() + with db.begin(): + session.add(bad_ban) # Since we got a transaction failure, we need to rollback. - session.rollback() + db.rollback() def test_banned(): diff --git a/test/test_db.py b/test/test_db.py index 9ece25ea..7798d2f6 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -278,18 +278,15 @@ def test_connection_execute_paramstyle_unsupported(): def test_create_delete(): - db.create(AccountType, AccountType="test") + with db.begin(): + db.create(AccountType, AccountType="test") + record = db.query(AccountType, AccountType.AccountType == "test").first() assert record is not None - db.delete(AccountType, AccountType.AccountType == "test") - record = db.query(AccountType, AccountType.AccountType == "test").first() - assert record is None - # Create and delete a record with autocommit=False. - db.create(AccountType, AccountType="test", autocommit=False) - db.commit() - db.delete(AccountType, AccountType.AccountType == "test", autocommit=False) - db.commit() + with db.begin(): + db.delete(AccountType, AccountType.AccountType == "test") + record = db.query(AccountType, AccountType.AccountType == "test").first() assert record is None @@ -297,8 +294,8 @@ def test_create_delete(): def test_add_commit(): # Use db.add and db.commit to add a temporary record. account_type = AccountType(AccountType="test") - db.add(account_type) - db.commit() + with db.begin(): + db.add(account_type) # Assert it got created in the DB. assert bool(account_type.ID) @@ -308,7 +305,8 @@ def test_add_commit(): assert record == account_type # Remove the record. - db.delete(AccountType, AccountType.ID == account_type.ID) + with db.begin(): + db.delete(AccountType, AccountType.ID == account_type.ID) def test_connection_executor_mysql_paramstyle(): diff --git a/test/test_dependency_type.py b/test/test_dependency_type.py index 6c37cc58..4d555123 100644 --- a/test/test_dependency_type.py +++ b/test/test_dependency_type.py @@ -1,6 +1,6 @@ import pytest -from aurweb.db import create, delete, query +from aurweb.db import begin, create, delete, query from aurweb.models.dependency_type import DependencyType from aurweb.testing import setup_test_db @@ -19,13 +19,17 @@ def test_dependency_types(): def test_dependency_type_creation(): - dependency_type = create(DependencyType, Name="Test Type") + with begin(): + dependency_type = create(DependencyType, Name="Test Type") assert bool(dependency_type.ID) assert dependency_type.Name == "Test Type" - delete(DependencyType, DependencyType.ID == dependency_type.ID) + with begin(): + delete(DependencyType, DependencyType.ID == dependency_type.ID) def test_dependency_type_null_name_uses_default(): - dependency_type = create(DependencyType) + with begin(): + dependency_type = create(DependencyType) assert dependency_type.Name == str() - delete(DependencyType, DependencyType.ID == dependency_type.ID) + with begin(): + delete(DependencyType, DependencyType.ID == dependency_type.ID) diff --git a/test/test_group.py b/test/test_group.py index da017a96..cea69b68 100644 --- a/test/test_group.py +++ b/test/test_group.py @@ -2,7 +2,7 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create +from aurweb import db from aurweb.models.group import Group from aurweb.testing import setup_test_db @@ -13,13 +13,14 @@ def setup(): def test_group_creation(): - group = create(Group, Name="Test Group") + with db.begin(): + group = db.create(Group, Name="Test Group") assert bool(group.ID) assert group.Name == "Test Group" def test_group_null_name_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(Group) - session.rollback() + with db.begin(): + db.create(Group) + db.rollback() diff --git a/test/test_homepage.py b/test/test_homepage.py index 2cd6682f..fef3532d 100644 --- a/test/test_homepage.py +++ b/test/test_homepage.py @@ -38,8 +38,10 @@ def setup(): @pytest.fixture def user(): - yield db.create(User, Username="test", Email="test@example.org", - Passwd="testPassword", AccountTypeID=USER_ID) + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + Passwd="testPassword", AccountTypeID=USER_ID) + yield user @pytest.fixture @@ -68,17 +70,14 @@ def packages(user): # For i..num_packages, create a package named pkg_{i}. pkgs = [] now = int(datetime.utcnow().timestamp()) - for i in range(num_packages): - pkgbase = db.create(PackageBase, Name=f"pkg_{i}", - Maintainer=user, Packager=user, - autocommit=False, SubmittedTS=now, - ModifiedTS=now) - pkg = db.create(Package, PackageBase=pkgbase, - Name=pkgbase.Name, autocommit=False) - pkgs.append(pkg) - now += 1 - - db.commit() + with db.begin(): + for i in range(num_packages): + pkgbase = db.create(PackageBase, Name=f"pkg_{i}", + Maintainer=user, Packager=user, + SubmittedTS=now, ModifiedTS=now) + pkg = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + pkgs.append(pkg) + now += 1 yield pkgs @@ -159,10 +158,11 @@ def test_homepage_updates(redis, packages): def test_homepage_dashboard(redis, packages, user): # Create Comaintainer records for all of the packages. - for pkg in packages: - db.create(PackageComaintainer, PackageBase=pkg.PackageBase, - User=user, Priority=1, autocommit=False) - db.commit() + with db.begin(): + for pkg in packages: + db.create(PackageComaintainer, + PackageBase=pkg.PackageBase, + User=user, Priority=1) cookies = {"AURSID": user.login(Request(), "testPassword")} with client as request: @@ -193,11 +193,12 @@ def test_homepage_dashboard_requests(redis, packages, user): pkg = packages[0] reqtype = db.query(RequestType, RequestType.ID == DELETION_ID).first() - pkgreq = db.create(PackageRequest, PackageBase=pkg.PackageBase, - PackageBaseName=pkg.PackageBase.Name, - User=user, Comments=str(), - ClosureComment=str(), RequestTS=now, - RequestType=reqtype) + with db.begin(): + pkgreq = db.create(PackageRequest, PackageBase=pkg.PackageBase, + PackageBaseName=pkg.PackageBase.Name, + User=user, Comments=str(), + ClosureComment=str(), RequestTS=now, + RequestType=reqtype) cookies = {"AURSID": user.login(Request(), "testPassword")} with client as request: @@ -213,8 +214,8 @@ def test_homepage_dashboard_requests(redis, packages, user): def test_homepage_dashboard_flagged_packages(redis, packages, user): # Set the first Package flagged by setting its OutOfDateTS column. pkg = packages[0] - pkg.PackageBase.OutOfDateTS = int(datetime.utcnow().timestamp()) - db.commit() + with db.begin(): + pkg.PackageBase.OutOfDateTS = int(datetime.utcnow().timestamp()) cookies = {"AURSID": user.login(Request(), "testPassword")} with client as request: diff --git a/test/test_license.py b/test/test_license.py index feb7a396..2c52f058 100644 --- a/test/test_license.py +++ b/test/test_license.py @@ -2,7 +2,7 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create +from aurweb import db from aurweb.models.license import License from aurweb.testing import setup_test_db @@ -13,13 +13,14 @@ def setup(): def test_license_creation(): - license = create(License, Name="Test License") + with db.begin(): + license = db.create(License, Name="Test License") assert bool(license.ID) assert license.Name == "Test License" def test_license_null_name_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(License) - session.rollback() + with db.begin(): + db.create(License) + db.rollback() diff --git a/test/test_official_provider.py b/test/test_official_provider.py index a1d3d54a..0aa4f1d1 100644 --- a/test/test_official_provider.py +++ b/test/test_official_provider.py @@ -2,7 +2,7 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create +from aurweb import db from aurweb.models.official_provider import OfficialProvider from aurweb.testing import setup_test_db @@ -13,10 +13,11 @@ def setup(): def test_official_provider_creation(): - oprovider = create(OfficialProvider, - Name="some-name", - Repo="some-repo", - Provides="some-provides") + with db.begin(): + oprovider = db.create(OfficialProvider, + Name="some-name", + Repo="some-repo", + Provides="some-provides") assert bool(oprovider.ID) assert oprovider.Name == "some-name" assert oprovider.Repo == "some-repo" @@ -25,16 +26,18 @@ def test_official_provider_creation(): def test_official_provider_cs(): """ Test case sensitivity of the database table. """ - oprovider = create(OfficialProvider, - Name="some-name", - Repo="some-repo", - Provides="some-provides") + with db.begin(): + oprovider = db.create(OfficialProvider, + Name="some-name", + Repo="some-repo", + Provides="some-provides") assert bool(oprovider.ID) - oprovider_cs = create(OfficialProvider, - Name="SOME-NAME", - Repo="SOME-REPO", - Provides="SOME-PROVIDES") + with db.begin(): + oprovider_cs = db.create(OfficialProvider, + Name="SOME-NAME", + Repo="SOME-REPO", + Provides="SOME-PROVIDES") assert bool(oprovider_cs.ID) assert oprovider.ID != oprovider_cs.ID @@ -49,27 +52,27 @@ def test_official_provider_cs(): def test_official_provider_null_name_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(OfficialProvider, - Repo="some-repo", - Provides="some-provides") - session.rollback() + with db.begin(): + db.create(OfficialProvider, + Repo="some-repo", + Provides="some-provides") + db.rollback() def test_official_provider_null_repo_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(OfficialProvider, - Name="some-name", - Provides="some-provides") - session.rollback() + with db.begin(): + db.create(OfficialProvider, + Name="some-name", + Provides="some-provides") + db.rollback() def test_official_provider_null_provides_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(OfficialProvider, - Name="some-name", - Repo="some-repo") - session.rollback() + with db.begin(): + db.create(OfficialProvider, + Name="some-name", + Repo="some-repo") + db.rollback() diff --git a/test/test_package.py b/test/test_package.py index 1e940164..112ca9b4 100644 --- a/test/test_package.py +++ b/test/test_package.py @@ -3,7 +3,7 @@ import pytest from sqlalchemy import and_ from sqlalchemy.exc import IntegrityError -from aurweb.db import create, query +from aurweb import db from aurweb.models.account_type import AccountType from aurweb.models.package import Package from aurweb.models.package_base import PackageBase @@ -19,25 +19,25 @@ def setup(): setup_test_db("Packages", "PackageBases", "Users") - account_type = query(AccountType, - AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + account_type = db.query(AccountType, + AccountType.AccountType == "User").first() - pkgbase = create(PackageBase, - Name="beautiful-package", - Maintainer=user) - package = create(Package, - PackageBase=pkgbase, - Name=pkgbase.Name, - Description="Test description.", - URL="https://test.package") + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + + pkgbase = db.create(PackageBase, + Name="beautiful-package", + Maintainer=user) + package = db.create(Package, + PackageBase=pkgbase, + Name=pkgbase.Name, + Description="Test description.", + URL="https://test.package") def test_package(): - from aurweb.db import session - assert pkgbase == package.PackageBase assert package.Name == "beautiful-package" assert package.Description == "Test description." @@ -45,33 +45,31 @@ def test_package(): assert package.URL == "https://test.package" # Update package Version. - package.Version = "1.2.3" - session.commit() + with db.begin(): + package.Version = "1.2.3" # Make sure it got updated in the database. - record = query(Package, - and_(Package.ID == package.ID, - Package.Version == "1.2.3")).first() + record = db.query(Package, + and_(Package.ID == package.ID, + Package.Version == "1.2.3")).first() assert record is not None def test_package_null_pkgbase_raises_exception(): - from aurweb.db import session - with pytest.raises(IntegrityError): - create(Package, - Name="some-package", - Description="Some description.", - URL="https://some.package") - session.rollback() + with db.begin(): + db.create(Package, + Name="some-package", + Description="Some description.", + URL="https://some.package") + db.rollback() def test_package_null_name_raises_exception(): - from aurweb.db import session - with pytest.raises(IntegrityError): - create(Package, - PackageBase=pkgbase, - Description="Some description.", - URL="https://some.package") - session.rollback() + with db.begin(): + db.create(Package, + PackageBase=pkgbase, + Description="Some description.", + URL="https://some.package") + db.rollback() diff --git a/test/test_package_base.py b/test/test_package_base.py index 0c0d0526..2bc6278f 100644 --- a/test/test_package_base.py +++ b/test/test_package_base.py @@ -4,7 +4,7 @@ from sqlalchemy.exc import IntegrityError import aurweb.config -from aurweb.db import create, query +from aurweb import db from aurweb.models.account_type import AccountType from aurweb.models.package_base import PackageBase from aurweb.models.user import User @@ -19,17 +19,19 @@ def setup(): setup_test_db("Users", "PackageBases") - account_type = query(AccountType, - AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + account_type = db.query(AccountType, + AccountType.AccountType == "User").first() + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) def test_package_base(): - pkgbase = create(PackageBase, - Name="beautiful-package", - Maintainer=user) + with db.begin(): + pkgbase = db.create(PackageBase, + Name="beautiful-package", + Maintainer=user) assert pkgbase in user.maintained_bases assert not pkgbase.OutOfDateTS @@ -38,7 +40,8 @@ def test_package_base(): # Set Popularity to a string, then get it by attribute to # exercise the string -> float conversion path. - pkgbase.Popularity = "0.0" + with db.begin(): + pkgbase.Popularity = "0.0" assert pkgbase.Popularity == 0.0 @@ -47,27 +50,28 @@ def test_package_base_ci(): if aurweb.config.get("database", "backend") == "sqlite": return None # SQLite doesn't seem handle this. - from aurweb.db import session - - pkgbase = create(PackageBase, - Name="beautiful-package", - Maintainer=user) + with db.begin(): + pkgbase = db.create(PackageBase, + Name="beautiful-package", + Maintainer=user) assert bool(pkgbase.ID) with pytest.raises(IntegrityError): - create(PackageBase, - Name="Beautiful-Package", - Maintainer=user) - session.rollback() + with db.begin(): + db.create(PackageBase, + Name="Beautiful-Package", + Maintainer=user) + db.rollback() def test_package_base_relationships(): - pkgbase = create(PackageBase, - Name="beautiful-package", - Flagger=user, - Maintainer=user, - Submitter=user, - Packager=user) + with db.begin(): + pkgbase = db.create(PackageBase, + Name="beautiful-package", + Flagger=user, + Maintainer=user, + Submitter=user, + Packager=user) assert pkgbase in user.flagged_bases assert pkgbase in user.maintained_bases assert pkgbase in user.submitted_bases @@ -75,8 +79,7 @@ def test_package_base_relationships(): def test_package_base_null_name_raises_exception(): - from aurweb.db import session - with pytest.raises(IntegrityError): - create(PackageBase) - session.rollback() + with db.begin(): + db.create(PackageBase) + db.rollback() diff --git a/test/test_package_blacklist.py b/test/test_package_blacklist.py index 3c64cc21..93f15de7 100644 --- a/test/test_package_blacklist.py +++ b/test/test_package_blacklist.py @@ -2,7 +2,7 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, rollback +from aurweb import db from aurweb.models.package_base import PackageBase from aurweb.models.package_blacklist import PackageBlacklist from aurweb.models.user import User @@ -17,18 +17,20 @@ def setup(): setup_test_db("PackageBlacklist", "PackageBases", "Users") - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword") - pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword") + pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) def test_package_blacklist_creation(): - package_blacklist = create(PackageBlacklist, Name="evil-package") + with db.begin(): + package_blacklist = db.create(PackageBlacklist, Name="evil-package") assert bool(package_blacklist.ID) assert package_blacklist.Name == "evil-package" def test_package_blacklist_null_name_raises_exception(): with pytest.raises(IntegrityError): - create(PackageBlacklist) - rollback() + with db.begin(): + db.create(PackageBlacklist) + db.rollback() diff --git a/test/test_package_comment.py b/test/test_package_comment.py index ca77b511..60f0333d 100644 --- a/test/test_package_comment.py +++ b/test/test_package_comment.py @@ -2,7 +2,7 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, query, rollback +from aurweb.db import begin, create, query, rollback from aurweb.models.account_type import AccountType from aurweb.models.package_base import PackageBase from aurweb.models.package_comment import PackageComment @@ -20,45 +20,52 @@ def setup(): account_type = query(AccountType, AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + with begin(): + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + pkgbase = create(PackageBase, Name="test-package", Maintainer=user) def test_package_comment_creation(): - package_comment = create(PackageComment, - PackageBase=pkgbase, - User=user, - Comments="Test comment.", - RenderedComment="Test rendered comment.") + with begin(): + package_comment = create(PackageComment, + PackageBase=pkgbase, + User=user, + Comments="Test comment.", + RenderedComment="Test rendered comment.") assert bool(package_comment.ID) def test_package_comment_null_package_base_raises_exception(): with pytest.raises(IntegrityError): - create(PackageComment, User=user, Comments="Test comment.", - RenderedComment="Test rendered comment.") + with begin(): + create(PackageComment, User=user, Comments="Test comment.", + RenderedComment="Test rendered comment.") rollback() def test_package_comment_null_user_raises_exception(): with pytest.raises(IntegrityError): - create(PackageComment, PackageBase=pkgbase, Comments="Test comment.", - RenderedComment="Test rendered comment.") + with begin(): + create(PackageComment, PackageBase=pkgbase, + Comments="Test comment.", + RenderedComment="Test rendered comment.") rollback() def test_package_comment_null_comments_raises_exception(): with pytest.raises(IntegrityError): - create(PackageComment, PackageBase=pkgbase, User=user, - RenderedComment="Test rendered comment.") + with begin(): + create(PackageComment, PackageBase=pkgbase, User=user, + RenderedComment="Test rendered comment.") rollback() def test_package_comment_null_renderedcomment_defaults(): - record = create(PackageComment, - PackageBase=pkgbase, - User=user, - Comments="Test comment.") + with begin(): + record = create(PackageComment, + PackageBase=pkgbase, + User=user, + Comments="Test comment.") assert record.RenderedComment == str() diff --git a/test/test_package_dependency.py b/test/test_package_dependency.py index e28f1781..2ddef68e 100644 --- a/test/test_package_dependency.py +++ b/test/test_package_dependency.py @@ -2,7 +2,8 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import commit, create, query +from aurweb import db +from aurweb.db import create, query from aurweb.models.account_type import AccountType from aurweb.models.dependency_type import DependencyType from aurweb.models.package import Package @@ -22,25 +23,28 @@ def setup(): account_type = query(AccountType, AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - pkgbase = create(PackageBase, - Name="test-package", - Maintainer=user) - package = create(Package, - PackageBase=pkgbase, - Name=pkgbase.Name, - Description="Test description.", - URL="https://test.package") + with db.begin(): + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + pkgbase = create(PackageBase, + Name="test-package", + Maintainer=user) + package = create(Package, + PackageBase=pkgbase, + Name=pkgbase.Name, + Description="Test description.", + URL="https://test.package") def test_package_dependencies(): depends = query(DependencyType, DependencyType.Name == "depends").first() - pkgdep = create(PackageDependency, Package=package, - DependencyType=depends, - DepName="test-dep") + + with db.begin(): + pkgdep = create(PackageDependency, Package=package, + DependencyType=depends, + DepName="test-dep") assert pkgdep.DepName == "test-dep" assert pkgdep.Package == package assert pkgdep.DependencyType == depends @@ -49,8 +53,8 @@ def test_package_dependencies(): makedepends = query(DependencyType, DependencyType.Name == "makedepends").first() - pkgdep.DependencyType = makedepends - commit() + with db.begin(): + pkgdep.DependencyType = makedepends assert pkgdep.DepName == "test-dep" assert pkgdep.Package == package assert pkgdep.DependencyType == makedepends @@ -59,8 +63,8 @@ def test_package_dependencies(): checkdepends = query(DependencyType, DependencyType.Name == "checkdepends").first() - pkgdep.DependencyType = checkdepends - commit() + with db.begin(): + pkgdep.DependencyType = checkdepends assert pkgdep.DepName == "test-dep" assert pkgdep.Package == package assert pkgdep.DependencyType == checkdepends @@ -69,8 +73,8 @@ def test_package_dependencies(): optdepends = query(DependencyType, DependencyType.Name == "optdepends").first() - pkgdep.DependencyType = optdepends - commit() + with db.begin(): + pkgdep.DependencyType = optdepends assert pkgdep.DepName == "test-dep" assert pkgdep.Package == package assert pkgdep.DependencyType == optdepends @@ -79,39 +83,37 @@ def test_package_dependencies(): assert not pkgdep.is_package() - base = create(PackageBase, Name=pkgdep.DepName, Maintainer=user) - create(Package, PackageBase=base, Name=pkgdep.DepName) + with db.begin(): + base = create(PackageBase, Name=pkgdep.DepName, Maintainer=user) + create(Package, PackageBase=base, Name=pkgdep.DepName) assert pkgdep.is_package() def test_package_dependencies_null_package_raises_exception(): - from aurweb.db import session - depends = query(DependencyType, DependencyType.Name == "depends").first() with pytest.raises(IntegrityError): - create(PackageDependency, - DependencyType=depends, - DepName="test-dep") - session.rollback() + with db.begin(): + create(PackageDependency, + DependencyType=depends, + DepName="test-dep") + db.rollback() def test_package_dependencies_null_dependency_type_raises_exception(): - from aurweb.db import session - with pytest.raises(IntegrityError): - create(PackageDependency, - Package=package, - DepName="test-dep") - session.rollback() + with db.begin(): + create(PackageDependency, + Package=package, + DepName="test-dep") + db.rollback() def test_package_dependencies_null_depname_raises_exception(): - from aurweb.db import session - depends = query(DependencyType, DependencyType.Name == "depends").first() with pytest.raises(IntegrityError): - create(PackageDependency, - Package=package, - DependencyType=depends) - session.rollback() + with db.begin(): + create(PackageDependency, + Package=package, + DependencyType=depends) + db.rollback() diff --git a/test/test_package_relation.py b/test/test_package_relation.py index 766d0017..edb67078 100644 --- a/test/test_package_relation.py +++ b/test/test_package_relation.py @@ -2,7 +2,8 @@ import pytest from sqlalchemy.exc import IntegrityError, OperationalError -from aurweb.db import commit, create, query +from aurweb import db +from aurweb.db import create, query from aurweb.models.account_type import AccountType from aurweb.models.package import Package from aurweb.models.package_base import PackageBase @@ -22,25 +23,28 @@ def setup(): account_type = query(AccountType, AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - pkgbase = create(PackageBase, - Name="test-package", - Maintainer=user) - package = create(Package, - PackageBase=pkgbase, - Name=pkgbase.Name, - Description="Test description.", - URL="https://test.package") + with db.begin(): + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + pkgbase = create(PackageBase, + Name="test-package", + Maintainer=user) + package = create(Package, + PackageBase=pkgbase, + Name=pkgbase.Name, + Description="Test description.", + URL="https://test.package") def test_package_relation(): conflicts = query(RelationType, RelationType.Name == "conflicts").first() - pkgrel = create(PackageRelation, Package=package, - RelationType=conflicts, - RelName="test-relation") + + with db.begin(): + pkgrel = create(PackageRelation, Package=package, + RelationType=conflicts, + RelName="test-relation") assert pkgrel.RelName == "test-relation" assert pkgrel.Package == package assert pkgrel.RelationType == conflicts @@ -48,8 +52,8 @@ def test_package_relation(): assert pkgrel in package.package_relations provides = query(RelationType, RelationType.Name == "provides").first() - pkgrel.RelationType = provides - commit() + with db.begin(): + pkgrel.RelationType = provides assert pkgrel.RelName == "test-relation" assert pkgrel.Package == package assert pkgrel.RelationType == provides @@ -57,8 +61,8 @@ def test_package_relation(): assert pkgrel in package.package_relations replaces = query(RelationType, RelationType.Name == "replaces").first() - pkgrel.RelationType = replaces - commit() + with db.begin(): + pkgrel.RelationType = replaces assert pkgrel.RelName == "test-relation" assert pkgrel.Package == package assert pkgrel.RelationType == replaces @@ -67,36 +71,33 @@ def test_package_relation(): def test_package_relation_null_package_raises_exception(): - from aurweb.db import session - conflicts = query(RelationType, RelationType.Name == "conflicts").first() assert conflicts is not None with pytest.raises(IntegrityError): - create(PackageRelation, - RelationType=conflicts, - RelName="test-relation") - session.rollback() + with db.begin(): + create(PackageRelation, + RelationType=conflicts, + RelName="test-relation") + db.rollback() def test_package_relation_null_relation_type_raises_exception(): - from aurweb.db import session - with pytest.raises(IntegrityError): - create(PackageRelation, - Package=package, - RelName="test-relation") - session.rollback() + with db.begin(): + create(PackageRelation, + Package=package, + RelName="test-relation") + db.rollback() def test_package_relation_null_relname_raises_exception(): - from aurweb.db import session - depends = query(RelationType, RelationType.Name == "conflicts").first() assert depends is not None with pytest.raises((OperationalError, IntegrityError)): - create(PackageRelation, - Package=package, - RelationType=depends) - session.rollback() + with db.begin(): + create(PackageRelation, + Package=package, + RelationType=depends) + db.rollback() diff --git a/test/test_package_request.py b/test/test_package_request.py index c28af6bd..1589ffc2 100644 --- a/test/test_package_request.py +++ b/test/test_package_request.py @@ -4,7 +4,8 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import commit, create, query, rollback +from aurweb import db +from aurweb.db import create, query, rollback from aurweb.models.package_base import PackageBase from aurweb.models.package_request import (ACCEPTED, ACCEPTED_ID, CLOSED, CLOSED_ID, PENDING, PENDING_ID, REJECTED, REJECTED_ID, PackageRequest) @@ -21,19 +22,21 @@ def setup(): setup_test_db("PackageRequests", "PackageBases", "Users") - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword") - pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + with db.begin(): + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword") + pkgbase = create(PackageBase, Name="test-package", Maintainer=user) def test_package_request_creation(): request_type = query(RequestType, RequestType.Name == "merge").first() assert request_type.Name == "merge" - package_request = create(PackageRequest, RequestType=request_type, - User=user, PackageBase=pkgbase, - PackageBaseName=pkgbase.Name, - Comments=str(), ClosureComment=str()) + with db.begin(): + package_request = create(PackageRequest, RequestType=request_type, + User=user, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str()) assert bool(package_request.ID) assert package_request.RequestType == request_type @@ -54,11 +57,12 @@ def test_package_request_closed(): assert request_type.Name == "merge" ts = int(datetime.utcnow().timestamp()) - package_request = create(PackageRequest, RequestType=request_type, - User=user, PackageBase=pkgbase, - PackageBaseName=pkgbase.Name, - Closer=user, ClosedTS=ts, - Comments=str(), ClosureComment=str()) + with db.begin(): + package_request = create(PackageRequest, RequestType=request_type, + User=user, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Closer=user, ClosedTS=ts, + Comments=str(), ClosureComment=str()) assert package_request.Closer == user assert package_request.ClosedTS == ts @@ -69,54 +73,60 @@ def test_package_request_closed(): def test_package_request_null_request_type_raises_exception(): with pytest.raises(IntegrityError): - create(PackageRequest, User=user, PackageBase=pkgbase, - PackageBaseName=pkgbase.Name, - Comments=str(), ClosureComment=str()) + with db.begin(): + create(PackageRequest, User=user, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str()) rollback() def test_package_request_null_user_raises_exception(): request_type = query(RequestType, RequestType.Name == "merge").first() with pytest.raises(IntegrityError): - create(PackageRequest, RequestType=request_type, PackageBase=pkgbase, - PackageBaseName=pkgbase.Name, - Comments=str(), ClosureComment=str()) + with db.begin(): + create(PackageRequest, RequestType=request_type, + PackageBase=pkgbase, PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str()) rollback() def test_package_request_null_package_base_raises_exception(): request_type = query(RequestType, RequestType.Name == "merge").first() with pytest.raises(IntegrityError): - create(PackageRequest, RequestType=request_type, - User=user, PackageBaseName=pkgbase.Name, - Comments=str(), ClosureComment=str()) + with db.begin(): + create(PackageRequest, RequestType=request_type, + User=user, PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str()) rollback() def test_package_request_null_package_base_name_raises_exception(): request_type = query(RequestType, RequestType.Name == "merge").first() with pytest.raises(IntegrityError): - create(PackageRequest, RequestType=request_type, - User=user, PackageBase=pkgbase, - Comments=str(), ClosureComment=str()) + with db.begin(): + create(PackageRequest, RequestType=request_type, + User=user, PackageBase=pkgbase, + Comments=str(), ClosureComment=str()) rollback() def test_package_request_null_comments_raises_exception(): request_type = query(RequestType, RequestType.Name == "merge").first() with pytest.raises(IntegrityError): - create(PackageRequest, RequestType=request_type, - User=user, PackageBase=pkgbase, PackageBaseName=pkgbase.Name, - ClosureComment=str()) + with db.begin(): + create(PackageRequest, RequestType=request_type, User=user, + PackageBase=pkgbase, PackageBaseName=pkgbase.Name, + ClosureComment=str()) rollback() def test_package_request_null_closure_comment_raises_exception(): request_type = query(RequestType, RequestType.Name == "merge").first() with pytest.raises(IntegrityError): - create(PackageRequest, RequestType=request_type, - User=user, PackageBase=pkgbase, PackageBaseName=pkgbase.Name, - Comments=str()) + with db.begin(): + create(PackageRequest, RequestType=request_type, User=user, + PackageBase=pkgbase, PackageBaseName=pkgbase.Name, + Comments=str()) rollback() @@ -124,26 +134,27 @@ def test_package_request_status_display(): """ Test status_display() based on the Status column value. """ request_type = query(RequestType, RequestType.Name == "merge").first() - pkgreq = create(PackageRequest, RequestType=request_type, - User=user, PackageBase=pkgbase, - PackageBaseName=pkgbase.Name, - Comments=str(), ClosureComment=str(), - Status=PENDING_ID) + with db.begin(): + pkgreq = create(PackageRequest, RequestType=request_type, + User=user, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str(), + Status=PENDING_ID) assert pkgreq.status_display() == PENDING - pkgreq.Status = CLOSED_ID - commit() + with db.begin(): + pkgreq.Status = CLOSED_ID assert pkgreq.status_display() == CLOSED - pkgreq.Status = ACCEPTED_ID - commit() + with db.begin(): + pkgreq.Status = ACCEPTED_ID assert pkgreq.status_display() == ACCEPTED - pkgreq.Status = REJECTED_ID - commit() + with db.begin(): + pkgreq.Status = REJECTED_ID assert pkgreq.status_display() == REJECTED - pkgreq.Status = 124 - commit() + with db.begin(): + pkgreq.Status = 124 with pytest.raises(KeyError): pkgreq.status_display() diff --git a/test/test_package_source.py b/test/test_package_source.py index 7453f756..d1adcf9c 100644 --- a/test/test_package_source.py +++ b/test/test_package_source.py @@ -2,7 +2,7 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, query, rollback +from aurweb.db import begin, create, query, rollback from aurweb.models.account_type import AccountType from aurweb.models.package import Package from aurweb.models.package_base import PackageBase @@ -21,17 +21,19 @@ def setup(): account_type = query(AccountType, AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - pkgbase = create(PackageBase, - Name="test-package", - Maintainer=user) - package = create(Package, PackageBase=pkgbase, Name="test-package") + with begin(): + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) + pkgbase = create(PackageBase, + Name="test-package", + Maintainer=user) + package = create(Package, PackageBase=pkgbase, Name="test-package") def test_package_source(): - pkgsource = create(PackageSource, Package=package) + with begin(): + pkgsource = create(PackageSource, Package=package) assert pkgsource.Package == package # By default, PackageSources.Source assigns the string '/dev/null'. assert pkgsource.Source == "/dev/null" @@ -40,5 +42,6 @@ def test_package_source(): def test_package_source_null_package_raises_exception(): with pytest.raises(IntegrityError): - create(PackageSource) + with begin(): + create(PackageSource) rollback() diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index ad07ec17..8a468c15 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -28,31 +28,25 @@ def package_endpoint(package: Package) -> str: return f"/packages/{package.Name}" -def create_package(pkgname: str, maintainer: User, - autocommit: bool = True) -> Package: +def create_package(pkgname: str, maintainer: User) -> Package: pkgbase = db.create(PackageBase, Name=pkgname, - Maintainer=maintainer, - autocommit=False) - return db.create(Package, Name=pkgbase.Name, PackageBase=pkgbase, - autocommit=autocommit) + Maintainer=maintainer) + return db.create(Package, Name=pkgbase.Name, PackageBase=pkgbase) def create_package_dep(package: Package, depname: str, - dep_type_name: str = "depends", - autocommit: bool = True) -> PackageDependency: + dep_type_name: str = "depends") -> PackageDependency: dep_type = db.query(DependencyType, DependencyType.Name == dep_type_name).first() return db.create(PackageDependency, DependencyType=dep_type, Package=package, - DepName=depname, - autocommit=autocommit) + DepName=depname) def create_package_rel(package: Package, - relname: str, - autocommit: bool = True) -> PackageRelation: + relname: str) -> PackageRelation: rel_type = db.query(RelationType, RelationType.ID == PROVIDES_ID).first() return db.create(PackageRelation, @@ -84,31 +78,37 @@ def client() -> TestClient: def user() -> User: """ Yield a user. """ account_type = db.query(AccountType, AccountType.ID == USER_ID).first() - yield db.create(User, Username="test", - Email="test@example.org", - Passwd="testPassword", - AccountType=account_type) + with db.begin(): + user = db.create(User, Username="test", + Email="test@example.org", + Passwd="testPassword", + AccountType=account_type) + yield user @pytest.fixture def maintainer() -> User: """ Yield a specific User used to maintain packages. """ account_type = db.query(AccountType, AccountType.ID == USER_ID).first() - yield db.create(User, Username="test_maintainer", - Email="test_maintainer@example.org", - Passwd="testPassword", - AccountType=account_type) + with db.begin(): + maintainer = db.create(User, Username="test_maintainer", + Email="test_maintainer@example.org", + Passwd="testPassword", + AccountType=account_type) + yield maintainer @pytest.fixture def package(maintainer: User) -> Package: """ Yield a Package created by user. """ - pkgbase = db.create(PackageBase, - Name="test-package", - Maintainer=maintainer) - yield db.create(Package, - PackageBase=pkgbase, - Name=pkgbase.Name) + with db.begin(): + pkgbase = db.create(PackageBase, + Name="test-package", + Maintainer=maintainer) + package = db.create(Package, + PackageBase=pkgbase, + Name=pkgbase.Name) + yield package def test_package_not_found(client: TestClient): @@ -121,10 +121,11 @@ def test_package_official_not_found(client: TestClient, package: Package): """ When a Package has a matching OfficialProvider record, it is not hosted on AUR, but in the official repositories. Getting a package with this kind of record should return a status code 404. """ - db.create(OfficialProvider, - Name=package.Name, - Repo="core", - Provides=package.Name) + with db.begin(): + db.create(OfficialProvider, + Name=package.Name, + Repo="core", + Provides=package.Name) with client as request: resp = request.get(package_endpoint(package)) @@ -157,8 +158,9 @@ def test_package(client: TestClient, package: Package): def test_package_comments(client: TestClient, user: User, package: Package): now = (datetime.utcnow().timestamp()) - comment = db.create(PackageComment, PackageBase=package.PackageBase, - User=user, Comments="Test comment", CommentTS=now) + with db.begin(): + comment = db.create(PackageComment, PackageBase=package.PackageBase, + User=user, Comments="Test comment", CommentTS=now) cookies = {"AURSID": user.login(Request(), "testPassword")} with client as request: @@ -178,11 +180,12 @@ def test_package_comments(client: TestClient, user: User, package: Package): def test_package_requests_display(client: TestClient, user: User, package: Package): type_ = db.query(RequestType, RequestType.ID == DELETION_ID).first() - db.create(PackageRequest, PackageBase=package.PackageBase, - PackageBaseName=package.PackageBase.Name, - User=user, RequestType=type_, - Comments="Test comment.", - ClosureComment=str()) + with db.begin(): + db.create(PackageRequest, PackageBase=package.PackageBase, + PackageBaseName=package.PackageBase.Name, + User=user, RequestType=type_, + Comments="Test comment.", + ClosureComment=str()) # Test that a single request displays "1 pending request". with client as request: @@ -195,11 +198,12 @@ def test_package_requests_display(client: TestClient, user: User, assert target.text.strip() == "1 pending request" type_ = db.query(RequestType, RequestType.ID == DELETION_ID).first() - db.create(PackageRequest, PackageBase=package.PackageBase, - PackageBaseName=package.PackageBase.Name, - User=user, RequestType=type_, - Comments="Test comment2.", - ClosureComment=str()) + with db.begin(): + db.create(PackageRequest, PackageBase=package.PackageBase, + PackageBaseName=package.PackageBase.Name, + User=user, RequestType=type_, + Comments="Test comment2.", + ClosureComment=str()) # Test that a two requests display "2 pending requests". with client as request: @@ -271,50 +275,43 @@ def test_package_authenticated_maintainer(client: TestClient, def test_package_dependencies(client: TestClient, maintainer: User, package: Package): # Create a normal dependency of type depends. - dep_pkg = create_package("test-dep-1", maintainer, autocommit=False) - dep = create_package_dep(package, dep_pkg.Name, autocommit=False) - dep.DepArch = "x86_64" + with db.begin(): + dep_pkg = create_package("test-dep-1", maintainer) + dep = create_package_dep(package, dep_pkg.Name) + dep.DepArch = "x86_64" - # Also, create a makedepends. - make_dep_pkg = create_package("test-dep-2", maintainer, autocommit=False) - make_dep = create_package_dep(package, make_dep_pkg.Name, - dep_type_name="makedepends", - autocommit=False) + # Also, create a makedepends. + make_dep_pkg = create_package("test-dep-2", maintainer) + make_dep = create_package_dep(package, make_dep_pkg.Name, + dep_type_name="makedepends") - # And... a checkdepends! - check_dep_pkg = create_package("test-dep-3", maintainer, autocommit=False) - check_dep = create_package_dep(package, check_dep_pkg.Name, - dep_type_name="checkdepends", - autocommit=False) + # And... a checkdepends! + check_dep_pkg = create_package("test-dep-3", maintainer) + check_dep = create_package_dep(package, check_dep_pkg.Name, + dep_type_name="checkdepends") - # Geez. Just stop. This is optdepends. - opt_dep_pkg = create_package("test-dep-4", maintainer, autocommit=False) - opt_dep = create_package_dep(package, opt_dep_pkg.Name, - dep_type_name="optdepends", - autocommit=False) + # Geez. Just stop. This is optdepends. + opt_dep_pkg = create_package("test-dep-4", maintainer) + opt_dep = create_package_dep(package, opt_dep_pkg.Name, + dep_type_name="optdepends") - # Heh. Another optdepends to test one with a description. - opt_desc_dep_pkg = create_package("test-dep-5", maintainer, - autocommit=False) - opt_desc_dep = create_package_dep(package, opt_desc_dep_pkg.Name, - dep_type_name="optdepends", - autocommit=False) - opt_desc_dep.DepDesc = "Test description." + # Heh. Another optdepends to test one with a description. + opt_desc_dep_pkg = create_package("test-dep-5", maintainer) + opt_desc_dep = create_package_dep(package, opt_desc_dep_pkg.Name, + dep_type_name="optdepends") + opt_desc_dep.DepDesc = "Test description." - broken_dep = create_package_dep(package, "test-dep-6", - dep_type_name="depends", - autocommit=False) + broken_dep = create_package_dep(package, "test-dep-6", + dep_type_name="depends") - # Create an official provider record. - db.create(OfficialProvider, Name="test-dep-99", - Repo="core", Provides="test-dep-99", - autocommit=False) - official_dep = create_package_dep(package, "test-dep-99", - autocommit=False) + # Create an official provider record. + db.create(OfficialProvider, Name="test-dep-99", + Repo="core", Provides="test-dep-99") + official_dep = create_package_dep(package, "test-dep-99") - # Also, create a provider who provides our test-dep-99. - provider = create_package("test-provider", maintainer, autocommit=False) - create_package_rel(provider, dep.DepName) + # Also, create a provider who provides our test-dep-99. + provider = create_package("test-provider", maintainer) + create_package_rel(provider, dep.DepName) with client as request: resp = request.get(package_endpoint(package)) @@ -358,8 +355,9 @@ def test_pkgbase_redirect(client: TestClient, package: Package): def test_pkgbase(client: TestClient, package: Package): - second = db.create(Package, Name="second-pkg", - PackageBase=package.PackageBase) + with db.begin(): + second = db.create(Package, Name="second-pkg", + PackageBase=package.PackageBase) expected = [package.Name, second.Name] with client as request: diff --git a/test/test_packages_util.py b/test/test_packages_util.py index bc6a941c..754e3b8d 100644 --- a/test/test_packages_util.py +++ b/test/test_packages_util.py @@ -26,17 +26,21 @@ def setup(): @pytest.fixture def maintainer() -> User: account_type = db.query(AccountType, AccountType.ID == USER_ID).first() - yield db.create(User, Username="test_maintainer", - Email="test_maintainer@examepl.org", - Passwd="testPassword", - AccountType=account_type) + with db.begin(): + maintainer = db.create(User, Username="test_maintainer", + Email="test_maintainer@examepl.org", + Passwd="testPassword", + AccountType=account_type) + yield maintainer @pytest.fixture def package(maintainer: User) -> Package: - pkgbase = db.create(PackageBase, Name="test-pkg", - Packager=maintainer, Maintainer=maintainer) - yield db.create(Package, Name=pkgbase.Name, PackageBase=pkgbase) + with db.begin(): + pkgbase = db.create(PackageBase, Name="test-pkg", + Packager=maintainer, Maintainer=maintainer) + package = db.create(Package, Name=pkgbase.Name, PackageBase=pkgbase) + yield package @pytest.fixture @@ -45,10 +49,11 @@ def client() -> TestClient: def test_package_link(client: TestClient, maintainer: User, package: Package): - db.create(OfficialProvider, - Name=package.Name, - Repo="core", - Provides=package.Name) + with db.begin(): + db.create(OfficialProvider, + Name=package.Name, + Repo="core", + Provides=package.Name) expected = f"{OFFICIAL_BASE}/packages/?q={package.Name}" assert util.package_link(package) == expected diff --git a/test/test_relation_type.py b/test/test_relation_type.py index bf23505c..fbc22c71 100644 --- a/test/test_relation_type.py +++ b/test/test_relation_type.py @@ -1,6 +1,6 @@ import pytest -from aurweb.db import create, delete, query +from aurweb import db from aurweb.models.relation_type import RelationType from aurweb.testing import setup_test_db @@ -11,22 +11,25 @@ def setup(): def test_relation_type_creation(): - relation_type = create(RelationType, Name="test-relation") + with db.begin(): + relation_type = db.create(RelationType, Name="test-relation") + assert bool(relation_type.ID) assert relation_type.Name == "test-relation" - delete(RelationType, RelationType.ID == relation_type.ID) + with db.begin(): + db.delete(RelationType, RelationType.ID == relation_type.ID) def test_relation_types(): - conflicts = query(RelationType, RelationType.Name == "conflicts").first() + conflicts = db.query(RelationType, RelationType.Name == "conflicts").first() assert conflicts is not None assert conflicts.Name == "conflicts" - provides = query(RelationType, RelationType.Name == "provides").first() + provides = db.query(RelationType, RelationType.Name == "provides").first() assert provides is not None assert provides.Name == "provides" - replaces = query(RelationType, RelationType.Name == "replaces").first() + replaces = db.query(RelationType, RelationType.Name == "replaces").first() assert replaces is not None assert replaces.Name == "replaces" diff --git a/test/test_request_type.py b/test/test_request_type.py index a3b3ccb8..8d21c2d9 100644 --- a/test/test_request_type.py +++ b/test/test_request_type.py @@ -1,6 +1,6 @@ import pytest -from aurweb.db import create, delete, query +from aurweb import db from aurweb.models.request_type import DELETION_ID, MERGE_ID, ORPHAN_ID, RequestType from aurweb.testing import setup_test_db @@ -11,25 +11,33 @@ def setup(): def test_request_type_creation(): - request_type = create(RequestType, Name="Test Request") + with db.begin(): + request_type = db.create(RequestType, Name="Test Request") + assert bool(request_type.ID) assert request_type.Name == "Test Request" - delete(RequestType, RequestType.ID == request_type.ID) + + with db.begin(): + db.delete(RequestType, RequestType.ID == request_type.ID) def test_request_type_null_name_returns_empty_string(): - request_type = create(RequestType) + with db.begin(): + request_type = db.create(RequestType) + assert bool(request_type.ID) assert request_type.Name == str() - delete(RequestType, RequestType.ID == request_type.ID) + + with db.begin(): + db.delete(RequestType, RequestType.ID == request_type.ID) def test_request_type_name_display(): - deletion = query(RequestType, RequestType.ID == DELETION_ID).first() + deletion = db.query(RequestType, RequestType.ID == DELETION_ID).first() assert deletion.name_display() == "Deletion" - orphan = query(RequestType, RequestType.ID == ORPHAN_ID).first() + orphan = db.query(RequestType, RequestType.ID == ORPHAN_ID).first() assert orphan.name_display() == "Orphan" - merge = query(RequestType, RequestType.ID == MERGE_ID).first() + merge = db.query(RequestType, RequestType.ID == MERGE_ID).first() assert merge.name_display() == "Merge" diff --git a/test/test_routes.py b/test/test_routes.py index a2d1786e..e3f69d7a 100644 --- a/test/test_routes.py +++ b/test/test_routes.py @@ -8,8 +8,8 @@ import pytest from fastapi.testclient import TestClient +from aurweb import db from aurweb.asgi import app -from aurweb.db import create, query from aurweb.models.account_type import AccountType from aurweb.models.user import User from aurweb.testing import setup_test_db @@ -24,11 +24,13 @@ def setup(): setup_test_db("Users", "Sessions") - account_type = query(AccountType, - AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + account_type = db.query(AccountType, + AccountType.AccountType == "User").first() + + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) client = TestClient(app) diff --git a/test/test_rss.py b/test/test_rss.py index 7dd5bb47..ce3bc71f 100644 --- a/test/test_rss.py +++ b/test/test_rss.py @@ -49,14 +49,13 @@ def packages(user): now = int(datetime.utcnow().timestamp()) # Create 101 packages; we limit 100 on RSS feeds. - for i in range(101): - pkgbase = db.create( - PackageBase, Maintainer=user, Name=f"test-package-{i}", - SubmittedTS=(now + i), ModifiedTS=(now + i), autocommit=False) - pkg = db.create(Package, Name=pkgbase.Name, PackageBase=pkgbase, - autocommit=False) - pkgs.append(pkg) - db.commit() + with db.begin(): + for i in range(101): + pkgbase = db.create( + PackageBase, Maintainer=user, Name=f"test-package-{i}", + SubmittedTS=(now + i), ModifiedTS=(now + i)) + pkg = db.create(Package, Name=pkgbase.Name, PackageBase=pkgbase) + pkgs.append(pkg) yield pkgs diff --git a/test/test_session.py b/test/test_session.py index 1ba11556..4e6f4db4 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -4,7 +4,7 @@ from unittest import mock import pytest -from aurweb.db import create, query +from aurweb import db from aurweb.models.account_type import AccountType from aurweb.models.session import Session, generate_unique_sid from aurweb.models.user import User @@ -19,13 +19,16 @@ def setup(): setup_test_db("Users", "Sessions") - account_type = query(AccountType, - AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - ResetKey="testReset", Passwd="testPassword", - AccountType=account_type) - session = create(Session, UsersID=user.ID, SessionID="testSession", - LastUpdateTS=datetime.utcnow().timestamp()) + account_type = db.query(AccountType, + AccountType.AccountType == "User").first() + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + ResetKey="testReset", Passwd="testPassword", + AccountType=account_type) + + with db.begin(): + session = db.create(Session, UsersID=user.ID, SessionID="testSession", + LastUpdateTS=datetime.utcnow().timestamp()) def test_session(): @@ -35,12 +38,15 @@ def test_session(): def test_session_cs(): """ Test case sensitivity of the database table. """ - user2 = create(User, Username="test2", Email="test2@example.org", - ResetKey="testReset2", Passwd="testPassword", - AccountType=account_type) - session_cs = create(Session, UsersID=user2.ID, - SessionID="TESTSESSION", - LastUpdateTS=datetime.utcnow().timestamp()) + with db.begin(): + user2 = db.create(User, Username="test2", Email="test2@example.org", + ResetKey="testReset2", Passwd="testPassword", + AccountType=account_type) + + with db.begin(): + session_cs = db.create(Session, UsersID=user2.ID, + SessionID="TESTSESSION", + LastUpdateTS=datetime.utcnow().timestamp()) assert session_cs.SessionID == "TESTSESSION" assert session.SessionID == "testSession" diff --git a/test/test_ssh_pub_key.py b/test/test_ssh_pub_key.py index 0793199a..12a3e1ce 100644 --- a/test/test_ssh_pub_key.py +++ b/test/test_ssh_pub_key.py @@ -1,6 +1,6 @@ import pytest -from aurweb.db import create, query +from aurweb import db from aurweb.models.account_type import AccountType from aurweb.models.ssh_pub_key import SSHPubKey, get_fingerprint from aurweb.models.user import User @@ -19,19 +19,18 @@ def setup(): setup_test_db("Users", "SSHPubKeys") - account_type = query(AccountType, - AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + account_type = db.query(AccountType, + AccountType.AccountType == "User").first() + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) - assert account_type == user.AccountType - assert account_type.ID == user.AccountTypeID - - ssh_pub_key = create(SSHPubKey, - UserID=user.ID, - Fingerprint="testFingerprint", - PubKey="testPubKey") + with db.begin(): + ssh_pub_key = db.create(SSHPubKey, + UserID=user.ID, + Fingerprint="testFingerprint", + PubKey="testPubKey") def test_ssh_pub_key(): @@ -43,9 +42,10 @@ def test_ssh_pub_key(): def test_ssh_pub_key_cs(): """ Test case sensitivity of the database table. """ - ssh_pub_key_cs = create(SSHPubKey, UserID=user.ID, - Fingerprint="TESTFINGERPRINT", - PubKey="TESTPUBKEY") + with db.begin(): + ssh_pub_key_cs = db.create(SSHPubKey, UserID=user.ID, + Fingerprint="TESTFINGERPRINT", + PubKey="TESTPUBKEY") assert ssh_pub_key_cs.Fingerprint == "TESTFINGERPRINT" assert ssh_pub_key_cs.PubKey == "TESTPUBKEY" diff --git a/test/test_term.py b/test/test_term.py index 25108419..3f28311f 100644 --- a/test/test_term.py +++ b/test/test_term.py @@ -2,7 +2,7 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create +from aurweb import db from aurweb.models.term import Term from aurweb.testing import setup_test_db @@ -18,8 +18,9 @@ def setup(): def test_term_creation(): - term = create(Term, Description="Term description", - URL="https://fake_url.io") + with db.begin(): + term = db.create(Term, Description="Term description", + URL="https://fake_url.io") assert bool(term.ID) assert term.Description == "Term description" assert term.URL == "https://fake_url.io" @@ -27,14 +28,14 @@ def test_term_creation(): def test_term_null_description_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(Term, URL="https://fake_url.io") - session.rollback() + with db.begin(): + db.create(Term, URL="https://fake_url.io") + db.rollback() def test_term_null_url_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(Term, Description="Term description") - session.rollback() + with db.begin(): + db.create(Term, Description="Term description") + db.rollback() diff --git a/test/test_trusted_user_routes.py b/test/test_trusted_user_routes.py index 0c33f958..67181db3 100644 --- a/test/test_trusted_user_routes.py +++ b/test/test_trusted_user_routes.py @@ -90,37 +90,37 @@ def client(): def tu_user(): tu_type = db.query(AccountType, AccountType.AccountType == "Trusted User").first() - yield db.create(User, Username="test_tu", Email="test_tu@example.org", - RealName="Test TU", Passwd="testPassword", - AccountType=tu_type) + with db.begin(): + tu_user = db.create(User, Username="test_tu", + Email="test_tu@example.org", + RealName="Test TU", Passwd="testPassword", + AccountType=tu_type) + yield tu_user @pytest.fixture def user(): user_type = db.query(AccountType, AccountType.AccountType == "User").first() - yield db.create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=user_type) + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=user_type) + yield user @pytest.fixture -def proposal(tu_user): +def proposal(user, tu_user): ts = int(datetime.utcnow().timestamp()) agenda = "Test proposal." start = ts - 5 end = ts + 1000 - user_type = db.query(AccountType, - AccountType.AccountType == "User").first() - user = db.create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=user_type) - - voteinfo = db.create(TUVoteInfo, - Agenda=agenda, Quorum=0.0, - User=user.Username, Submitter=tu_user, - Submitted=start, End=end) + with db.begin(): + voteinfo = db.create(TUVoteInfo, + Agenda=agenda, Quorum=0.0, + User=user.Username, Submitter=tu_user, + Submitted=start, End=end) yield (tu_user, user, voteinfo) @@ -170,20 +170,22 @@ def test_tu_index(client, tu_user): ("Test agenda 2", ts - 1000, ts - 5) # Not running anymore. ] vote_records = [] - for vote in votes: - agenda, start, end = vote - vote_records.append( - db.create(TUVoteInfo, Agenda=agenda, - User=tu_user.Username, - Submitted=start, End=end, - Quorum=0.0, - Submitter=tu_user)) + with db.begin(): + for vote in votes: + agenda, start, end = vote + vote_records.append( + db.create(TUVoteInfo, Agenda=agenda, + User=tu_user.Username, + Submitted=start, End=end, + Quorum=0.0, + Submitter=tu_user)) - # Vote on an ended proposal. - vote_record = vote_records[1] - vote_record.Yes += 1 - vote_record.ActiveTUs += 1 - db.create(TUVote, VoteInfo=vote_record, User=tu_user) + with db.begin(): + # Vote on an ended proposal. + vote_record = vote_records[1] + vote_record.Yes += 1 + vote_record.ActiveTUs += 1 + db.create(TUVote, VoteInfo=vote_record, User=tu_user) cookies = {"AURSID": tu_user.login(Request(), "testPassword")} with client as request: @@ -255,22 +257,22 @@ def test_tu_index(client, tu_user): def test_tu_index_table_paging(client, tu_user): ts = int(datetime.utcnow().timestamp()) - for i in range(25): - # Create 25 current votes. - db.create(TUVoteInfo, Agenda=f"Agenda #{i}", - User=tu_user.Username, - Submitted=(ts - 5), End=(ts + 1000), - Quorum=0.0, - Submitter=tu_user, autocommit=False) + with db.begin(): + for i in range(25): + # Create 25 current votes. + db.create(TUVoteInfo, Agenda=f"Agenda #{i}", + User=tu_user.Username, + Submitted=(ts - 5), End=(ts + 1000), + Quorum=0.0, + Submitter=tu_user) - for i in range(25): - # Create 25 past votes. - db.create(TUVoteInfo, Agenda=f"Agenda #{25 + i}", - User=tu_user.Username, - Submitted=(ts - 1000), End=(ts - 5), - Quorum=0.0, - Submitter=tu_user, autocommit=False) - db.commit() + for i in range(25): + # Create 25 past votes. + db.create(TUVoteInfo, Agenda=f"Agenda #{25 + i}", + User=tu_user.Username, + Submitted=(ts - 1000), End=(ts - 5), + Quorum=0.0, + Submitter=tu_user) cookies = {"AURSID": tu_user.login(Request(), "testPassword")} with client as request: @@ -363,18 +365,19 @@ def test_tu_index_table_paging(client, tu_user): def test_tu_index_sorting(client, tu_user): ts = int(datetime.utcnow().timestamp()) - for i in range(2): - # Create 'Agenda #1' and 'Agenda #2'. - db.create(TUVoteInfo, Agenda=f"Agenda #{i + 1}", - User=tu_user.Username, - Submitted=(ts + 5), End=(ts + 1000), - Quorum=0.0, - Submitter=tu_user, autocommit=False) + with db.begin(): + for i in range(2): + # Create 'Agenda #1' and 'Agenda #2'. + db.create(TUVoteInfo, Agenda=f"Agenda #{i + 1}", + User=tu_user.Username, + Submitted=(ts + 5), End=(ts + 1000), + Quorum=0.0, + Submitter=tu_user) - # Let's order each vote one day after the other. - # This will allow us to test the sorting nature - # of the tables. - ts += 86405 + # Let's order each vote one day after the other. + # This will allow us to test the sorting nature + # of the tables. + ts += 86405 # Make a default request to /tu. cookies = {"AURSID": tu_user.login(Request(), "testPassword")} @@ -432,18 +435,19 @@ def test_tu_index_sorting(client, tu_user): def test_tu_index_last_votes(client, tu_user, user): ts = int(datetime.utcnow().timestamp()) - # Create a proposal which has ended. - voteinfo = db.create(TUVoteInfo, Agenda="Test agenda", - User=user.Username, - Submitted=(ts - 1000), - End=(ts - 5), - Yes=1, - ActiveTUs=1, - Quorum=0.0, - Submitter=tu_user) + with db.begin(): + # Create a proposal which has ended. + voteinfo = db.create(TUVoteInfo, Agenda="Test agenda", + User=user.Username, + Submitted=(ts - 1000), + End=(ts - 5), + Yes=1, + ActiveTUs=1, + Quorum=0.0, + Submitter=tu_user) - # Create a vote on it from tu_user. - db.create(TUVote, VoteInfo=voteinfo, User=tu_user) + # Create a vote on it from tu_user. + db.create(TUVote, VoteInfo=voteinfo, User=tu_user) # Now, check that tu_user got populated in the .last-votes table. cookies = {"AURSID": tu_user.login(Request(), "testPassword")} @@ -529,10 +533,10 @@ def test_tu_running_proposal(client, proposal): assert abstain.attrib["value"] == "Abstain" # Create a vote. - db.create(TUVote, VoteInfo=voteinfo, User=tu_user) - voteinfo.ActiveTUs += 1 - voteinfo.Yes += 1 - db.commit() + with db.begin(): + db.create(TUVote, VoteInfo=voteinfo, User=tu_user) + voteinfo.ActiveTUs += 1 + voteinfo.Yes += 1 # Make another request now that we've voted. with client as request: @@ -556,8 +560,8 @@ def test_tu_ended_proposal(client, proposal): tu_user, user, voteinfo = proposal ts = int(datetime.utcnow().timestamp()) - voteinfo.End = ts - 5 # 5 seconds ago. - db.commit() + with db.begin(): + voteinfo.End = ts - 5 # 5 seconds ago. # Initiate an authenticated GET request to /tu/{proposal_id}. proposal_id = voteinfo.ID @@ -635,8 +639,8 @@ def test_tu_proposal_vote_unauthorized(client, proposal): dev_type = db.query(AccountType, AccountType.AccountType == "Developer").first() - tu_user.AccountType = dev_type - db.commit() + with db.begin(): + tu_user.AccountType = dev_type cookies = {"AURSID": tu_user.login(Request(), "testPassword")} with client as request: @@ -664,8 +668,8 @@ def test_tu_proposal_vote_cant_self_vote(client, proposal): tu_user, user, voteinfo = proposal # Update voteinfo.User. - voteinfo.User = tu_user.Username - db.commit() + with db.begin(): + voteinfo.User = tu_user.Username cookies = {"AURSID": tu_user.login(Request(), "testPassword")} with client as request: @@ -692,10 +696,10 @@ def test_tu_proposal_vote_cant_self_vote(client, proposal): def test_tu_proposal_vote_already_voted(client, proposal): tu_user, user, voteinfo = proposal - db.create(TUVote, VoteInfo=voteinfo, User=tu_user) - voteinfo.Yes += 1 - voteinfo.ActiveTUs += 1 - db.commit() + with db.begin(): + db.create(TUVote, VoteInfo=voteinfo, User=tu_user) + voteinfo.Yes += 1 + voteinfo.ActiveTUs += 1 cookies = {"AURSID": tu_user.login(Request(), "testPassword")} with client as request: diff --git a/test/test_tu_voteinfo.py b/test/test_tu_voteinfo.py index 494300c5..b60e2e6a 100644 --- a/test/test_tu_voteinfo.py +++ b/test/test_tu_voteinfo.py @@ -4,7 +4,8 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import commit, create, query, rollback +from aurweb import db +from aurweb.db import create, query, rollback from aurweb.models.account_type import AccountType from aurweb.models.tu_voteinfo import TUVoteInfo from aurweb.models.user import User @@ -21,19 +22,21 @@ def setup(): tu_type = query(AccountType, AccountType.AccountType == "Trusted User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=tu_type) + with db.begin(): + user = create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=tu_type) def test_tu_voteinfo_creation(): ts = int(datetime.utcnow().timestamp()) - tu_voteinfo = create(TUVoteInfo, - Agenda="Blah blah.", - User=user.Username, - Submitted=ts, End=ts + 5, - Quorum=0.5, - Submitter=user) + with db.begin(): + tu_voteinfo = create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=ts, End=ts + 5, + Quorum=0.5, + Submitter=user) assert bool(tu_voteinfo.ID) assert tu_voteinfo.Agenda == "Blah blah." assert tu_voteinfo.User == user.Username @@ -51,32 +54,33 @@ def test_tu_voteinfo_creation(): def test_tu_voteinfo_is_running(): ts = int(datetime.utcnow().timestamp()) - tu_voteinfo = create(TUVoteInfo, - Agenda="Blah blah.", - User=user.Username, - Submitted=ts, End=ts + 1000, - Quorum=0.5, - Submitter=user) + with db.begin(): + tu_voteinfo = create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=ts, End=ts + 1000, + Quorum=0.5, + Submitter=user) assert tu_voteinfo.is_running() is True - tu_voteinfo.End = ts - 5 - commit() + with db.begin(): + tu_voteinfo.End = ts - 5 assert tu_voteinfo.is_running() is False def test_tu_voteinfo_total_votes(): ts = int(datetime.utcnow().timestamp()) - tu_voteinfo = create(TUVoteInfo, - Agenda="Blah blah.", - User=user.Username, - Submitted=ts, End=ts + 1000, - Quorum=0.5, - Submitter=user) + with db.begin(): + tu_voteinfo = create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=ts, End=ts + 1000, + Quorum=0.5, + Submitter=user) - tu_voteinfo.Yes = 1 - tu_voteinfo.No = 3 - tu_voteinfo.Abstain = 5 - commit() + tu_voteinfo.Yes = 1 + tu_voteinfo.No = 3 + tu_voteinfo.Abstain = 5 # total_votes() should be the sum of Yes, No and Abstain: 1 + 3 + 5 = 9. assert tu_voteinfo.total_votes() == 9 @@ -84,61 +88,67 @@ def test_tu_voteinfo_total_votes(): def test_tu_voteinfo_null_submitter_raises_exception(): with pytest.raises(IntegrityError): - create(TUVoteInfo, - Agenda="Blah blah.", - User=user.Username, - Submitted=0, End=0, - Quorum=0.50) + with db.begin(): + create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=0, End=0, + Quorum=0.50) rollback() def test_tu_voteinfo_null_agenda_raises_exception(): with pytest.raises(IntegrityError): - create(TUVoteInfo, - User=user.Username, - Submitted=0, End=0, - Quorum=0.50, - Submitter=user) + with db.begin(): + create(TUVoteInfo, + User=user.Username, + Submitted=0, End=0, + Quorum=0.50, + Submitter=user) rollback() def test_tu_voteinfo_null_user_raises_exception(): with pytest.raises(IntegrityError): - create(TUVoteInfo, - Agenda="Blah blah.", - Submitted=0, End=0, - Quorum=0.50, - Submitter=user) + with db.begin(): + create(TUVoteInfo, + Agenda="Blah blah.", + Submitted=0, End=0, + Quorum=0.50, + Submitter=user) rollback() def test_tu_voteinfo_null_submitted_raises_exception(): with pytest.raises(IntegrityError): - create(TUVoteInfo, - Agenda="Blah blah.", - User=user.Username, - End=0, - Quorum=0.50, - Submitter=user) + with db.begin(): + create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + End=0, + Quorum=0.50, + Submitter=user) rollback() def test_tu_voteinfo_null_end_raises_exception(): with pytest.raises(IntegrityError): - create(TUVoteInfo, - Agenda="Blah blah.", - User=user.Username, - Submitted=0, - Quorum=0.50, - Submitter=user) + with db.begin(): + create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=0, + Quorum=0.50, + Submitter=user) rollback() def test_tu_voteinfo_null_quorum_raises_exception(): with pytest.raises(IntegrityError): - create(TUVoteInfo, - Agenda="Blah blah.", - User=user.Username, - Submitted=0, End=0, - Submitter=user) + with db.begin(): + create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=0, End=0, + Submitter=user) rollback() diff --git a/test/test_user.py b/test/test_user.py index 7756cff3..70eac079 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -9,7 +9,7 @@ import pytest import aurweb.auth import aurweb.config -from aurweb.db import commit, create, query +from aurweb import db from aurweb.models.account_type import AccountType from aurweb.models.ban import Ban from aurweb.models.package import Package @@ -40,12 +40,13 @@ def setup(): PackageNotification.__tablename__ ) - account_type = query(AccountType, - AccountType.AccountType == "User").first() + account_type = db.query(AccountType, + AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) def test_user_login_logout(): @@ -70,14 +71,14 @@ def test_user_login_logout(): assert "AURSID" in request.cookies # Expect that User session relationships work right. - user_session = query(Session, - Session.UsersID == user.ID).first() + user_session = db.query(Session, + Session.UsersID == user.ID).first() assert user_session == user.session assert user.session.SessionID == sid assert user.session.User == user # Search for the user via query API. - result = query(User, User.ID == user.ID).first() + result = db.query(User, User.ID == user.ID).first() # Compare the result and our original user. assert result == user @@ -114,7 +115,8 @@ def test_user_login_twice(): def test_user_login_banned(): # Add ban for the next 30 seconds. banned_timestamp = datetime.utcnow() + timedelta(seconds=30) - create(Ban, IPAddress="127.0.0.1", BanTS=banned_timestamp) + with db.begin(): + db.create(Ban, IPAddress="127.0.0.1", BanTS=banned_timestamp) request = Request() request.client.host = "127.0.0.1" @@ -122,18 +124,17 @@ def test_user_login_banned(): def test_user_login_suspended(): - from aurweb.db import session - user.Suspended = True - session.commit() + with db.begin(): + user.Suspended = True assert not user.login(Request(), "testPassword") def test_legacy_user_authentication(): - from aurweb.db import session - - user.Salt = bcrypt.gensalt().decode() - user.Passwd = hashlib.md5(f"{user.Salt}testPassword".encode()).hexdigest() - session.commit() + with db.begin(): + user.Salt = bcrypt.gensalt().decode() + user.Passwd = hashlib.md5( + f"{user.Salt}testPassword".encode() + ).hexdigest() assert not user.valid_password("badPassword") assert user.valid_password("testPassword") @@ -145,8 +146,9 @@ def test_legacy_user_authentication(): def test_user_login_with_outdated_sid(): # Make a session with a LastUpdateTS 5 seconds ago, causing # user.login to update it with a new sid. - create(Session, UsersID=user.ID, SessionID="stub", - LastUpdateTS=datetime.utcnow().timestamp() - 5) + with db.begin(): + db.create(Session, UsersID=user.ID, SessionID="stub", + LastUpdateTS=datetime.utcnow().timestamp() - 5) sid = user.login(Request(), "testPassword") assert sid and user.is_authenticated() assert sid != "stub" @@ -171,43 +173,42 @@ def test_user_has_credential(): def test_user_ssh_pub_key(): assert user.ssh_pub_key is None - ssh_pub_key = create(SSHPubKey, UserID=user.ID, - Fingerprint="testFingerprint", - PubKey="testPubKey") + with db.begin(): + ssh_pub_key = db.create(SSHPubKey, UserID=user.ID, + Fingerprint="testFingerprint", + PubKey="testPubKey") assert user.ssh_pub_key == ssh_pub_key def test_user_credential_types(): - from aurweb.db import session - assert aurweb.auth.user_developer_or_trusted_user(user) assert not aurweb.auth.trusted_user(user) assert not aurweb.auth.developer(user) assert not aurweb.auth.trusted_user_or_dev(user) - trusted_user_type = query(AccountType, - AccountType.AccountType == "Trusted User")\ - .first() - user.AccountType = trusted_user_type - session.commit() + trusted_user_type = db.query(AccountType).filter( + AccountType.AccountType == "Trusted User" + ).first() + with db.begin(): + user.AccountType = trusted_user_type assert aurweb.auth.trusted_user(user) assert aurweb.auth.trusted_user_or_dev(user) - developer_type = query(AccountType, - AccountType.AccountType == "Developer").first() - user.AccountType = developer_type - session.commit() + developer_type = db.query(AccountType, + AccountType.AccountType == "Developer").first() + with db.begin(): + user.AccountType = developer_type assert aurweb.auth.developer(user) assert aurweb.auth.trusted_user_or_dev(user) type_str = "Trusted User & Developer" - elevated_type = query(AccountType, - AccountType.AccountType == type_str).first() - user.AccountType = elevated_type - session.commit() + elevated_type = db.query(AccountType, + AccountType.AccountType == type_str).first() + with db.begin(): + user.AccountType = elevated_type assert aurweb.auth.trusted_user(user) assert aurweb.auth.developer(user) @@ -233,53 +234,56 @@ def test_user_as_dict(): def test_user_is_trusted_user(): - tu_type = query(AccountType, - AccountType.AccountType == "Trusted User").first() - user.AccountType = tu_type - commit() + tu_type = db.query(AccountType, + AccountType.AccountType == "Trusted User").first() + with db.begin(): + user.AccountType = tu_type assert user.is_trusted_user() is True # Do it again with the combined role. - tu_type = query( + tu_type = db.query( AccountType, AccountType.AccountType == "Trusted User & Developer").first() - user.AccountType = tu_type - commit() + with db.begin(): + user.AccountType = tu_type assert user.is_trusted_user() is True def test_user_is_developer(): - dev_type = query(AccountType, - AccountType.AccountType == "Developer").first() - user.AccountType = dev_type - commit() + dev_type = db.query(AccountType, + AccountType.AccountType == "Developer").first() + with db.begin(): + user.AccountType = dev_type assert user.is_developer() is True # Do it again with the combined role. - dev_type = query( + dev_type = db.query( AccountType, AccountType.AccountType == "Trusted User & Developer").first() - user.AccountType = dev_type - commit() + with db.begin(): + user.AccountType = dev_type assert user.is_developer() is True def test_user_voted_for(): now = int(datetime.utcnow().timestamp()) - pkgbase = create(PackageBase, Name="pkg1", Maintainer=user) - pkg = create(Package, PackageBase=pkgbase, Name=pkgbase.Name) - create(PackageVote, PackageBase=pkgbase, User=user, VoteTS=now) + with db.begin(): + pkgbase = db.create(PackageBase, Name="pkg1", Maintainer=user) + pkg = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + db.create(PackageVote, PackageBase=pkgbase, User=user, VoteTS=now) assert user.voted_for(pkg) def test_user_notified(): - pkgbase = create(PackageBase, Name="pkg1", Maintainer=user) - pkg = create(Package, PackageBase=pkgbase, Name=pkgbase.Name) - create(PackageNotification, PackageBase=pkgbase, User=user) + with db.begin(): + pkgbase = db.create(PackageBase, Name="pkg1", Maintainer=user) + pkg = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + db.create(PackageNotification, PackageBase=pkgbase, User=user) assert user.notified(pkg) def test_user_packages(): - pkgbase = create(PackageBase, Name="pkg1", Maintainer=user) - pkg = create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + with db.begin(): + pkgbase = db.create(PackageBase, Name="pkg1", Maintainer=user) + pkg = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) assert pkg in user.packages() From 1b452d126471df269605ed4e39edcdd38f9e59d8 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 3 Sep 2021 21:02:35 -0700 Subject: [PATCH 318/844] Add GPL 2.0 LICENSE file This was missing from the project and really needs to be here. Closes #107 Signed-off-by: Kevin Morris --- LICENSE | 339 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 339 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..d511905c --- /dev/null +++ b/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. From 5c7e76ef891af78dbe4e3cacb42675b86d9f3a35 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 3 Sep 2021 21:02:35 -0700 Subject: [PATCH 319/844] Add GPL 2.0 LICENSE file This was missing from the project and really needs to be here. Closes #107 Signed-off-by: Kevin Morris --- LICENSE | 339 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 339 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..d511905c --- /dev/null +++ b/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. From 5e6f0cb8d71266d8ba29ea24db267914b1e33973 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Sep 2021 10:02:40 -0700 Subject: [PATCH 320/844] Revert "Add GPL 2.0 LICENSE file" This was already in the repository in ./COPYING This reverts commit 1b452d126471df269605ed4e39edcdd38f9e59d8. --- LICENSE | 339 -------------------------------------------------------- 1 file changed, 339 deletions(-) delete mode 100644 LICENSE diff --git a/LICENSE b/LICENSE deleted file mode 100644 index d511905c..00000000 --- a/LICENSE +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. From 4e5b67f0a6164a237bbab17f56db9c037cc365f0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Sep 2021 10:02:40 -0700 Subject: [PATCH 321/844] Revert "Add GPL 2.0 LICENSE file" This was already in the repository in ./COPYING This reverts commit 1b452d126471df269605ed4e39edcdd38f9e59d8. --- LICENSE | 339 -------------------------------------------------------- 1 file changed, 339 deletions(-) delete mode 100644 LICENSE diff --git a/LICENSE b/LICENSE deleted file mode 100644 index d511905c..00000000 --- a/LICENSE +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. From 2f9994807becf152cc6617e5e0e6d6ba24b2c363 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Sep 2021 10:02:13 -0700 Subject: [PATCH 322/844] use Poetry to deal with deps and package install As the new-age Python package manager, Poetry brings a lot of good additions to the table. It allows us to more easily deal with virtualenvs for the project and resolve dependencies. As of this commit, `requirements.txt` is replaced by Poetry, configured at `pyproject.toml`. In Docker and GitLab, we currently use Poetry in a root fashion. We should work toward purely using virtualenvs in Docker, but, for now we'd like to move forward with other things. The project can still be installed to a virtualenv and used on a user's system through Poetry; it is just not yet doing so in Docker. Modifications: * docker/scripts/install-deps.sh * Remove python dependencies. * conf/config.defaults * Script paths have been updated to use '/usr/bin'. * docker/git-entrypoint.sh * Use '/usr/bin/aurweb-git-auth' instead of '/usr/local/bin/aurweb-git-auth'. Additions: * docker/scripts/install-python-deps.sh * A script used purely to install Python dependencies with Poetry. This has to be used within the aurweb project directory and requires system-wide dependencies are installed beforehand. * Also upgrades system-wide pip. Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 4 +- Dockerfile | 15 +- INSTALL | 54 +- conf/config.defaults | 10 +- docker/git-entrypoint.sh | 2 +- docker/scripts/install-deps.sh | 14 +- docker/scripts/install-python-deps.sh | 14 + poetry.lock | 1577 +++++++++++++++++++++++++ pyproject.toml | 100 ++ requirements.txt | 36 - setup.py | 36 - 11 files changed, 1756 insertions(+), 106 deletions(-) create mode 100755 docker/scripts/install-python-deps.sh create mode 100644 poetry.lock create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d360d483..ffea5308 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,8 +11,9 @@ variables: DB_HOST: localhost before_script: + - export PATH="$HOME/.poetry/bin:${PATH}" - ./docker/scripts/install-deps.sh - - pip install -r requirements.txt + - ./docker/scripts/install-python-deps.sh - useradd -U -d /aurweb -c 'AUR User' aur - ./docker/mariadb-entrypoint.sh - (cd '/usr' && /usr/bin/mysqld_safe --datadir='/var/lib/mysql') & @@ -20,7 +21,6 @@ before_script: - ./docker/test-mysql-entrypoint.sh # Create mysql AUR_CONFIG. - ./docker/test-sqlite-entrypoint.sh # Create sqlite AUR_CONFIG. - make -C po all install - - python setup.py install --install-scripts=/usr/local/bin - python -m aurweb.initdb # Initialize MySQL tables. - AUR_CONFIG=conf/config.sqlite python -m aurweb.initdb - make -C test clean diff --git a/Dockerfile b/Dockerfile index 6539bd94..76da62f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,25 @@ FROM archlinux:base-devel +ENV PATH="$HOME/.poetry/bin:${PATH}" ENV PYTHONPATH=/aurweb ENV AUR_CONFIG=conf/config +# Install system-wide dependencies. +COPY ./docker/scripts/install-deps.sh /install-deps.sh +RUN /install-deps.sh + # Copy Docker scripts COPY ./docker /docker COPY ./docker/scripts/*.sh /usr/local/bin/ -# Install system-wide dependencies. -RUN /docker/scripts/install-deps.sh - # Copy over all aurweb files. COPY . /aurweb # Working directory is aurweb root @ /aurweb. WORKDIR /aurweb -# Install pip directories now that we have access to /aurweb. -RUN pip install -r requirements.txt +# Install Python dependencies. +RUN /docker/scripts/install-python-deps.sh # Add our aur user. RUN useradd -U -d /aurweb -c 'AUR User' aur @@ -27,6 +29,3 @@ RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime # Install translations. RUN make -C po all install - -# Install package and scripts. -RUN python setup.py install --install-scripts=/usr/local/bin diff --git a/INSTALL b/INSTALL index 4df59bd2..e14b9f31 100644 --- a/INSTALL +++ b/INSTALL @@ -45,22 +45,54 @@ read the instructions below. if the defaults file does not exist) and adjust the configuration (pay attention to disable_http_login, enable_maintenance and aur_location). -4) Install Python modules and dependencies: +4) Install dependencies. - # pacman -S python-mysql-connector python-pygit2 python-srcinfo python-sqlalchemy \ - python-bleach python-markdown python-alembic hypercorn \ - python-itsdangerous python-authlib python-httpx \ - python-jinja python-aiofiles python-python-multipart \ - python-requests hypercorn python-bcrypt python-email-validator \ - python-lxml python-feedgen - # python3 setup.py install +4a) Install system-wide dependencies: -(FastAPI-Specific) + # pacman -S git gpgme cgit pyalpm python-srcinfo curl openssh \ + uwsgi uwsgi-plugin-cgi php php-fpm - # pacman -S redis python-redis python-fakeredis python-orjson +4b) Install Python dependencies via poetry (required): + +**NOTE** Users do not need to install pip or poetry dependencies system-wide. +You may take advantage of Poetry's virtualenv integration to manage +dependencies. This is merely a demonstration to show users how to without +a virtualenv. In Docker and CI, we don't yet use a virtualenv. + + ## Install Poetry dependencies system-wide, if not using a virtualenv. + # pacman -S python-pip + + ## Ensure pip is upgraded. Poetry depends on it being up to date. + # pip install --upgrade pip + + ## Install Poetry. + # curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - + # export PATH="$HOME/.poetry/bin:${PATH}" + + ## Use Poetry to install dependencies and the aurweb package. + # poetry lock # Resolve dependencies + # poetry update # Install/update dependencies + # poetry build # Build the aurweb package + # poetry install # Install the aurweb package and scripts + +When installing in a virtualenv, config.defaults must contain the correct +absolute paths to aurweb scripts, which requires modification. + +4c) Setup FastAPI Redis cache (optional). + +First, install Redis and start its service. + + # pacman -S redis # systemctl enable --now redis -5) Create a new MySQL database and a user and import the aurweb SQL schema: +Now that Redis is running, ensure that you configure aurweb to use +the Redis cache by setting `cache = redis` in your AUR config. + +In `conf/config.defaults`, the `redis_address` configuration is set +to `redis://localhost`. This can be set to point to any Redis server +and will be used as long as `cache = redis`. + +5) Create a new database and a user and import the aurweb SQL schema: $ python -m aurweb.initdb diff --git a/conf/config.defaults b/conf/config.defaults index 1b4c3a74..1c96a55d 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -34,8 +34,8 @@ commit_uri = /cgit/aur.git/commit/?h=%s&id=%s snapshot_uri = /cgit/aur.git/snapshot/%s.tar.gz enable-maintenance = 1 maintenance-exceptions = 127.0.0.1 -render-comment-cmd = /usr/local/bin/aurweb-rendercomment -localedir = /srv/http/aurweb/aur.git/web/locale/ +render-comment-cmd = /usr/bin/aurweb-rendercomment +localedir = /srv/http/aurweb/web/locale/ ; memcache, apc, or redis ; memcache/apc are supported in PHP, redis is supported in Python. cache = none @@ -49,7 +49,7 @@ request_limit = 4000 window_length = 86400 [notifications] -notify-cmd = /usr/local/bin/aurweb-notify +notify-cmd = /usr/bin/aurweb-notify sendmail = smtp-server = localhost smtp-port = 25 @@ -68,7 +68,7 @@ RSA = SHA256:Ju+yWiMb/2O+gKQ9RJCDqvRg7l+Q95KFAeqM5sr6l2s [auth] valid-keytypes = ssh-rsa ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 sk-ssh-ecdsa@openssh.com sk-ssh-ed25519@openssh.com username-regex = [a-zA-Z0-9]+[.\-_]?[a-zA-Z0-9]+$ -git-serve-cmd = /usr/local/bin/aurweb-git-serve +git-serve-cmd = /usr/bin/aurweb-git-serve ssh-options = restrict [sso] @@ -83,7 +83,7 @@ session_secret = repo-path = /srv/http/aurweb/aur.git/ repo-regex = [a-z0-9][a-z0-9.+_-]*$ git-shell-cmd = /usr/bin/git-shell -git-update-cmd = /usr/local/bin/aurweb-git-update +git-update-cmd = /usr/bin/aurweb-git-update ssh-cmdline = ssh aur@aur.archlinux.org [update] diff --git a/docker/git-entrypoint.sh b/docker/git-entrypoint.sh index 57752ac5..cfd159c9 100755 --- a/docker/git-entrypoint.sh +++ b/docker/git-entrypoint.sh @@ -25,7 +25,7 @@ chmod 755 /app cat >> $AUTH_SCRIPT << EOF #!/usr/bin/env bash export AUR_CONFIG="$AUR_CONFIG" -exec /usr/local/bin/aurweb-git-auth "\$@" +exec /usr/bin/aurweb-git-auth "\$@" EOF chmod 755 $AUTH_SCRIPT diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh index 4985fe85..f8881d05 100755 --- a/docker/scripts/install-deps.sh +++ b/docker/scripts/install-deps.sh @@ -5,12 +5,12 @@ set -eou pipefail pacman -Syu --noconfirm --noprogressbar \ - --cachedir .pkg-cache git gpgme \ - nginx redis openssh \ - mariadb mariadb-libs \ - cgit uwsgi uwsgi-plugin-cgi \ - php php-fpm \ - memcached php-memcached \ - python-pip pyalpm python-srcinfo + --cachedir .pkg-cache git gpgme nginx redis openssh \ + mariadb mariadb-libs cgit uwsgi uwsgi-plugin-cgi \ + php php-fpm memcached php-memcached python-pip pyalpm \ + python-srcinfo curl + +# https://python-poetry.org/docs/ Installation section. +curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - exec "$@" diff --git a/docker/scripts/install-python-deps.sh b/docker/scripts/install-python-deps.sh new file mode 100755 index 00000000..df9b5997 --- /dev/null +++ b/docker/scripts/install-python-deps.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -eou pipefail + +# Upgrade PIP; Arch Linux's version of pip is outdated for Poetry. +pip install --upgrade pip + +# Install the aurweb package and deps system-wide via poetry. +poetry config virtualenvs.create false +poetry lock +poetry update +poetry build +poetry install --no-interaction --no-ansi + +exec "$@" diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 00000000..3cc84361 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,1577 @@ +[[package]] +name = "aiofiles" +version = "0.7.0" +description = "File support for asyncio." +category = "main" +optional = false +python-versions = ">=3.6,<4.0" + +[[package]] +name = "alembic" +version = "1.6.5" +description = "A database migration tool for SQLAlchemy." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" + +[package.dependencies] +Mako = "*" +python-dateutil = "*" +python-editor = ">=0.3" +SQLAlchemy = ">=1.3.0" + +[[package]] +name = "anyio" +version = "3.3.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "main" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"] +test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=6.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"] +trio = ["trio (>=0.16)"] + +[[package]] +name = "asgiref" +version = "3.4.1" +description = "ASGI specs, helper code, and adapters" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +tests = ["pytest", "pytest-asyncio", "mypy (>=0.800)"] + +[[package]] +name = "atomicwrites" +version = "1.4.0" +description = "Atomic file writes." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "attrs" +version = "21.2.0" +description = "Classes Without Boilerplate" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] + +[[package]] +name = "authlib" +version = "0.15.2" +description = "The ultimate Python library in building OAuth and OpenID Connect servers." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +cryptography = "*" + +[package.extras] +client = ["requests"] + +[[package]] +name = "bcrypt" +version = "3.2.0" +description = "Modern password hashing for your software and your servers" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = ">=1.1" +six = ">=1.4.1" + +[package.extras] +tests = ["pytest (>=3.2.1,!=3.3.0)"] +typecheck = ["mypy"] + +[[package]] +name = "bleach" +version = "3.3.1" +description = "An easy safelist-based HTML-sanitizing tool." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +packaging = "*" +six = ">=1.9.0" +webencodings = "*" + +[[package]] +name = "certifi" +version = "2021.5.30" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "cffi" +version = "1.14.6" +description = "Foreign Function Interface for Python calling C code." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "2.0.4" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.5.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + +[[package]] +name = "click" +version = "8.0.1" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "coverage" +version = "5.5" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.extras] +toml = ["toml"] + +[[package]] +name = "cryptography" +version = "3.4.8" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pytest (>=6.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] + +[[package]] +name = "dnspython" +version = "2.1.0" +description = "DNS toolkit" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +dnssec = ["cryptography (>=2.6)"] +doh = ["requests", "requests-toolbelt"] +idna = ["idna (>=2.1)"] +curio = ["curio (>=1.2)", "sniffio (>=1.1)"] +trio = ["trio (>=0.14.0)", "sniffio (>=1.1)"] + +[[package]] +name = "dunamai" +version = "1.6.0" +description = "Dynamic version generation" +category = "main" +optional = false +python-versions = ">=3.5,<4.0" + +[[package]] +name = "email-validator" +version = "1.1.3" +description = "A robust email syntax and deliverability validation library for Python 2.x/3.x." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +dnspython = ">=1.15.0" +idna = ">=2.0.0" + +[[package]] +name = "fakeredis" +version = "1.6.0" +description = "Fake implementation of redis API for testing purposes." +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +redis = "<3.6.0" +six = ">=1.12" +sortedcontainers = "*" + +[package.extras] +aioredis = ["aioredis"] +lua = ["lupa"] + +[[package]] +name = "fastapi" +version = "0.66.0" +description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0" +starlette = "0.14.2" + +[package.extras] +all = ["requests (>=2.24.0,<3.0.0)", "aiofiles (>=0.5.0,<0.6.0)", "jinja2 (>=2.11.2,<3.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "itsdangerous (>=1.1.0,<2.0.0)", "pyyaml (>=5.3.1,<6.0.0)", "graphene (>=2.1.8,<3.0.0)", "ujson (>=4.0.1,<5.0.0)", "orjson (>=3.2.1,<4.0.0)", "email_validator (>=1.1.1,<2.0.0)", "uvicorn[standard] (>=0.12.0,<0.14.0)", "async_exit_stack (>=1.0.1,<2.0.0)", "async_generator (>=1.10,<2.0.0)"] +dev = ["python-jose[cryptography] (>=3.1.0,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "uvicorn[standard] (>=0.12.0,<0.14.0)", "graphene (>=2.1.8,<3.0.0)"] +doc = ["mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=7.1.9,<8.0.0)", "markdown-include (>=0.6.0,<0.7.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.2.0)", "typer-cli (>=0.0.12,<0.0.13)", "pyyaml (>=5.3.1,<6.0.0)"] +test = ["pytest (==5.4.3)", "pytest-cov (==2.10.0)", "pytest-asyncio (>=0.14.0,<0.15.0)", "mypy (==0.812)", "flake8 (>=3.8.3,<4.0.0)", "black (==20.8b1)", "isort (>=5.0.6,<6.0.0)", "requests (>=2.24.0,<3.0.0)", "httpx (>=0.14.0,<0.15.0)", "email_validator (>=1.1.1,<2.0.0)", "sqlalchemy (>=1.3.18,<1.4.0)", "peewee (>=3.13.3,<4.0.0)", "databases[sqlite] (>=0.3.2,<0.4.0)", "orjson (>=3.2.1,<4.0.0)", "ujson (>=4.0.1,<5.0.0)", "async_exit_stack (>=1.0.1,<2.0.0)", "async_generator (>=1.10,<2.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "aiofiles (>=0.5.0,<0.6.0)", "flask (>=1.1.2,<2.0.0)"] + +[[package]] +name = "feedgen" +version = "0.9.0" +description = "Feed Generator (ATOM, RSS, Podcasts)" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +lxml = "*" +python-dateutil = "*" + +[[package]] +name = "flake8" +version = "3.9.2" +description = "the modular source code checker: pep8 pyflakes and co" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +mccabe = ">=0.6.0,<0.7.0" +pycodestyle = ">=2.7.0,<2.8.0" +pyflakes = ">=2.3.0,<2.4.0" + +[[package]] +name = "h11" +version = "0.12.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "h2" +version = "4.0.0" +description = "HTTP/2 State-Machine based protocol implementation" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +hpack = ">=4.0,<5" +hyperframe = ">=6.0,<7" + +[[package]] +name = "hpack" +version = "4.0.0" +description = "Pure-Python HPACK header compression" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[[package]] +name = "httpcore" +version = "0.13.6" +description = "A minimal low-level HTTP client." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +anyio = ">=3.0.0,<4.0.0" +h11 = ">=0.11,<0.13" +sniffio = ">=1.0.0,<2.0.0" + +[package.extras] +http2 = ["h2 (>=3,<5)"] + +[[package]] +name = "httpx" +version = "0.18.2" +description = "The next generation HTTP client." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +certifi = "*" +httpcore = ">=0.13.3,<0.14.0" +rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]} +sniffio = "*" + +[package.extras] +brotli = ["brotlicffi (>=1.0.0,<2.0.0)"] +http2 = ["h2 (>=3.0.0,<4.0.0)"] + +[[package]] +name = "hypercorn" +version = "0.11.2" +description = "A ASGI Server based on Hyper libraries and inspired by Gunicorn." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +h11 = "*" +h2 = ">=3.1.0" +priority = "*" +toml = "*" +wsproto = ">=0.14.0" + +[package.extras] +h3 = ["aioquic (>=0.9.0,<1.0)"] +tests = ["hypothesis", "mock", "pytest", "pytest-asyncio", "pytest-cov", "pytest-trio", "trio"] +trio = ["trio (>=0.11.0)"] +uvloop = ["uvloop"] + +[[package]] +name = "hyperframe" +version = "6.0.1" +description = "HTTP/2 framing layer for Python" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[[package]] +name = "idna" +version = "3.2" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "isort" +version = "5.9.3" +description = "A Python utility / library to sort Python imports." +category = "dev" +optional = false +python-versions = ">=3.6.1,<4.0" + +[package.extras] +pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +requirements_deprecated_finder = ["pipreqs", "pip-api"] +colors = ["colorama (>=0.4.3,<0.5.0)"] +plugins = ["setuptools"] + +[[package]] +name = "itsdangerous" +version = "2.0.1" +description = "Safely pass data to untrusted environments and back." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "jinja2" +version = "3.0.1" +description = "A very fast and expressive template engine." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "lxml" +version = "4.6.3" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html5 = ["html5lib"] +htmlsoup = ["beautifulsoup4"] +source = ["Cython (>=0.29.7)"] + +[[package]] +name = "mako" +version = "1.1.5" +description = "A super-fast templating language that borrows the best ideas from the existing templating languages." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +MarkupSafe = ">=0.9.2" + +[package.extras] +babel = ["babel"] +lingua = ["lingua"] + +[[package]] +name = "markdown" +version = "3.3.4" +description = "Python implementation of Markdown." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +testing = ["coverage", "pyyaml"] + +[[package]] +name = "markupsafe" +version = "2.0.1" +description = "Safely add untrusted strings to HTML/XML markup." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "mccabe" +version = "0.6.1" +description = "McCabe checker, plugin for flake8" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "mysqlclient" +version = "2.0.3" +description = "Python interface to MySQL" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "orjson" +version = "3.6.3" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "packaging" +version = "21.0" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pyparsing = ">=2.0.2" + +[[package]] +name = "pluggy" +version = "0.13.1" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +dev = ["pre-commit", "tox"] + +[[package]] +name = "poetry-dynamic-versioning" +version = "0.13.1" +description = "Plugin for Poetry to enable dynamic versioning based on VCS tags" +category = "main" +optional = false +python-versions = ">=3.5,<4.0" + +[package.dependencies] +dunamai = ">=1.5,<2.0" +jinja2 = {version = ">=2.11.1,<4", markers = "python_version >= \"3.6\" and python_version < \"4.0\""} +tomlkit = ">=0.4" + +[[package]] +name = "priority" +version = "2.0.0" +description = "A pure-Python implementation of the HTTP/2 priority tree" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[[package]] +name = "protobuf" +version = "3.17.3" +description = "Protocol Buffers" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.9" + +[[package]] +name = "py" +version = "1.10.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pycodestyle" +version = "2.7.0" +description = "Python style guide checker" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pycparser" +version = "2.20" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pydantic" +version = "1.8.2" +description = "Data validation and settings management using python 3.6 type hinting" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +typing-extensions = ">=3.7.4.3" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyflakes" +version = "2.3.1" +description = "passive checker of Python programs" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pygit2" +version = "1.6.1" +description = "Python bindings for libgit2." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +cffi = ">=1.4.0" + +[[package]] +name = "pyparsing" +version = "2.4.7" +description = "Python parsing module" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "pytest" +version = "6.2.4" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<1.0.0a1" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-asyncio" +version = "0.15.1" +description = "Pytest support for asyncio." +category = "dev" +optional = false +python-versions = ">= 3.6" + +[package.dependencies] +pytest = ">=5.4.0" + +[package.extras] +testing = ["coverage", "hypothesis (>=5.7.1)"] + +[[package]] +name = "pytest-cov" +version = "2.12.1" +description = "Pytest plugin for measuring coverage." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +coverage = ">=5.2.1" +pytest = ">=4.6" +toml = "*" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"] + +[[package]] +name = "pytest-tap" +version = "3.2" +description = "Test Anything Protocol (TAP) reporting plugin for pytest" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +pytest = ">=3.0" +"tap.py" = ">=3.0,<4.0" + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-editor" +version = "1.0.4" +description = "Programmatically open an editor, capture the result." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "python-multipart" +version = "0.0.5" +description = "A streaming multipart parser for Python" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.4.0" + +[[package]] +name = "redis" +version = "3.5.3" +description = "Python client for Redis key-value store" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +hiredis = ["hiredis (>=0.1.3)"] + +[[package]] +name = "requests" +version = "2.26.0" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] + +[[package]] +name = "rfc3986" +version = "1.5.0" +description = "Validating URI References per RFC 3986" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +idna = {version = "*", optional = true, markers = "extra == \"idna2008\""} + +[package.extras] +idna2008 = ["idna"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "sniffio" +version = "1.2.0" +description = "Sniff out which async library your code is running under" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "sqlalchemy" +version = "1.3.23" +description = "Database Abstraction Library" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +mssql = ["pyodbc"] +mssql_pymssql = ["pymssql"] +mssql_pyodbc = ["pyodbc"] +mysql = ["mysqlclient"] +oracle = ["cx-oracle"] +postgresql = ["psycopg2"] +postgresql_pg8000 = ["pg8000 (<1.16.6)"] +postgresql_psycopg2binary = ["psycopg2-binary"] +postgresql_psycopg2cffi = ["psycopg2cffi"] +pymysql = ["pymysql (<1)", "pymysql"] + +[[package]] +name = "starlette" +version = "0.14.2" +description = "The little ASGI library that shines." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +full = ["aiofiles", "graphene", "itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"] + +[[package]] +name = "tap.py" +version = "3.0" +description = "Test Anything Protocol (TAP) tools" +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +yaml = ["more-itertools", "PyYAML (>=5.1)"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "tomlkit" +version = "0.7.2" +description = "Style preserving TOML library" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +typing = {version = ">=3.6,<4.0", markers = "python_version >= \"2.7\" and python_version < \"2.8\" or python_version >= \"3.4\" and python_version < \"3.5\""} + +[[package]] +name = "typing" +version = "3.7.4.3" +description = "Type Hints for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "typing-extensions" +version = "3.10.0.2" +description = "Backported and Experimental Type Hints for Python 3.5+" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "urllib3" +version = "1.26.6" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.extras] +brotli = ["brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "uvicorn" +version = "0.15.0" +description = "The lightning-fast ASGI server." +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +asgiref = ">=3.4.0" +click = ">=7.0" +h11 = ">=0.8" + +[package.extras] +standard = ["websockets (>=9.1)", "httptools (>=0.2.0,<0.3.0)", "watchgod (>=0.6)", "python-dotenv (>=0.13)", "PyYAML (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "colorama (>=0.4)"] + +[[package]] +name = "webencodings" +version = "0.5.1" +description = "Character encoding aliases for legacy web content" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "werkzeug" +version = "2.0.1" +description = "The comprehensive WSGI web application library." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +watchdog = ["watchdog"] + +[[package]] +name = "wsproto" +version = "1.0.0" +description = "WebSockets state-machine based protocol implementation" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +h11 = ">=0.9.0,<1" + +[metadata] +lock-version = "1.1" +python-versions = "*" +content-hash = "96112731ca21a6ff5d0657c6c40979642bb992ae660ba8d6135421718737c6b0" + +[metadata.files] +aiofiles = [ + {file = "aiofiles-0.7.0-py3-none-any.whl", hash = "sha256:c67a6823b5f23fcab0a2595a289cec7d8c863ffcb4322fb8cd6b90400aedfdbc"}, + {file = "aiofiles-0.7.0.tar.gz", hash = "sha256:a1c4fc9b2ff81568c83e21392a82f344ea9d23da906e4f6a52662764545e19d4"}, +] +alembic = [ + {file = "alembic-1.6.5-py2.py3-none-any.whl", hash = "sha256:e78be5b919f5bb184e3e0e2dd1ca986f2362e29a2bc933c446fe89f39dbe4e9c"}, + {file = "alembic-1.6.5.tar.gz", hash = "sha256:a21fedebb3fb8f6bbbba51a11114f08c78709377051384c9c5ead5705ee93a51"}, +] +anyio = [ + {file = "anyio-3.3.0-py3-none-any.whl", hash = "sha256:929a6852074397afe1d989002aa96d457e3e1e5441357c60d03e7eea0e65e1b0"}, + {file = "anyio-3.3.0.tar.gz", hash = "sha256:ae57a67583e5ff8b4af47666ff5651c3732d45fd26c929253748e796af860374"}, +] +asgiref = [ + {file = "asgiref-3.4.1-py3-none-any.whl", hash = "sha256:ffc141aa908e6f175673e7b1b3b7af4fdb0ecb738fc5c8b88f69f055c2415214"}, + {file = "asgiref-3.4.1.tar.gz", hash = "sha256:4ef1ab46b484e3c706329cedeff284a5d40824200638503f5768edb6de7d58e9"}, +] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] +attrs = [ + {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, + {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, +] +authlib = [ + {file = "Authlib-0.15.2-py2.py3-none-any.whl", hash = "sha256:078b900fa9fbebf9f8dae1d5dc1ca857b6a742493093ef9b0b36ad926f36e41f"}, + {file = "Authlib-0.15.2.tar.gz", hash = "sha256:21b34625c83ca48150684bbeca8f7c884cd281913c72d146dbf0e9d2fbfdec4e"}, +] +bcrypt = [ + {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux1_x86_64.whl", hash = "sha256:63d4e3ff96188e5898779b6057878fecf3f11cfe6ec3b313ea09955d587ec7a7"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d"}, + {file = "bcrypt-3.2.0-cp36-abi3-win32.whl", hash = "sha256:a67fb841b35c28a59cebed05fbd3e80eea26e6d75851f0574a9273c80f3e9b55"}, + {file = "bcrypt-3.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:81fec756feff5b6818ea7ab031205e1d323d8943d237303baca2c5f9c7846f34"}, + {file = "bcrypt-3.2.0.tar.gz", hash = "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29"}, +] +bleach = [ + {file = "bleach-3.3.1-py2.py3-none-any.whl", hash = "sha256:ae976d7174bba988c0b632def82fdc94235756edfb14e6558a9c5be555c9fb78"}, + {file = "bleach-3.3.1.tar.gz", hash = "sha256:306483a5a9795474160ad57fce3ddd1b50551e981eed8e15a582d34cef28aafa"}, +] +certifi = [ + {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, + {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, +] +cffi = [ + {file = "cffi-1.14.6-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c"}, + {file = "cffi-1.14.6-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99"}, + {file = "cffi-1.14.6-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819"}, + {file = "cffi-1.14.6-cp27-cp27m-win32.whl", hash = "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20"}, + {file = "cffi-1.14.6-cp27-cp27m-win_amd64.whl", hash = "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224"}, + {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7"}, + {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33"}, + {file = "cffi-1.14.6-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534"}, + {file = "cffi-1.14.6-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a"}, + {file = "cffi-1.14.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5"}, + {file = "cffi-1.14.6-cp35-cp35m-win32.whl", hash = "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca"}, + {file = "cffi-1.14.6-cp35-cp35m-win_amd64.whl", hash = "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218"}, + {file = "cffi-1.14.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f"}, + {file = "cffi-1.14.6-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872"}, + {file = "cffi-1.14.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195"}, + {file = "cffi-1.14.6-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d"}, + {file = "cffi-1.14.6-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b"}, + {file = "cffi-1.14.6-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb"}, + {file = "cffi-1.14.6-cp36-cp36m-win32.whl", hash = "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a"}, + {file = "cffi-1.14.6-cp36-cp36m-win_amd64.whl", hash = "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e"}, + {file = "cffi-1.14.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5"}, + {file = "cffi-1.14.6-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf"}, + {file = "cffi-1.14.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69"}, + {file = "cffi-1.14.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56"}, + {file = "cffi-1.14.6-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c"}, + {file = "cffi-1.14.6-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762"}, + {file = "cffi-1.14.6-cp37-cp37m-win32.whl", hash = "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771"}, + {file = "cffi-1.14.6-cp37-cp37m-win_amd64.whl", hash = "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a"}, + {file = "cffi-1.14.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0"}, + {file = "cffi-1.14.6-cp38-cp38-manylinux1_i686.whl", hash = "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e"}, + {file = "cffi-1.14.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346"}, + {file = "cffi-1.14.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc"}, + {file = "cffi-1.14.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd"}, + {file = "cffi-1.14.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc"}, + {file = "cffi-1.14.6-cp38-cp38-win32.whl", hash = "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548"}, + {file = "cffi-1.14.6-cp38-cp38-win_amd64.whl", hash = "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156"}, + {file = "cffi-1.14.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d"}, + {file = "cffi-1.14.6-cp39-cp39-manylinux1_i686.whl", hash = "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e"}, + {file = "cffi-1.14.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c"}, + {file = "cffi-1.14.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202"}, + {file = "cffi-1.14.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f"}, + {file = "cffi-1.14.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87"}, + {file = "cffi-1.14.6-cp39-cp39-win32.whl", hash = "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728"}, + {file = "cffi-1.14.6-cp39-cp39-win_amd64.whl", hash = "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2"}, + {file = "cffi-1.14.6.tar.gz", hash = "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd"}, +] +charset-normalizer = [ + {file = "charset-normalizer-2.0.4.tar.gz", hash = "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"}, + {file = "charset_normalizer-2.0.4-py3-none-any.whl", hash = "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b"}, +] +click = [ + {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, + {file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +coverage = [ + {file = "coverage-5.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf"}, + {file = "coverage-5.5-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b"}, + {file = "coverage-5.5-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:c2723d347ab06e7ddad1a58b2a821218239249a9e4365eaff6649d31180c1669"}, + {file = "coverage-5.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:900fbf7759501bc7807fd6638c947d7a831fc9fdf742dc10f02956ff7220fa90"}, + {file = "coverage-5.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:004d1880bed2d97151facef49f08e255a20ceb6f9432df75f4eef018fdd5a78c"}, + {file = "coverage-5.5-cp27-cp27m-win32.whl", hash = "sha256:06191eb60f8d8a5bc046f3799f8a07a2d7aefb9504b0209aff0b47298333302a"}, + {file = "coverage-5.5-cp27-cp27m-win_amd64.whl", hash = "sha256:7501140f755b725495941b43347ba8a2777407fc7f250d4f5a7d2a1050ba8e82"}, + {file = "coverage-5.5-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:372da284cfd642d8e08ef606917846fa2ee350f64994bebfbd3afb0040436905"}, + {file = "coverage-5.5-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:8963a499849a1fc54b35b1c9f162f4108017b2e6db2c46c1bed93a72262ed083"}, + {file = "coverage-5.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:869a64f53488f40fa5b5b9dcb9e9b2962a66a87dab37790f3fcfb5144b996ef5"}, + {file = "coverage-5.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:4a7697d8cb0f27399b0e393c0b90f0f1e40c82023ea4d45d22bce7032a5d7b81"}, + {file = "coverage-5.5-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:8d0a0725ad7c1a0bcd8d1b437e191107d457e2ec1084b9f190630a4fb1af78e6"}, + {file = "coverage-5.5-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:51cb9476a3987c8967ebab3f0fe144819781fca264f57f89760037a2ea191cb0"}, + {file = "coverage-5.5-cp310-cp310-win_amd64.whl", hash = "sha256:c0891a6a97b09c1f3e073a890514d5012eb256845c451bd48f7968ef939bf4ae"}, + {file = "coverage-5.5-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:3487286bc29a5aa4b93a072e9592f22254291ce96a9fbc5251f566b6b7343cdb"}, + {file = "coverage-5.5-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:deee1077aae10d8fa88cb02c845cfba9b62c55e1183f52f6ae6a2df6a2187160"}, + {file = "coverage-5.5-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f11642dddbb0253cc8853254301b51390ba0081750a8ac03f20ea8103f0c56b6"}, + {file = "coverage-5.5-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:6c90e11318f0d3c436a42409f2749ee1a115cd8b067d7f14c148f1ce5574d701"}, + {file = "coverage-5.5-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:30c77c1dc9f253283e34c27935fded5015f7d1abe83bc7821680ac444eaf7793"}, + {file = "coverage-5.5-cp35-cp35m-win32.whl", hash = "sha256:9a1ef3b66e38ef8618ce5fdc7bea3d9f45f3624e2a66295eea5e57966c85909e"}, + {file = "coverage-5.5-cp35-cp35m-win_amd64.whl", hash = "sha256:972c85d205b51e30e59525694670de6a8a89691186012535f9d7dbaa230e42c3"}, + {file = "coverage-5.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:af0e781009aaf59e25c5a678122391cb0f345ac0ec272c7961dc5455e1c40066"}, + {file = "coverage-5.5-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:74d881fc777ebb11c63736622b60cb9e4aee5cace591ce274fb69e582a12a61a"}, + {file = "coverage-5.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:92b017ce34b68a7d67bd6d117e6d443a9bf63a2ecf8567bb3d8c6c7bc5014465"}, + {file = "coverage-5.5-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:d636598c8305e1f90b439dbf4f66437de4a5e3c31fdf47ad29542478c8508bbb"}, + {file = "coverage-5.5-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:41179b8a845742d1eb60449bdb2992196e211341818565abded11cfa90efb821"}, + {file = "coverage-5.5-cp36-cp36m-win32.whl", hash = "sha256:040af6c32813fa3eae5305d53f18875bedd079960822ef8ec067a66dd8afcd45"}, + {file = "coverage-5.5-cp36-cp36m-win_amd64.whl", hash = "sha256:5fec2d43a2cc6965edc0bb9e83e1e4b557f76f843a77a2496cbe719583ce8184"}, + {file = "coverage-5.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:18ba8bbede96a2c3dde7b868de9dcbd55670690af0988713f0603f037848418a"}, + {file = "coverage-5.5-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:2910f4d36a6a9b4214bb7038d537f015346f413a975d57ca6b43bf23d6563b53"}, + {file = "coverage-5.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:f0b278ce10936db1a37e6954e15a3730bea96a0997c26d7fee88e6c396c2086d"}, + {file = "coverage-5.5-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:796c9c3c79747146ebd278dbe1e5c5c05dd6b10cc3bcb8389dfdf844f3ead638"}, + {file = "coverage-5.5-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:53194af30d5bad77fcba80e23a1441c71abfb3e01192034f8246e0d8f99528f3"}, + {file = "coverage-5.5-cp37-cp37m-win32.whl", hash = "sha256:184a47bbe0aa6400ed2d41d8e9ed868b8205046518c52464fde713ea06e3a74a"}, + {file = "coverage-5.5-cp37-cp37m-win_amd64.whl", hash = "sha256:2949cad1c5208b8298d5686d5a85b66aae46d73eec2c3e08c817dd3513e5848a"}, + {file = "coverage-5.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:217658ec7187497e3f3ebd901afdca1af062b42cfe3e0dafea4cced3983739f6"}, + {file = "coverage-5.5-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1aa846f56c3d49205c952d8318e76ccc2ae23303351d9270ab220004c580cfe2"}, + {file = "coverage-5.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:24d4a7de75446be83244eabbff746d66b9240ae020ced65d060815fac3423759"}, + {file = "coverage-5.5-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d1f8bf7b90ba55699b3a5e44930e93ff0189aa27186e96071fac7dd0d06a1873"}, + {file = "coverage-5.5-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:970284a88b99673ccb2e4e334cfb38a10aab7cd44f7457564d11898a74b62d0a"}, + {file = "coverage-5.5-cp38-cp38-win32.whl", hash = "sha256:01d84219b5cdbfc8122223b39a954820929497a1cb1422824bb86b07b74594b6"}, + {file = "coverage-5.5-cp38-cp38-win_amd64.whl", hash = "sha256:2e0d881ad471768bf6e6c2bf905d183543f10098e3b3640fc029509530091502"}, + {file = "coverage-5.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d1f9ce122f83b2305592c11d64f181b87153fc2c2bbd3bb4a3dde8303cfb1a6b"}, + {file = "coverage-5.5-cp39-cp39-manylinux1_i686.whl", hash = "sha256:13c4ee887eca0f4c5a247b75398d4114c37882658300e153113dafb1d76de529"}, + {file = "coverage-5.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:52596d3d0e8bdf3af43db3e9ba8dcdaac724ba7b5ca3f6358529d56f7a166f8b"}, + {file = "coverage-5.5-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:2cafbbb3af0733db200c9b5f798d18953b1a304d3f86a938367de1567f4b5bff"}, + {file = "coverage-5.5-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:44d654437b8ddd9eee7d1eaee28b7219bec228520ff809af170488fd2fed3e2b"}, + {file = "coverage-5.5-cp39-cp39-win32.whl", hash = "sha256:d314ed732c25d29775e84a960c3c60808b682c08d86602ec2c3008e1202e3bb6"}, + {file = "coverage-5.5-cp39-cp39-win_amd64.whl", hash = "sha256:13034c4409db851670bc9acd836243aeee299949bd5673e11844befcb0149f03"}, + {file = "coverage-5.5-pp36-none-any.whl", hash = "sha256:f030f8873312a16414c0d8e1a1ddff2d3235655a2174e3648b4fa66b3f2f1079"}, + {file = "coverage-5.5-pp37-none-any.whl", hash = "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4"}, + {file = "coverage-5.5.tar.gz", hash = "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c"}, +] +cryptography = [ + {file = "cryptography-3.4.8-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14"}, + {file = "cryptography-3.4.8-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb"}, + {file = "cryptography-3.4.8-cp36-abi3-win32.whl", hash = "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7"}, + {file = "cryptography-3.4.8-cp36-abi3-win_amd64.whl", hash = "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e"}, + {file = "cryptography-3.4.8.tar.gz", hash = "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c"}, +] +dnspython = [ + {file = "dnspython-2.1.0-py3-none-any.whl", hash = "sha256:95d12f6ef0317118d2a1a6fc49aac65ffec7eb8087474158f42f26a639135216"}, + {file = "dnspython-2.1.0.zip", hash = "sha256:e4a87f0b573201a0f3727fa18a516b055fd1107e0e5477cded4a2de497df1dd4"}, +] +dunamai = [ + {file = "dunamai-1.6.0-py3-none-any.whl", hash = "sha256:44a94a4edebb145bb6198a2f26de957b12b77d43b7c9c0646be814c60cf5d8df"}, + {file = "dunamai-1.6.0.tar.gz", hash = "sha256:6f1111f47e869ed58d44a7d37f112e3e7c761dce3c71f2c5464526928d7e9896"}, +] +email-validator = [ + {file = "email_validator-1.1.3-py2.py3-none-any.whl", hash = "sha256:5675c8ceb7106a37e40e2698a57c056756bf3f272cfa8682a4f87ebd95d8440b"}, + {file = "email_validator-1.1.3.tar.gz", hash = "sha256:aa237a65f6f4da067119b7df3f13e89c25c051327b2b5b66dc075f33d62480d7"}, +] +fakeredis = [ + {file = "fakeredis-1.6.0-py3-none-any.whl", hash = "sha256:3449b306f3a85102b28f8180c24722ef966fcb1e3c744758b6f635ec80321a5c"}, + {file = "fakeredis-1.6.0.tar.gz", hash = "sha256:11ccfc9769d718d37e45b382e64a6ba02586b622afa0371a6bd85766d72255f3"}, +] +fastapi = [ + {file = "fastapi-0.66.0-py3-none-any.whl", hash = "sha256:85d8aee8c3c46171f4cb7bb3651425a42c07cb9183345d100ef55d88ca2ce15f"}, + {file = "fastapi-0.66.0.tar.gz", hash = "sha256:6ea4225448786f3d6fae737713789f87631a7455f65580de0a4a2e50471060d9"}, +] +feedgen = [ + {file = "feedgen-0.9.0.tar.gz", hash = "sha256:8e811bdbbed6570034950db23a4388453628a70e689a6e8303ccec430f5a804a"}, +] +flake8 = [ + {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, + {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, +] +h11 = [ + {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, + {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"}, +] +h2 = [ + {file = "h2-4.0.0-py3-none-any.whl", hash = "sha256:ac9e293a1990b339d5d71b19c5fe630e3dd4d768c620d1730d355485323f1b25"}, + {file = "h2-4.0.0.tar.gz", hash = "sha256:bb7ac7099dd67a857ed52c815a6192b6b1f5ba6b516237fc24a085341340593d"}, +] +hpack = [ + {file = "hpack-4.0.0-py3-none-any.whl", hash = "sha256:84a076fad3dc9a9f8063ccb8041ef100867b1878b25ef0ee63847a5d53818a6c"}, + {file = "hpack-4.0.0.tar.gz", hash = "sha256:fc41de0c63e687ebffde81187a948221294896f6bdc0ae2312708df339430095"}, +] +httpcore = [ + {file = "httpcore-0.13.6-py3-none-any.whl", hash = "sha256:db4c0dcb8323494d01b8c6d812d80091a31e520033e7b0120883d6f52da649ff"}, + {file = "httpcore-0.13.6.tar.gz", hash = "sha256:b0d16f0012ec88d8cc848f5a55f8a03158405f4bca02ee49bc4ca2c1fda49f3e"}, +] +httpx = [ + {file = "httpx-0.18.2-py3-none-any.whl", hash = "sha256:979afafecb7d22a1d10340bafb403cf2cb75aff214426ff206521fc79d26408c"}, + {file = "httpx-0.18.2.tar.gz", hash = "sha256:9f99c15d33642d38bce8405df088c1c4cfd940284b4290cacbfb02e64f4877c6"}, +] +hypercorn = [ + {file = "Hypercorn-0.11.2-py3-none-any.whl", hash = "sha256:8007c10f81566920f8ae12c0e26e146f94ca70506da964b5a727ad610aa1d821"}, + {file = "Hypercorn-0.11.2.tar.gz", hash = "sha256:5ba1e719c521080abd698ff5781a2331e34ef50fc1c89a50960538115a896a9a"}, +] +hyperframe = [ + {file = "hyperframe-6.0.1-py3-none-any.whl", hash = "sha256:0ec6bafd80d8ad2195c4f03aacba3a8265e57bc4cff261e802bf39970ed02a15"}, + {file = "hyperframe-6.0.1.tar.gz", hash = "sha256:ae510046231dc8e9ecb1a6586f63d2347bf4c8905914aa84ba585ae85f28a914"}, +] +idna = [ + {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, + {file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +isort = [ + {file = "isort-5.9.3-py3-none-any.whl", hash = "sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2"}, + {file = "isort-5.9.3.tar.gz", hash = "sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899"}, +] +itsdangerous = [ + {file = "itsdangerous-2.0.1-py3-none-any.whl", hash = "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c"}, + {file = "itsdangerous-2.0.1.tar.gz", hash = "sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0"}, +] +jinja2 = [ + {file = "Jinja2-3.0.1-py3-none-any.whl", hash = "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4"}, + {file = "Jinja2-3.0.1.tar.gz", hash = "sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4"}, +] +lxml = [ + {file = "lxml-4.6.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:df7c53783a46febb0e70f6b05df2ba104610f2fb0d27023409734a3ecbb78fb2"}, + {file = "lxml-4.6.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1b7584d421d254ab86d4f0b13ec662a9014397678a7c4265a02a6d7c2b18a75f"}, + {file = "lxml-4.6.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:079f3ae844f38982d156efce585bc540c16a926d4436712cf4baee0cce487a3d"}, + {file = "lxml-4.6.3-cp27-cp27m-win32.whl", hash = "sha256:bc4313cbeb0e7a416a488d72f9680fffffc645f8a838bd2193809881c67dd106"}, + {file = "lxml-4.6.3-cp27-cp27m-win_amd64.whl", hash = "sha256:8157dadbb09a34a6bd95a50690595e1fa0af1a99445e2744110e3dca7831c4ee"}, + {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7728e05c35412ba36d3e9795ae8995e3c86958179c9770e65558ec3fdfd3724f"}, + {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:4bff24dfeea62f2e56f5bab929b4428ae6caba2d1eea0c2d6eb618e30a71e6d4"}, + {file = "lxml-4.6.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:64812391546a18896adaa86c77c59a4998f33c24788cadc35789e55b727a37f4"}, + {file = "lxml-4.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c1a40c06fd5ba37ad39caa0b3144eb3772e813b5fb5b084198a985431c2f1e8d"}, + {file = "lxml-4.6.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:74f7d8d439b18fa4c385f3f5dfd11144bb87c1da034a466c5b5577d23a1d9b51"}, + {file = "lxml-4.6.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f90ba11136bfdd25cae3951af8da2e95121c9b9b93727b1b896e3fa105b2f586"}, + {file = "lxml-4.6.3-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:4c61b3a0db43a1607d6264166b230438f85bfed02e8cff20c22e564d0faff354"}, + {file = "lxml-4.6.3-cp35-cp35m-manylinux2014_x86_64.whl", hash = "sha256:5c8c163396cc0df3fd151b927e74f6e4acd67160d6c33304e805b84293351d16"}, + {file = "lxml-4.6.3-cp35-cp35m-win32.whl", hash = "sha256:f2380a6376dfa090227b663f9678150ef27543483055cc327555fb592c5967e2"}, + {file = "lxml-4.6.3-cp35-cp35m-win_amd64.whl", hash = "sha256:c4f05c5a7c49d2fb70223d0d5bcfbe474cf928310ac9fa6a7c6dddc831d0b1d4"}, + {file = "lxml-4.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d2e35d7bf1c1ac8c538f88d26b396e73dd81440d59c1ef8522e1ea77b345ede4"}, + {file = "lxml-4.6.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:289e9ca1a9287f08daaf796d96e06cb2bc2958891d7911ac7cae1c5f9e1e0ee3"}, + {file = "lxml-4.6.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:bccbfc27563652de7dc9bdc595cb25e90b59c5f8e23e806ed0fd623755b6565d"}, + {file = "lxml-4.6.3-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:d916d31fd85b2f78c76400d625076d9124de3e4bda8b016d25a050cc7d603f24"}, + {file = "lxml-4.6.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:820628b7b3135403540202e60551e741f9b6d3304371712521be939470b454ec"}, + {file = "lxml-4.6.3-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:c47ff7e0a36d4efac9fd692cfa33fbd0636674c102e9e8d9b26e1b93a94e7617"}, + {file = "lxml-4.6.3-cp36-cp36m-win32.whl", hash = "sha256:5a0a14e264069c03e46f926be0d8919f4105c1623d620e7ec0e612a2e9bf1c04"}, + {file = "lxml-4.6.3-cp36-cp36m-win_amd64.whl", hash = "sha256:92e821e43ad382332eade6812e298dc9701c75fe289f2a2d39c7960b43d1e92a"}, + {file = "lxml-4.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:efd7a09678fd8b53117f6bae4fa3825e0a22b03ef0a932e070c0bdbb3a35e654"}, + {file = "lxml-4.6.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:efac139c3f0bf4f0939f9375af4b02c5ad83a622de52d6dfa8e438e8e01d0eb0"}, + {file = "lxml-4.6.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:0fbcf5565ac01dff87cbfc0ff323515c823081c5777a9fc7703ff58388c258c3"}, + {file = "lxml-4.6.3-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:36108c73739985979bf302006527cf8a20515ce444ba916281d1c43938b8bb96"}, + {file = "lxml-4.6.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:122fba10466c7bd4178b07dba427aa516286b846b2cbd6f6169141917283aae2"}, + {file = "lxml-4.6.3-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:cdaf11d2bd275bf391b5308f86731e5194a21af45fbaaaf1d9e8147b9160ea92"}, + {file = "lxml-4.6.3-cp37-cp37m-win32.whl", hash = "sha256:3439c71103ef0e904ea0a1901611863e51f50b5cd5e8654a151740fde5e1cade"}, + {file = "lxml-4.6.3-cp37-cp37m-win_amd64.whl", hash = "sha256:4289728b5e2000a4ad4ab8da6e1db2e093c63c08bdc0414799ee776a3f78da4b"}, + {file = "lxml-4.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b007cbb845b28db4fb8b6a5cdcbf65bacb16a8bd328b53cbc0698688a68e1caa"}, + {file = "lxml-4.6.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:76fa7b1362d19f8fbd3e75fe2fb7c79359b0af8747e6f7141c338f0bee2f871a"}, + {file = "lxml-4.6.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:26e761ab5b07adf5f555ee82fb4bfc35bf93750499c6c7614bd64d12aaa67927"}, + {file = "lxml-4.6.3-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:e1cbd3f19a61e27e011e02f9600837b921ac661f0c40560eefb366e4e4fb275e"}, + {file = "lxml-4.6.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:66e575c62792c3f9ca47cb8b6fab9e35bab91360c783d1606f758761810c9791"}, + {file = "lxml-4.6.3-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:1b38116b6e628118dea5b2186ee6820ab138dbb1e24a13e478490c7db2f326ae"}, + {file = "lxml-4.6.3-cp38-cp38-win32.whl", hash = "sha256:89b8b22a5ff72d89d48d0e62abb14340d9e99fd637d046c27b8b257a01ffbe28"}, + {file = "lxml-4.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:2a9d50e69aac3ebee695424f7dbd7b8c6d6eb7de2a2eb6b0f6c7db6aa41e02b7"}, + {file = "lxml-4.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ce256aaa50f6cc9a649c51be3cd4ff142d67295bfc4f490c9134d0f9f6d58ef0"}, + {file = "lxml-4.6.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:7610b8c31688f0b1be0ef882889817939490a36d0ee880ea562a4e1399c447a1"}, + {file = "lxml-4.6.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f8380c03e45cf09f8557bdaa41e1fa7c81f3ae22828e1db470ab2a6c96d8bc23"}, + {file = "lxml-4.6.3-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:3082c518be8e97324390614dacd041bb1358c882d77108ca1957ba47738d9d59"}, + {file = "lxml-4.6.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:884ab9b29feaca361f7f88d811b1eea9bfca36cf3da27768d28ad45c3ee6f969"}, + {file = "lxml-4.6.3-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:6f12e1427285008fd32a6025e38e977d44d6382cf28e7201ed10d6c1698d2a9a"}, + {file = "lxml-4.6.3-cp39-cp39-win32.whl", hash = "sha256:33bb934a044cf32157c12bfcfbb6649807da20aa92c062ef51903415c704704f"}, + {file = "lxml-4.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:542d454665a3e277f76954418124d67516c5f88e51a900365ed54a9806122b83"}, + {file = "lxml-4.6.3.tar.gz", hash = "sha256:39b78571b3b30645ac77b95f7c69d1bffc4cf8c3b157c435a34da72e78c82468"}, +] +mako = [ + {file = "Mako-1.1.5-py2.py3-none-any.whl", hash = "sha256:6804ee66a7f6a6416910463b00d76a7b25194cd27f1918500c5bd7be2a088a23"}, + {file = "Mako-1.1.5.tar.gz", hash = "sha256:169fa52af22a91900d852e937400e79f535496191c63712e3b9fda5a9bed6fc3"}, +] +markdown = [ + {file = "Markdown-3.3.4-py3-none-any.whl", hash = "sha256:96c3ba1261de2f7547b46a00ea8463832c921d3f9d6aba3f255a6f71386db20c"}, + {file = "Markdown-3.3.4.tar.gz", hash = "sha256:31b5b491868dcc87d6c24b7e3d19a0d730d59d3e46f4eea6430a321bed387a49"}, +] +markupsafe = [ + {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-win32.whl", hash = "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-win32.whl", hash = "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-win32.whl", hash = "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-win32.whl", hash = "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-win32.whl", hash = "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, + {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, +] +mccabe = [ + {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, + {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, +] +mysqlclient = [ + {file = "mysqlclient-2.0.3-cp36-cp36m-win_amd64.whl", hash = "sha256:3381ca1a4f37ff1155fcfde20836b46416d66531add8843f6aa6d968982731c3"}, + {file = "mysqlclient-2.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0ac0dd759c4ca02c35a9fedc24bc982cf75171651e8187c2495ec957a87dfff7"}, + {file = "mysqlclient-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:71c4b330cf2313bbda0307fc858cc9055e64493ba9bf28454d25cf8b3ee8d7f5"}, + {file = "mysqlclient-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:fc575093cf81b6605bed84653e48b277318b880dc9becf42dd47fa11ffd3e2b6"}, + {file = "mysqlclient-2.0.3.tar.gz", hash = "sha256:f6ebea7c008f155baeefe16c56cd3ee6239f7a5a9ae42396c2f1860f08a7c432"}, +] +orjson = [ + {file = "orjson-3.6.3-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:5f78ed46b179585272a5670537f2203dbb7b3e2f8e4db1be72839cc423e2daef"}, + {file = "orjson-3.6.3-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:a99f310960e3acdda72ba1e98df8bf8c9145d90a0f72719786f43f4ea6937846"}, + {file = "orjson-3.6.3-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:8a5e46418f51f03060f91d743b59aed70c8d02a5012428365cfa20b7f670e903"}, + {file = "orjson-3.6.3-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:084de43ca9b19ad58c618c9f1ff93784e0190df2d88a02ae24c3cdebe9f2e9f7"}, + {file = "orjson-3.6.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b68a601f49c0328bf16498309e56ab87c1d6c2bb0287abf70329eb958d565c62"}, + {file = "orjson-3.6.3-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:9e4a26212851ea8ff81dee7e4e0da7e1e63b5b4f4330a8b4f27e99f1ba3f758b"}, + {file = "orjson-3.6.3-cp37-cp37m-manylinux_2_24_x86_64.whl", hash = "sha256:5eb9d7f2f45e12cbc7500da4176f2d3221a73891b4be505fe79c52cbb800e872"}, + {file = "orjson-3.6.3-cp37-none-win_amd64.whl", hash = "sha256:39aa7d42c9760fba36c37adb1d9c6752696ce9443c5dcb65222dd0994b5735e1"}, + {file = "orjson-3.6.3-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:8d4430e0cc390c1d745aea3827fd0c6fd7aa5f0690de30a2fe25c406aa5efa20"}, + {file = "orjson-3.6.3-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:be79e0ddea7f3a47332ec9573365c0b8a8cce4357e9682050f53c1bc75c1571f"}, + {file = "orjson-3.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fce5ada0f8dd7c9e16c675626a29dfc5cc766e1eb67d8021b1e77d0861e4e850"}, + {file = "orjson-3.6.3-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:1014a6f514b39dc414fce60568c9e7f635de97a1f1f5972ebc38f88a6160944a"}, + {file = "orjson-3.6.3-cp38-cp38-manylinux_2_24_x86_64.whl", hash = "sha256:c3beff02a339f194274ec1fcf03e2c1563e84f297b568eb3d45751722454a52e"}, + {file = "orjson-3.6.3-cp38-none-win_amd64.whl", hash = "sha256:8f105e9290f901a618a0ced87f785fce2fcf6ab753699de081d82ee05c90f038"}, + {file = "orjson-3.6.3-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:7936bef5589c9955ebee3423df51709d5f3b37ef54b830239bddb9fa5ead99f4"}, + {file = "orjson-3.6.3-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:82e3afbf404cb91774f894ed7bf52fd83bb1cc6bd72221711f4ce4e7774f0560"}, + {file = "orjson-3.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c702c78c33416fc8a138c5ec36eef5166ecfe8990c8f99c97551cd37c396e4d"}, + {file = "orjson-3.6.3-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:4606907b9aaec9fea6159ac14f838dbd2851f18b05fb414c4b3143bff9f2bb0d"}, + {file = "orjson-3.6.3-cp39-cp39-manylinux_2_24_x86_64.whl", hash = "sha256:4ebb464b8b557a1401a03da6f41761544886db95b52280e60d25549da7427453"}, + {file = "orjson-3.6.3-cp39-none-win_amd64.whl", hash = "sha256:720a7d7ba1dcf32bbd8fb380370b1fdd06ed916caea48403edd64f2ccf7883c1"}, + {file = "orjson-3.6.3.tar.gz", hash = "sha256:353cc079cedfe990ea2d2186306f766e0d47bba63acd072e22d6df96c67be993"}, +] +packaging = [ + {file = "packaging-21.0-py3-none-any.whl", hash = "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"}, + {file = "packaging-21.0.tar.gz", hash = "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7"}, +] +pluggy = [ + {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, + {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, +] +poetry-dynamic-versioning = [ + {file = "poetry-dynamic-versioning-0.13.1.tar.gz", hash = "sha256:5c0e7b22560db76812057ef95dadad662ecc63eb270145787eabe73da7c222f9"}, + {file = "poetry_dynamic_versioning-0.13.1-py3-none-any.whl", hash = "sha256:6d79f76436c624653fc06eb9bb54fb4f39b1d54362bc366ad2496855711d3a78"}, +] +priority = [ + {file = "priority-2.0.0-py3-none-any.whl", hash = "sha256:6f8eefce5f3ad59baf2c080a664037bb4725cd0a790d53d59ab4059288faf6aa"}, + {file = "priority-2.0.0.tar.gz", hash = "sha256:c965d54f1b8d0d0b19479db3924c7c36cf672dbf2aec92d43fbdaf4492ba18c0"}, +] +protobuf = [ + {file = "protobuf-3.17.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ab6bb0e270c6c58e7ff4345b3a803cc59dbee19ddf77a4719c5b635f1d547aa8"}, + {file = "protobuf-3.17.3-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:13ee7be3c2d9a5d2b42a1030976f760f28755fcf5863c55b1460fd205e6cd637"}, + {file = "protobuf-3.17.3-cp35-cp35m-macosx_10_9_intel.whl", hash = "sha256:1556a1049ccec58c7855a78d27e5c6e70e95103b32de9142bae0576e9200a1b0"}, + {file = "protobuf-3.17.3-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:f0e59430ee953184a703a324b8ec52f571c6c4259d496a19d1cabcdc19dabc62"}, + {file = "protobuf-3.17.3-cp35-cp35m-win32.whl", hash = "sha256:a981222367fb4210a10a929ad5983ae93bd5a050a0824fc35d6371c07b78caf6"}, + {file = "protobuf-3.17.3-cp35-cp35m-win_amd64.whl", hash = "sha256:6d847c59963c03fd7a0cd7c488cadfa10cda4fff34d8bc8cba92935a91b7a037"}, + {file = "protobuf-3.17.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:145ce0af55c4259ca74993ddab3479c78af064002ec8227beb3d944405123c71"}, + {file = "protobuf-3.17.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6ce4d8bf0321e7b2d4395e253f8002a1a5ffbcfd7bcc0a6ba46712c07d47d0b4"}, + {file = "protobuf-3.17.3-cp36-cp36m-win32.whl", hash = "sha256:7a4c97961e9e5b03a56f9a6c82742ed55375c4a25f2692b625d4087d02ed31b9"}, + {file = "protobuf-3.17.3-cp36-cp36m-win_amd64.whl", hash = "sha256:a22b3a0dbac6544dacbafd4c5f6a29e389a50e3b193e2c70dae6bbf7930f651d"}, + {file = "protobuf-3.17.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ffea251f5cd3c0b9b43c7a7a912777e0bc86263436a87c2555242a348817221b"}, + {file = "protobuf-3.17.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:9b7a5c1022e0fa0dbde7fd03682d07d14624ad870ae52054849d8960f04bc764"}, + {file = "protobuf-3.17.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8727ee027157516e2c311f218ebf2260a18088ffb2d29473e82add217d196b1c"}, + {file = "protobuf-3.17.3-cp37-cp37m-win32.whl", hash = "sha256:14c1c9377a7ffbeaccd4722ab0aa900091f52b516ad89c4b0c3bb0a4af903ba5"}, + {file = "protobuf-3.17.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c56c050a947186ba51de4f94ab441d7f04fcd44c56df6e922369cc2e1a92d683"}, + {file = "protobuf-3.17.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2ae692bb6d1992afb6b74348e7bb648a75bb0d3565a3f5eea5bec8f62bd06d87"}, + {file = "protobuf-3.17.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:99938f2a2d7ca6563c0ade0c5ca8982264c484fdecf418bd68e880a7ab5730b1"}, + {file = "protobuf-3.17.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6902a1e4b7a319ec611a7345ff81b6b004b36b0d2196ce7a748b3493da3d226d"}, + {file = "protobuf-3.17.3-cp38-cp38-win32.whl", hash = "sha256:59e5cf6b737c3a376932fbfb869043415f7c16a0cf176ab30a5bbc419cd709c1"}, + {file = "protobuf-3.17.3-cp38-cp38-win_amd64.whl", hash = "sha256:ebcb546f10069b56dc2e3da35e003a02076aaa377caf8530fe9789570984a8d2"}, + {file = "protobuf-3.17.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4ffbd23640bb7403574f7aff8368e2aeb2ec9a5c6306580be48ac59a6bac8bde"}, + {file = "protobuf-3.17.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:26010f693b675ff5a1d0e1bdb17689b8b716a18709113288fead438703d45539"}, + {file = "protobuf-3.17.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e76d9686e088fece2450dbc7ee905f9be904e427341d289acbe9ad00b78ebd47"}, + {file = "protobuf-3.17.3-cp39-cp39-win32.whl", hash = "sha256:a38bac25f51c93e4be4092c88b2568b9f407c27217d3dd23c7a57fa522a17554"}, + {file = "protobuf-3.17.3-cp39-cp39-win_amd64.whl", hash = "sha256:85d6303e4adade2827e43c2b54114d9a6ea547b671cb63fafd5011dc47d0e13d"}, + {file = "protobuf-3.17.3-py2.py3-none-any.whl", hash = "sha256:2bfb815216a9cd9faec52b16fd2bfa68437a44b67c56bee59bc3926522ecb04e"}, + {file = "protobuf-3.17.3.tar.gz", hash = "sha256:72804ea5eaa9c22a090d2803813e280fb273b62d5ae497aaf3553d141c4fdd7b"}, +] +py = [ + {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, + {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, +] +pycodestyle = [ + {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, + {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, +] +pycparser = [ + {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, + {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, +] +pydantic = [ + {file = "pydantic-1.8.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:05ddfd37c1720c392f4e0d43c484217b7521558302e7069ce8d318438d297739"}, + {file = "pydantic-1.8.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a7c6002203fe2c5a1b5cbb141bb85060cbff88c2d78eccbc72d97eb7022c43e4"}, + {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:589eb6cd6361e8ac341db97602eb7f354551482368a37f4fd086c0733548308e"}, + {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:10e5622224245941efc193ad1d159887872776df7a8fd592ed746aa25d071840"}, + {file = "pydantic-1.8.2-cp36-cp36m-win_amd64.whl", hash = "sha256:99a9fc39470010c45c161a1dc584997f1feb13f689ecf645f59bb4ba623e586b"}, + {file = "pydantic-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a83db7205f60c6a86f2c44a61791d993dff4b73135df1973ecd9eed5ea0bda20"}, + {file = "pydantic-1.8.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:41b542c0b3c42dc17da70554bc6f38cbc30d7066d2c2815a94499b5684582ecb"}, + {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:ea5cb40a3b23b3265f6325727ddfc45141b08ed665458be8c6285e7b85bd73a1"}, + {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:18b5ea242dd3e62dbf89b2b0ec9ba6c7b5abaf6af85b95a97b00279f65845a23"}, + {file = "pydantic-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:234a6c19f1c14e25e362cb05c68afb7f183eb931dd3cd4605eafff055ebbf287"}, + {file = "pydantic-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:021ea0e4133e8c824775a0cfe098677acf6fa5a3cbf9206a376eed3fc09302cd"}, + {file = "pydantic-1.8.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e710876437bc07bd414ff453ac8ec63d219e7690128d925c6e82889d674bb505"}, + {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:ac8eed4ca3bd3aadc58a13c2aa93cd8a884bcf21cb019f8cfecaae3b6ce3746e"}, + {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:4a03cbbe743e9c7247ceae6f0d8898f7a64bb65800a45cbdc52d65e370570820"}, + {file = "pydantic-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:8621559dcf5afacf0069ed194278f35c255dc1a1385c28b32dd6c110fd6531b3"}, + {file = "pydantic-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8b223557f9510cf0bfd8b01316bf6dd281cf41826607eada99662f5e4963f316"}, + {file = "pydantic-1.8.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:244ad78eeb388a43b0c927e74d3af78008e944074b7d0f4f696ddd5b2af43c62"}, + {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:05ef5246a7ffd2ce12a619cbb29f3307b7c4509307b1b49f456657b43529dc6f"}, + {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:54cd5121383f4a461ff7644c7ca20c0419d58052db70d8791eacbbe31528916b"}, + {file = "pydantic-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:4be75bebf676a5f0f87937c6ddb061fa39cbea067240d98e298508c1bda6f3f3"}, + {file = "pydantic-1.8.2-py3-none-any.whl", hash = "sha256:fec866a0b59f372b7e776f2d7308511784dace622e0992a0b59ea3ccee0ae833"}, + {file = "pydantic-1.8.2.tar.gz", hash = "sha256:26464e57ccaafe72b7ad156fdaa4e9b9ef051f69e175dbbb463283000c05ab7b"}, +] +pyflakes = [ + {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, + {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, +] +pygit2 = [ + {file = "pygit2-1.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:547429774c11f5bc9d20a49aa86e4bd13c90a55140504ef05f55cf424470ee34"}, + {file = "pygit2-1.6.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e75865d7b6fc161d93b16f10365eaad353cd546e302a98f2de2097ddea1066b"}, + {file = "pygit2-1.6.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be4a64b6090308ffd1c82e2dd4316cb79483715387b13818156d516134a5b17c"}, + {file = "pygit2-1.6.1-cp36-cp36m-win32.whl", hash = "sha256:2666a3970b2ea1222a9f0463b466f98c8d564f29ec84cf0a58d9b0d3865dbaaf"}, + {file = "pygit2-1.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b2de12ca2d3b7eb86106223b40b2edc0c61103c71e7962e53092c6ddef71a194"}, + {file = "pygit2-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9c1d96c66fb6e69ec710078a73c19edff420bc1db430caa9e03a825eede3f25c"}, + {file = "pygit2-1.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:454d42550fa6a6cd0e6a6ad9ab3f3262135fd157f57bad245ce156c36ee93370"}, + {file = "pygit2-1.6.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce0827b77dd2f8a3465bdc181c4e65f27dd12dbd92635c038e58030cc90c2de0"}, + {file = "pygit2-1.6.1-cp37-cp37m-win32.whl", hash = "sha256:b0161a141888d450eb821472fdcdadd14a072ddeda841fee9984956d34d3e19d"}, + {file = "pygit2-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:af2fa259b6f7899227611ab978c600695724e85965836cb607d8b1e70cfea9b3"}, + {file = "pygit2-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0e1e02c28983ddc004c0f54063f3e46fca388225d468e32e16689cfb750e0bd6"}, + {file = "pygit2-1.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5dadc4844feb76cde5cc9a37656326a361dd8b5c8e8f8674dcd4a5ecf395db3"}, + {file = "pygit2-1.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef07458e4172a31318663295083b43f957d611145738ff56aa76db593542a6e8"}, + {file = "pygit2-1.6.1-cp38-cp38-win32.whl", hash = "sha256:7a0c0a1f11fd41f57e8c6c64d903cc7fa4ec95d15592270be3217ed7f78eb023"}, + {file = "pygit2-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:2fd5c1b2d84dc6084f1bda836607afe37e95186a53a5a827a69083415e57fe4f"}, + {file = "pygit2-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b9b88b7e9a5286a71be0b6c307f0523c9606aeedff6b61eb9c440e18817fa641"}, + {file = "pygit2-1.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac12d32b714c3383ebccffee5eb6aff0b69a2542a40a664fd5ad370afcb28ee7"}, + {file = "pygit2-1.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe682ed6afd2ab31127f6a502cf3e002dc1cc8d26c36a5d49dfd180250351eb6"}, + {file = "pygit2-1.6.1-cp39-cp39-win32.whl", hash = "sha256:dbbf66a23860aa899949068ac9b503b4bc21e6063e8f53870440adbdc909405e"}, + {file = "pygit2-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:f90775afb11f69376e2af21ab56fcfbb52f6bc84117059ddf0355f81e5e36352"}, + {file = "pygit2-1.6.1.tar.gz", hash = "sha256:c3303776f774d3e0115c1c4f6e1fc35470d15f113a7ae9401a0b90acfa1661ac"}, +] +pyparsing = [ + {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, + {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, +] +pytest = [ + {file = "pytest-6.2.4-py3-none-any.whl", hash = "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"}, + {file = "pytest-6.2.4.tar.gz", hash = "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b"}, +] +pytest-asyncio = [ + {file = "pytest-asyncio-0.15.1.tar.gz", hash = "sha256:2564ceb9612bbd560d19ca4b41347b54e7835c2f792c504f698e05395ed63f6f"}, + {file = "pytest_asyncio-0.15.1-py3-none-any.whl", hash = "sha256:3042bcdf1c5d978f6b74d96a151c4cfb9dcece65006198389ccd7e6c60eb1eea"}, +] +pytest-cov = [ + {file = "pytest-cov-2.12.1.tar.gz", hash = "sha256:261ceeb8c227b726249b376b8526b600f38667ee314f910353fa318caa01f4d7"}, + {file = "pytest_cov-2.12.1-py2.py3-none-any.whl", hash = "sha256:261bb9e47e65bd099c89c3edf92972865210c36813f80ede5277dceb77a4a62a"}, +] +pytest-tap = [ + {file = "pytest-tap-3.2.tar.gz", hash = "sha256:1b585c4a636458dbd958d136381bbabb1752c5877d05fac7d6a6001a8a9ddc29"}, + {file = "pytest_tap-3.2-py3-none-any.whl", hash = "sha256:18f59047f8bc68247d37f807fae7f2f8897d2c7397aea2fd2870f0421dc566cb"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +python-editor = [ + {file = "python-editor-1.0.4.tar.gz", hash = "sha256:51fda6bcc5ddbbb7063b2af7509e43bd84bfc32a4ff71349ec7847713882327b"}, + {file = "python_editor-1.0.4-py2-none-any.whl", hash = "sha256:5f98b069316ea1c2ed3f67e7f5df6c0d8f10b689964a4a811ff64f0106819ec8"}, + {file = "python_editor-1.0.4-py2.7.egg", hash = "sha256:ea87e17f6ec459e780e4221f295411462e0d0810858e055fc514684350a2f522"}, + {file = "python_editor-1.0.4-py3-none-any.whl", hash = "sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d"}, + {file = "python_editor-1.0.4-py3.5.egg", hash = "sha256:c3da2053dbab6b29c94e43c486ff67206eafbe7eb52dbec7390b5e2fb05aac77"}, +] +python-multipart = [ + {file = "python-multipart-0.0.5.tar.gz", hash = "sha256:f7bb5f611fc600d15fa47b3974c8aa16e93724513b49b5f95c81e6624c83fa43"}, +] +redis = [ + {file = "redis-3.5.3-py2.py3-none-any.whl", hash = "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24"}, + {file = "redis-3.5.3.tar.gz", hash = "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2"}, +] +requests = [ + {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, + {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, +] +rfc3986 = [ + {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"}, + {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +sniffio = [ + {file = "sniffio-1.2.0-py3-none-any.whl", hash = "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663"}, + {file = "sniffio-1.2.0.tar.gz", hash = "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"}, +] +sortedcontainers = [ + {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, + {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, +] +sqlalchemy = [ + {file = "SQLAlchemy-1.3.23-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:fd3b96f8c705af8e938eaa99cbd8fd1450f632d38cad55e7367c33b263bf98ec"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:29cccc9606750fe10c5d0e8bd847f17a97f3850b8682aef1f56f5d5e1a5a64b1"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:927ce09e49bff3104459e1451ce82983b0a3062437a07d883a4c66f0b344c9b5"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27m-win32.whl", hash = "sha256:b4b0e44d586cd64b65b507fa116a3814a1a53d55dce4836d7c1a6eb2823ff8d1"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27m-win_amd64.whl", hash = "sha256:6b8b8c80c7f384f06825612dd078e4a31f0185e8f1f6b8c19e188ff246334205"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:9e9c25522933e569e8b53ccc644dc993cab87e922fb7e142894653880fdd419d"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:a0e306e9bb76fd93b29ae3a5155298e4c1b504c7cbc620c09c20858d32d16234"}, + {file = "SQLAlchemy-1.3.23-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:6c9e6cc9237de5660bcddea63f332428bb83c8e2015c26777281f7ffbd2efb84"}, + {file = "SQLAlchemy-1.3.23-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:94f667d86be82dd4cb17d08de0c3622e77ca865320e0b95eae6153faa7b4ecaf"}, + {file = "SQLAlchemy-1.3.23-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:751934967f5336a3e26fc5993ccad1e4fee982029f9317eb6153bc0bc3d2d2da"}, + {file = "SQLAlchemy-1.3.23-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:63677d0c08524af4c5893c18dbe42141de7178001360b3de0b86217502ed3601"}, + {file = "SQLAlchemy-1.3.23-cp35-cp35m-win32.whl", hash = "sha256:ddfb511e76d016c3a160910642d57f4587dc542ce5ee823b0d415134790eeeb9"}, + {file = "SQLAlchemy-1.3.23-cp35-cp35m-win_amd64.whl", hash = "sha256:040bdfc1d76a9074717a3f43455685f781c581f94472b010cd6c4754754e1862"}, + {file = "SQLAlchemy-1.3.23-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:d1a85dfc5dee741bf49cb9b6b6b8d2725a268e4992507cf151cba26b17d97c37"}, + {file = "SQLAlchemy-1.3.23-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:639940bbe1108ac667dcffc79925db2966826c270112e9159439ab6bb14f8d80"}, + {file = "SQLAlchemy-1.3.23-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:e8a1750b44ad6422ace82bf3466638f1aa0862dbb9689690d5f2f48cce3476c8"}, + {file = "SQLAlchemy-1.3.23-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:e5bb3463df697279e5459a7316ad5a60b04b0107f9392e88674d0ece70e9cf70"}, + {file = "SQLAlchemy-1.3.23-cp36-cp36m-win32.whl", hash = "sha256:e273367f4076bd7b9a8dc2e771978ef2bfd6b82526e80775a7db52bff8ca01dd"}, + {file = "SQLAlchemy-1.3.23-cp36-cp36m-win_amd64.whl", hash = "sha256:ac2244e64485c3778f012951fdc869969a736cd61375fde6096d08850d8be729"}, + {file = "SQLAlchemy-1.3.23-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:23927c3981d1ec6b4ea71eb99d28424b874d9c696a21e5fbd9fa322718be3708"}, + {file = "SQLAlchemy-1.3.23-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d90010304abb4102123d10cbad2cdf2c25a9f2e66a50974199b24b468509bad5"}, + {file = "SQLAlchemy-1.3.23-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:a8bfc1e1afe523e94974132d7230b82ca7fa2511aedde1f537ec54db0399541a"}, + {file = "SQLAlchemy-1.3.23-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:269990b3ab53cb035d662dcde51df0943c1417bdab707dc4a7e4114a710504b4"}, + {file = "SQLAlchemy-1.3.23-cp37-cp37m-win32.whl", hash = "sha256:fdd2ed7395df8ac2dbb10cefc44737b66c6a5cd7755c92524733d7a443e5b7e2"}, + {file = "SQLAlchemy-1.3.23-cp37-cp37m-win_amd64.whl", hash = "sha256:6a939a868fdaa4b504e8b9d4a61f21aac11e3fecc8a8214455e144939e3d2aea"}, + {file = "SQLAlchemy-1.3.23-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:24f9569e82a009a09ce2d263559acb3466eba2617203170e4a0af91e75b4f075"}, + {file = "SQLAlchemy-1.3.23-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2578dbdbe4dbb0e5126fb37ffcd9793a25dcad769a95f171a2161030bea850ff"}, + {file = "SQLAlchemy-1.3.23-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:1fe5d8d39118c2b018c215c37b73fd6893c3e1d4895be745ca8ff6eb83333ed3"}, + {file = "SQLAlchemy-1.3.23-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:c7dc052432cd5d060d7437e217dd33c97025287f99a69a50e2dc1478dd610d64"}, + {file = "SQLAlchemy-1.3.23-cp38-cp38-win32.whl", hash = "sha256:ecce8c021894a77d89808222b1ff9687ad84db54d18e4bd0500ca766737faaf6"}, + {file = "SQLAlchemy-1.3.23-cp38-cp38-win_amd64.whl", hash = "sha256:37b83bf81b4b85dda273aaaed5f35ea20ad80606f672d94d2218afc565fb0173"}, + {file = "SQLAlchemy-1.3.23-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:8be835aac18ec85351385e17b8665bd4d63083a7160a017bef3d640e8e65cadb"}, + {file = "SQLAlchemy-1.3.23-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6ec1044908414013ebfe363450c22f14698803ce97fbb47e53284d55c5165848"}, + {file = "SQLAlchemy-1.3.23-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:eab063a70cca4a587c28824e18be41d8ecc4457f8f15b2933584c6c6cccd30f0"}, + {file = "SQLAlchemy-1.3.23-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:baeb451ee23e264de3f577fee5283c73d9bbaa8cb921d0305c0bbf700094b65b"}, + {file = "SQLAlchemy-1.3.23-cp39-cp39-win32.whl", hash = "sha256:94208867f34e60f54a33a37f1c117251be91a47e3bfdb9ab8a7847f20886ad06"}, + {file = "SQLAlchemy-1.3.23-cp39-cp39-win_amd64.whl", hash = "sha256:f4d972139d5000105fcda9539a76452039434013570d6059993120dc2a65e447"}, + {file = "SQLAlchemy-1.3.23.tar.gz", hash = "sha256:6fca33672578666f657c131552c4ef8979c1606e494f78cd5199742dfb26918b"}, +] +starlette = [ + {file = "starlette-0.14.2-py3-none-any.whl", hash = "sha256:3c8e48e52736b3161e34c9f0e8153b4f32ec5d8995a3ee1d59410d92f75162ed"}, + {file = "starlette-0.14.2.tar.gz", hash = "sha256:7d49f4a27f8742262ef1470608c59ddbc66baf37c148e938c7038e6bc7a998aa"}, +] +"tap.py" = [ + {file = "tap.py-3.0-py2.py3-none-any.whl", hash = "sha256:a598bfaa2e224d71f2e86147c2ef822c18ff2e1b8ef006397e5056b08f92f699"}, + {file = "tap.py-3.0.tar.gz", hash = "sha256:f5eeeeebfd64e53d32661752bb4c288589a3babbb96db3f391a4ec29f1359c70"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +tomlkit = [ + {file = "tomlkit-0.7.2-py2.py3-none-any.whl", hash = "sha256:173ad840fa5d2aac140528ca1933c29791b79a374a0861a80347f42ec9328117"}, + {file = "tomlkit-0.7.2.tar.gz", hash = "sha256:d7a454f319a7e9bd2e249f239168729327e4dd2d27b17dc68be264ad1ce36754"}, +] +typing = [ + {file = "typing-3.7.4.3-py2-none-any.whl", hash = "sha256:283d868f5071ab9ad873e5e52268d611e851c870a2ba354193026f2dfb29d8b5"}, + {file = "typing-3.7.4.3.tar.gz", hash = "sha256:1187fb9c82fd670d10aa07bbb6cfcfe4bdda42d6fab8d5134f04e8c4d0b71cc9"}, +] +typing-extensions = [ + {file = "typing_extensions-3.10.0.2-py2-none-any.whl", hash = "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7"}, + {file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"}, + {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"}, +] +urllib3 = [ + {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, + {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, +] +uvicorn = [ + {file = "uvicorn-0.15.0-py3-none-any.whl", hash = "sha256:17f898c64c71a2640514d4089da2689e5db1ce5d4086c2d53699bf99513421c1"}, + {file = "uvicorn-0.15.0.tar.gz", hash = "sha256:d9a3c0dd1ca86728d3e235182683b4cf94cd53a867c288eaeca80ee781b2caff"}, +] +webencodings = [ + {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, + {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, +] +werkzeug = [ + {file = "Werkzeug-2.0.1-py3-none-any.whl", hash = "sha256:6c1ec500dcdba0baa27600f6a22f6333d8b662d22027ff9f6202e3367413caa8"}, + {file = "Werkzeug-2.0.1.tar.gz", hash = "sha256:1de1db30d010ff1af14a009224ec49ab2329ad2cde454c8a708130642d579c42"}, +] +wsproto = [ + {file = "wsproto-1.0.0-py3-none-any.whl", hash = "sha256:d8345d1808dd599b5ffb352c25a367adb6157e664e140dbecba3f9bc007edb9f"}, + {file = "wsproto-1.0.0.tar.gz", hash = "sha256:868776f8456997ad0d9720f7322b746bbe9193751b5b290b7f924659377c8c38"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..8cb276ce --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,100 @@ +# Poetry build configuration for the aurweb project. +# +# Dependencies: +# * python >= 3.9 +# * pip +# * poetry +# * poetry-dynamic-versioning +# +[tool.poetry] +name = "aurweb" +version = "5.0.0" # Updated via poetry-dynamic-versioning +license = "GPL-2.0-only" +description = "Source code for the Arch User Repository's website" +homepage = "https://aur.archlinux.org" +repository = "https://gitlab.archlinux.org/archlinux/aurweb" +documentation = "https://gitlab.archlinux.org/archlinux/aurweb/-/blob/master/README.md" +keywords = ["aurweb", "aur", "Arch", "Linux"] +authors = [ + "Lucas Fleischer ", + "Eli Schwartz ", + "Kevin Morris " +] +maintainers = [ + "Eli Schwartz " +] +packages = [ + { include = "aurweb" } +] + +[tool.poetry-dynamic-versioning] +enable = true +vcs = "git" + +[build-system] +requires = ["poetry>=1.1.8", "poetry-dynamic-versioning"] +build-backend = "poetry.masonry.api" + +[tool.poetry.urls] +"Repository" = "https://gitlab.archlinux.org/archlinux/aurweb" +"Bug Tracker" = "https://gitlab.archlinux.org/archlinux/aurweb/-/issues" +"Development Mailing List" = "https://lists.archlinux.org/listinfo/aur-dev" +"General Mailing List" = "https://lists.archlinux.org/listinfo/aur-general" +"Request Mailing List" = "https://lists.archlinux.org/listinfo/aur-requests" + +[tool.poetry.dependencies] +# poetry-dynamic-versioning is used to produce tool.poetry.version +# based on git tags. +poetry-dynamic-versioning = { version = "0.13.1", python = "^3.9" } + +# General +authlib = { version = "0.15.2", python = "^3.9" } +aiofiles = { version = "0.7.0", python = "^3.9" } +asgiref = { version = "3.4.1", python = "^3.9" } +bcrypt = { version = "3.2.0", python = "^3.9" } +bleach = { version = "3.3.1", python = "^3.9" } +email-validator = { version = "1.1.3", python = "^3.9" } +fakeredis = { version = "1.6.0", python = "^3.9" } +fastapi = { version = "0.66.0", python = "^3.9" } +feedgen = { version = "0.9.0", python = "^3.9" } +httpx = { version = "0.18.2", python = "^3.9" } +hypercorn = { version = "0.11.2", python = "^3.9" } +itsdangerous = { version = "2.0.1", python = "^3.9" } +jinja2 = { version = "3.0.1", python = "^3.9" } +lxml = { version = "4.6.3", python = "^3.9" } +markdown = { version = "3.3.4", python = "^3.9" } +orjson = { version = "3.6.3", python = "^3.9" } +protobuf = { version = "3.17.3", python = "^3.9" } +pygit2 = { version = "1.6.1", python = "^3.9" } +python-multipart = { version = "0.0.5", python = "^3.9" } +redis = { version = "3.5.3", python = "^3.9" } +requests = { version = "2.26.0", python = "^3.9" } +werkzeug = { version = "2.0.1", python = "^3.9" } + +# SQL +alembic = { version = "1.6.5", python = "^3.9" } +sqlalchemy = { version = "1.3.23", python = "^3.9" } +mysqlclient = { version = "2.0.3", python = "^3.9" } + +[tool.poetry.dev-dependencies] +flake8 = { version = "3.9.2", python = "^3.9" } +isort = { version = "5.9.3", python = "^3.9" } +coverage = { version = "5.5", python = "^3.9" } +pytest = { version = "6.2.4", python = "^3.9" } +pytest-asyncio = { version = "0.15.1", python = "^3.9" } +pytest-cov = { version = "2.12.1", python = "^3.9" } +pytest-tap = { version = "3.2", python = "^3.9" } +uvicorn = { version = "0.15.0", python = "^3.9" } + +[tool.poetry.scripts] +aurweb-git-auth = "aurweb.git.auth:main" +aurweb-git-serve = "aurweb.git.serve:main" +aurweb-git-update = "aurweb.git.update:main" +aurweb-aurblup = "aurweb.scripts.aurblup:main" +aurweb-mkpkglists = "aurweb.scripts.mkpkglists:main" +aurweb-notify = "aurweb.scripts.notify:main" +aurweb-pkgmaint = "aurweb.scripts.pkgmaint:main" +aurweb-popupdate = "aurweb.scripts.popupdate:main" +aurweb-rendercomment = "aurweb.scripts.rendercomment:main" +aurweb-tuvotereminder = "aurweb.scripts.tuvotereminder:main" +aurweb-usermaint = "aurweb.scripts.usermaint:main" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 37a12f61..00000000 --- a/requirements.txt +++ /dev/null @@ -1,36 +0,0 @@ -# General -authlib==0.15.2 -aiofiles==0.7.0 -asgiref==3.4.1 -bcrypt==3.2.0 -bleach==3.3.1 -coverage==5.5 -email-validator==1.1.3 -fakeredis==1.6.0 -fastapi==0.66.0 -feedgen==0.9.0 -flake8==3.9.2 -httpx==0.18.2 -hypercorn==0.11.2 -isort==5.9.3 -itsdangerous==2.0.1 -jinja2==3.0.1 -lxml==4.6.3 -markdown==3.3.4 -orjson==3.6.3 -protobuf==3.17.3 -pygit2==1.6.1 -pytest==6.2.4 -pytest-asyncio==0.15.1 -pytest-cov==2.12.1 -pytest-tap==3.2 -python-multipart==0.0.5 -redis==3.5.3 -requests==2.26.0 -uvicorn==0.15.0 -werkzeug==2.0.1 - -# SQL -alembic==1.6.5 -sqlalchemy==1.3.23 -mysqlclient==2.0.3 diff --git a/setup.py b/setup.py deleted file mode 100644 index cf88488c..00000000 --- a/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -import re -import sys - -from setuptools import find_packages, setup - -version = None -with open('web/lib/version.inc.php', 'r') as f: - for line in f.readlines(): - match = re.match(r'^define\("AURWEB_VERSION", "v([0-9.]+)"\);$', line) - if match: - version = match.group(1) - -if not version: - sys.stderr.write('error: Failed to parse version file!') - sys.exit(1) - -setup( - name="aurweb", - version=version, - packages=find_packages(), - entry_points={ - 'console_scripts': [ - 'aurweb-git-auth = aurweb.git.auth:main', - 'aurweb-git-serve = aurweb.git.serve:main', - 'aurweb-git-update = aurweb.git.update:main', - 'aurweb-aurblup = aurweb.scripts.aurblup:main', - 'aurweb-mkpkglists = aurweb.scripts.mkpkglists:main', - 'aurweb-notify = aurweb.scripts.notify:main', - 'aurweb-pkgmaint = aurweb.scripts.pkgmaint:main', - 'aurweb-popupdate = aurweb.scripts.popupdate:main', - 'aurweb-rendercomment = aurweb.scripts.rendercomment:main', - 'aurweb-tuvotereminder = aurweb.scripts.tuvotereminder:main', - 'aurweb-usermaint = aurweb.scripts.usermaint:main', - ], - }, -) From 3f034ac1287ec7533589807e23240d0d136aaeca Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Sep 2021 18:59:24 -0700 Subject: [PATCH 323/844] Docker: Fix incorrect ENV PATH specification As root, seems that $HOME doesn't work like I expected it to. Tested this before, but I apparently had some cache still holding on. Fixing the issue in this commit here. Signed-off-by: Kevin Morris --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 76da62f7..b490d2fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM archlinux:base-devel -ENV PATH="$HOME/.poetry/bin:${PATH}" +ENV PATH="/root/.poetry/bin:${PATH}" ENV PYTHONPATH=/aurweb ENV AUR_CONFIG=conf/config From fa07f940514fd30886acf3d47c3a37d79940adfa Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Sep 2021 19:08:10 -0700 Subject: [PATCH 324/844] Docker: Fix FastAPI db initialization PHP was doing this correctly, but FastAPI was doing this in it's exec script @ docker/scripts/run-fastapi.sh. Modify the fastapi service so that it does the same thing as PHP, and the existing "fastapi restart quirk" is no more. Signed-off-by: Kevin Morris --- docker/fastapi-entrypoint.sh | 3 +++ docker/scripts/run-fastapi.sh | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/fastapi-entrypoint.sh b/docker/fastapi-entrypoint.sh index 41a88206..83a2cda8 100755 --- a/docker/fastapi-entrypoint.sh +++ b/docker/fastapi-entrypoint.sh @@ -14,4 +14,7 @@ sed -ri 's|^(redis_address) = .+|\1 = redis://redis|' conf/config sed -ri "s|^(git_clone_uri_anon) = .+|\1 = https://localhost:8444/%s.git|" conf/config.defaults sed -ri "s|^(git_clone_uri_priv) = .+|\1 = ssh://aur@localhost:2222/%s.git|" conf/config.defaults +# Initialize the new database; ignore errors. +python -m aurweb.initdb 2>/dev/null || /bin/true + exec "$@" diff --git a/docker/scripts/run-fastapi.sh b/docker/scripts/run-fastapi.sh index 1db4c505..bb1a01a7 100755 --- a/docker/scripts/run-fastapi.sh +++ b/docker/scripts/run-fastapi.sh @@ -1,8 +1,5 @@ #!/bin/bash -# Initialize the new database; ignore errors. -python -m aurweb.initdb 2>/dev/null || /bin/true - if [ "$1" == "uvicorn" ] || [ "$1" == "" ]; then exec uvicorn --reload \ --ssl-certfile /cache/localhost.cert.pem \ From e93b0a9b452da9db9b28b1c734fb323e367c991d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 5 Sep 2021 00:08:47 -0700 Subject: [PATCH 325/844] Docker: expose fastapi (18000) and php-fpm (19000) Signed-off-by: Kevin Morris --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 0e91d6eb..e4eccb12 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -166,6 +166,8 @@ services: - ./web/template:/aurweb/web/template - ./web/lib:/aurweb/web/lib - ./templates:/aurweb/templates + ports: + - "19000:9000" fastapi: image: aurweb:latest @@ -197,6 +199,8 @@ services: - ./web/template:/aurweb/web/template - ./web/lib:/aurweb/web/lib - ./templates:/aurweb/templates + ports: + - "18000:8000" nginx: image: aurweb:latest From 95357687f9e0a6c9519e7dc2475059a3506abcd3 Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Sun, 5 Sep 2021 16:13:45 -0500 Subject: [PATCH 326/844] Added ability to specify fortune file via an environment variable --- schema/gendummydata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/gendummydata.py b/schema/gendummydata.py index 9224b051..275b3601 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -41,7 +41,7 @@ CLOSE_PROPOSALS = int(os.environ.get("CLOSE_PROPOSALS", 50)) RANDOM_TLDS = ("edu", "com", "org", "net", "tw", "ru", "pl", "de", "es") RANDOM_URL = ("http://www.", "ftp://ftp.", "http://", "ftp://") RANDOM_LOCS = ("pub", "release", "files", "downloads", "src") -FORTUNE_FILE = "/usr/share/fortune/cookie" +FORTUNE_FILE = os.environ.get("FORTUNE_FILE", "/usr/share/fortune/cookie") # setup logging logformat = "%(levelname)s: %(message)s" From 2e3f69ab126e6ac6ffa92b4e149faab61bfa3374 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 8 Sep 2021 17:10:14 -0700 Subject: [PATCH 327/844] fix(docker): Fix git service's update hook The update hook was incorrectly linked to /usr/local/bin/aurweb-git-update, which was neglected during the original patch regarding dependency conversion to `poetry`. Signed-off-by: Kevin Morris --- docker/git-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/git-entrypoint.sh b/docker/git-entrypoint.sh index cfd159c9..f07a5577 100755 --- a/docker/git-entrypoint.sh +++ b/docker/git-entrypoint.sh @@ -78,7 +78,7 @@ if [ ! -f $GIT_REPO/config ]; then git config --local transfer.hideRefs '^refs/' git config --local --add transfer.hideRefs '!refs/' git config --local --add transfer.hideRefs '!HEAD' - ln -sf /usr/local/bin/aurweb-git-update hooks/update + ln -sf /usr/bin/aurweb-git-update hooks/update cd $curdir chown -R aur:aur $GIT_REPO fi From 0fd31b8d368a4e4a267b5e83d608bc088bb13b4d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 8 Sep 2021 17:14:55 -0700 Subject: [PATCH 328/844] refactor(docker): New mariadb_init service Provides a single source of truth for mariadb database initialization. Previously, php-fpm and fastapi were racing against each other; while this wasn't an issue, it was very messy. Signed-off-by: Kevin Morris --- docker-compose.yml | 45 +++++++++++++------------------ docker/fastapi-entrypoint.sh | 19 ++++++++----- docker/mariadb-entrypoint.sh | 2 -- docker/mariadb-init-entrypoint.sh | 20 ++++++++++++++ docker/php-entrypoint.sh | 18 +++++++++---- 5 files changed, 64 insertions(+), 40 deletions(-) create mode 100755 docker/mariadb-init-entrypoint.sh diff --git a/docker-compose.yml b/docker-compose.yml index e4eccb12..309e95fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,8 +49,6 @@ services: mariadb: image: aurweb:latest init: true - environment: - - DB_HOST="%" entrypoint: /docker/mariadb-entrypoint.sh command: /usr/bin/mysqld_safe --datadir=/var/lib/mysql ports: @@ -63,11 +61,23 @@ services: healthcheck: test: "bash /docker/health/mariadb.sh" + mariadb_init: + image: aurweb:latest + init: true + environment: + - DB_HOST=mariadb + entrypoint: /docker/mariadb-init-entrypoint.sh + command: echo "MariaDB tables initialized." + depends_on: + mariadb: + condition: service_healthy + git: image: aurweb:latest init: true environment: - AUR_CONFIG=/aurweb/conf/config + - DB_HOST=mariadb entrypoint: /docker/git-entrypoint.sh command: /docker/scripts/run-sshd.sh ports: @@ -75,11 +85,9 @@ services: healthcheck: test: "bash /docker/health/sshd.sh" depends_on: - mariadb: - condition: service_healthy + mariadb_init: + condition: service_started volumes: - - mariadb_run:/var/run/mysqld - - mariadb_data:/var/lib/mysql - git_data:/aurweb/aur.git - ./cache:/cache @@ -96,8 +104,6 @@ services: mariadb: condition: service_healthy volumes: - - mariadb_run:/var/run/mysqld - - mariadb_data:/var/lib/mysql - git_data:/aurweb/aur.git - ./cache:/cache - smartgit_run:/var/run/smartgit @@ -114,8 +120,6 @@ services: depends_on: git: condition: service_healthy - php-fpm: - condition: service_healthy volumes: - git_data:/aurweb/aur.git @@ -131,8 +135,6 @@ services: depends_on: git: condition: service_healthy - fastapi: - condition: service_healthy volumes: - git_data:/aurweb/aur.git @@ -151,13 +153,9 @@ services: condition: service_started git: condition: service_healthy - mariadb: - condition: service_healthy memcached: condition: service_healthy volumes: - - mariadb_run:/var/run/mysqld # Bind socket in this volume. - - mariadb_data:/var/lib/mysql - ./cache:/cache - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations @@ -186,11 +184,7 @@ services: condition: service_healthy redis: condition: service_healthy - mariadb: - condition: service_healthy volumes: - - mariadb_run:/var/run/mysqld # Bind socket in this volume. - - mariadb_data:/var/lib/mysql - ./cache:/cache - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations @@ -268,10 +262,9 @@ services: stdin_open: true tty: true depends_on: - mariadb: - condition: service_healthy + mariadb_init: + condition: service_started volumes: - - mariadb_run:/var/run/mysqld - git_data:/aurweb/aur.git - ./cache:/cache - ./aurweb:/aurweb/aurweb @@ -292,7 +285,6 @@ services: stdin_open: true tty: true volumes: - - mariadb_run:/var/run/mysqld - git_data:/aurweb/aur.git - ./cache:/cache - ./aurweb:/aurweb/aurweb @@ -314,10 +306,9 @@ services: stdin_open: true tty: true depends_on: - mariadb: - condition: service_healthy + mariadb_init: + condition: service_started volumes: - - mariadb_run:/var/run/mysqld - git_data:/aurweb/aur.git - ./cache:/cache - ./aurweb:/aurweb/aurweb diff --git a/docker/fastapi-entrypoint.sh b/docker/fastapi-entrypoint.sh index 83a2cda8..3829b0bf 100755 --- a/docker/fastapi-entrypoint.sh +++ b/docker/fastapi-entrypoint.sh @@ -1,11 +1,21 @@ #!/bin/bash set -eou pipefail -dir="$(dirname $0)" -bash $dir/test-mysql-entrypoint.sh +[[ -z "$DB_HOST" ]] && echo 'Error: $DB_HOST required but missing.' && exit 1 + +DB_NAME="aurweb" +DB_USER="aur" +DB_PASS="aur" + +# Setup a config for our mysql db. +cp -vf conf/config.dev conf/config +sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config +sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" conf/config +sed -ri "s/^(host) = .+/\1 = ${DB_HOST}/" conf/config +sed -ri "s/^(user) = .+/\1 = ${DB_USER}/" conf/config +sed -ri "s/^;?(password) = .+/\1 = ${DB_PASS}/" conf/config sed -ri "s;^(aur_location) = .+;\1 = https://localhost:8444;" conf/config -sed -ri 's/^(name) = .+/\1 = aurweb/' conf/config # Setup Redis for FastAPI. sed -ri 's/^(cache) = .+/\1 = redis/' conf/config @@ -14,7 +24,4 @@ sed -ri 's|^(redis_address) = .+|\1 = redis://redis|' conf/config sed -ri "s|^(git_clone_uri_anon) = .+|\1 = https://localhost:8444/%s.git|" conf/config.defaults sed -ri "s|^(git_clone_uri_priv) = .+|\1 = ssh://aur@localhost:2222/%s.git|" conf/config.defaults -# Initialize the new database; ignore errors. -python -m aurweb.initdb 2>/dev/null || /bin/true - exec "$@" diff --git a/docker/mariadb-entrypoint.sh b/docker/mariadb-entrypoint.sh index 945a4b82..e1ebfa6a 100755 --- a/docker/mariadb-entrypoint.sh +++ b/docker/mariadb-entrypoint.sh @@ -3,8 +3,6 @@ set -eou pipefail MYSQL_DATA=/var/lib/mysql -[[ -z "$DB_HOST" ]] && DB_HOST="localhost" - mariadb-install-db --user=mysql --basedir=/usr --datadir=$MYSQL_DATA # Start it up. diff --git a/docker/mariadb-init-entrypoint.sh b/docker/mariadb-init-entrypoint.sh new file mode 100755 index 00000000..4cd6f46c --- /dev/null +++ b/docker/mariadb-init-entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -eou pipefail + +[[ -z "$DB_HOST" ]] && echo 'Error: $DB_HOST required but missing.' && exit 1 + +DB_NAME="aurweb" +DB_USER="aur" +DB_PASS="aur" + +# Setup a config for our mysql db. +cp -vf conf/config.dev conf/config +sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config +sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" conf/config +sed -ri "s/^(host) = .+/\1 = ${DB_HOST}/" conf/config +sed -ri "s/^(user) = .+/\1 = ${DB_USER}/" conf/config +sed -ri "s/^;?(password) = .+/\1 = ${DB_PASS}/" conf/config + +python -m aurweb.initdb 2>/dev/null || /bin/true + +exec "$@" diff --git a/docker/php-entrypoint.sh b/docker/php-entrypoint.sh index 1f3ed82b..8fda1830 100755 --- a/docker/php-entrypoint.sh +++ b/docker/php-entrypoint.sh @@ -1,11 +1,21 @@ #!/bin/bash set -eou pipefail -dir="$(dirname $0)" -bash $dir/test-mysql-entrypoint.sh +[[ -z "$DB_HOST" ]] && echo 'Error: $DB_HOST required but missing.' && exit 1 + +DB_NAME="aurweb" +DB_USER="aur" +DB_PASS="aur" + +# Setup a config for our mysql db. +cp -vf conf/config.dev conf/config +sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config +sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" conf/config +sed -ri "s/^(host) = .+/\1 = ${DB_HOST}/" conf/config +sed -ri "s/^(user) = .+/\1 = ${DB_USER}/" conf/config +sed -ri "s/^;?(password) = .+/\1 = ${DB_PASS}/" conf/config sed -ri "s;^(aur_location) = .+;\1 = https://localhost:8443;" conf/config -sed -ri 's/^(name) = .+/\1 = aurweb/' conf/config # Enable memcached. sed -ri 's/^(cache) = .+$/\1 = memcache/' conf/config @@ -27,6 +37,4 @@ sed -ri 's/^;?(open_basedir).*$/\1 = \//' /etc/php/php.ini # Use the sqlite3 extension line for memcached. sed -ri 's/^;(extension)=sqlite3$/\1=memcached/' /etc/php/php.ini -python -m aurweb.initdb 2>/dev/null || /bin/true - exec "$@" From ad3016ef4f98a3131af2680d8d7a80cf0ce6ac74 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 8 Sep 2021 17:36:37 -0700 Subject: [PATCH 329/844] fix: /account/{name}/edit Account Type selection The "Account Type" selection was not properly being rendered due to an incorrect equality. This has been fixed in templates/partials/account_form.html. Signed-off-by: Kevin Morris --- templates/partials/account_form.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/partials/account_form.html b/templates/partials/account_form.html index 6374fd5e..f166c230 100644 --- a/templates/partials/account_form.html +++ b/templates/partials/account_form.html @@ -59,7 +59,7 @@ {% if request.user.is_authenticated() %} diff --git a/test/test_packages_util.py b/test/test_packages_util.py index 754e3b8d..1396734b 100644 --- a/test/test_packages_util.py +++ b/test/test_packages_util.py @@ -1,3 +1,5 @@ +from datetime import datetime + import pytest from fastapi.testclient import TestClient @@ -7,6 +9,8 @@ from aurweb.models.account_type import USER_ID, AccountType from aurweb.models.official_provider import OFFICIAL_BASE, OfficialProvider from aurweb.models.package import Package from aurweb.models.package_base import PackageBase +from aurweb.models.package_notification import PackageNotification +from aurweb.models.package_vote import PackageVote from aurweb.models.user import User from aurweb.packages import util from aurweb.redis import kill_redis @@ -19,6 +23,8 @@ def setup(): User.__tablename__, Package.__tablename__, PackageBase.__tablename__, + PackageVote.__tablename__, + PackageNotification.__tablename__, OfficialProvider.__tablename__ ) @@ -71,3 +77,24 @@ def test_updated_packages(maintainer: User, package: Package): assert util.updated_packages(1, 0) == [expected] assert util.updated_packages(1, 600) == [expected] kill_redis() # Kill it again, in case other tests use a real instance. + + +def test_query_voted(maintainer: User, package: Package): + now = int(datetime.utcnow().timestamp()) + with db.begin(): + db.create(PackageVote, User=maintainer, VoteTS=now, + PackageBase=package.PackageBase) + + query = db.query(Package).filter(Package.ID == package.ID).all() + query_voted = util.query_voted(query, maintainer) + assert query_voted[package.PackageBase.ID] + + +def test_query_notified(maintainer: User, package: Package): + with db.begin(): + db.create(PackageNotification, User=maintainer, + PackageBase=package.PackageBase) + + query = db.query(Package).filter(Package.ID == package.ID).all() + query_notified = util.query_notified(query, maintainer) + assert query_notified[package.PackageBase.ID] From aee1390e2c0aab59b2008d575c5f1750ca664733 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 19 Sep 2021 11:48:19 -0700 Subject: [PATCH 342/844] fix(FastAPI): registration sends WelcomeNotification Signed-off-by: Kevin Morris --- aurweb/routers/accounts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index ef4b99af..3c799938 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -22,7 +22,7 @@ from aurweb.models.ban import Ban from aurweb.models.ssh_pub_key import SSHPubKey, get_fingerprint from aurweb.models.term import Term from aurweb.models.user import User -from aurweb.scripts.notify import ResetKeyNotification +from aurweb.scripts.notify import ResetKeyNotification, WelcomeNotification from aurweb.templates import make_context, make_variable_context, render_template router = APIRouter() @@ -414,7 +414,7 @@ async def account_register_post(request: Request, # Send a reset key notification to the new user. executor = db.ConnectionExecutor(db.get_engine().raw_connection()) - ResetKeyNotification(executor, user.ID).send() + WelcomeNotification(executor, user.ID).send() context["complete"] = True context["user"] = user From b59601a8b7af137c0f31cd92dbff10f52afe82f6 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Sep 2021 19:14:47 -0700 Subject: [PATCH 343/844] feat(poetry): add paginate==0.5.6 With upstream at https://github.com/Pylons/paginate, this module helps us deal with pagination without reinventing the wheel. Signed-off-by: Kevin Morris --- poetry.lock | 13 ++++++++++++- pyproject.toml | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 3cc84361..322e250f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -513,6 +513,14 @@ python-versions = ">=3.6" [package.dependencies] pyparsing = ">=2.0.2" +[[package]] +name = "paginate" +version = "0.5.6" +description = "Divides large result sets into pages for easier browsing" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "pluggy" version = "0.13.1" @@ -921,7 +929,7 @@ h11 = ">=0.9.0,<1" [metadata] lock-version = "1.1" python-versions = "*" -content-hash = "96112731ca21a6ff5d0657c6c40979642bb992ae660ba8d6135421718737c6b0" +content-hash = "c262ac1160b83593377fb7520d35c4b8ad81e5acff9d0a2060b2b048e3865b78" [metadata.files] aiofiles = [ @@ -1328,6 +1336,9 @@ packaging = [ {file = "packaging-21.0-py3-none-any.whl", hash = "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"}, {file = "packaging-21.0.tar.gz", hash = "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7"}, ] +paginate = [ + {file = "paginate-0.5.6.tar.gz", hash = "sha256:5e6007b6a9398177a7e1648d04fdd9f8c9766a1a945bceac82f1929e8c78af2d"}, +] pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, diff --git a/pyproject.toml b/pyproject.toml index 8cb276ce..4b530493 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ python-multipart = { version = "0.0.5", python = "^3.9" } redis = { version = "3.5.3", python = "^3.9" } requests = { version = "2.26.0", python = "^3.9" } werkzeug = { version = "2.0.1", python = "^3.9" } +paginate = { version = "0.5.6", python = "^3.9" } # SQL alembic = { version = "1.6.5", python = "^3.9" } From c006386079b3f1dff893ba359eb978132800627c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 28 Aug 2021 16:14:07 -0700 Subject: [PATCH 344/844] add User.is_elevated() This one returns true if the user is either a Trusted User or a Developer. Signed-off-by: Kevin Morris --- aurweb/models/user.py | 9 +++++++++ test/test_user.py | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 70d15f88..28aa613e 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -165,6 +165,15 @@ class User(Base): aurweb.models.account_type.TRUSTED_USER_AND_DEV_ID } + def is_elevated(self): + """ A User is 'elevated' when they have either a + Trusted User or Developer AccountType. """ + return self.AccountType.ID in { + aurweb.models.account_type.TRUSTED_USER_ID, + aurweb.models.account_type.DEVELOPER_ID, + aurweb.models.account_type.TRUSTED_USER_AND_DEV_ID, + } + def can_edit_user(self, user): """ Can this account record edit the target user? It must either be the target user or a user with enough permissions to do so. diff --git a/test/test_user.py b/test/test_user.py index 70eac079..43cbf58a 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -214,6 +214,11 @@ def test_user_credential_types(): assert aurweb.auth.developer(user) assert aurweb.auth.trusted_user_or_dev(user) + # Some model authorization checks. + assert user.is_elevated() + assert user.is_trusted_user() + assert user.is_developer() + def test_user_json(): data = json.loads(user.json()) From 741cbfaa4e4d08dc02544140faf7bd1470ca7692 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 28 Aug 2021 16:15:48 -0700 Subject: [PATCH 345/844] auth: add several AnonymousUser method stubs We'll need to use these, so this commit implements them here with tests for coverage. Signed-off-by: Kevin Morris --- aurweb/auth.py | 20 ++++++++++++++++++++ test/test_auth.py | 27 ++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index 26e4073d..2e6674b0 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -53,10 +53,30 @@ class AnonymousUser: def is_authenticated(): return False + @staticmethod + def is_trusted_user(): + return False + + @staticmethod + def is_developer(): + return False + + @staticmethod + def is_elevated(): + return False + @staticmethod def has_credential(credential): return False + @staticmethod + def voted_for(package): + return False + + @staticmethod + def notified(package): + return False + class BasicAuthBackend(AuthenticationBackend): async def authenticate(self, conn: HTTPConnection): diff --git a/test/test_auth.py b/test/test_auth.py index caa39468..ced64064 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -5,7 +5,7 @@ import pytest from sqlalchemy.exc import IntegrityError from aurweb import db -from aurweb.auth import BasicAuthBackend, account_type_required, has_credential +from aurweb.auth import AnonymousUser, BasicAuthBackend, account_type_required, has_credential from aurweb.db import create, query from aurweb.models.account_type import USER, USER_ID, AccountType from aurweb.models.session import Session @@ -92,3 +92,28 @@ def test_account_type_required(): # But this one should! We have no "FAKE" key. with pytest.raises(KeyError): account_type_required({'FAKE'}) + + +def test_is_trusted_user(): + user_ = AnonymousUser() + assert not user_.is_trusted_user() + + +def test_is_developer(): + user_ = AnonymousUser() + assert not user_.is_developer() + + +def test_is_elevated(): + user_ = AnonymousUser() + assert not user_.is_elevated() + + +def test_voted_for(): + user_ = AnonymousUser() + assert not user_.voted_for(None) + + +def test_notified(): + user_ = AnonymousUser() + assert not user_.notified(None) From 6298b1228a7313de4375a567d79a7ab046bc2a26 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 15 Sep 2021 11:31:55 -0700 Subject: [PATCH 346/844] feat(FastAPI): add templates/partials/widgets/pager.html A pager that can be used for paginated result tables. Signed-off-by: Kevin Morris --- aurweb/filters.py | 50 +++++++++++++++++++++++++++ templates/partials/widgets/pager.html | 26 ++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 aurweb/filters.py create mode 100644 templates/partials/widgets/pager.html diff --git a/aurweb/filters.py b/aurweb/filters.py new file mode 100644 index 00000000..bb56c656 --- /dev/null +++ b/aurweb/filters.py @@ -0,0 +1,50 @@ +from typing import Any, Dict + +import paginate + +from jinja2 import pass_context + +from aurweb import util +from aurweb.templates import register_filter + + +@register_filter("pager_nav") +@pass_context +def pager_nav(context: Dict[str, Any], + page: int, total: int, prefix: str) -> str: + page = int(page) # Make sure this is an int. + + pp = context.get("PP", 50) + + # Setup a local query string dict, optionally passed by caller. + q = context.get("q", dict()) + + search_by = context.get("SeB", None) + if search_by: + q["SeB"] = search_by + + sort_by = context.get("SB", None) + if sort_by: + q["SB"] = sort_by + + def create_url(page: int): + nonlocal q + offset = max(page * pp - pp, 0) + qs = util.to_qs(util.extend_query(q, ["O", offset])) + return f"{prefix}?{qs}" + + # Use the paginate module to produce our linkage. + pager = paginate.Page([], page=page + 1, + items_per_page=pp, + item_count=total, + url_maker=create_url) + + return pager.pager( + link_attr={"class": "page"}, + curpage_attr={"class": "page"}, + separator=" ", + format="$link_first $link_previous ~5~ $link_next $link_last", + symbol_first="« First", + symbol_previous="‹ Previous", + symbol_next="Next ›", + symbol_last="Last »") diff --git a/templates/partials/widgets/pager.html b/templates/partials/widgets/pager.html new file mode 100644 index 00000000..4809accf --- /dev/null +++ b/templates/partials/widgets/pager.html @@ -0,0 +1,26 @@ +{# A pager widget that can be used for navigation of a number of results. + +Inputs required: + + prefix: Request URI prefix used to produce navigation offsets + singular: Singular sentence to be translated via tn + plural: Plural sentence to be translated via tn + PP: The number of results per page + O: The current offset value + total: The total number of results +#} + +{% set page = ((O / PP) | int) %} +{% set pages = ((total / PP) | ceil) %} + +
    +

    + {{ total | tn(singular, plural) | format(total) }} + {{ "Page %d of %d." | tr | format(page + 1, pages) }} +

    + {% if pages > 1 %} +

    + {{ page | pager_nav(total, prefix) | safe }} +

    + {% endif %} +

    From 5cf70620921848050ed3ac92dff7f84df1dbf979 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 29 Aug 2021 22:21:39 -0700 Subject: [PATCH 347/844] feat(FastAPI): add /packages (get) search In terms of performance, most queries on this page win over PHP in query times, with the exception of sorting by Voted or Notify (https://gitlab.archlinux.org/archlinux/aurweb/-/issues/102). Otherwise, there are a few modifications: described below. * Pagination * The `paginate` Python module has been used in the FastAPI project here to implement paging on the packages search page. This changes how pagination is displayed, however it serves the same purpose. We'll take advantage of this module in other places as well. * Form action * The form action for actions now use `POST /packages` to perform. This is currently implemented and will be addressed in a follow-up commit. * Input names and values * Input names and values have been modified to satisfy the snake_case naming convention we'd like to use as much as possible. * Some input names and values were modified to comply with FastAPI Forms: (IDs[]) -> (IDs, ). Signed-off-by: Kevin Morris --- aurweb/packages/search.py | 195 +++++++ aurweb/routers/packages.py | 83 ++- aurweb/templates.py | 2 + conf/config.defaults | 1 + setup.cfg | 2 + templates/packages.html | 84 +++ templates/partials/packages/search.html | 58 +- .../partials/packages/search_actions.html | 25 + .../partials/packages/search_results.html | 114 ++++ test/test_packages_routes.py | 540 +++++++++++++++++- web/html/css/aurweb.css | 7 + 11 files changed, 1081 insertions(+), 30 deletions(-) create mode 100644 aurweb/packages/search.py create mode 100644 templates/packages.html create mode 100644 templates/partials/packages/search_actions.html create mode 100644 templates/partials/packages/search_results.html diff --git a/aurweb/packages/search.py b/aurweb/packages/search.py new file mode 100644 index 00000000..854834ee --- /dev/null +++ b/aurweb/packages/search.py @@ -0,0 +1,195 @@ +from sqlalchemy import and_, case, or_, orm + +from aurweb import config, db +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.package_comaintainer import PackageComaintainer +from aurweb.models.package_keyword import PackageKeyword +from aurweb.models.package_notification import PackageNotification +from aurweb.models.package_vote import PackageVote +from aurweb.models.user import User + +DEFAULT_MAX_RESULTS = 2500 + + +class PackageSearch: + """ A Package search query builder. """ + + # A constant mapping of short to full name sort orderings. + FULL_SORT_ORDER = {"d": "desc", "a": "asc"} + + def __init__(self, user: User): + """ Construct an instance of PackageSearch. + + This constructors performs several steps during initialization: + 1. Setup self.query: an ORM query of Package joined by PackageBase. + """ + self.user = user + self.query = db.query(Package).join(PackageBase).join( + PackageVote, + and_(PackageVote.PackageBaseID == PackageBase.ID, + PackageVote.UsersID == self.user.ID), + isouter=True + ).join( + PackageNotification, + and_(PackageNotification.PackageBaseID == PackageBase.ID, + PackageNotification.UserID == self.user.ID), + isouter=True + ) + self.ordering = "d" + + # Setup SeB (Search By) callbacks. + self.search_by_cb = { + "nd": self._search_by_namedesc, + "n": self._search_by_name, + "b": self._search_by_pkgbase, + "N": self._search_by_exact_name, + "B": self._search_by_exact_pkgbase, + "k": self._search_by_keywords, + "m": self._search_by_maintainer, + "c": self._search_by_comaintainer, + "M": self._search_by_co_or_maintainer, + "s": self._search_by_submitter + } + + # Setup SB (Sort By) callbacks. + self.sort_by_cb = { + "n": self._sort_by_name, + "v": self._sort_by_votes, + "p": self._sort_by_popularity, + "w": self._sort_by_voted, + "o": self._sort_by_notify, + "m": self._sort_by_maintainer, + "l": self._sort_by_last_modified + } + + def _search_by_namedesc(self, keywords: str) -> orm.Query: + self.query = self.query.filter( + or_(Package.Name.like(f"%{keywords}%"), + Package.Description.like(f"%{keywords}%")) + ) + return self + + def _search_by_name(self, keywords: str) -> orm.Query: + self.query = self.query.filter(Package.Name.like(f"%{keywords}%")) + return self + + def _search_by_exact_name(self, keywords: str) -> orm.Query: + self.query = self.query.filter(Package.Name == keywords) + return self + + def _search_by_pkgbase(self, keywords: str) -> orm.Query: + self.query = self.query.filter(PackageBase.Name.like(f"%{keywords}%")) + return self + + def _search_by_exact_pkgbase(self, keywords: str) -> orm.Query: + self.query = self.query.filter(PackageBase.Name == keywords) + return self + + def _search_by_keywords(self, keywords: str) -> orm.Query: + self.query = self.query.join(PackageKeyword).filter( + PackageKeyword.Keyword == keywords + ) + return self + + def _search_by_maintainer(self, keywords: str) -> orm.Query: + self.query = self.query.join( + User, User.ID == PackageBase.MaintainerUID + ).filter(User.Username == keywords) + return self + + def _search_by_comaintainer(self, keywords: str) -> orm.Query: + self.query = self.query.join(PackageComaintainer).join( + User, User.ID == PackageComaintainer.UsersID + ).filter(User.Username == keywords) + return self + + def _search_by_co_or_maintainer(self, keywords: str) -> orm.Query: + self.query = self.query.join( + PackageComaintainer, + isouter=True + ).join( + User, or_(User.ID == PackageBase.MaintainerUID, + User.ID == PackageComaintainer.UsersID) + ).filter(User.Username == keywords) + return self + + def _search_by_submitter(self, keywords: str) -> orm.Query: + self.query = self.query.join( + User, User.ID == PackageBase.SubmitterUID + ).filter(User.Username == keywords) + return self + + def search_by(self, search_by: str, keywords: str) -> orm.Query: + if search_by not in self.search_by_cb: + search_by = "nd" # Default: Name, Description + callback = self.search_by_cb.get(search_by) + result = callback(keywords) + return result + + def _sort_by_name(self, order: str): + column = getattr(Package.Name, order) + self.query = self.query.order_by(column()) + return self + + def _sort_by_votes(self, order: str): + column = getattr(PackageBase.NumVotes, order) + self.query = self.query.order_by(column()) + return self + + def _sort_by_popularity(self, order: str): + column = getattr(PackageBase.Popularity, order) + self.query = self.query.order_by(column()) + return self + + def _sort_by_voted(self, order: str): + # FIXME: Currently, PHP is destroying this implementation + # in terms of performance. We should improve this; there's no + # reason it should take _longer_. + column = getattr( + case([(PackageVote.UsersID == self.user.ID, 1)], else_=0), + order + ) + self.query = self.query.order_by(column(), Package.Name.desc()) + return self + + def _sort_by_notify(self, order: str): + # FIXME: Currently, PHP is destroying this implementation + # in terms of performance. We should improve this; there's no + # reason it should take _longer_. + column = getattr( + case([(PackageNotification.UserID == self.user.ID, 1)], else_=0), + order + ) + self.query = self.query.order_by(column(), Package.Name.desc()) + return self + + def _sort_by_maintainer(self, order: str): + column = getattr(User.Username, order) + self.query = self.query.join( + User, User.ID == PackageBase.MaintainerUID, isouter=True + ).order_by(column()) + return self + + def _sort_by_last_modified(self, order: str): + column = getattr(PackageBase.ModifiedTS, order) + self.query = self.query.order_by(column()) + return self + + def sort_by(self, sort_by: str, ordering: str = "d") -> orm.Query: + if sort_by not in self.sort_by_cb: + sort_by = "n" # Default: Name. + callback = self.sort_by_cb.get(sort_by) + if ordering not in self.FULL_SORT_ORDER: + ordering = "d" # Default: Descending. + ordering = self.FULL_SORT_ORDER.get(ordering) + return callback(ordering) + + def results(self) -> orm.Query: + # Store the total count of all records found up to limit. + limit = (config.getint("options", "max_search_results") + or DEFAULT_MAX_RESULTS) + self.total_count = self.query.limit(limit).count() + + # Return the query to the user. + return self.query diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index a20c97b1..3eda2539 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -5,6 +5,7 @@ from fastapi import APIRouter, Request, Response from fastapi.responses import RedirectResponse from sqlalchemy import and_ +import aurweb.filters import aurweb.models.package_comment import aurweb.models.package_keyword import aurweb.packages.util @@ -21,12 +22,92 @@ from aurweb.models.package_request import PackageRequest from aurweb.models.package_source import PackageSource from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import CONFLICTS_ID -from aurweb.packages.util import get_pkgbase +from aurweb.packages.search import PackageSearch +from aurweb.packages.util import get_pkgbase, query_notified, query_voted from aurweb.templates import make_context, render_template router = APIRouter() +async def packages_get(request: Request, context: Dict[str, Any]): + # Query parameters used in this request. + context["q"] = dict(request.query_params) + + # Per page and offset. + per_page = context["PP"] = int(request.query_params.get("PP", 50)) + offset = context["O"] = int(request.query_params.get("O", 0)) + + # Query search by. + search_by = context["SeB"] = request.query_params.get("SeB", "nd") + + # Query sort by. + sort_by = context["SB"] = request.query_params.get("SB", "n") + + # Query sort order. + sort_order = request.query_params.get("SO", None) + + # Apply ordering, limit and offset. + search = PackageSearch(request.user) + + # For each keyword found in K, apply a search_by filter. + # This means that for any sentences separated by spaces, + # they are used as if they were ANDed. + keywords = context["K"] = request.query_params.get("K", str()) + keywords = keywords.split(" ") + for keyword in keywords: + search.search_by(search_by, keyword) + + flagged = request.query_params.get("outdated", None) + if flagged: + # If outdated was given, set it up in the context. + context["outdated"] = flagged + + # When outdated is set to "on," we filter records which do have + # an OutOfDateTS. When it's set to "off," we filter out any which + # do **not** have OutOfDateTS. + criteria = None + if flagged == "on": + criteria = PackageBase.OutOfDateTS.isnot + else: + criteria = PackageBase.OutOfDateTS.is_ + + # Apply the flag criteria to our PackageSearch.query. + search.query = search.query.filter(criteria(None)) + + submit = request.query_params.get("submit", "Go") + if submit == "Orphans": + # If the user clicked the "Orphans" button, we only want + # orphaned packages. + search.query = search.query.filter(PackageBase.MaintainerUID.is_(None)) + + # Apply user-specified specified sort column and ordering. + search.sort_by(sort_by, sort_order) + + # If no SO was given, default the context SO to 'a' (Ascending). + # By default, if no SO is given, the search should sort by 'd' + # (Descending), but display "Ascending" for the Sort order select. + if sort_order is None: + sort_order = "a" + context["SO"] = sort_order + + # Insert search results into the context. + results = search.results() + context["packages"] = results.limit(per_page).offset(offset) + context["packages_voted"] = query_voted( + context.get("packages"), request.user) + context["packages_notified"] = query_notified( + context.get("packages"), request.user) + context["packages_count"] = search.total_count + + return render_template(request, "packages.html", context) + + +@router.get("/packages") +async def packages(request: Request) -> Response: + context = make_context(request, "Packages") + return await packages_get(request, context) + + async def make_single_context(request: Request, pkgbase: PackageBase) -> Dict[str, Any]: """ Make a basic context for package or pkgbase. diff --git a/aurweb/templates.py b/aurweb/templates.py index 6a1b6a1c..09be049c 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -1,5 +1,6 @@ import copy import functools +import math import os import zoneinfo @@ -35,6 +36,7 @@ _env.filters["urlencode"] = util.to_qs _env.filters["quote_plus"] = quote_plus _env.filters["get_vote"] = util.get_vote _env.filters["number_format"] = util.number_format +_env.filters["ceil"] = math.ceil # Add captcha filters. _env.filters["captcha_salt"] = captcha.captcha_salt_filter diff --git a/conf/config.defaults b/conf/config.defaults index 1c96a55d..988859a0 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -22,6 +22,7 @@ aur_location = https://aur.archlinux.org git_clone_uri_anon = https://aur.archlinux.org/%s.git git_clone_uri_priv = ssh://aur@aur.archlinux.org/%s.git max_rpc_results = 5000 +max_search_results = 2500 max_depends = 1000 aur_request_ml = aur-requests@lists.archlinux.org request_idle_time = 1209600 diff --git a/setup.cfg b/setup.cfg index 1d67ca96..4f2bdf7d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,6 @@ [pycodestyle] max-line-length = 127 +ignore = E741, W503 [flake8] max-line-length = 127 @@ -25,6 +26,7 @@ max-complexity = 10 per-file-ignores = aurweb/routers/accounts.py:E741,C901 test/test_ssh_pub_key.py:E501 + aurweb/routers/packages.py:E741 [isort] line_length = 127 diff --git a/templates/packages.html b/templates/packages.html new file mode 100644 index 00000000..8b5b06d1 --- /dev/null +++ b/templates/packages.html @@ -0,0 +1,84 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} + {% if errors %} + +
      + {% for error in errors %} +
    • {{ error | tr }}
    • + {% endfor %} +
    + {% include "partials/packages/search.html" %} + + {% else %} + + {% set pages = (packages_count / PP) | ceil %} + {% set page = O / PP %} + + {% if success %} +
      + {% for message in success %} +
    • {{ message | tr }}
    • + {% endfor %} +
    + {% endif %} + + {# Search form #} + {% include "partials/packages/search.html" %} +
    + + {# /packages does things a bit roundabout-wise: + + If SeB is not given, "nd" is the default. + If SB is not given, "n" is the default. + If SO is not given, "d" is the default. + + However, we depend on flipping SO for column sorting. + + This section sets those defaults for the context if + they are not already setup. #} + {% if not SeB %} + {% set SeB = "nd" %} + {% endif %} + {% if not SB %} + {% set SB = "n" %} + {% endif %} + {% if not SO %} + {% set SO = "d" %} + {% endif %} + + {# Pagination widget #} + {% with total = packages_count, + singular = "%d package found.", + plural = "%d packages found.", + prefix = "/packages" %} + {% include "partials/widgets/pager.html" %} + {% endwith %} + + {# Package action form #} +
    + + {# Search results #} + {% with voted = packages_voted, notified = packages_notified %} + {% include "partials/packages/search_results.html" %} + {% endwith %} + + {# Pagination widget #} + {% with total = packages_count, + singular = "%d package found.", + plural = "%d packages found.", + prefix = "/packages" %} + {% include "partials/widgets/pager.html" %} + {% endwith %} + + {% if request.user.is_authenticated() %} + {# Package actions #} + {% include "partials/packages/search_actions.html" %} + {% endif %} + +
    + + {% endif %} +{% endblock %} diff --git a/templates/partials/packages/search.html b/templates/partials/packages/search.html index c4488b95..bb6fdb50 100644 --- a/templates/partials/packages/search.html +++ b/templates/partials/packages/search.html @@ -8,61 +8,65 @@
    - +
    - - + +
    diff --git a/templates/partials/packages/search_actions.html b/templates/partials/packages/search_actions.html new file mode 100644 index 00000000..2f5fe2e7 --- /dev/null +++ b/templates/partials/packages/search_actions.html @@ -0,0 +1,25 @@ +

    + + + {% if request.user.is_trusted_user() or request.user.is_developer() %} + + + {% endif %} + + + + +

    diff --git a/templates/partials/packages/search_results.html b/templates/partials/packages/search_results.html new file mode 100644 index 00000000..28cf0b48 --- /dev/null +++ b/templates/partials/packages/search_results.html @@ -0,0 +1,114 @@ +
    {{ "Git Clone URL" | tr }}:
    {{ "Description" | tr }}:{{ pkgbase.packages.first().Description }}{{ pkg.Description }}
    {{ "Upstream URL" | tr }}: - {% set pkg = pkgbase.packages.first() %} {% if pkg.URL %} {{ pkg.URL }} {% else %} @@ -33,7 +33,7 @@
    {{ "Keywords" | tr }}:
    {{ "Licenses" | tr }}: {{ licenses | join(', ', attribute='Name') | default('None' | tr) }}
    {{ "Conflicts" | tr }}: From ae0f69a5e463b5cdb502aa3777eb5da011112eba Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 16 Aug 2021 17:18:29 -0700 Subject: [PATCH 282/844] Docker: remove intervals and timeouts These weren't needed at all and provided false negatives in general. Removed them to let Docker deal with them. Additionally. 'exit 0' -> 'echo' for ca's command; 'exit 0' happens to depend on the shell running Docker (it seems). echo is quite a bit more agnostic. Moreso, added mariadb deps to php-fpm and fastapi. Signed-off-by: Kevin Morris --- docker-compose.yml | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ab8d7c41..3500b8e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: image: aurweb:latest init: true entrypoint: /docker/ca-entrypoint.sh - command: exit 0 + command: echo volumes: - ./cache:/cache @@ -45,8 +45,6 @@ services: - mariadb_data:/var/lib/mysql healthcheck: test: "bash /docker/health/mariadb.sh" - interval: 2s - timeout: 60s git: image: aurweb:latest @@ -59,8 +57,6 @@ services: - "2222:2222" healthcheck: test: "bash /docker/health/sshd.sh" - interval: 2s - timeout: 30s depends_on: mariadb: condition: service_healthy @@ -79,8 +75,6 @@ services: command: /docker/scripts/run-smartgit.sh healthcheck: test: "bash /docker/health/smartgit.sh" - interval: 2s - timeout: 30s depends_on: mariadb: condition: service_healthy @@ -100,11 +94,11 @@ services: command: /docker/scripts/run-cgit.sh 3000 "https://localhost:8443/cgit" healthcheck: test: "bash /docker/health/cgit.sh 3000" - interval: 2s - timeout: 30s depends_on: git: condition: service_healthy + php-fpm: + condition: service_healthy volumes: - git_data:/aurweb/aur.git @@ -117,11 +111,11 @@ services: command: /docker/scripts/run-cgit.sh 3000 "https://localhost:8444/cgit" healthcheck: test: "bash /docker/health/cgit.sh 3000" - interval: 2s - timeout: 30s depends_on: git: condition: service_healthy + fastapi: + condition: service_healthy volumes: - git_data:/aurweb/aur.git @@ -135,8 +129,6 @@ services: command: /docker/scripts/run-php.sh healthcheck: test: "bash /docker/health/php.sh" - interval: 2s - timeout: 30s depends_on: ca: condition: service_started @@ -166,8 +158,6 @@ services: command: /docker/scripts/run-fastapi.sh "${FASTAPI_BACKEND}" healthcheck: test: "bash /docker/health/fastapi.sh ${FASTAPI_BACKEND}" - interval: 2s - timeout: 30s depends_on: ca: condition: service_started @@ -199,8 +189,6 @@ services: - "8444:8444" # FastAPI healthcheck: test: "bash /docker/health/nginx.sh" - interval: 2s - timeout: 30s depends_on: cgit-php: condition: service_healthy From 35851d553348383903c17d98cd049ac1f07bfaab Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 8 Aug 2021 18:35:49 -0700 Subject: [PATCH 283/844] Docker: add service 'memcached' Additionally, setup memcached for php-fpm. Signed-off-by: Kevin Morris --- conf/config.dev | 4 ++++ docker-compose.yml | 9 +++++++++ docker/health/memcached.sh | 2 ++ docker/php-entrypoint.sh | 6 ++++++ docker/scripts/install-deps.sh | 2 +- docker/scripts/run-memcached.sh | 2 ++ 6 files changed, 24 insertions(+), 1 deletion(-) create mode 100755 docker/health/memcached.sh create mode 100755 docker/scripts/run-memcached.sh diff --git a/conf/config.dev b/conf/config.dev index fc3bde91..566b655e 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -28,6 +28,10 @@ enable-maintenance = 0 localedir = YOUR_AUR_ROOT/web/locale ; In production, salt_rounds should be higher; suggested: 12. salt_rounds = 4 +cache = none +; In docker, the memcached host is available. On a user's system, +; this should be set to localhost (most likely). +memcache_servers = memcached:11211 [notifications] ; For development/testing, use /usr/bin/sendmail diff --git a/docker-compose.yml b/docker-compose.yml index 3500b8e9..56eff570 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,13 @@ services: volumes: - ./cache:/cache + memcached: + image: aurweb:latest + init: true + command: /docker/scripts/run-memcached.sh + healthcheck: + test: "bash /docker/health/memcached.sh" + mariadb: image: aurweb:latest init: true @@ -136,6 +143,8 @@ services: condition: service_healthy mariadb: condition: service_healthy + memcached: + condition: service_healthy volumes: - mariadb_run:/var/run/mysqld # Bind socket in this volume. - mariadb_data:/var/lib/mysql diff --git a/docker/health/memcached.sh b/docker/health/memcached.sh new file mode 100755 index 00000000..00f8cd98 --- /dev/null +++ b/docker/health/memcached.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec pgrep memcached diff --git a/docker/php-entrypoint.sh b/docker/php-entrypoint.sh index b4f6c631..1f3ed82b 100755 --- a/docker/php-entrypoint.sh +++ b/docker/php-entrypoint.sh @@ -7,6 +7,9 @@ bash $dir/test-mysql-entrypoint.sh sed -ri "s;^(aur_location) = .+;\1 = https://localhost:8443;" conf/config sed -ri 's/^(name) = .+/\1 = aurweb/' conf/config +# Enable memcached. +sed -ri 's/^(cache) = .+$/\1 = memcache/' conf/config + sed -ri "s|^(git_clone_uri_anon) = .+|\1 = https://localhost:8443/%s.git|" conf/config.defaults sed -ri "s|^(git_clone_uri_priv) = .+|\1 = ssh://aur@localhost:2222/%s.git|" conf/config.defaults @@ -21,6 +24,9 @@ sed -ri 's|^;?(access\.log) = .*$|\1 = /proc/self/fd/2|g' \ sed -ri 's/^;?(extension=pdo_mysql)/\1/' /etc/php/php.ini sed -ri 's/^;?(open_basedir).*$/\1 = \//' /etc/php/php.ini +# Use the sqlite3 extension line for memcached. +sed -ri 's/^;(extension)=sqlite3$/\1=memcached/' /etc/php/php.ini + python -m aurweb.initdb 2>/dev/null || /bin/true exec "$@" diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh index 8d4525de..6edbff5a 100755 --- a/docker/scripts/install-deps.sh +++ b/docker/scripts/install-deps.sh @@ -14,6 +14,6 @@ pacman -Syu --noconfirm --noprogressbar \ python-pytest-asyncio python-coverage hypercorn python-bcrypt \ python-email-validator openssh python-lxml mariadb mariadb-libs \ python-isort flake8 cgit uwsgi uwsgi-plugin-cgi php php-fpm \ - python-asgiref uvicorn python-feedgen + python-asgiref uvicorn python-feedgen memcached php-memcached exec "$@" diff --git a/docker/scripts/run-memcached.sh b/docker/scripts/run-memcached.sh new file mode 100755 index 00000000..90784b0f --- /dev/null +++ b/docker/scripts/run-memcached.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec /usr/bin/memcached -u memcached -m 64 -c 1024 -l 0.0.0.0 From 96d1af936381a959090d5362ee2ee49b4c91eced Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 28 Jun 2021 08:30:12 -0700 Subject: [PATCH 284/844] docker-compose: add redis service Now, the fastapi docker-compose service uses the new redis service for a cache option. Signed-off-by: Kevin Morris --- INSTALL | 5 +++++ docker-compose.yml | 12 ++++++++++++ docker/fastapi-entrypoint.sh | 4 ++++ docker/health/redis.sh | 2 ++ docker/redis-entrypoint.sh | 6 ++++++ docker/scripts/install-deps.sh | 3 ++- docker/scripts/run-redis.sh | 2 ++ 7 files changed, 33 insertions(+), 1 deletion(-) create mode 100755 docker/health/redis.sh create mode 100755 docker/redis-entrypoint.sh create mode 100755 docker/scripts/run-redis.sh diff --git a/INSTALL b/INSTALL index f192f9f5..c41a5c8e 100644 --- a/INSTALL +++ b/INSTALL @@ -55,6 +55,11 @@ read the instructions below. python-lxml python-feedgen # python3 setup.py install +(FastAPI-Specific) + + # pacman -S redis python-redis + # systemctl enable --now redis + 5) Create a new MySQL database and a user and import the aurweb SQL schema: $ python -m aurweb.initdb diff --git a/docker-compose.yml b/docker-compose.yml index 56eff570..0e91d6eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,16 @@ services: healthcheck: test: "bash /docker/health/memcached.sh" + redis: + image: aurweb:latest + init: true + entrypoint: /docker/redis-entrypoint.sh + command: /docker/scripts/run-redis.sh + healthcheck: + test: "bash /docker/health/redis.sh" + ports: + - "16379:6379" + mariadb: image: aurweb:latest init: true @@ -172,6 +182,8 @@ services: condition: service_started git: condition: service_healthy + redis: + condition: service_healthy mariadb: condition: service_healthy volumes: diff --git a/docker/fastapi-entrypoint.sh b/docker/fastapi-entrypoint.sh index c46a33eb..41a88206 100755 --- a/docker/fastapi-entrypoint.sh +++ b/docker/fastapi-entrypoint.sh @@ -7,6 +7,10 @@ bash $dir/test-mysql-entrypoint.sh sed -ri "s;^(aur_location) = .+;\1 = https://localhost:8444;" conf/config sed -ri 's/^(name) = .+/\1 = aurweb/' conf/config +# Setup Redis for FastAPI. +sed -ri 's/^(cache) = .+/\1 = redis/' conf/config +sed -ri 's|^(redis_address) = .+|\1 = redis://redis|' conf/config + sed -ri "s|^(git_clone_uri_anon) = .+|\1 = https://localhost:8444/%s.git|" conf/config.defaults sed -ri "s|^(git_clone_uri_priv) = .+|\1 = ssh://aur@localhost:2222/%s.git|" conf/config.defaults diff --git a/docker/health/redis.sh b/docker/health/redis.sh new file mode 100755 index 00000000..b5b442e8 --- /dev/null +++ b/docker/health/redis.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec pgrep redis-server diff --git a/docker/redis-entrypoint.sh b/docker/redis-entrypoint.sh new file mode 100755 index 00000000..e92be6c5 --- /dev/null +++ b/docker/redis-entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -eou pipefail + +sed -ri 's/^bind .*$/bind 0.0.0.0 -::1/g' /etc/redis/redis.conf + +exec "$@" diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh index 6edbff5a..6b0ec48b 100755 --- a/docker/scripts/install-deps.sh +++ b/docker/scripts/install-deps.sh @@ -14,6 +14,7 @@ pacman -Syu --noconfirm --noprogressbar \ python-pytest-asyncio python-coverage hypercorn python-bcrypt \ python-email-validator openssh python-lxml mariadb mariadb-libs \ python-isort flake8 cgit uwsgi uwsgi-plugin-cgi php php-fpm \ - python-asgiref uvicorn python-feedgen memcached php-memcached + python-asgiref uvicorn python-feedgen memcached php-memcached \ + python-redis redis exec "$@" diff --git a/docker/scripts/run-redis.sh b/docker/scripts/run-redis.sh new file mode 100755 index 00000000..8dc98b10 --- /dev/null +++ b/docker/scripts/run-redis.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec /usr/bin/redis-server /etc/redis/redis.conf From 91e769f6033867daaa951334ff51380275de5c84 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 28 Jun 2021 08:49:02 -0700 Subject: [PATCH 285/844] FastAPI: add redis integration This includes the addition of the python-fakeredis package, used for stubbing python-redis when a user does not have a configured cache. Signed-off-by: Kevin Morris --- INSTALL | 2 +- aurweb/redis.py | 57 ++++++++++++++++++++++++++++++++++ conf/config.defaults | 4 ++- conf/config.dev | 3 ++ docker/scripts/install-deps.sh | 2 +- test/test_asgi.py | 18 +++++++++++ test/test_redis.py | 40 ++++++++++++++++++++++++ 7 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 aurweb/redis.py create mode 100644 test/test_redis.py diff --git a/INSTALL b/INSTALL index c41a5c8e..fdeb64ca 100644 --- a/INSTALL +++ b/INSTALL @@ -57,7 +57,7 @@ read the instructions below. (FastAPI-Specific) - # pacman -S redis python-redis + # pacman -S redis python-redis python-fakeredis # systemctl enable --now redis 5) Create a new MySQL database and a user and import the aurweb SQL schema: diff --git a/aurweb/redis.py b/aurweb/redis.py new file mode 100644 index 00000000..6b8dede4 --- /dev/null +++ b/aurweb/redis.py @@ -0,0 +1,57 @@ +import logging + +import fakeredis + +from redis import ConnectionPool, Redis + +import aurweb.config + +logger = logging.getLogger(__name__) +pool = None + + +class FakeConnectionPool: + """ A fake ConnectionPool class which holds an internal reference + to a fakeredis handle. + + We normally deal with Redis by keeping its ConnectionPool globally + referenced so we can persist connection state through different calls + to redis_connection(), and since FakeRedis does not offer a ConnectionPool, + we craft one up here to hang onto the same handle instance as long as the + same instance is alive; this allows us to use a similar flow from the + redis_connection() user's perspective. + """ + + def __init__(self): + self.handle = fakeredis.FakeStrictRedis() + + def disconnect(self): + pass + + +def redis_connection(): # pragma: no cover + global pool + + disabled = aurweb.config.get("options", "cache") != "redis" + + # If we haven't initialized redis yet, construct a pool. + if disabled: + logger.debug("Initializing fake Redis instance.") + if pool is None: + pool = FakeConnectionPool() + return pool.handle + else: + logger.debug("Initializing real Redis instance.") + if pool is None: + redis_addr = aurweb.config.get("options", "redis_address") + pool = ConnectionPool.from_url(redis_addr) + + # Create a connection to the pool. + return Redis(connection_pool=pool) + + +def kill_redis(): + global pool + if pool: + pool.disconnect() + pool = None diff --git a/conf/config.defaults b/conf/config.defaults index ebc21e51..1b4c3a74 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -36,11 +36,13 @@ enable-maintenance = 1 maintenance-exceptions = 127.0.0.1 render-comment-cmd = /usr/local/bin/aurweb-rendercomment localedir = /srv/http/aurweb/aur.git/web/locale/ -# memcache or apc +; memcache, apc, or redis +; memcache/apc are supported in PHP, redis is supported in Python. cache = none cache_pkginfo_ttl = 86400 memcache_servers = 127.0.0.1:11211 salt_rounds = 12 +redis_address = redis://localhost [ratelimit] request_limit = 4000 diff --git a/conf/config.dev b/conf/config.dev index 566b655e..94a9630b 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -28,10 +28,13 @@ enable-maintenance = 0 localedir = YOUR_AUR_ROOT/web/locale ; In production, salt_rounds should be higher; suggested: 12. salt_rounds = 4 +; See config.defaults comment about cache. cache = none ; In docker, the memcached host is available. On a user's system, ; this should be set to localhost (most likely). memcache_servers = memcached:11211 +; If cache = 'redis' this address is used to connect to Redis. +redis_address = redis://127.0.0.1 [notifications] ; For development/testing, use /usr/bin/sendmail diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh index 6b0ec48b..0405f29b 100755 --- a/docker/scripts/install-deps.sh +++ b/docker/scripts/install-deps.sh @@ -15,6 +15,6 @@ pacman -Syu --noconfirm --noprogressbar \ python-email-validator openssh python-lxml mariadb mariadb-libs \ python-isort flake8 cgit uwsgi uwsgi-plugin-cgi php php-fpm \ python-asgiref uvicorn python-feedgen memcached php-memcached \ - python-redis redis + python-redis redis python-fakeredis exec "$@" diff --git a/test/test_asgi.py b/test/test_asgi.py index 79b34daf..b8856741 100644 --- a/test/test_asgi.py +++ b/test/test_asgi.py @@ -9,6 +9,24 @@ from fastapi import HTTPException import aurweb.asgi import aurweb.config +import aurweb.redis + + +@pytest.mark.asyncio +async def test_asgi_startup_session_secret_exception(monkeypatch): + """ Test that we get an IOError on app_startup when we cannot + connect to options.redis_address. """ + + redis_addr = aurweb.config.get("options", "redis_address") + + def mock_get(section: str, key: str): + if section == "fastapi" and key == "session_secret": + return None + return redis_addr + + with mock.patch("aurweb.config.get", side_effect=mock_get): + with pytest.raises(Exception): + await aurweb.asgi.app_startup() @pytest.mark.asyncio diff --git a/test/test_redis.py b/test/test_redis.py new file mode 100644 index 00000000..82aebb57 --- /dev/null +++ b/test/test_redis.py @@ -0,0 +1,40 @@ +from unittest import mock + +import pytest + +import aurweb.config + +from aurweb.redis import redis_connection + + +@pytest.fixture +def rediss(): + """ Create a RedisStub. """ + def mock_get(section, key): + return "none" + + with mock.patch("aurweb.config.get", side_effect=mock_get): + aurweb.config.rehash() + redis = redis_connection() + aurweb.config.rehash() + + yield redis + + +def test_redis_stub(rediss): + # We don't yet have a test key set. + assert rediss.get("test") is None + + # Set the test key to abc. + rediss.set("test", "abc") + assert rediss.get("test").decode() == "abc" + + # Test expire. + rediss.expire("test", 0) + assert rediss.get("test") is None + + # Now, set the test key again and use delete() on it. + rediss.set("test", "abc") + assert rediss.get("test").decode() == "abc" + rediss.delete("test") + assert rediss.get("test") is None From 968ed736c16f92de6ebca1a77fe1e7b2d78ec4e5 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 8 Aug 2021 20:42:00 -0700 Subject: [PATCH 286/844] add python-orjson dependency python-orjson speeds up a lot of JSON serialization steps, so we choose to use it over the standard library json module. Signed-off-by: Kevin Morris --- INSTALL | 2 +- docker/scripts/install-deps.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL b/INSTALL index fdeb64ca..4df59bd2 100644 --- a/INSTALL +++ b/INSTALL @@ -57,7 +57,7 @@ read the instructions below. (FastAPI-Specific) - # pacman -S redis python-redis python-fakeredis + # pacman -S redis python-redis python-fakeredis python-orjson # systemctl enable --now redis 5) Create a new MySQL database and a user and import the aurweb SQL schema: diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh index 0405f29b..a532a6b2 100755 --- a/docker/scripts/install-deps.sh +++ b/docker/scripts/install-deps.sh @@ -15,6 +15,6 @@ pacman -Syu --noconfirm --noprogressbar \ python-email-validator openssh python-lxml mariadb mariadb-libs \ python-isort flake8 cgit uwsgi uwsgi-plugin-cgi php php-fpm \ python-asgiref uvicorn python-feedgen memcached php-memcached \ - python-redis redis python-fakeredis + python-redis redis python-fakeredis python-orjson exec "$@" From 9e73936c4e52a862f392947005d6ed29668969de Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 6 Aug 2021 22:44:19 -0700 Subject: [PATCH 287/844] add aurweb.cache, a redis caching utility module Signed-off-by: Kevin Morris --- aurweb/cache.py | 20 +++++++++++++ test/test_cache.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 aurweb/cache.py create mode 100644 test/test_cache.py diff --git a/aurweb/cache.py b/aurweb/cache.py new file mode 100644 index 00000000..697473b8 --- /dev/null +++ b/aurweb/cache.py @@ -0,0 +1,20 @@ +from redis import Redis +from sqlalchemy import orm + + +async def db_count_cache(redis: Redis, key: str, query: orm.Query, + expire: int = None) -> int: + """ Store and retrieve a query.count() via redis cache. + + :param redis: Redis handle + :param key: Redis key + :param query: SQLAlchemy ORM query + :param expire: Optional expiration in seconds + :return: query.count() + """ + result = redis.get(key) + if result is None: + redis.set(key, (result := int(query.count()))) + if expire: + redis.expire(key, expire) + return int(result) diff --git a/test/test_cache.py b/test/test_cache.py new file mode 100644 index 00000000..35346e52 --- /dev/null +++ b/test/test_cache.py @@ -0,0 +1,74 @@ +import pytest + +from aurweb import cache, db +from aurweb.models.account_type import USER_ID +from aurweb.models.user import User +from aurweb.testing import setup_test_db + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db( + User.__tablename__ + ) + + +class StubRedis: + """ A class which acts as a RedisConnection without using Redis. """ + + cache = dict() + expires = dict() + + def get(self, key, *args): + if "key" not in self.cache: + self.cache[key] = None + return self.cache[key] + + def set(self, key, *args): + self.cache[key] = list(args)[0] + + def expire(self, key, *args): + self.expires[key] = list(args)[0] + + async def execute(self, command, key, *args): + f = getattr(self, command) + return f(key, *args) + + +@pytest.fixture +def redis(): + yield StubRedis() + + +@pytest.mark.asyncio +async def test_db_count_cache(redis): + db.create(User, Username="user1", + Email="user1@example.org", + Passwd="testPassword", + AccountTypeID=USER_ID) + + query = db.query(User) + + # Now, perform several checks that db_count_cache matches query.count(). + + # We have no cached value yet. + assert await cache.db_count_cache(redis, "key1", query) == query.count() + + # It's cached now. + assert await cache.db_count_cache(redis, "key1", query) == query.count() + + +@pytest.mark.asyncio +async def test_db_count_cache_expires(redis): + db.create(User, Username="user1", + Email="user1@example.org", + Passwd="testPassword", + AccountTypeID=USER_ID) + + query = db.query(User) + + # Cache a query with an expire. + value = await cache.db_count_cache(redis, "key1", query, 100) + assert value == query.count() + + assert redis.expires["key1"] == 100 From d9cdd5faeff608f4191872cc808be5c3ce0ce4b4 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 28 Jul 2021 13:28:17 -0700 Subject: [PATCH 288/844] [FastAPI] Modularize homepage and add side panel This puts one more toward completion of the homepage overall; we'll need to still implement the authenticated user dashboard after this. Signed-off-by: Kevin Morris --- aurweb/packages/util.py | 55 ++++++++- aurweb/routers/html.py | 74 ++++++++++- templates/home.html | 101 +++++++++++++++ templates/index.html | 107 ++-------------- .../partials/packages/widgets/search.html | 14 +++ .../partials/packages/widgets/statistics.html | 55 +++++++++ .../partials/packages/widgets/updates.html | 35 ++++++ templates/partials/widgets/statistics.html | 27 ++++ test/test_homepage.py | 115 ++++++++++++++++++ test/test_packages_util.py | 19 ++- 10 files changed, 500 insertions(+), 102 deletions(-) create mode 100644 templates/home.html create mode 100644 templates/partials/packages/widgets/search.html create mode 100644 templates/partials/packages/widgets/statistics.html create mode 100644 templates/partials/packages/widgets/updates.html create mode 100644 templates/partials/widgets/statistics.html diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index 60db2962..036e3441 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -1,7 +1,10 @@ from http import HTTPStatus +from typing import List + +import orjson from fastapi import HTTPException -from sqlalchemy import and_ +from sqlalchemy import and_, orm from aurweb import db from aurweb.models.official_provider import OFFICIAL_BASE, OfficialProvider @@ -10,6 +13,7 @@ from aurweb.models.package_base import PackageBase from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_relation import PackageRelation from aurweb.models.relation_type import PROVIDES_ID, RelationType +from aurweb.redis import redis_connection from aurweb.templates import register_filter @@ -111,3 +115,52 @@ def get_pkgbase(name: str) -> PackageBase: raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) return pkgbase + + +@register_filter("out_of_date") +def out_of_date(packages: orm.Query) -> orm.Query: + return packages.filter(PackageBase.OutOfDateTS.isnot(None)) + + +def updated_packages(limit: int = 0, cache_ttl: int = 600) -> List[Package]: + """ Return a list of valid Package objects ordered by their + ModifiedTS column in descending order from cache, after setting + the cache when no key yet exists. + + :param limit: Optional record limit + :param cache_ttl: Cache expiration time (in seconds) + :return: A list of Packages + """ + redis = redis_connection() + packages = redis.get("package_updates") + if packages: + # If we already have a cache, deserialize it and return. + return orjson.loads(packages) + + query = db.query(Package).join(PackageBase).filter( + PackageBase.PackagerUID.isnot(None) + ).order_by( + PackageBase.ModifiedTS.desc() + ) + + if limit: + query = query.limit(limit) + + packages = [] + for pkg in query: + # For each Package returned by the query, append a dict + # containing Package columns we're interested in. + packages.append({ + "Name": pkg.Name, + "Version": pkg.Version, + "PackageBase": { + "ModifiedTS": pkg.PackageBase.ModifiedTS + } + }) + + # Store the JSON serialization of the package_updates key into Redis. + redis.set("package_updates", orjson.dumps(packages)) + redis.expire("package_updates", cache_ttl) + + # Return the deserialized list of packages. + return packages diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index f6f1a54e..ae012901 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -1,14 +1,21 @@ """ AURWeb's primary routing module. Define all routes via @app.app.{get,post} decorators in some way; more complex routes should be defined in their own modules and imported here. """ +from datetime import datetime from http import HTTPStatus from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse +from sqlalchemy import and_, or_ import aurweb.config -from aurweb import util +from aurweb import db, util +from aurweb.cache import db_count_cache +from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID +from aurweb.models.package_base import PackageBase +from aurweb.models.user import User +from aurweb.packages.util import updated_packages from aurweb.templates import make_context, render_template router = APIRouter() @@ -60,6 +67,71 @@ async def index(request: Request): context = make_context(request, "Home") context['ssh_fingerprints'] = util.get_ssh_fingerprints() + bases = db.query(PackageBase) + + redis = aurweb.redis.redis_connection() + stats_expire = 300 # Five minutes. + updates_expire = 600 # Ten minutes. + + # Package statistics. + query = bases.filter(PackageBase.PackagerUID.isnot(None)) + context["package_count"] = await db_count_cache( + redis, "package_count", query, expire=stats_expire) + + query = bases.filter( + and_(PackageBase.MaintainerUID.is_(None), + PackageBase.PackagerUID.isnot(None)) + ) + context["orphan_count"] = await db_count_cache( + redis, "orphan_count", query, expire=stats_expire) + + query = db.query(User) + context["user_count"] = await db_count_cache( + redis, "user_count", query, expire=stats_expire) + + query = query.filter( + or_(User.AccountTypeID == TRUSTED_USER_ID, + User.AccountTypeID == TRUSTED_USER_AND_DEV_ID)) + context["trusted_user_count"] = await db_count_cache( + redis, "trusted_user_count", query, expire=stats_expire) + + # Current timestamp. + now = int(datetime.utcnow().timestamp()) + + seven_days = 86400 * 7 # Seven days worth of seconds. + seven_days_ago = now - seven_days + + one_hour = 3600 + updated = bases.filter( + and_(PackageBase.ModifiedTS - PackageBase.SubmittedTS >= one_hour, + PackageBase.PackagerUID.isnot(None)) + ) + + query = bases.filter( + and_(PackageBase.SubmittedTS >= seven_days_ago, + PackageBase.PackagerUID.isnot(None)) + ) + context["seven_days_old_added"] = await db_count_cache( + redis, "seven_days_old_added", query, expire=stats_expire) + + query = updated.filter(PackageBase.ModifiedTS >= seven_days_ago) + context["seven_days_old_updated"] = await db_count_cache( + redis, "seven_days_old_updated", query, expire=stats_expire) + + year = seven_days * 52 # Fifty two weeks worth: one year. + year_ago = now - year + query = updated.filter(PackageBase.ModifiedTS >= year_ago) + context["year_old_updated"] = await db_count_cache( + redis, "year_old_updated", query, expire=stats_expire) + + query = bases.filter( + PackageBase.ModifiedTS - PackageBase.SubmittedTS < 3600) + context["never_updated"] = await db_count_cache( + redis, "never_updated", query, expire=stats_expire) + + # Get the 15 most recently updated packages. + context["package_updates"] = updated_packages(15, updates_expire) + return render_template(request, "index.html", context) diff --git a/templates/home.html b/templates/home.html new file mode 100644 index 00000000..a8cae5b8 --- /dev/null +++ b/templates/home.html @@ -0,0 +1,101 @@ +
    +

    AUR {% trans %}Home{% endtrans %}

    +

    + {{ "Welcome to the AUR! Please read the %sAUR User Guidelines%s and %sAUR TU Guidelines%s for more information." + | tr + | format('', "", + '', "") + | safe + }} + {{ "Contributed PKGBUILDs %smust%s conform to the %sArch Packaging Standards%s otherwise they will be deleted!" + | tr + | format("", "", + '', + "") + | safe + }} + {% trans %}Remember to vote for your favourite packages!{% endtrans %} + {% trans %}Some packages may be provided as binaries in [community].{% endtrans %} +

    +

    + {% trans %}DISCLAIMER{% endtrans %}: + {% trans %}AUR packages are user produced content. Any use of the provided files is at your own risk.{% endtrans %} +

    +

    {% trans %}Learn more...{% endtrans %}

    +
    +
    +

    {% trans %}Support{% endtrans %}

    +

    {% trans %}Package Requests{% endtrans %}

    +
    +

    + {{ "There are three types of requests that can be filed in the %sPackage Actions%s box on the package details page:" + | tr + | format("", "") + | safe + }} +

    +
      +
    • {% trans %}Orphan Request{% endtrans %}: {% trans %}Request a package to be disowned, e.g. when the maintainer is inactive and the package has been flagged out-of-date for a long time.{% endtrans %}
    • +
    • {% trans %}Deletion Request{% endtrans %}: {%trans %}Request a package to be removed from the Arch User Repository. Please do not use this if a package is broken and can be fixed easily. Instead, contact the package maintainer and file orphan request if necessary.{% endtrans %}
    • +
    • {% trans %}Merge Request{% endtrans %}: {% trans %}Request a package to be merged into another one. Can be used when a package needs to be renamed or replaced by a split package.{% endtrans %}
    • +
    +

    + {{ "If you want to discuss a request, you can use the %saur-requests%s mailing list. However, please do not use that list to file requests." + | tr + | format('', "") + | safe + }} +

    +
    +

    {% trans %}Submitting Packages{% endtrans %}

    +
    +

    + {{ "Git over SSH is now used to submit packages to the AUR. See the %sSubmitting packages%s section of the Arch User Repository ArchWiki page for more details." + | tr + | format('', "") + | safe + }} +

    + {% if ssh_fingerprints %} +

    + {% trans %}The following SSH fingerprints are used for the AUR:{% endtrans %} +

    +

      + {% for keytype in ssh_fingerprints %} +
    • {{ keytype }}: {{ ssh_fingerprints[keytype] }} + {% endfor %} +
    + {% endif %} +
    +

    {% trans %}Discussion{% endtrans %}

    +
    +

    + {{ "General discussion regarding the Arch User Repository (AUR) and Trusted User structure takes place on %saur-general%s. For discussion relating to the development of the AUR web interface, use the %saur-dev%s mailing list." + | tr + | format('', "", + '', "") + | safe + }} +

    +

    +

    {% trans %}Bug Reporting{% endtrans %}

    +
    +

    + {{ "If you find a bug in the AUR web interface, please fill out a bug report on our %sbug tracker%s. Use the tracker to report bugs in the AUR web interface %sonly%s. To report packaging bugs contact the package maintainer or leave a comment on the appropriate package page." + | tr + | format('', "", + "", "") + | safe + }} +

    +
    +
    + + + + + + diff --git a/templates/index.html b/templates/index.html index f8745f33..e50a99cd 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,106 +1,15 @@ {% extends 'partials/layout.html' %} {% block pageContent %} -
    -

    AUR {% trans %}Home{% endtrans %}

    -

    - {{ "Welcome to the AUR! Please read the %sAUR User Guidelines%s and %sAUR TU Guidelines%s for more information." - | tr - | format('', "", - '', "") - | safe - }} - {{ "Contributed PKGBUILDs %smust%s conform to the %sArch Packaging Standards%s otherwise they will be deleted!" - | tr - | format("", "", - '', - "") - | safe - }} - {% trans %}Remember to vote for your favourite packages!{% endtrans %} - {% trans %}Some packages may be provided as binaries in [community].{% endtrans %} -

    - {% trans %}DISCLAIMER{% endtrans %}: - {% trans %}AUR packages are user produced content. Any use of the provided files is at your own risk.{% endtrans %} -

    -

    {% trans %}Learn more...{% endtrans %}

    -

    -
    -
    -

    {% trans %}Support{% endtrans %}

    -

    {% trans %}Package Requests{% endtrans %}

    -
    -

    - {{ "There are three types of requests that can be filed in the %sPackage Actions%s box on the package details page:" - | tr - | format("", "") - | safe - }} -

    -
      -
    • {% trans %}Orphan Request{% endtrans %}: {% trans %}Request a package to be disowned, e.g. when the maintainer is inactive and the package has been flagged out-of-date for a long time.{% endtrans %}
    • -
    • {% trans %}Deletion Request{% endtrans %}: {%trans %}Request a package to be removed from the Arch User Repository. Please do not use this if a package is broken and can be fixed easily. Instead, contact the package maintainer and file orphan request if necessary.{% endtrans %}
    • -
    • {% trans %}Merge Request{% endtrans %}: {% trans %}Request a package to be merged into another one. Can be used when a package needs to be renamed or replaced by a split package.{% endtrans %}
    • -
    -

    - {{ "If you want to discuss a request, you can use the %saur-requests%s mailing list. However, please do not use that list to file requests." - | tr - | format('', "") - | safe - }} -

    +
    +
    + {% include 'home.html' %} +
    -

    {% trans %}Submitting Packages{% endtrans %}

    -
    -

    - {{ "Git over SSH is now used to submit packages to the AUR. See the %sSubmitting packages%s section of the Arch User Repository ArchWiki page for more details." - | tr - | format('', "") - | safe - }} -

    - {% if ssh_fingerprints %} -

    - {% trans %}The following SSH fingerprints are used for the AUR:{% endtrans %} -

    -

      - {% for keytype in ssh_fingerprints %} -
    • {{ keytype }}: {{ ssh_fingerprints[keytype] }} - {% endfor %} -
    - {% endif %} +
    + {% include 'partials/packages/widgets/search.html' %} + {% include 'partials/packages/widgets/updates.html' %} + {% include 'partials/packages/widgets/statistics.html' %}
    -

    {% trans %}Discussion{% endtrans %}

    -
    -

    - {{ "General discussion regarding the Arch User Repository (AUR) and Trusted User structure takes place on %saur-general%s. For discussion relating to the development of the AUR web interface, use the %saur-dev%s mailing list." - | tr - | format('', "", - '', "") - | safe - }} -

    -

    -

    {% trans %}Bug Reporting{% endtrans %}

    -
    -

    - {{ "If you find a bug in the AUR web interface, please fill out a bug report on our %sbug tracker%s. Use the tracker to report bugs in the AUR web interface %sonly%s. To report packaging bugs contact the package maintainer or leave a comment on the appropriate package page." - | tr - | format('', "", - "", "") - | safe - }} -

    -
    -
    - - - - - - {% endblock %} diff --git a/templates/partials/packages/widgets/search.html b/templates/partials/packages/widgets/search.html new file mode 100644 index 00000000..106b93ea --- /dev/null +++ b/templates/partials/packages/widgets/search.html @@ -0,0 +1,14 @@ +
    +
    +
    + + + +
    +
    +
    diff --git a/templates/partials/packages/widgets/statistics.html b/templates/partials/packages/widgets/statistics.html new file mode 100644 index 00000000..f841ae0e --- /dev/null +++ b/templates/partials/packages/widgets/statistics.html @@ -0,0 +1,55 @@ +
    +

    {{ "Statistics" | tr }}

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    {{ "Packages" | tr }}{{ package_count }}
    {{ "Orphan Packages" | tr }}{{ orphan_count }}
    + {{ "Packages added in the past 7 days" | tr }} + {{ seven_days_old_added }}
    + {{ "Packages updated in the past 7 days" | tr }} + {{ seven_days_old_updated }}
    + {{ "Packages updated in the past year" | tr }} + {{ year_old_updated }}
    + {{ "Packages never updated" | tr }} + {{ never_updated }}
    + {{ "Registered Users" | tr }} + {{ user_count }}
    + {{ "Trusted Users" | tr }} + {{ trusted_user_count }}
    +
    + +{% if request.user.is_authenticated() %} + + {% include 'partials/widgets/statistics.html' %} +{% endif %} diff --git a/templates/partials/packages/widgets/updates.html b/templates/partials/packages/widgets/updates.html new file mode 100644 index 00000000..3ee1b98e --- /dev/null +++ b/templates/partials/packages/widgets/updates.html @@ -0,0 +1,35 @@ +
    +

    + {{ "Recent Updates" | tr }} + + ({{ "more" | tr }}) + +

    + + RSS Feed + + + RSS Feed + + + + + {% for pkg in package_updates %} + + + + + {% endfor %} + +
    + + {{ pkg.Name }} {{ pkg.Version }} + + + {% set modified = pkg.PackageBase.ModifiedTS | dt | as_timezone(timezone) %} + {{ modified.strftime("%Y-%m-%d %H:%M") }} +
    + +
    diff --git a/templates/partials/widgets/statistics.html b/templates/partials/widgets/statistics.html new file mode 100644 index 00000000..0bf844b6 --- /dev/null +++ b/templates/partials/widgets/statistics.html @@ -0,0 +1,27 @@ +
    +

    {{ "My Statistics" | tr }}

    + + {% set bases = request.user.maintained_bases %} + + + + + + + {% set out_of_date_packages = bases | out_of_date %} + + + + + +
    + + {{ "Packages" | tr }} + + {{ bases.count() }}
    + + {{ "Out of Date" | tr }} + + {{ out_of_date_packages.count() }}
    + +
    diff --git a/test/test_homepage.py b/test/test_homepage.py index 23d7185f..a629b98c 100644 --- a/test/test_homepage.py +++ b/test/test_homepage.py @@ -1,13 +1,82 @@ +import re + +from datetime import datetime from http import HTTPStatus from unittest.mock import patch +import pytest + from fastapi.testclient import TestClient +from aurweb import db from aurweb.asgi import app +from aurweb.models.account_type import USER_ID +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.user import User +from aurweb.redis import redis_connection +from aurweb.testing import setup_test_db +from aurweb.testing.html import parse_root client = TestClient(app) +@pytest.fixture(autouse=True) +def setup(): + yield setup_test_db( + User.__tablename__, + Package.__tablename__, + PackageBase.__tablename__ + ) + + +@pytest.fixture +def user(): + yield db.create(User, Username="test", Email="test@example.org", + Passwd="testPassword", AccountTypeID=USER_ID) + + +@pytest.fixture +def redis(): + redis = redis_connection() + + def delete_keys(): + # Cleanup keys if they exist. + for key in ("package_count", "orphan_count", "user_count", + "trusted_user_count", "seven_days_old_added", + "seven_days_old_updated", "year_old_updated", + "never_updated", "package_updates"): + if redis.get(key) is not None: + redis.delete(key) + + delete_keys() + yield redis + delete_keys() + + +@pytest.fixture +def packages(user): + """ Yield a list of num_packages Package objects maintained by user. """ + num_packages = 50 # Tunable + + # For i..num_packages, create a package named pkg_{i}. + pkgs = [] + now = int(datetime.utcnow().timestamp()) + for i in range(num_packages): + pkgbase = db.create(PackageBase, Name=f"pkg_{i}", + Maintainer=user, Packager=user, + autocommit=False, SubmittedTS=now, + ModifiedTS=now) + pkg = db.create(Package, PackageBase=pkgbase, + Name=pkgbase.Name, autocommit=False) + pkgs.append(pkg) + now += 1 + + db.commit() + + yield pkgs + + def test_homepage(): with client as request: response = request.get("/") @@ -34,3 +103,49 @@ def test_homepage_no_ssh_fingerprints(get_ssh_fingerprints_mock): response = request.get("/") assert 'The following SSH fingerprints are used for the AUR' not in response.content.decode() + + +def test_homepage_stats(redis, packages): + with client as request: + response = request.get("/") + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + + expectations = [ + ("Packages", r'\d+'), + ("Orphan Packages", r'\d+'), + ("Packages added in the past 7 days", r'\d+'), + ("Packages updated in the past 7 days", r'\d+'), + ("Packages updated in the past year", r'\d+'), + ("Packages never updated", r'\d+'), + ("Registered Users", r'\d+'), + ("Trusted Users", r'\d+') + ] + + stats = root.xpath('//div[@id="pkg-stats"]//tr') + for i, expected in enumerate(expectations): + expected_key, expected_regex = expected + key, value = stats[i].xpath('./td') + assert key.text.strip() == expected_key + assert re.match(expected_regex, value.text.strip()) + + +def test_homepage_updates(redis, packages): + with client as request: + response = request.get("/") + assert response.status_code == int(HTTPStatus.OK) + # Run the request a second time to exercise the Redis path. + response = request.get("/") + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + + # We expect to see the latest 15 packages, which happens to be + # pkg_49 .. pkg_34. So, create a list of expectations using a range + # starting at 49, stepping down to 49 - 15, -1 step at a time. + expectations = [f"pkg_{i}" for i in range(50 - 1, 50 - 1 - 15, -1)] + updates = root.xpath('//div[@id="pkg-updates"]/table/tbody/tr') + for i, expected in enumerate(expectations): + pkgname = updates[i].xpath('./td/a').pop(0) + assert pkgname.text.strip() == expected diff --git a/test/test_packages_util.py b/test/test_packages_util.py index 17978490..bc6a941c 100644 --- a/test/test_packages_util.py +++ b/test/test_packages_util.py @@ -9,6 +9,7 @@ from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.user import User from aurweb.packages import util +from aurweb.redis import kill_redis from aurweb.testing import setup_test_db @@ -33,7 +34,8 @@ def maintainer() -> User: @pytest.fixture def package(maintainer: User) -> Package: - pkgbase = db.create(PackageBase, Name="test-pkg", Maintainer=maintainer) + pkgbase = db.create(PackageBase, Name="test-pkg", + Packager=maintainer, Maintainer=maintainer) yield db.create(Package, Name=pkgbase.Name, PackageBase=pkgbase) @@ -49,3 +51,18 @@ def test_package_link(client: TestClient, maintainer: User, package: Package): Provides=package.Name) expected = f"{OFFICIAL_BASE}/packages/?q={package.Name}" assert util.package_link(package) == expected + + +def test_updated_packages(maintainer: User, package: Package): + expected = { + "Name": package.Name, + "Version": package.Version, + "PackageBase": { + "ModifiedTS": package.PackageBase.ModifiedTS + } + } + + kill_redis() # Kill it here to ensure we're on a fake instance. + assert util.updated_packages(1, 0) == [expected] + assert util.updated_packages(1, 600) == [expected] + kill_redis() # Kill it again, in case other tests use a real instance. From 469c141f6b541ea4b6d6aadbbd32fdb9822f6975 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 17 Aug 2021 20:58:41 -0700 Subject: [PATCH 289/844] [FastAPI] bugfix: remove use of scalar() in plural context Anything where we can have more than one of something, scalar() cannot be used. Signed-off-by: Kevin Morris --- templates/partials/packages/comments.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/partials/packages/comments.html b/templates/partials/packages/comments.html index 051849b0..39cfb363 100644 --- a/templates/partials/packages/comments.html +++ b/templates/partials/packages/comments.html @@ -49,7 +49,7 @@
    {% endif %} -{% if comments.scalar() %} +{% if comments %}

    From eb8ea53a4483d3a9d42b9b5cb0bfcfcc87d2cb4e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 7 Aug 2021 14:57:24 -0700 Subject: [PATCH 290/844] PackageRequest: add status_display() A helper function which provides a textual string conversion of a particular Status column. In a PackageRequest, Status is split up into four different types: - PENDING : "Pending", PENDING_ID: 0 - CLOSED : "Closed", CLOSED_ID: 1 - ACCEPTED : "Accepted", ACCEPTED_ID: 2 - REJECTED : "Rejected", REJECTED_ID: 3 This commit adds constants for the textual strings and the IDs. It also adds a PackageRequest.status_display() function which grabs the proper display string for a particular Status ID. Signed-off-by: Kevin Morris --- aurweb/models/package_request.py | 22 +++++++++++++++++++++ test/test_package_request.py | 34 ++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/aurweb/models/package_request.py b/aurweb/models/package_request.py index 00f46ce2..a5125cee 100644 --- a/aurweb/models/package_request.py +++ b/aurweb/models/package_request.py @@ -8,6 +8,17 @@ import aurweb.models.user from aurweb.models.declarative import Base +PENDING = "Pending" +CLOSED = "Closed" +ACCEPTED = "Accepted" +REJECTED = "Rejected" + +# Integer values used for the Status column of PackageRequest. +PENDING_ID = 0 +CLOSED_ID = 1 +ACCEPTED_ID = 2 +REJECTED_ID = 3 + class PackageRequest(Base): __tablename__ = "PackageRequests" @@ -40,6 +51,13 @@ class PackageRequest(Base): __mapper_args__ = {"primary_key": [ID]} + STATUS_DISPLAY = { + PENDING_ID: PENDING, + CLOSED_ID: CLOSED, + ACCEPTED_ID: ACCEPTED, + REJECTED_ID: REJECTED + } + def __init__(self, RequestType: aurweb.models.request_type.RequestType = None, PackageBase: aurweb.models.package_base.PackageBase = None, @@ -91,3 +109,7 @@ class PackageRequest(Base): statement="Column ClosureComment cannot be null.", orig="PackageRequests.ClosureComment", params=("NULL")) + + def status_display(self) -> str: + """ Return a display string for the Status column. """ + return self.STATUS_DISPLAY[self.Status] diff --git a/test/test_package_request.py b/test/test_package_request.py index fc839836..c28af6bd 100644 --- a/test/test_package_request.py +++ b/test/test_package_request.py @@ -4,9 +4,10 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, query, rollback +from aurweb.db import commit, create, query, rollback from aurweb.models.package_base import PackageBase -from aurweb.models.package_request import PackageRequest +from aurweb.models.package_request import (ACCEPTED, ACCEPTED_ID, CLOSED, CLOSED_ID, PENDING, PENDING_ID, REJECTED, + REJECTED_ID, PackageRequest) from aurweb.models.request_type import RequestType from aurweb.models.user import User from aurweb.testing import setup_test_db @@ -117,3 +118,32 @@ def test_package_request_null_closure_comment_raises_exception(): User=user, PackageBase=pkgbase, PackageBaseName=pkgbase.Name, Comments=str()) rollback() + + +def test_package_request_status_display(): + """ Test status_display() based on the Status column value. """ + request_type = query(RequestType, RequestType.Name == "merge").first() + + pkgreq = create(PackageRequest, RequestType=request_type, + User=user, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str(), + Status=PENDING_ID) + assert pkgreq.status_display() == PENDING + + pkgreq.Status = CLOSED_ID + commit() + assert pkgreq.status_display() == CLOSED + + pkgreq.Status = ACCEPTED_ID + commit() + assert pkgreq.status_display() == ACCEPTED + + pkgreq.Status = REJECTED_ID + commit() + assert pkgreq.status_display() == REJECTED + + pkgreq.Status = 124 + commit() + with pytest.raises(KeyError): + pkgreq.status_display() From 5bd3a7bbabd06bf846991a9dd4c74d61c9a8d0a0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 7 Aug 2021 15:43:44 -0700 Subject: [PATCH 291/844] RequestType: add name_display() and record constants Just like some of the other tables, we have some constant records that we use to denote types of things. This commit adds constants which correlate with these record constants. Signed-off-by: Kevin Morris --- aurweb/models/request_type.py | 15 +++++++++++++++ test/test_request_type.py | 15 +++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/aurweb/models/request_type.py b/aurweb/models/request_type.py index 2c8276e8..a26dcf9a 100644 --- a/aurweb/models/request_type.py +++ b/aurweb/models/request_type.py @@ -1,7 +1,12 @@ from sqlalchemy import Column, Integer +from aurweb import db from aurweb.models.declarative import Base +DELETION = "deletion" +ORPHAN = "orphan" +MERGE = "merge" + class RequestType(Base): __tablename__ = "RequestTypes" @@ -9,3 +14,13 @@ class RequestType(Base): ID = Column(Integer, primary_key=True) __mapper_args__ = {"primary_key": [ID]} + + def name_display(self) -> str: + """ Return the Name column with its first char capitalized. """ + name = self.Name + return name[0].upper() + name[1:] + + +DELETION_ID = db.query(RequestType, RequestType.Name == DELETION).first().ID +ORPHAN_ID = db.query(RequestType, RequestType.Name == ORPHAN).first().ID +MERGE_ID = db.query(RequestType, RequestType.Name == MERGE).first().ID diff --git a/test/test_request_type.py b/test/test_request_type.py index a470a60b..a3b3ccb8 100644 --- a/test/test_request_type.py +++ b/test/test_request_type.py @@ -1,7 +1,7 @@ import pytest -from aurweb.db import create, delete -from aurweb.models.request_type import RequestType +from aurweb.db import create, delete, query +from aurweb.models.request_type import DELETION_ID, MERGE_ID, ORPHAN_ID, RequestType from aurweb.testing import setup_test_db @@ -22,3 +22,14 @@ def test_request_type_null_name_returns_empty_string(): assert bool(request_type.ID) assert request_type.Name == str() delete(RequestType, RequestType.ID == request_type.ID) + + +def test_request_type_name_display(): + deletion = query(RequestType, RequestType.ID == DELETION_ID).first() + assert deletion.name_display() == "Deletion" + + orphan = query(RequestType, RequestType.ID == ORPHAN_ID).first() + assert orphan.name_display() == "Orphan" + + merge = query(RequestType, RequestType.ID == MERGE_ID).first() + assert merge.name_display() == "Merge" From af51b5c4604636408d78d015b49bfbc0e8d7de27 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 7 Aug 2021 19:35:50 -0700 Subject: [PATCH 292/844] User: add several utility methods Added: - User.voted_for(package) - Has a user voted for a particular package? - User.notified(package) - Is a user being notified about a particular package? - User.packages() - Entire collection of Package objects related to User. Signed-off-by: Kevin Morris --- aurweb/models/user.py | 36 +++++++++++++++++++++++++++++++++++- test/test_user.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 4705c050..0ccf7329 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -5,13 +5,14 @@ from datetime import datetime import bcrypt from fastapi import Request -from sqlalchemy import Column, ForeignKey, Integer, String, text +from sqlalchemy import Column, ForeignKey, Integer, String, or_, text from sqlalchemy.orm import backref, relationship import aurweb.config import aurweb.models.account_type import aurweb.schema +from aurweb import db from aurweb.models.ban import is_banned from aurweb.models.declarative import Base @@ -177,6 +178,39 @@ class User(Base): """ return self == user or self.is_trusted_user() or self.is_developer() + def voted_for(self, package) -> bool: + """ Has this User voted for package? """ + from aurweb.models.package_vote import PackageVote + return bool(package.PackageBase.package_votes.filter( + PackageVote.UsersID == self.ID + ).scalar()) + + def notified(self, package) -> bool: + """ Is this User being notified about package? """ + from aurweb.models.package_notification import PackageNotification + return bool(package.PackageBase.package_notifications.filter( + PackageNotification.UserID == self.ID + ).scalar()) + + def packages(self): + """ Returns an ORM query to Package objects owned by this user. + + This should really be replaced with an internal ORM join + configured for the User model. This has not been done yet + due to issues I've been encountering in the process, so + sticking with this function until we can properly implement it. + + :return: ORM query of User-packaged or maintained Package objects + """ + from aurweb.models.package import Package + from aurweb.models.package_base import PackageBase + return db.query(Package).join(PackageBase).filter( + or_( + PackageBase.PackagerUID == self.ID, + PackageBase.MaintainerUID == self.ID + ) + ) + def __repr__(self): return "" % ( self.ID, str(self.AccountType), self.Username) diff --git a/test/test_user.py b/test/test_user.py index 9ab40801..7756cff3 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -12,6 +12,10 @@ import aurweb.config from aurweb.db import commit, create, query from aurweb.models.account_type import AccountType from aurweb.models.ban import Ban +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.package_notification import PackageNotification +from aurweb.models.package_vote import PackageVote from aurweb.models.session import Session from aurweb.models.ssh_pub_key import SSHPubKey from aurweb.models.user import User @@ -25,7 +29,16 @@ account_type = user = None def setup(): global account_type, user - setup_test_db("Users", "Sessions", "Bans", "SSHPubKeys") + setup_test_db( + User.__tablename__, + Session.__tablename__, + Ban.__tablename__, + SSHPubKey.__tablename__, + Package.__tablename__, + PackageBase.__tablename__, + PackageVote.__tablename__, + PackageNotification.__tablename__ + ) account_type = query(AccountType, AccountType.AccountType == "User").first() @@ -249,3 +262,24 @@ def test_user_is_developer(): user.AccountType = dev_type commit() assert user.is_developer() is True + + +def test_user_voted_for(): + now = int(datetime.utcnow().timestamp()) + pkgbase = create(PackageBase, Name="pkg1", Maintainer=user) + pkg = create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + create(PackageVote, PackageBase=pkgbase, User=user, VoteTS=now) + assert user.voted_for(pkg) + + +def test_user_notified(): + pkgbase = create(PackageBase, Name="pkg1", Maintainer=user) + pkg = create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + create(PackageNotification, PackageBase=pkgbase, User=user) + assert user.notified(pkg) + + +def test_user_packages(): + pkgbase = create(PackageBase, Name="pkg1", Maintainer=user) + pkg = create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + assert pkg in user.packages() From 5a175bd92a791b45ad1d224cca133645abfcaa0d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 9 Aug 2021 23:43:48 -0700 Subject: [PATCH 293/844] routers.html: add authenticated dashboard to homepage Signed-off-by: Kevin Morris --- aurweb/routers/html.py | 39 +++++++++++ templates/dashboard.html | 54 ++++++++++++++++ templates/index.html | 6 +- templates/partials/packages/requests.html | 38 +++++++++++ templates/partials/packages/results.html | 55 ++++++++++++++++ test/test_homepage.py | 79 ++++++++++++++++++++++- 6 files changed, 269 insertions(+), 2 deletions(-) create mode 100644 templates/dashboard.html create mode 100644 templates/partials/packages/requests.html create mode 100644 templates/partials/packages/results.html diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index ae012901..c2375f69 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -9,11 +9,15 @@ from fastapi.responses import HTMLResponse, RedirectResponse from sqlalchemy import and_, or_ import aurweb.config +import aurweb.models.package_request from aurweb import db, util from aurweb.cache import db_count_cache from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID +from aurweb.models.package import Package from aurweb.models.package_base import PackageBase +from aurweb.models.package_comaintainer import PackageComaintainer +from aurweb.models.package_request import PackageRequest from aurweb.models.user import User from aurweb.packages.util import updated_packages from aurweb.templates import make_context, render_template @@ -132,6 +136,41 @@ async def index(request: Request): # Get the 15 most recently updated packages. context["package_updates"] = updated_packages(15, updates_expire) + if request.user.is_authenticated(): + # Authenticated users get a few extra pieces of data for + # the dashboard display. + packages = db.query(Package).join(PackageBase) + + maintained = packages.join( + User, PackageBase.MaintainerUID == User.ID + ).filter( + PackageBase.MaintainerUID == request.user.ID + ) + + context["flagged_packages"] = maintained.filter( + PackageBase.OutOfDateTS.isnot(None) + ).order_by( + PackageBase.ModifiedTS.desc(), Package.Name.asc() + ).limit(50).all() + + archive_time = aurweb.config.getint('options', 'request_archive_time') + start = now - archive_time + context["package_requests"] = request.user.package_requests.filter( + PackageRequest.RequestTS >= start + ).limit(50).all() + + # Packages that the request user maintains or comaintains. + context["packages"] = maintained.order_by( + PackageBase.ModifiedTS.desc(), Package.Name.desc() + ).limit(50).all() + + # Any packages that the request user comaintains. + context["comaintained"] = packages.join( + PackageComaintainer).filter( + PackageComaintainer.UsersID == request.user.ID).order_by( + PackageBase.ModifiedTS.desc(), Package.Name.desc() + ).limit(50).all() + return render_template(request, "index.html", context) diff --git a/templates/dashboard.html b/templates/dashboard.html new file mode 100644 index 00000000..5ad89992 --- /dev/null +++ b/templates/dashboard.html @@ -0,0 +1,54 @@ +
    +

    {{ "Dashboard" | tr }}

    + +

    {{ "My Flagged Packages" | tr }}

    + {% if not flagged_packages %} +

    {{ "No packages matched your search criteria." | tr }}

    + {% else %} + {% with table_id = "flagged-packages", packages = flagged_packages %} + {% include 'partials/packages/results.html' %} + {% endwith %} + {% endif %} + +

    {{ "My Requests" | tr }}

    + {% if not package_requests %} +

    {{ "No requests matched your search criteria." | tr }}

    + {% else %} + {% with requests = package_requests %} + {% include 'partials/packages/requests.html' %} + {% endwith %} + {% endif %} +
    + +
    +

    {{ "My Packages" | tr }}

    +

    + + {{ "Search for packages I maintain" | tr }} + +

    + {% if not packages %} +

    {{ "No packages matched your search criteria." | tr }}

    + {% else %} + {% with table_id = "my-packages" %} + {% include 'partials/packages/results.html' %} + {% endwith %} + {% endif %} +
    + +
    +

    {{ "Co-Maintained Packages" | tr }}

    +

    + + {{ "Search for packages I co-maintain" | tr }} + +

    + {% if not comaintained %} +

    {{ "No packages matched your search criteria." | tr }}

    + {% else %} + {% with table_id = "comaintained-packages", packages = comaintained %} + {% include 'partials/packages/results.html' %} + {% endwith %} + {% endif %} +
    + diff --git a/templates/index.html b/templates/index.html index e50a99cd..0b6eda50 100644 --- a/templates/index.html +++ b/templates/index.html @@ -3,7 +3,11 @@ {% block pageContent %}
    - {% include 'home.html' %} + {% if request.user.is_authenticated() %} + {% include 'dashboard.html' %} + {% else %} + {% include 'home.html' %} + {% endif %}
    diff --git a/templates/partials/packages/requests.html b/templates/partials/packages/requests.html new file mode 100644 index 00000000..5239ca72 --- /dev/null +++ b/templates/partials/packages/requests.html @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + {% for request in requests %} + {% set requested = request.RequestTS | dt | as_timezone(timezone) %} + + + + + + + + + {% endfor %} + + + +
    {{ "Package" | tr }}{{ "Type" | tr }}{{ "Comments" | tr }}{{ "Filed by" | tr }}{{ "Date" | tr }}{{ "Status" | tr }}
    + + {{ request.PackageBase.Name }} + + {{ request.RequestType.name_display() | tr }}{{ request.Comments }} + + {{ request.User.Username }} + + {{ requested.strftime("%Y-%m-%d %H:%M") }}{{ request.status_display() | tr }}
    diff --git a/templates/partials/packages/results.html b/templates/partials/packages/results.html new file mode 100644 index 00000000..005bd5a9 --- /dev/null +++ b/templates/partials/packages/results.html @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + {% for pkg in packages %} + {% set flagged = pkg.PackageBase.OutOfDateTS %} + + + {% if flagged %} + + {% else %} + + {% endif %} + + + + + + + + {% endfor %} + +
    {{ "Name" | tr }}{{ "Version" | tr }}{{ "Votes" | tr }}{{ "Popularity" | tr }}{{ "Voted" | tr }}{{ "Notify" | tr }}{{ "Description" | tr }}{{ "Maintainer" | tr }}
    + + {{ pkg.Name }} + + {{ pkg.Version }}{{ pkg.Version }}{{ pkg.PackageBase.NumVotes }} + {{ pkg.PackageBase.Popularity | number_format(2) }} + + + {% if request.user.voted_for(pkg) %} + {{ "Yes" | tr }} + {% endif %} + + + {% if request.user.notified(pkg) %} + {{ "Yes" | tr }} + {% endif %} + {{ pkg.Description or '' }} + {% set maintainer = pkg.PackageBase.Maintainer %} + + {{ maintainer.Username }} + +
    diff --git a/test/test_homepage.py b/test/test_homepage.py index a629b98c..2cd6682f 100644 --- a/test/test_homepage.py +++ b/test/test_homepage.py @@ -13,10 +13,14 @@ from aurweb.asgi import app from aurweb.models.account_type import USER_ID from aurweb.models.package import Package from aurweb.models.package_base import PackageBase +from aurweb.models.package_comaintainer import PackageComaintainer +from aurweb.models.package_request import PackageRequest +from aurweb.models.request_type import DELETION_ID, RequestType from aurweb.models.user import User from aurweb.redis import redis_connection from aurweb.testing import setup_test_db from aurweb.testing.html import parse_root +from aurweb.testing.requests import Request client = TestClient(app) @@ -26,7 +30,9 @@ def setup(): yield setup_test_db( User.__tablename__, Package.__tablename__, - PackageBase.__tablename__ + PackageBase.__tablename__, + PackageComaintainer.__tablename__, + PackageRequest.__tablename__ ) @@ -149,3 +155,74 @@ def test_homepage_updates(redis, packages): for i, expected in enumerate(expectations): pkgname = updates[i].xpath('./td/a').pop(0) assert pkgname.text.strip() == expected + + +def test_homepage_dashboard(redis, packages, user): + # Create Comaintainer records for all of the packages. + for pkg in packages: + db.create(PackageComaintainer, PackageBase=pkg.PackageBase, + User=user, Priority=1, autocommit=False) + db.commit() + + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + response = request.get("/", cookies=cookies) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + + # Assert some expectations that we end up getting all fifty + # packages in the "My Packages" table. + expectations = [f"pkg_{i}" for i in range(50 - 1, 0, -1)] + my_packages = root.xpath('//table[@id="my-packages"]/tbody/tr') + for i, expected in enumerate(expectations): + name, version, votes, pop, voted, notify, desc, maint \ + = my_packages[i].xpath('./td') + assert name.xpath('./a').pop(0).text.strip() == expected + + # Do the same for the Comaintained Packages table. + my_packages = root.xpath('//table[@id="comaintained-packages"]/tbody/tr') + for i, expected in enumerate(expectations): + name, version, votes, pop, voted, notify, desc, maint \ + = my_packages[i].xpath('./td') + assert name.xpath('./a').pop(0).text.strip() == expected + + +def test_homepage_dashboard_requests(redis, packages, user): + now = int(datetime.utcnow().timestamp()) + + pkg = packages[0] + reqtype = db.query(RequestType, RequestType.ID == DELETION_ID).first() + pkgreq = db.create(PackageRequest, PackageBase=pkg.PackageBase, + PackageBaseName=pkg.PackageBase.Name, + User=user, Comments=str(), + ClosureComment=str(), RequestTS=now, + RequestType=reqtype) + + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + response = request.get("/", cookies=cookies) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + request = root.xpath('//table[@id="pkgreq-results"]/tbody/tr').pop(0) + pkgname = request.xpath('./td/a').pop(0) + assert pkgname.text.strip() == pkgreq.PackageBaseName + + +def test_homepage_dashboard_flagged_packages(redis, packages, user): + # Set the first Package flagged by setting its OutOfDateTS column. + pkg = packages[0] + pkg.PackageBase.OutOfDateTS = int(datetime.utcnow().timestamp()) + db.commit() + + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + response = request.get("/", cookies=cookies) + assert response.status_code == int(HTTPStatus.OK) + + # Check to see that the package showed up in the Flagged Packages table. + root = parse_root(response.text) + flagged_pkg = root.xpath('//table[@id="flagged-packages"]/tbody/tr').pop(0) + flagged_name = flagged_pkg.xpath('./td/a').pop(0) + assert flagged_name.text.strip() == pkg.Name From f086457741574867f30109dc58ae656684968e2e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 17 Aug 2021 21:52:59 -0700 Subject: [PATCH 294/844] aurweb.redis: Reduce logging Signed-off-by: Kevin Morris --- aurweb/redis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/redis.py b/aurweb/redis.py index 6b8dede4..6d3cff38 100644 --- a/aurweb/redis.py +++ b/aurweb/redis.py @@ -36,13 +36,13 @@ def redis_connection(): # pragma: no cover # If we haven't initialized redis yet, construct a pool. if disabled: - logger.debug("Initializing fake Redis instance.") if pool is None: + logger.debug("Initializing fake Redis instance.") pool = FakeConnectionPool() return pool.handle else: - logger.debug("Initializing real Redis instance.") if pool is None: + logger.debug("Initializing real Redis instance.") redis_addr = aurweb.config.get("options", "redis_address") pool = ConnectionPool.from_url(redis_addr) From 6eafb457ec18cefd9341e27d84b7be76abbcca29 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 20 Aug 2021 16:36:10 -0700 Subject: [PATCH 295/844] aurweb.util: fix code style violation Signed-off-by: Kevin Morris --- aurweb/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/util.py b/aurweb/util.py index d4a0b221..860bdd12 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -20,8 +20,8 @@ import aurweb.config def make_random_string(length): - return ''.join(random.choices(string.ascii_lowercase + - string.digits, k=length)) + return ''.join(random.choices(string.ascii_lowercase + + string.digits, k=length)) def make_nonce(length: int = 8): From a72ab61902961562048f487c2e102249b4a33964 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 25 Aug 2021 16:34:37 -0700 Subject: [PATCH 296/844] [FastAPI] fix dashboard template Some columns should only be shown when a user is authenticated. Signed-off-by: Kevin Morris --- templates/partials/packages/results.html | 52 ++++++++++++------------ 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/templates/partials/packages/results.html b/templates/partials/packages/results.html index 005bd5a9..80029d10 100644 --- a/templates/partials/packages/results.html +++ b/templates/partials/packages/results.html @@ -6,8 +6,10 @@

    {{ "Version" | tr }} {{ "Votes" | tr }} {{ "Popularity" | tr }}{{ "Voted" | tr }}{{ "Notify" | tr }}{{ "Voted" | tr }}{{ "Notify" | tr }}{{ "Description" | tr }} {{ "Maintainer" | tr }}
    {{ pkg.Version }}{{ pkg.Version }}{{ pkg.PackageBase.NumVotes }} - {{ pkg.PackageBase.Popularity | number_format(2) }} - - - {% if request.user.voted_for(pkg) %} - {{ "Yes" | tr }} - {% endif %} - - - {% if request.user.notified(pkg) %} - {{ "Yes" | tr }} - {% endif %} - {{ pkg.PackageBase.Popularity | number_format(2) }} + + {% if request.user.voted_for(pkg) %} + {{ "Yes" | tr }} + {% endif %} + + + {% if request.user.notified(pkg) %} + {{ "Yes" | tr }} + {% endif %} + {{ pkg.Description or '' }} {% set maintainer = pkg.PackageBase.Maintainer %} - - {{ maintainer.Username }} - + {% if maintainer %} + + {{ maintainer.Username }} + + {% else %} + {{ "orphan" | tr }} + {% endif %}
    {{ "Proposal" | tr }} {% set off_qs = "%s=%d" | format(off_param, off) %} - {% set by_qs = "%s=%s" | format(by_param, by_next | urlencode) %} + {% set by_qs = "%s=%s" | format(by_param, by_next | quote_plus) %} {{ "Start" | tr }} @@ -95,7 +95,7 @@ {% if off > 0 %} {% set off_qs = "%s=%d" | format(off_param, off - 10) %} - {% set by_qs = "%s=%s" | format(by_param, by | urlencode) %} + {% set by_qs = "%s=%s" | format(by_param, by | quote_plus) %} ‹ Back @@ -104,7 +104,7 @@ {% if off < total_votes - pp %} {% set off_qs = "%s=%d" | format(off_param, off + 10) %} - {% set by_qs = "%s=%s" | format(by_param, by | urlencode) %} + {% set by_qs = "%s=%s" | format(by_param, by | quote_plus) %} Next › From a114bd3e16e5df00cc2542e37cedb82bf99e9a84 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 28 Aug 2021 16:33:33 -0700 Subject: [PATCH 312/844] aurweb.util: add extend_query and to_qs helpers The first addition, extend_query, can be used to take an existing query parameter dictionary and inject an *additions as replacement key/value pairs. The second, to_qs, converts a query parameter dictionary to a query string in the form "a=b&c=d". These two functions simplify and make dedupe_qs and quote_plus more efficient in terms of constructing custom query string overrides. Signed-off-by: Kevin Morris --- aurweb/util.py | 14 +++++++++++++- test/test_util.py | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/aurweb/util.py b/aurweb/util.py index 494a988d..e993c440 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -8,7 +8,8 @@ import string from collections import OrderedDict from datetime import datetime -from urllib.parse import quote_plus, urlparse +from typing import Any, Dict +from urllib.parse import quote_plus, urlencode, urlparse from zoneinfo import ZoneInfo import fastapi @@ -148,6 +149,17 @@ def dedupe_qs(query_string: str, *additions): return '&'.join([f"{k}={quote_plus(v)}" for k, v in reversed(qs.items())]) +def extend_query(query: Dict[str, Any], *additions) -> Dict[str, Any]: + """ Add additionally key value pairs to query. """ + for k, v in list(additions): + query[k] = v + return query + + +def to_qs(query: Dict[str, Any]) -> str: + return urlencode(query, doseq=True) + + def get_vote(voteinfo, request: fastapi.Request): from aurweb.models.tu_vote import TUVote return voteinfo.tu_votes.filter(TUVote.User == request.user).first() diff --git a/test/test_util.py b/test/test_util.py index f54a98a0..06fc08d3 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -35,3 +35,18 @@ def test_dedupe_qs(): def test_number_format(): assert util.number_format(0.222, 2) == "0.22" assert util.number_format(0.226, 2) == "0.23" + + +def test_extend_query(): + """ Test extension of a query via extend_query. """ + query = {"a": "b"} + extended = util.extend_query(query, ("a", "c"), ("b", "d")) + assert extended.get("a") == "c" + assert extended.get("b") == "d" + + +def test_to_qs(): + """ Test conversion from a query dictionary to a query string. """ + query = {"a": "b", "c": [1, 2, 3]} + qs = util.to_qs(query) + assert qs == "a=b&c=1&c=2&c=3" From c9374732c013ba336cd1b3d6b3991fece9b2c934 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 31 Aug 2021 14:25:58 -0700 Subject: [PATCH 313/844] add filters for extend_query, to_qs New jinja2 filters: * `extend_query` -> `aurweb.util.extend_query` * `urlencode` -> `aurweb.util.to_qs` Signed-off-by: Kevin Morris --- aurweb/templates.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aurweb/templates.py b/aurweb/templates.py index a648d5a1..4a9927fb 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -31,6 +31,8 @@ _env.filters["tn"] = l10n.tn _env.filters["dt"] = util.timestamp_to_datetime _env.filters["as_timezone"] = util.as_timezone _env.filters["dedupe_qs"] = util.dedupe_qs +_env.filters["extend_query"] = util.extend_query +_env.filters["urlencode"] = util.to_qs _env.filters["quote_plus"] = quote_plus _env.filters["get_vote"] = util.get_vote _env.filters["number_format"] = util.number_format From 210e459ba90e274e585a6928585aa2362d168944 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 31 Aug 2021 14:27:16 -0700 Subject: [PATCH 314/844] Eradicate the `dedupe_qs` filter The new `extend_query` and `urlencode` filters are way cleaner ways to achieve what we did with `dedupe_qs`. Signed-off-by: Kevin Morris --- aurweb/routers/trusted_user.py | 13 ++++++------- aurweb/templates.py | 1 - aurweb/util.py | 28 ++-------------------------- templates/partials/tu/proposals.html | 6 +++--- test/test_util.py | 16 ---------------- 5 files changed, 11 insertions(+), 53 deletions(-) diff --git a/aurweb/routers/trusted_user.py b/aurweb/routers/trusted_user.py index fd5ebb04..61cfec6c 100644 --- a/aurweb/routers/trusted_user.py +++ b/aurweb/routers/trusted_user.py @@ -5,7 +5,6 @@ import typing from datetime import datetime from http import HTTPStatus -from urllib.parse import quote_plus from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import RedirectResponse, Response @@ -106,12 +105,12 @@ async def trusted_user(request: Request, context["current_by_next"] = "asc" if current_by == "desc" else "desc" context["past_by_next"] = "asc" if past_by == "desc" else "desc" - context["q"] = '&'.join([ - f"coff={current_off}", - f"cby={quote_plus(current_by)}", - f"poff={past_off}", - f"pby={quote_plus(past_by)}" - ]) + context["q"] = { + "coff": current_off, + "cby": current_by, + "poff": past_off, + "pby": past_by + } return render_template(request, "tu/index.html", context) diff --git a/aurweb/templates.py b/aurweb/templates.py index 4a9927fb..6a1b6a1c 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -30,7 +30,6 @@ _env.filters["tn"] = l10n.tn # Utility filters. _env.filters["dt"] = util.timestamp_to_datetime _env.filters["as_timezone"] = util.as_timezone -_env.filters["dedupe_qs"] = util.dedupe_qs _env.filters["extend_query"] = util.extend_query _env.filters["urlencode"] = util.to_qs _env.filters["quote_plus"] = quote_plus diff --git a/aurweb/util.py b/aurweb/util.py index e993c440..f9181811 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -6,10 +6,9 @@ import re import secrets import string -from collections import OrderedDict from datetime import datetime from typing import Any, Dict -from urllib.parse import quote_plus, urlencode, urlparse +from urllib.parse import urlencode, urlparse from zoneinfo import ZoneInfo import fastapi @@ -126,31 +125,8 @@ def as_timezone(dt: datetime, timezone: str): return dt.astimezone(tz=ZoneInfo(timezone)) -def dedupe_qs(query_string: str, *additions): - """ Dedupe keys found in a query string by rewriting it without - duplicates found while iterating from the end to the beginning, - using an ordered memo to track keys found and persist locations. - - That is, query string 'a=1&b=1&a=2' will be deduped and converted - to 'b=1&a=2'. - - :param query_string: An HTTP URL query string. - :param *additions: Optional additional fields to add to the query string. - :return: Deduped query string, including *additions at the tail. - """ - for addition in list(additions): - query_string += f"&{addition}" - - qs = OrderedDict() - for item in reversed(query_string.split('&')): - key, value = item.split('=') - if key not in qs: - qs[key] = value - return '&'.join([f"{k}={quote_plus(v)}" for k, v in reversed(qs.items())]) - - def extend_query(query: Dict[str, Any], *additions) -> Dict[str, Any]: - """ Add additionally key value pairs to query. """ + """ Add additional key value pairs to query. """ for k, v in list(additions): query[k] = v return query diff --git a/templates/partials/tu/proposals.html b/templates/partials/tu/proposals.html index ab90444e..40eba22b 100644 --- a/templates/partials/tu/proposals.html +++ b/templates/partials/tu/proposals.html @@ -24,7 +24,7 @@ {% set off_qs = "%s=%d" | format(off_param, off) %} {% set by_qs = "%s=%s" | format(by_param, by_next | quote_plus) %} - + {{ "Start" | tr }} {{ pkg.PackageBase.Popularity | number_format(2) }} - - {% if request.user.voted_for(pkg) %} + {# If I voted, display "Yes". #} + {% if pkg.PackageBase.ID in votes %} {{ "Yes" | tr }} {% endif %} - - {% if request.user.notified(pkg) %} + {# If I'm being notified, display "Yes". #} + {% if pkg.PackageBase.ID in notified %} {{ "Yes" | tr }} {% endif %}
    + + + {% if request.user.is_authenticated() %} + + {% endif %} + + + + + {% if request.user.is_authenticated() %} + + + {% endif %} + + + + + + {% for pkg in packages %} + {% set flagged = pkg.PackageBase.OutOfDateTS %} + + {% if request.user.is_authenticated() %} + + {% endif %} + + {% if flagged %} + + {% else %} + + {% endif %} + + + {% if request.user.is_authenticated() %} + + + {% endif %} + + + + {% endfor %} + +
    + {% set order = SO %} + {% if SB == "n" %} + {% set order = "d" if order == "a" else "a" %} + {% endif %} + + {{ "Name" | tr }} + + {{ "Version" | tr }} + {% set order = SO %} + {% if SB == "v" %} + {% set order = "d" if order == "a" else "a" %} + {% endif %} + + {{ "Votes" | tr }} + + + {% set order = SO %} + {% if SB == "p" %} + {% set order = "d" if order == "a" else "a" %} + {% endif %} + {{ "Popularity" | tr }}? + + {% set order = SO %} + {% if SB == "w" %} + {% set order = "d" if order == "a" else "a" %} + {% endif %} + + {{ "Voted" | tr }} + + + {% set order = SO %} + {% if SB == "o" %} + {% set order = "d" if order == "a" else "a" %} + {% endif %} + + {{ "Notify" | tr }} + + {{ "Description" | tr }} + {% set order = SO %} + {% if SB == "m" %} + {% set order = "d" if order == "a" else "a" %} + {% endif %} + + {{ "Maintainer" | tr }} + +
    + + + + {{ pkg.Name }} + + {{ pkg.Version }}{{ pkg.Version }}{{ pkg.PackageBase.NumVotes }} + {{ pkg.PackageBase.Popularity | number_format(2) }} + + {% if pkg.PackageBase.ID in voted %} + {{ "Yes" | tr }} + {% endif %} + + {% if pkg.PackageBase.ID in notified %} + {{ "Yes" | tr }} + {% endif %} + {{ pkg.Description or '' }} + {% set maintainer = pkg.PackageBase.Maintainer %} + {% if maintainer %} + + {{ maintainer.Username }} + + {% else %} + {{ "orphan" | tr }} + {% endif %} +
    diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 8a468c15..fb45af88 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1,5 +1,8 @@ +import re + from datetime import datetime from http import HTTPStatus +from typing import List import pytest @@ -11,11 +14,14 @@ from aurweb.models.dependency_type import DependencyType from aurweb.models.official_provider import OfficialProvider from aurweb.models.package import Package from aurweb.models.package_base import PackageBase +from aurweb.models.package_comaintainer import PackageComaintainer from aurweb.models.package_comment import PackageComment from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_keyword import PackageKeyword +from aurweb.models.package_notification import PackageNotification from aurweb.models.package_relation import PackageRelation from aurweb.models.package_request import PackageRequest +from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import PROVIDES_ID, RelationType from aurweb.models.request_type import DELETION_ID, RequestType from aurweb.models.user import User @@ -64,6 +70,9 @@ def setup(): PackageDependency.__tablename__, PackageRelation.__tablename__, PackageKeyword.__tablename__, + PackageVote.__tablename__, + PackageNotification.__tablename__, + PackageComaintainer.__tablename__, OfficialProvider.__tablename__ ) @@ -101,16 +110,41 @@ def maintainer() -> User: @pytest.fixture def package(maintainer: User) -> Package: """ Yield a Package created by user. """ + now = int(datetime.utcnow().timestamp()) with db.begin(): pkgbase = db.create(PackageBase, Name="test-package", - Maintainer=maintainer) + Maintainer=maintainer, + Packager=maintainer, + Submitter=maintainer, + ModifiedTS=now) package = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) yield package +@pytest.fixture +def packages(maintainer: User) -> List[Package]: + """ Yield 55 packages named pkg_0 .. pkg_54. """ + packages_ = [] + now = int(datetime.utcnow().timestamp()) + with db.begin(): + for i in range(55): + pkgbase = db.create(PackageBase, + Name=f"pkg_{i}", + Maintainer=maintainer, + Packager=maintainer, + Submitter=maintainer, + ModifiedTS=now) + package = db.create(Package, + PackageBase=pkgbase, + Name=f"pkg_{i}") + packages_.append(package) + + yield packages_ + + def test_package_not_found(client: TestClient): with client as request: resp = request.get("/packages/not_found") @@ -133,7 +167,7 @@ def test_package_official_not_found(client: TestClient, package: Package): def test_package(client: TestClient, package: Package): - """ Test a single /packages/{name} route. """ + """ Test a single / packages / {name} route. """ with client as request: resp = request.get(package_endpoint(package)) @@ -376,3 +410,505 @@ def test_pkgbase(client: TestClient, package: Package): pkgs = root.findall('.//div[@id="pkgs"]/ul/li/a') for i, name in enumerate(expected): assert pkgs[i].text.strip() == name + + +def test_packages(client: TestClient, packages: List[Package]): + """ Test the / packages route with defaults. + + Defaults: + 50 results per page + offset of 0 + """ + with client as request: + response = request.get("/packages", params={ + "SeB": "X" # "X" isn't valid, defaults to "nd" + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + stats = root.xpath('//div[@class="pkglist-stats"]/p')[0] + pager_text = re.sub(r'\s+', " ", stats.text.replace("\n", "").strip()) + assert pager_text == "55 packages found. Page 1 of 2." + + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 50 # Default per-page + + +def test_packages_search_by_name(client: TestClient, packages: List[Package]): + with client as request: + response = request.get("/packages", params={ + "SeB": "n", + "K": "pkg_" + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 50 # Default per-page + + +def test_packages_search_by_exact_name(client: TestClient, + packages: List[Package]): + with client as request: + response = request.get("/packages", params={ + "SeB": "N", + "K": "pkg_" + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + + # There is no package named exactly 'pkg_', we get 0 results. + assert len(rows) == 0 + + with client as request: + response = request.get("/packages", params={ + "SeB": "N", + "K": "pkg_1" + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + + # There's just one package named 'pkg_1', we get 1 result. + assert len(rows) == 1 + + +def test_packages_search_by_pkgbase(client: TestClient, + packages: List[Package]): + with client as request: + response = request.get("/packages", params={ + "SeB": "b", + "K": "pkg_" + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 50 + + +def test_packages_search_by_exact_pkgbase(client: TestClient, + packages: List[Package]): + with client as request: + response = request.get("/packages", params={ + "SeB": "B", + "K": "pkg_" + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 0 + + with client as request: + response = request.get("/packages", params={ + "SeB": "B", + "K": "pkg_1" + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 + + +def test_packages_search_by_keywords(client: TestClient, + packages: List[Package]): + # None of our packages have keywords, so this query should return nothing. + with client as request: + response = request.get("/packages", params={ + "SeB": "k", + "K": "testKeyword" + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 0 + + # But now, let's create the keyword for the first package. + package = packages[0] + with db.begin(): + db.create(PackageKeyword, + PackageBase=package.PackageBase, + Keyword="testKeyword") + + # And request packages with that keyword, we should get 1 result. + with client as request: + response = request.get("/packages", params={ + "SeB": "k", + "K": "testKeyword" + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 + + +def test_packages_search_by_maintainer(client: TestClient, + maintainer: User, + package: Package): + with client as request: + response = request.get("/packages", params={ + "SeB": "m", + "K": maintainer.Username + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 + + +def test_packages_search_by_comaintainer(client: TestClient, + maintainer: User, + package: Package): + # Nobody's a comaintainer yet. + with client as request: + response = request.get("/packages", params={ + "SeB": "c", + "K": maintainer.Username + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 0 + + # Now, we create a comaintainer. + with db.begin(): + db.create(PackageComaintainer, + PackageBase=package.PackageBase, + User=maintainer, + Priority=1) + + # Then test that it's returned by our search. + with client as request: + response = request.get("/packages", params={ + "SeB": "c", + "K": maintainer.Username + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 + + +def test_packages_search_by_co_or_maintainer(client: TestClient, + maintainer: User, + package: Package): + with client as request: + response = request.get("/packages", params={ + "SeB": "M", + "SB": "BLAH", # Invalid SB; gets reset to default "n". + "K": maintainer.Username + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 + + with db.begin(): + user = db.create(User, Username="comaintainer", + Email="comaintainer@example.org", + Passwd="testPassword") + db.create(PackageComaintainer, + PackageBase=package.PackageBase, + User=user, + Priority=1) + + with client as request: + response = request.get("/packages", params={ + "SeB": "M", + "K": user.Username + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 + + +def test_packages_search_by_submitter(client: TestClient, + maintainer: User, + package: Package): + with client as request: + response = request.get("/packages", params={ + "SeB": "s", + "K": maintainer.Username + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 + + +def test_packages_sort_by_votes(client: TestClient, + maintainer: User, + packages: List[Package]): + # Set the first package's NumVotes to 1. + with db.begin(): + packages[0].PackageBase.NumVotes = 1 + + # Test that, by default, the first result is what we just set above. + with client as request: + response = request.get("/packages", params={ + "SB": "v" # Votes. + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + votes = rows[0].xpath('./td')[2] # The third column of the first row. + assert votes.text.strip() == "1" + + # Now, test that with an ascending order, the last result is + # the one we set, since the default (above) is descending. + with client as request: + response = request.get("/packages", params={ + "SB": "v", # Votes. + "SO": "a", # Ascending. + "O": "50" # Second page. + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + votes = rows[-1].xpath('./td')[2] # The third column of the last row. + assert votes.text.strip() == "1" + + +def test_packages_sort_by_popularity(client: TestClient, + maintainer: User, + packages: List[Package]): + # Set the first package's Popularity to 0.50. + with db.begin(): + packages[0].PackageBase.Popularity = "0.50" + + # Test that, by default, the first result is what we just set above. + with client as request: + response = request.get("/packages", params={ + "SB": "p" # Popularity + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + pop = rows[0].xpath('./td')[3] # The fourth column of the first row. + assert pop.text.strip() == "0.50" + + +def test_packages_sort_by_voted(client: TestClient, + maintainer: User, + packages: List[Package]): + now = int(datetime.utcnow().timestamp()) + with db.begin(): + db.create(PackageVote, PackageBase=packages[0].PackageBase, + User=maintainer, VoteTS=now) + + # Test that, by default, the first result is what we just set above. + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + with client as request: + response = request.get("/packages", params={ + "SB": "w", # Voted + "SO": "d" # Descending, Voted first. + }, cookies=cookies) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + voted = rows[0].xpath('./td')[5] # The sixth column of the first row. + assert voted.text.strip() == "Yes" + + # Conversely, everything else was not voted on. + voted = rows[1].xpath('./td')[5] # The sixth column of the second row. + assert voted.text.strip() == str() # Empty. + + +def test_packages_sort_by_notify(client: TestClient, + maintainer: User, + packages: List[Package]): + db.create(PackageNotification, + PackageBase=packages[0].PackageBase, + User=maintainer) + + # Test that, by default, the first result is what we just set above. + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + with client as request: + response = request.get("/packages", params={ + "SB": "o", # Voted + "SO": "d" # Descending, Voted first. + }, cookies=cookies) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + notify = rows[0].xpath('./td')[6] # The sixth column of the first row. + assert notify.text.strip() == "Yes" + + # Conversely, everything else was not voted on. + notify = rows[1].xpath('./td')[6] # The sixth column of the second row. + assert notify.text.strip() == str() # Empty. + + +def test_packages_sort_by_maintainer(client: TestClient, + maintainer: User, + package: Package): + """ Sort a package search by the maintainer column. """ + + # Create a second package, so the two can be ordered and checked. + with db.begin(): + maintainer2 = db.create(User, Username="maintainer2", + Email="maintainer2@example.org", + Passwd="testPassword") + base2 = db.create(PackageBase, Name="pkg_2", Maintainer=maintainer2, + Submitter=maintainer2, Packager=maintainer2) + db.create(Package, Name="pkg_2", PackageBase=base2) + + # Check the descending order route. + with client as request: + response = request.get("/packages", params={ + "SB": "m", + "SO": "d" + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + col = rows[0].xpath('./td')[5].xpath('./a')[0] # Last column. + + assert col.text.strip() == maintainer.Username + + # On the other hand, with ascending, we should get reverse ordering. + with client as request: + response = request.get("/packages", params={ + "SB": "m", + "SO": "a" + }) + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + col = rows[0].xpath('./td')[5].xpath('./a')[0] # Last column. + + assert col.text.strip() == maintainer2.Username + + +def test_packages_sort_by_last_modified(client: TestClient, + packages: List[Package]): + now = int(datetime.utcnow().timestamp()) + # Set the first package's ModifiedTS to be 1000 seconds before now. + package = packages[0] + with db.begin(): + package.PackageBase.ModifiedTS = now - 1000 + + with client as request: + response = request.get("/packages", params={ + "SB": "l", + "SO": "a" # Ascending; oldest modification first. + }) + assert response.status_code == int(HTTPStatus.OK) + + # We should have 50 (default per page) results. + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 50 + + # Let's assert that the first item returned was the one we modified above. + row = rows[0] + col = row.xpath('./td')[0].xpath('./a')[0] + assert col.text.strip() == package.Name + + +def test_packages_flagged(client: TestClient, maintainer: User, + packages: List[Package]): + package = packages[0] + + now = int(datetime.utcnow().timestamp()) + + with db.begin(): + package.PackageBase.OutOfDateTS = now + package.PackageBase.Flagger = maintainer + + with client as request: + response = request.get("/packages", params={ + "outdated": "on" + }) + assert response.status_code == int(HTTPStatus.OK) + + # We should only get one result from this query; the package we flagged. + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 + + with client as request: + response = request.get("/packages", params={ + "outdated": "off" + }) + assert response.status_code == int(HTTPStatus.OK) + + # In this case, we should get 54 results, which means that the first + # page will have 50 results (55 packages - 1 outdated = 54 not outdated). + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 50 + + +def test_packages_orphans(client: TestClient, packages: List[Package]): + package = packages[0] + with db.begin(): + package.PackageBase.Maintainer = None + + with client as request: + response = request.get("/packages", params={"submit": "Orphans"}) + assert response.status_code == int(HTTPStatus.OK) + + # We only have one orphan. Let's make sure that's what is returned. + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 + + +def test_packages_per_page(client: TestClient, maintainer: User): + """ Test the ability for /packages to deal with the PP query + argument specifications (50, 100, 250; default: 50). """ + with db.begin(): + for i in range(255): + base = db.create(PackageBase, Name=f"pkg_{i}", + Maintainer=maintainer, + Submitter=maintainer, + Packager=maintainer) + db.create(Package, PackageBase=base, Name=base.Name) + + # Test default case, PP of 50. + with client as request: + response = request.get("/packages", params={"PP": 50}) + assert response.status_code == int(HTTPStatus.OK) + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 50 + + # Alright, test the next case, PP of 100. + with client as request: + response = request.get("/packages", params={"PP": 100}) + assert response.status_code == int(HTTPStatus.OK) + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 100 + + # And finally, the last case, a PP of 250. + with client as request: + response = request.get("/packages", params={"PP": 250}) + assert response.status_code == int(HTTPStatus.OK) + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 250 diff --git a/web/html/css/aurweb.css b/web/html/css/aurweb.css index b36dbd4d..62179769 100644 --- a/web/html/css/aurweb.css +++ b/web/html/css/aurweb.css @@ -222,3 +222,10 @@ button[type="reset"] { text-align: right; } +input#search-action-submit { + width: 80px; +} + +.success { + color: green; +} From 7e589863561a0777f8feb9f4b11f505715d80841 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 20 Sep 2021 01:30:12 -0700 Subject: [PATCH 348/844] feat: add util/adduser.py database tooling script We'll need to add tests for these things at some point. However, I'd like to include this script in here immediately for ease of testing or administration in general. Signed-off-by: Kevin Morris --- util/adduser.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 util/adduser.py diff --git a/util/adduser.py b/util/adduser.py new file mode 100644 index 00000000..7e35d13d --- /dev/null +++ b/util/adduser.py @@ -0,0 +1,67 @@ +import argparse +import sys +import traceback + +from aurweb import db +from aurweb.models.account_type import AccountType +from aurweb.models.ssh_pub_key import SSHPubKey, get_fingerprint +from aurweb.models.user import User + + +def parse_args(): + parser = argparse.ArgumentParser(description="aurweb-adduser options") + + parser.add_argument("-u", "--username", help="Username", required=True) + parser.add_argument("-e", "--email", help="Email", required=True) + parser.add_argument("-p", "--password", help="Password", required=True) + parser.add_argument("-r", "--realname", help="Real Name") + parser.add_argument("-i", "--ircnick", help="IRC Nick") + parser.add_argument("--pgp-key", help="PGP Key Fingerprint") + parser.add_argument("--ssh-pubkey", help="SSH PubKey") + + parser.add_argument("-t", "--type", help="Account Type", + choices=[ + "User", + "Trusted User", + "Developer", + "Trusted User & Developer" + ], default="User") + + return parser.parse_args() + + +def main(): + args = parse_args() + + type = db.query(AccountType, + AccountType.AccountType == args.type).first() + with db.begin(): + user = db.create(User, Username=args.username, + Email=args.email, Passwd=args.password, + RealName=args.realname, IRCNick=args.ircnick, + PGPKey=args.pgp_key, AccountType=type) + + if args.ssh_pubkey: + pubkey = args.ssh_pubkey.strip() + + # Remove host from the pubkey if it's there. + pubkey = ' '.join(pubkey.split(' ')[:2]) + + with db.begin(): + db.create(SSHPubKey, + User=user, + PubKey=pubkey, + Fingerprint=get_fingerprint(pubkey)) + + print(user.json()) + return 0 + + +if __name__ == "__main__": + e = 1 + try: + e = main() + except Exception: + traceback.print_exc() + e = 1 + sys.exit(e) From dc5dc233ec120ace46e870a057b632b4d5c8149c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 21 Sep 2021 00:01:39 -0700 Subject: [PATCH 349/844] .gitlab-ci.yml: add coverage regex This was required for the GitLab coverage badge to get the % of coverage. Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ffea5308..a8ddf08f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,6 +36,7 @@ test: - isort --check-only aurweb # Assert no isort violations in aurweb. - isort --check-only test # Assert no flake8 violations in test. - isort --check-only migrations # Assert no flake8 violations in migrations. + coverage: '/TOTAL.*\s+(\d+\%)/' artifacts: reports: cobertura: coverage.xml From fbd91f346a69e41f44407fc568343c469ea59460 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 22 Sep 2021 18:33:58 -0700 Subject: [PATCH 350/844] feat(FastAPI): add /pkgbase/{name}/voters (get) Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 9 +++++++++ templates/pkgbase/voters.html | 27 +++++++++++++++++++++++++++ test/test_packages_routes.py | 18 ++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 templates/pkgbase/voters.html diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 3eda2539..72cd8c99 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -195,3 +195,12 @@ async def package_base(request: Request, name: str) -> Response: context["packages"] = pkgbase.packages.all() return render_template(request, "pkgbase.html", context) + + +@router.get("/pkgbase/{name}/voters") +async def package_base_voters(request: Request, name: str) -> Response: + # Get the PackageBase. + pkgbase = get_pkgbase(name) + context = make_context(request, "Voters") + context["pkgbase"] = pkgbase + return render_template(request, "pkgbase/voters.html", context) diff --git a/templates/pkgbase/voters.html b/templates/pkgbase/voters.html new file mode 100644 index 00000000..be86f01f --- /dev/null +++ b/templates/pkgbase/voters.html @@ -0,0 +1,27 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} +
    +

    + {{ "Votes" | tr }} for + + {{ pkgbase.Name }} + +

    + +
    +
      + {% for pkg_vote in pkgbase.package_votes %} +
    • + + {{ pkg_vote.User.Username }} + + + {% set voted_at = pkg_vote.VoteTS | dt | as_timezone(timezone) %} + ({{ voted_at.strftime("%Y-%m-%d %H:%M") }}) +
    • + {% endfor %} +
    +
    +
    +{% endblock %} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index fb45af88..2190dc18 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -912,3 +912,21 @@ def test_packages_per_page(client: TestClient, maintainer: User): root = parse_root(response.text) rows = root.xpath('//table[@class="results"]/tbody/tr') assert len(rows) == 250 + + +def test_pkgbase_voters(client: TestClient, maintainer: User, package: Package): + pkgbase = package.PackageBase + endpoint = f"/pkgbase/{pkgbase.Name}/voters" + + now = int(datetime.utcnow().timestamp()) + with db.begin(): + db.create(PackageVote, User=maintainer, PackageBase=pkgbase, + VoteTS=now) + + with client as request: + resp = request.get(endpoint) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + rows = root.xpath('//div[@class="box"]//ul/li') + assert len(rows) == 1 From ad9997c48ff39d4412e888470b980330ea28e3ea Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 2 Oct 2021 12:59:49 -0700 Subject: [PATCH 351/844] feat(Docker): build aurweb:latest via docker-compose build Users can now build the required image by running (in the aurweb root): $ docker-compose build Signed-off-by: Kevin Morris --- docker-compose.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 309e95fe..8e2d91d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,7 @@ version: "3.8" services: ca: + build: . image: aurweb:latest init: true entrypoint: /docker/ca-entrypoint.sh @@ -30,6 +31,7 @@ services: - ./cache:/cache memcached: + build: . image: aurweb:latest init: true command: /docker/scripts/run-memcached.sh @@ -37,6 +39,7 @@ services: test: "bash /docker/health/memcached.sh" redis: + build: . image: aurweb:latest init: true entrypoint: /docker/redis-entrypoint.sh @@ -47,6 +50,7 @@ services: - "16379:6379" mariadb: + build: . image: aurweb:latest init: true entrypoint: /docker/mariadb-entrypoint.sh @@ -62,6 +66,7 @@ services: test: "bash /docker/health/mariadb.sh" mariadb_init: + build: . image: aurweb:latest init: true environment: @@ -73,6 +78,7 @@ services: condition: service_healthy git: + build: . image: aurweb:latest init: true environment: @@ -92,6 +98,7 @@ services: - ./cache:/cache smartgit: + build: . image: aurweb:latest init: true environment: @@ -109,6 +116,7 @@ services: - smartgit_run:/var/run/smartgit cgit-php: + build: . image: aurweb:latest init: true environment: @@ -124,6 +132,7 @@ services: - git_data:/aurweb/aur.git cgit-fastapi: + build: . image: aurweb:latest init: true environment: @@ -139,6 +148,7 @@ services: - git_data:/aurweb/aur.git php-fpm: + build: . image: aurweb:latest init: true environment: @@ -168,6 +178,7 @@ services: - "19000:9000" fastapi: + build: . image: aurweb:latest init: true environment: @@ -197,6 +208,7 @@ services: - "18000:8000" nginx: + build: . image: aurweb:latest init: true environment: @@ -229,6 +241,7 @@ services: - smartgit_run:/var/run/smartgit sharness: + build: . image: aurweb:latest init: true environment: @@ -252,6 +265,7 @@ services: - ./templates:/aurweb/templates pytest-mysql: + build: . image: aurweb:latest init: true environment: @@ -276,6 +290,7 @@ services: - ./templates:/aurweb/templates pytest-sqlite: + build: . image: aurweb:latest init: true environment: @@ -296,6 +311,7 @@ services: - ./templates:/aurweb/templates test: + build: . image: aurweb:latest init: true environment: From 3b1809e2ea8d7adcafaff09664569c75c35791e3 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 2 Oct 2021 13:26:05 -0700 Subject: [PATCH 352/844] feat(Docker): allow custom certificates for fastapi/nginx Now, when a `./cache/production.{cert,key}.pem` pair is found, it is used in place of any certificates generated by the `ca` service. This allows users to customize the certificate that the FastAPI ASGI server uses as well as the front-end nginx certificates. Optional: - ./cache/production.cert.pem - ./cache/production.key.pem Fallback: - ./cache/localhost.cert.pem + ./cache/root.ca.pem (chain) - ./cache/localhost.key.pem Signed-off-by: Kevin Morris --- docker/config/nginx.conf | 8 ++++---- docker/nginx-entrypoint.sh | 20 +++++++++++++++++--- docker/scripts/run-fastapi.sh | 20 ++++++++++++++++---- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/docker/config/nginx.conf b/docker/config/nginx.conf index 3a8de801..4288a57d 100644 --- a/docker/config/nginx.conf +++ b/docker/config/nginx.conf @@ -43,8 +43,8 @@ http { listen 8443 ssl http2; server_name localhost default_server; - ssl_certificate /etc/ssl/certs/localhost.cert.pem; - ssl_certificate_key /etc/ssl/private/localhost.key.pem; + ssl_certificate /etc/ssl/certs/web.cert.pem; + ssl_certificate_key /etc/ssl/private/web.key.pem; root /aurweb/web/html; index index.php; @@ -91,8 +91,8 @@ http { listen 8444 ssl http2; server_name localhost default_server; - ssl_certificate /etc/ssl/certs/localhost.cert.pem; - ssl_certificate_key /etc/ssl/private/localhost.key.pem; + ssl_certificate /etc/ssl/certs/web.cert.pem; + ssl_certificate_key /etc/ssl/private/web.key.pem; root /aurweb/web/html; diff --git a/docker/nginx-entrypoint.sh b/docker/nginx-entrypoint.sh index 226ded8f..63307948 100755 --- a/docker/nginx-entrypoint.sh +++ b/docker/nginx-entrypoint.sh @@ -1,6 +1,16 @@ #!/bin/bash set -eou pipefail +# If production.{cert,key}.pem exists, prefer them. This allows +# user customization of the certificates that FastAPI uses. +# Otherwise, fallback to localhost.{cert,key}.pem, generated by `ca`. + +CERT=/cache/production.cert.pem +KEY=/cache/production.key.pem + +DEST_CERT=/etc/ssl/certs/web.cert.pem +DEST_KEY=/etc/ssl/private/web.key.pem + # Setup a config for our mysql db. cp -vf conf/config.dev conf/config sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config @@ -12,9 +22,13 @@ sed -ri 's/^;?(password) = .+/\1 = aur/' conf/config sed -ri "s|^(aur_location) = .+|\1 = https://localhost:8444|" conf/config sed -ri 's/^(disable_http_login) = .+/\1 = 1/' conf/config -cat /cache/localhost.cert.pem /cache/ca.root.pem \ - > /etc/ssl/certs/localhost.cert.pem -cp -vf /cache/localhost.key.pem /etc/ssl/private/localhost.key.pem +if [ -f "$CERT" ]; then + cp -vf "$CERT" "$DEST_CERT" + cp -vf "$KEY" "$DEST_KEY" +else + cat /cache/localhost.cert.pem /cache/ca.root.pem > "$DEST_CERT" + cp -vf /cache/localhost.key.pem "$DEST_KEY" +fi cp -vf /docker/config/nginx.conf /etc/nginx/nginx.conf diff --git a/docker/scripts/run-fastapi.sh b/docker/scripts/run-fastapi.sh index bb1a01a7..4dcc1d96 100755 --- a/docker/scripts/run-fastapi.sh +++ b/docker/scripts/run-fastapi.sh @@ -1,17 +1,29 @@ #!/bin/bash +CERT=/cache/localhost.cert.pem +KEY=/cache/localhost.key.pem + +# If production.{cert,key}.pem exists, prefer them. This allows +# user customization of the certificates that FastAPI uses. +if [ -f /cache/production.cert.pem ]; then + CERT=/cache/production.cert.pem +fi +if [ -f /cache/production.key.pem ]; then + KEY=/cache/production.key.pem +fi + if [ "$1" == "uvicorn" ] || [ "$1" == "" ]; then exec uvicorn --reload \ - --ssl-certfile /cache/localhost.cert.pem \ - --ssl-keyfile /cache/localhost.key.pem \ + --ssl-certfile "$CERT" \ + --ssl-keyfile "$KEY" \ --log-config /docker/logging.conf \ --host "0.0.0.0" \ --port 8000 \ aurweb.asgi:app else exec hypercorn --reload \ - --certfile /cache/localhost.cert.pem \ - --keyfile /cache/localhost.key.pem \ + --certfile "$CERT" \ + --keyfile "$KEY" \ --log-config /docker/logging.conf \ -b "0.0.0.0:8000" \ aurweb.asgi:app From ef0c2d5a285f185143ec8d5c0632f53f28c230a6 Mon Sep 17 00:00:00 2001 From: Kristian Klausen Date: Sat, 2 Oct 2021 23:54:10 +0200 Subject: [PATCH 353/844] magic --- docker-compose.override.yml | 47 +++++++++++++++++++++++++++++++++++++ docker-compose.prod.yml | 40 +++++++++++++++++++++++++++++++ docker-compose.yml | 39 ++++-------------------------- 3 files changed, 91 insertions(+), 35 deletions(-) create mode 100644 docker-compose.override.yml create mode 100644 docker-compose.prod.yml diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 00000000..c0bee88c --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,47 @@ +services: + ca: + volumes: + - ./cache:/cache + + git: + volumes: + - git_data:/aurweb/aur.git + - ./cache:/cache + + smartgit: + volumes: + - git_data:/aurweb/aur.git + - ./cache:/cache + - smartgit_run:/var/run/smartgit + + php-fpm: + volumes: + - ./cache:/cache + - ./aurweb:/aurweb/aurweb + - ./migrations:/aurweb/migrations + - ./test:/aurweb/test + - ./web/html:/aurweb/web/html + - ./web/template:/aurweb/web/template + - ./web/lib:/aurweb/web/lib + - ./templates:/aurweb/templates + + fastapi: + volumes: + - ./cache:/cache + - ./aurweb:/aurweb/aurweb + - ./migrations:/aurweb/migrations + - ./test:/aurweb/test + - ./web/html:/aurweb/web/html + - ./web/template:/aurweb/web/template + - ./web/lib:/aurweb/web/lib + - ./templates:/aurweb/templates + + nginx: + volumes: + - git_data:/aurweb/aur.git + - ./cache:/cache + - ./logs:/var/log/nginx + - ./web/html:/aurweb/web/html + - ./web/template:/aurweb/web/template + - ./web/lib:/aurweb/web/lib + - smartgit_run:/var/run/smartgit diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 00000000..1addecb2 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,40 @@ +version: "3.8" + +services: + ca: + volumes: + - cache:/cache + + git: + volumes: + - git_data:/aurweb/aur.git + - cache:/cache + + smartgit: + volumes: + - git_data:/aurweb/aur.git + - cache:/cache + - smartgit_run:/var/run/smartgit + + php-fpm: + volumes: + - cache:/cache + + fastapi: + volumes: + - cache:/cache + + nginx: + volumes: + - git_data:/aurweb/aur.git + - cache:/cache + - logs:/var/log/nginx + - smartgit_run:/var/run/smartgit + +volumes: + mariadb_run: {} # Share /var/run/mysqld/mysqld.sock + mariadb_data: {} # Share /var/lib/mysql + git_data: {} # Share aurweb/aur.git + smartgit_run: {} + cache: {} + logs: {} diff --git a/docker-compose.yml b/docker-compose.yml index 8e2d91d8..f19485e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,8 +27,6 @@ services: init: true entrypoint: /docker/ca-entrypoint.sh command: echo - volumes: - - ./cache:/cache memcached: build: . @@ -93,9 +91,6 @@ services: depends_on: mariadb_init: condition: service_started - volumes: - - git_data:/aurweb/aur.git - - ./cache:/cache smartgit: build: . @@ -110,10 +105,6 @@ services: depends_on: mariadb: condition: service_healthy - volumes: - - git_data:/aurweb/aur.git - - ./cache:/cache - - smartgit_run:/var/run/smartgit cgit-php: build: . @@ -165,15 +156,6 @@ services: condition: service_healthy memcached: condition: service_healthy - volumes: - - ./cache:/cache - - ./aurweb:/aurweb/aurweb - - ./migrations:/aurweb/migrations - - ./test:/aurweb/test - - ./web/html:/aurweb/web/html - - ./web/template:/aurweb/web/template - - ./web/lib:/aurweb/web/lib - - ./templates:/aurweb/templates ports: - "19000:9000" @@ -195,15 +177,6 @@ services: condition: service_healthy redis: condition: service_healthy - volumes: - - ./cache:/cache - - ./aurweb:/aurweb/aurweb - - ./migrations:/aurweb/migrations - - ./test:/aurweb/test - - ./web/html:/aurweb/web/html - - ./web/template:/aurweb/web/template - - ./web/lib:/aurweb/web/lib - - ./templates:/aurweb/templates ports: - "18000:8000" @@ -231,18 +204,11 @@ services: condition: service_healthy php-fpm: condition: service_healthy - volumes: - - git_data:/aurweb/aur.git - - ./cache:/cache - - ./logs:/var/log/nginx - - ./web/html:/aurweb/web/html - - ./web/template:/aurweb/web/template - - ./web/lib:/aurweb/web/lib - - smartgit_run:/var/run/smartgit sharness: build: . image: aurweb:latest + profiles: ["dev"] init: true environment: - AUR_CONFIG=conf/config.sqlite @@ -267,6 +233,7 @@ services: pytest-mysql: build: . image: aurweb:latest + profiles: ["dev"] init: true environment: - AUR_CONFIG=conf/config @@ -292,6 +259,7 @@ services: pytest-sqlite: build: . image: aurweb:latest + profiles: ["dev"] init: true environment: - AUR_CONFIG=conf/config.sqlite @@ -313,6 +281,7 @@ services: test: build: . image: aurweb:latest + profiles: ["dev"] init: true environment: - AUR_CONFIG=conf/config From 438080827ab7e3700ad982b91d177ec6cf20dcc9 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 2 Oct 2021 15:13:14 -0700 Subject: [PATCH 354/844] fix(Docker): add production config overrides Additionally, `up -d` will no longer run tests unless `--profile dev` is specified by the Docker user. People should now be running docker with two files: $ docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d nginx $ docker-compose -f docker-compose.yml -f docker-compose.dev.yml run test Contributed by @klausenbusk. Thanks! From a3cb81962f6ad29e28f15102cff4aaba4f4b21db Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 2 Oct 2021 16:18:18 -0700 Subject: [PATCH 355/844] add: added aur_request_ml setting to config.dev For the dev environment, we use a no-op address. We don't want to be spamming aur-requests with development notifications. Signed-off-by: Kevin Morris --- conf/config.dev | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/config.dev b/conf/config.dev index 94a9630b..9f837171 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -35,6 +35,7 @@ cache = none memcache_servers = memcached:11211 ; If cache = 'redis' this address is used to connect to Redis. redis_address = redis://127.0.0.1 +aur_request_ml = aur-requests@example-noop.org [notifications] ; For development/testing, use /usr/bin/sendmail From 4abbf9a917a381141d7adce142df5f158325354c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 2 Oct 2021 16:34:34 -0700 Subject: [PATCH 356/844] fix: use @localhost for dev email addresses Signed-off-by: Kevin Morris --- conf/config.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/config.dev b/conf/config.dev index 9f837171..ec0b33dc 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -35,7 +35,7 @@ cache = none memcache_servers = memcached:11211 ; If cache = 'redis' this address is used to connect to Redis. redis_address = redis://127.0.0.1 -aur_request_ml = aur-requests@example-noop.org +aur_request_ml = aur-requests@localhost [notifications] ; For development/testing, use /usr/bin/sendmail From f849e8b696416933282185df2d7581890e748f5a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 27 Sep 2021 13:49:33 -0700 Subject: [PATCH 357/844] change(FastAPI): allow User.notified to accept a Package OR PackageBase In addition, shorten the `package_notifications` relationship to `notifications`. Signed-off-by: Kevin Morris --- aurweb/auth.py | 2 +- aurweb/models/package_notification.py | 4 ++-- aurweb/models/user.py | 22 +++++++++++++++++++--- aurweb/routers/packages.py | 4 +--- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index 2e6674b0..19c3a276 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -40,7 +40,7 @@ class AnonymousUser: ssh_pub_key = None # Add stubbed relationship backrefs. - package_notifications = StubQuery() + notifications = StubQuery() package_votes = StubQuery() # A nonce attribute, needed for all browser sessions; set in __init__. diff --git a/aurweb/models/package_notification.py b/aurweb/models/package_notification.py index ab23a212..803c0496 100644 --- a/aurweb/models/package_notification.py +++ b/aurweb/models/package_notification.py @@ -15,7 +15,7 @@ class PackageNotification(Base): Integer, ForeignKey("Users.ID", ondelete="CASCADE"), nullable=False) User = relationship( - "User", backref=backref("package_notifications", lazy="dynamic"), + "User", backref=backref("notifications", lazy="dynamic"), foreign_keys=[UserID]) PackageBaseID = Column( @@ -23,7 +23,7 @@ class PackageNotification(Base): nullable=False) PackageBase = relationship( "PackageBase", - backref=backref("package_notifications", lazy="dynamic"), + backref=backref("notifications", lazy="dynamic"), foreign_keys=[PackageBaseID]) __mapper_args__ = {"primary_key": [UserID, PackageBaseID]} diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 28aa613e..5f848304 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -191,10 +191,26 @@ class User(Base): ).scalar()) def notified(self, package) -> bool: - """ Is this User being notified about package? """ + """ Is this User being notified about package (or package base)? + + :param package: Package or PackageBase instance + :return: Boolean indicating state of package notification + in relation to this User + """ + from aurweb.models.package import Package + from aurweb.models.package_base import PackageBase from aurweb.models.package_notification import PackageNotification - return bool(package.PackageBase.package_notifications.filter( - PackageNotification.UserID == self.ID + + query = None + if isinstance(package, Package): + query = package.PackageBase.notifications + elif isinstance(package, PackageBase): + query = package.notifications + + # Run an exists() query where a pkgbase-related + # PackageNotification exists for self (a user). + return bool(db.query( + query.filter(PackageNotification.UserID == self.ID).exists() ).scalar()) def packages(self): diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 72cd8c99..aa20e5fa 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -127,9 +127,7 @@ async def make_single_context(request: Request, context["comments"] = pkgbase.comments context["is_maintainer"] = (request.user.is_authenticated() and request.user.ID == pkgbase.MaintainerUID) - context["notified"] = request.user.package_notifications.filter( - PackageNotification.PackageBaseID == pkgbase.ID - ).scalar() + context["notified"] = request.user.notified(pkgbase) context["out_of_date"] = bool(pkgbase.OutOfDateTS) From 5e95cfbc8a844c11682db57186ac01d9732e631d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 27 Sep 2021 14:58:07 -0700 Subject: [PATCH 358/844] fix(FastAPI): get_pkgbase -> get_pkg_or_base `get_pkgbase` has been replaced with `get_pkg_or_base`, which is quite similar, but it does take a new `cls` keyword argument which is to be the model class which we search for via its `Name` column. Additionally, this change fixes a bug in the `/packages/{name}` route by supplying the Package object in question to the context and modifying the template to use it instead of a hacky through-base workaround. Examples: pkgbase = get_pkg_or_base("some_pkgbase_name", PackageBase) pkg = get_pkg_or_base("some_package_name", Package) Signed-off-by: Kevin Morris --- aurweb/packages/util.py | 22 +++++++++++----------- aurweb/routers/packages.py | 11 +++++++---- templates/packages/show.html | 2 +- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index 18ac7a5a..696c158f 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -1,6 +1,6 @@ from collections import defaultdict from http import HTTPStatus -from typing import Dict, List +from typing import Dict, List, Union import orjson @@ -101,24 +101,24 @@ def provides_list(package: Package, depname: str) -> list: return string -def get_pkgbase(name: str) -> PackageBase: +def get_pkg_or_base(name: str, cls: Union[Package, PackageBase] = PackageBase): """ Get a PackageBase instance by its name or raise a 404 if - it can't be foudn in the database. + it can't be found in the database. - :param name: PackageBase.Name - :raises HTTPException: With status code 404 if PackageBase doesn't exist - :return: PackageBase instance + :param name: {Package,PackageBase}.Name + :raises HTTPException: With status code 404 if record doesn't exist + :return: {Package,PackageBase} instance """ - pkgbase = db.query(PackageBase).filter(PackageBase.Name == name).first() - if not pkgbase: - raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) - provider = db.query(OfficialProvider).filter( OfficialProvider.Name == name).first() if provider: raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) - return pkgbase + instance = db.query(cls).filter(cls.Name == name).first() + if cls == PackageBase and not instance: + raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) + + return instance @register_filter("out_of_date") diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index aa20e5fa..8fd7717b 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -23,7 +23,7 @@ from aurweb.models.package_source import PackageSource from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import CONFLICTS_ID from aurweb.packages.search import PackageSearch -from aurweb.packages.util import get_pkgbase, query_notified, query_voted +from aurweb.packages.util import get_pkg_or_base, query_notified, query_voted from aurweb.templates import make_context, render_template router = APIRouter() @@ -143,11 +143,14 @@ async def make_single_context(request: Request, @router.get("/packages/{name}") async def package(request: Request, name: str) -> Response: - # Get the PackageBase. - pkgbase = get_pkgbase(name) + # Get the Package. + pkg = get_pkg_or_base(name, Package) + pkgbase = (get_pkg_or_base(name, PackageBase) + if not pkg else pkg.PackageBase) # Add our base information. context = await make_single_context(request, pkgbase) + context["package"] = pkg # Package sources. context["sources"] = db.query(PackageSource).join(Package).join( @@ -181,7 +184,7 @@ async def package(request: Request, name: str) -> Response: @router.get("/pkgbase/{name}") async def package_base(request: Request, name: str) -> Response: # Get the PackageBase. - pkgbase = get_pkgbase(name) + pkgbase = get_pkg_or_base(name, PackageBase) # If this is not a split package, redirect to /packages/{name}. if pkgbase.packages.count() == 1: diff --git a/templates/packages/show.html b/templates/packages/show.html index 7480f573..0bf8d9fd 100644 --- a/templates/packages/show.html +++ b/templates/packages/show.html @@ -3,7 +3,7 @@ {% block pageContent %} {% include "partials/packages/search.html" %}
    -

    {{ 'Package Details' | tr }}: {{ pkgbase.Name }} {{ pkgbase.packages.first().Version }}

    +

    {{ 'Package Details' | tr }}: {{ package.Name }} {{ package.Version }}

    {% set result = pkgbase %} {% include "partials/packages/actions.html" %} From 7961fa932a92b3743411a3b5307e5ee6636d6904 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 27 Sep 2021 15:01:45 -0700 Subject: [PATCH 359/844] feat(FastAPI): add templates.render_raw_template This function is now used as `render_template`'s underlying implementation of rendering a template, and uses that render in its HTMLResponse path. This separation allows users to directly render a template without producing a Response object. Signed-off-by: Kevin Morris --- aurweb/templates.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/aurweb/templates.py b/aurweb/templates.py index 09be049c..ef020bdf 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -102,12 +102,8 @@ async def make_variable_context(request: Request, title: str, next: str = None): return context -def render_template(request: Request, - path: str, - context: dict, - status_code: HTTPStatus = HTTPStatus.OK): +def render_raw_template(request: Request, path: str, context: dict): """ Render a Jinja2 multi-lingual template with some context. """ - # Create a deep copy of our jinja2 _environment. The _environment in # total by itself is 48 bytes large (according to sys.getsizeof). # This is done so we can install gettext translations on the template @@ -119,8 +115,15 @@ def render_template(request: Request, templates.install_gettext_translations(translator) template = templates.get_template(path) - rendered = template.render(context) + return template.render(context) + +def render_template(request: Request, + path: str, + context: dict, + status_code: HTTPStatus = HTTPStatus.OK): + """ Render a template as an HTMLResponse. """ + rendered = render_raw_template(request, path, context) response = HTMLResponse(rendered, status_code=status_code) secure_cookies = aurweb.config.getboolean("options", "disable_http_login") response.set_cookie("AURLANG", context.get("language"), From 0d8216e8eabc96faa48d855a596597a510145cd7 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 1 Oct 2021 17:50:23 -0700 Subject: [PATCH 360/844] change(FastAPI): decouple rendercomment logic from main This commit decouples most of the rendercomment.py logic into a function, `update_comment_render`, which can be used by other Python modules to perform comment rendering. In addition, we silence some deprecation warnings from python-markdown by removing `md_globals` parameters from python-markdown callbacks. Signed-off-by: Kevin Morris --- aurweb/scripts/rendercomment.py | 43 +++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index f6dfd058..dd5da4f9 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import logging import sys import bleach @@ -9,6 +10,7 @@ import pygit2 import aurweb.config import aurweb.db +logger = logging.getLogger(__name__) repo_path = aurweb.config.get('serve', 'repo-path') commit_uri = aurweb.config.get('options', 'commit_uri') @@ -24,7 +26,7 @@ class LinkifyExtension(markdown.extensions.Extension): _urlre = (r'(\b(?:https?|ftp):\/\/[\w\/\#~:.?+=&%@!\-;,]+?' r'(?=[.:?\-;,]*(?:[^\w\/\#~:.?+=&%@!\-;,]|$)))') - def extendMarkdown(self, md, md_globals): + def extendMarkdown(self, md): processor = markdown.inlinepatterns.AutolinkInlineProcessor(self._urlre, md) # Register it right after the default <>-link processor (priority 120). md.inlinePatterns.register(processor, 'linkify', 119) @@ -46,7 +48,7 @@ class FlysprayLinksInlineProcessor(markdown.inlinepatterns.InlineProcessor): class FlysprayLinksExtension(markdown.extensions.Extension): - def extendMarkdown(self, md, md_globals): + def extendMarkdown(self, md): processor = FlysprayLinksInlineProcessor(r'\bFS#(\d+)\b', md) md.inlinePatterns.register(processor, 'flyspray-links', 118) @@ -90,9 +92,12 @@ class GitCommitsExtension(markdown.extensions.Extension): self._head = head super(markdown.extensions.Extension, self).__init__() - def extendMarkdown(self, md, md_globals): - processor = GitCommitsInlineProcessor(md, self._head) - md.inlinePatterns.register(processor, 'git-commits', 117) + def extendMarkdown(self, md): + try: + processor = GitCommitsInlineProcessor(md, self._head) + md.inlinePatterns.register(processor, 'git-commits', 117) + except pygit2.GitError: + logger.error(f"No git repository found for '{self._head}'.") class HeadingTreeprocessor(markdown.treeprocessors.Treeprocessor): @@ -105,7 +110,7 @@ class HeadingTreeprocessor(markdown.treeprocessors.Treeprocessor): class HeadingExtension(markdown.extensions.Extension): - def extendMarkdown(self, md, md_globals): + def extendMarkdown(self, md): # Priority doesn't matter since we don't conflict with other processors. md.treeprocessors.register(HeadingTreeprocessor(md), 'heading', 30) @@ -123,19 +128,20 @@ def save_rendered_comment(conn, commentid, html): [html, commentid]) -def main(): - commentid = int(sys.argv[1]) - +def update_comment_render(commentid): conn = aurweb.db.Connection() text, pkgbase = get_comment(conn, commentid) - html = markdown.markdown(text, extensions=['fenced_code', - LinkifyExtension(), - FlysprayLinksExtension(), - GitCommitsExtension(pkgbase), - HeadingExtension()]) - allowed_tags = (bleach.sanitizer.ALLOWED_TAGS + - ['p', 'pre', 'h4', 'h5', 'h6', 'br', 'hr']) + html = markdown.markdown(text, extensions=[ + 'fenced_code', + LinkifyExtension(), + FlysprayLinksExtension(), + GitCommitsExtension(pkgbase), + HeadingExtension() + ]) + + allowed_tags = (bleach.sanitizer.ALLOWED_TAGS + + ['p', 'pre', 'h4', 'h5', 'h6', 'br', 'hr']) html = bleach.clean(html, tags=allowed_tags) save_rendered_comment(conn, commentid, html) @@ -143,5 +149,10 @@ def main(): conn.close() +def main(): + commentid = int(sys.argv[1]) + update_comment_render(commentid) + + if __name__ == '__main__': main() From fc28aad245a0350ec3190fc484543bae4612883d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 27 Sep 2021 18:46:20 -0700 Subject: [PATCH 361/844] feat(FastAPI): add pkgbase comments (new, edit) In PHP, this was implemented using an /rpc type 'get-comment-form'. With FastAPI, we've decided to reorganize this into a non-RPC route: `/pkgbase/{name}/comments/{id}/form`, rendered via the new `templates/partials/packages/comment_form.html` template. When the comment_form.html template is provided a `comment` object, it will produce an edit comment form. Otherwise, it will produce a new comment form. A few new FastAPI routes have been introduced: - GET `/pkgbase/{name}/comments/{id}/form` - Produces a JSON response based on {"form": ""}. - POST `/pkgbase/{name}/comments' - Creates a new comment. - POST `/pkgbase/{name}/comments/{id}` - Edits an existing comment. In addition, some Javascript has been modified for our new routes. Signed-off-by: Kevin Morris --- aurweb/packages/util.py | 8 ++ aurweb/routers/packages.py | 105 +++++++++++++- templates/partials/packages/comment.html | 50 ++++--- templates/partials/packages/comment_form.html | 46 ++++++ templates/partials/packages/comments.html | 70 +++------ test/test_packages_routes.py | 133 ++++++++++++++++++ 6 files changed, 333 insertions(+), 79 deletions(-) create mode 100644 templates/partials/packages/comment_form.html diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index 696c158f..55149127 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -11,6 +11,7 @@ from aurweb import db from aurweb.models.official_provider import OFFICIAL_BASE, OfficialProvider from aurweb.models.package import Package from aurweb.models.package_base import PackageBase +from aurweb.models.package_comment import PackageComment from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_notification import PackageNotification from aurweb.models.package_relation import PackageRelation @@ -121,6 +122,13 @@ def get_pkg_or_base(name: str, cls: Union[Package, PackageBase] = PackageBase): return instance +def get_pkgbase_comment(pkgbase: PackageBase, id: int) -> PackageComment: + comment = pkgbase.comments.filter(PackageComment.ID == id).first() + if not comment: + raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) + return comment + + @register_filter("out_of_date") def out_of_date(packages: orm.Query) -> orm.Query: return packages.filter(PackageBase.OutOfDateTS.isnot(None)) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 8fd7717b..d5c99e8d 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -1,8 +1,9 @@ +from datetime import datetime from http import HTTPStatus from typing import Any, Dict -from fastapi import APIRouter, Request, Response -from fastapi.responses import RedirectResponse +from fastapi import APIRouter, Form, HTTPException, Request, Response +from fastapi.responses import JSONResponse, RedirectResponse from sqlalchemy import and_ import aurweb.filters @@ -11,9 +12,11 @@ import aurweb.models.package_keyword import aurweb.packages.util from aurweb import db +from aurweb.auth import auth_required from aurweb.models.license import License from aurweb.models.package import Package from aurweb.models.package_base import PackageBase +from aurweb.models.package_comment import PackageComment from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_license import PackageLicense from aurweb.models.package_notification import PackageNotification @@ -23,8 +26,9 @@ from aurweb.models.package_source import PackageSource from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import CONFLICTS_ID from aurweb.packages.search import PackageSearch -from aurweb.packages.util import get_pkg_or_base, query_notified, query_voted -from aurweb.templates import make_context, render_template +from aurweb.packages.util import get_pkg_or_base, get_pkgbase_comment, query_notified, query_voted +from aurweb.scripts.rendercomment import update_comment_render +from aurweb.templates import make_context, render_raw_template, render_template router = APIRouter() @@ -124,7 +128,9 @@ async def make_single_context(request: Request, context["pkgbase"] = pkgbase context["packages_count"] = pkgbase.packages.count() context["keywords"] = pkgbase.keywords - context["comments"] = pkgbase.comments + context["comments"] = pkgbase.comments.order_by( + PackageComment.CommentTS.desc() + ) context["is_maintainer"] = (request.user.is_authenticated() and request.user.ID == pkgbase.MaintainerUID) context["notified"] = request.user.notified(pkgbase) @@ -201,7 +207,94 @@ async def package_base(request: Request, name: str) -> Response: @router.get("/pkgbase/{name}/voters") async def package_base_voters(request: Request, name: str) -> Response: # Get the PackageBase. - pkgbase = get_pkgbase(name) + pkgbase = get_pkg_or_base(name, PackageBase) context = make_context(request, "Voters") context["pkgbase"] = pkgbase return render_template(request, "pkgbase/voters.html", context) + + +@router.post("/pkgbase/{name}/comments") +@auth_required(True) +async def pkgbase_comments_post( + request: Request, name: str, + comment: str = Form(default=str()), + enable_notifications: bool = Form(default=False)): + """ Add a new comment. """ + pkgbase = get_pkg_or_base(name, PackageBase) + + if not comment: + raise HTTPException(status_code=int(HTTPStatus.EXPECTATION_FAILED)) + + # If the provided comment is different than the record's version, + # update the db record. + now = int(datetime.utcnow().timestamp()) + with db.begin(): + comment = db.create(PackageComment, User=request.user, + PackageBase=pkgbase, + Comments=comment, RenderedComment=str(), + CommentTS=now) + + if enable_notifications and not request.user.notified(pkgbase): + db.create(PackageNotification, + User=request.user, + PackageBase=pkgbase) + update_comment_render(comment.ID) + + # Redirect to the pkgbase page. + return RedirectResponse(f"/pkgbase/{pkgbase.Name}#comment-{comment.ID}", + status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.get("/pkgbase/{name}/comments/{id}/form") +@auth_required(True) +async def pkgbase_comment_form(request: Request, name: str, id: int): + """ Produce a comment form for comment {id}. """ + pkgbase = get_pkg_or_base(name, PackageBase) + comment = pkgbase.comments.filter(PackageComment.ID == id).first() + if not comment: + return JSONResponse({}, status_code=int(HTTPStatus.NOT_FOUND)) + + if not request.user.is_elevated() and request.user != comment.User: + return JSONResponse({}, status_code=int(HTTPStatus.UNAUTHORIZED)) + + context = await make_single_context(request, pkgbase) + context["comment"] = comment + + form = render_raw_template( + request, "partials/packages/comment_form.html", context) + return JSONResponse({"form": form}) + + +@router.post("/pkgbase/{name}/comments/{id}") +@auth_required(True) +async def pkgbase_comment_post( + request: Request, name: str, id: int, + comment: str = Form(default=str()), + enable_notifications: bool = Form(default=False)): + pkgbase = get_pkg_or_base(name, PackageBase) + db_comment = get_pkgbase_comment(pkgbase, id) + + if not comment: + raise HTTPException(status_code=int(HTTPStatus.EXPECTATION_FAILED)) + + # If the provided comment is different than the record's version, + # update the db record. + now = int(datetime.utcnow().timestamp()) + if db_comment.Comments != comment: + with db.begin(): + db_comment.Comments = comment + db_comment.Editor = request.user + db_comment.EditedTS = now + + db_notif = request.user.notifications.filter( + PackageNotification.PackageBaseID == pkgbase.ID + ).first() + if enable_notifications and not db_notif: + db.create(PackageNotification, + User=request.user, + PackageBase=pkgbase) + update_comment_render(db_comment.ID) + + # Redirect to the pkgbase page anchored to the updated comment. + return RedirectResponse(f"/pkgbase/{pkgbase.Name}#comment-{db_comment.ID}", + status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/templates/partials/packages/comment.html b/templates/partials/packages/comment.html index 6cf5f319..36696215 100644 --- a/templates/partials/packages/comment.html +++ b/templates/partials/packages/comment.html @@ -16,26 +16,38 @@ ) | safe }} - {% if is_maintainer %} -
    -
    - - - - -
    -
    - Edit comment + {% if comment.Editor %} + {% set edited_on = comment.EditedTS | dt | as_timezone(timezone) %} + + ({{ "edited on %s by %s" | tr + | format(edited_on.strftime('%Y-%m-%d %H:%M'), + '%s' | format( + comment.Editor.Username, comment.Editor.Username)) + | safe + }}) + + {% endif %} + {% if request.user.is_elevated() or pkgbase.Maintainer == request.user %} +
    +
    + + + + +
    +
    + Edit comment + +
    +
    + + + + + +
    +
    {% endif %} -
    -
    - - - - - -
    -

    diff --git a/templates/partials/packages/comment_form.html b/templates/partials/packages/comment_form.html new file mode 100644 index 00000000..c1c25f87 --- /dev/null +++ b/templates/partials/packages/comment_form.html @@ -0,0 +1,46 @@ +{# `action` is assigned the proper route to use for the form action. +When `comment` is provided (PackageComment), we display an edit form +for the comment. Otherwise, we display a new form. + +Routes: + new comment - /pkgbase/{name}/comments + edit comment - /pkgbase/{name}/comments/{id} +#} +{% set action = "/pkgbase/%s/comments" | format(pkgbase.Name) %} +{% if comment %} + {% set action = "/pkgbase/%s/comments/%d" | format(pkgbase.Name, comment.ID) %} +{% endif %} + +
    +
    +

    + {{ "Git commit identifiers referencing commits in the AUR package " + "repository and URLs are converted to links automatically." | tr }} + {{ "%sMarkdown syntax%s is partially supported." | tr + | format('', + "") + | safe }} +

    +

    + +

    +

    + + {% if comment and not request.user.notified(pkgbase) %} + + + + + {% endif %} +

    +
    +
    diff --git a/templates/partials/packages/comments.html b/templates/partials/packages/comments.html index 39cfb363..7c8a32e5 100644 --- a/templates/partials/packages/comments.html +++ b/templates/partials/packages/comments.html @@ -8,44 +8,7 @@ {% if request.user.is_authenticated() %}

    Add Comment

    -
    -
    -
    - - -
    -

    - {{ - "Git commit identifiers referencing commits in the AUR package" - " repository and URLs are converted to links automatically." - | tr - }} - {{ - "%sMarkdown Syntax%s is partially supported." - | tr - | format('', '') - | safe - }} -

    -

    - -

    -

    - - {% if not notifications_enabled %} - - - - - {% endif %} -

    -
    -
    + {% include "partials/packages/comment_form.html" %}
    {% endif %} @@ -99,29 +62,28 @@ function handleEditCommentClick(event) { // The div class="article-content" which contains the comment const edit_form = parent_element.nextElementSibling; - const params = new URLSearchParams({ - type: "get-comment-form", - arg: comment_id, - base_id: {{ pkgbase.ID }}, - pkgbase_name: {{ pkgbase.Name }} - }); - - const url = '/rpc?' + params.toString(); + const url = "/pkgbase/{{ pkgbase.Name }}/comments/" + comment_id + "/form"; add_busy_indicator(event.target); fetch(url, { - method: 'GET' + method: 'GET', + credentials: 'same-origin' + }) + .then(function(response) { + if (!response.ok) { + throw Error(response.statusText); + } + return response.json(); }) - .then(function(response) { return response.json(); }) .then(function(data) { remove_busy_indicator(event.target); - if (data.success) { - edit_form.innerHTML = data.form; - edit_form.querySelector('textarea').focus(); - } else { - alert(data.error); - } + edit_form.innerHTML = data.form; + edit_form.querySelector('textarea').focus(); + }) + .catch(function(error) { + remove_busy_indicator(event.target); + console.error(error); }); } diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 2190dc18..1bfa5fc0 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -73,6 +73,7 @@ def setup(): PackageVote.__tablename__, PackageNotification.__tablename__, PackageComaintainer.__tablename__, + PackageComment.__tablename__, OfficialProvider.__tablename__ ) @@ -930,3 +931,135 @@ def test_pkgbase_voters(client: TestClient, maintainer: User, package: Package): root = parse_root(resp.text) rows = root.xpath('//div[@class="box"]//ul/li') assert len(rows) == 1 + + +def test_pkgbase_comment_not_found(client: TestClient, maintainer: User, + package: Package): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + comment_id = 12345 # A non-existing comment. + endpoint = f"/pkgbase/{package.PackageBase.Name}/comments/{comment_id}" + with client as request: + resp = request.post(endpoint, data={ + "comment": "Failure" + }, cookies=cookies) + assert resp.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_pkgbase_comment_form_unauthorized(client: TestClient, user: User, + maintainer: User, package: Package): + now = int(datetime.utcnow().timestamp()) + with db.begin(): + comment = db.create(PackageComment, PackageBase=package.PackageBase, + User=maintainer, Comments="Test", + RenderedComment=str(), CommentTS=now) + + cookies = {"AURSID": user.login(Request(), "testPassword")} + pkgbasename = package.PackageBase.Name + endpoint = f"/pkgbase/{pkgbasename}/comments/{comment.ID}/form" + with client as request: + resp = request.get(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.UNAUTHORIZED) + + +def test_pkgbase_comment_form_not_found(client: TestClient, maintainer: User, + package: Package): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + comment_id = 12345 # A non-existing comment. + pkgbasename = package.PackageBase.Name + endpoint = f"/pkgbase/{pkgbasename}/comments/{comment_id}/form" + with client as request: + resp = request.get(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_pkgbase_comments_missing_comment(client: TestClient, maintainer: User, + package: Package): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + endpoint = f"/pkgbase/{package.PackageBase.Name}/comments" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.EXPECTATION_FAILED) + + +def test_pkgbase_comments(client: TestClient, maintainer: User, + package: Package): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + pkgbasename = package.PackageBase.Name + endpoint = f"/pkgbase/{pkgbasename}/comments" + with client as request: + resp = request.post(endpoint, data={ + "comment": "Test comment.", + "enable_notifications": True + }, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + expected_prefix = f"/pkgbase/{pkgbasename}" + prefix_len = len(expected_prefix) + assert resp.headers.get("location")[:prefix_len] == expected_prefix + + with client as request: + resp = request.get(resp.headers.get("location")) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + headers = root.xpath('//h4[@class="comment-header"]') + bodies = root.xpath('//div[@class="article-content"]/div/p') + + assert len(headers) == 1 + assert len(bodies) == 1 + + assert bodies[0].text.strip() == "Test comment." + + # Clear up the PackageNotification. This doubles as testing + # that the notification was created and clears it up so we can + # test enabling it during edit. + pkgbase = package.PackageBase + db_notif = pkgbase.notifications.filter( + PackageNotification.UserID == maintainer.ID + ).first() + with db.begin(): + db.session.delete(db_notif) + + # Now, let's edit the comment we just created. + comment_id = int(headers[0].attrib["id"].split("-")[-1]) + endpoint = f"/pkgbase/{pkgbasename}/comments/{comment_id}" + with client as request: + resp = request.post(endpoint, data={ + "comment": "Edited comment.", + "enable_notifications": True + }, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + with client as request: + resp = request.get(resp.headers.get("location")) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + headers = root.xpath('//h4[@class="comment-header"]') + bodies = root.xpath('//div[@class="article-content"]/div/p') + + assert len(headers) == 1 + assert len(bodies) == 1 + + assert bodies[0].text.strip() == "Edited comment." + + # Ensure that a notification was created. + db_notif = pkgbase.notifications.filter( + PackageNotification.UserID == maintainer.ID + ).first() + assert db_notif is not None + + # Don't supply a comment; should return EXPECTATION_FAILED. + with client as request: + fail_resp = request.post(endpoint, cookies=cookies) + assert fail_resp.status_code == int(HTTPStatus.EXPECTATION_FAILED) + + # Now, test the form route, which should return form markup + # via JSON. + endpoint = f"{endpoint}/form" + with client as request: + resp = request.get(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + data = resp.json() + assert "form" in data From 59d04d6e0c50d3b0443dae29715de63f197be890 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 30 Sep 2021 13:53:31 -0700 Subject: [PATCH 362/844] fix(FastAPI): comment.html template rendering Deleters and edits were not previously taken into account. This fix addresses that issue using User.has_credential. Signed-off-by: Kevin Morris --- templates/partials/packages/comment.html | 148 ++++++++++++++--------- 1 file changed, 89 insertions(+), 59 deletions(-) diff --git a/templates/partials/packages/comment.html b/templates/partials/packages/comment.html index 36696215..6af5cd9e 100644 --- a/templates/partials/packages/comment.html +++ b/templates/partials/packages/comment.html @@ -1,60 +1,90 @@ -

    - {% set commented_at = comment.CommentTS | dt | as_timezone(timezone) %} - {% set view_account_info = 'View account information for %s' | tr | format(comment.User.Username) %} - {{ - "%s commented on %s" | tr | format( - ('%s' | format( - comment.User.Username, - view_account_info, - comment.User.Username - )) if request.user.is_authenticated() else - (comment.User.Username), - '%s' | format( - comment.ID, - commented_at.strftime("%Y-%m-%d %H:%M") - ) - ) - | safe - }} - {% if comment.Editor %} - {% set edited_on = comment.EditedTS | dt | as_timezone(timezone) %} - - ({{ "edited on %s by %s" | tr - | format(edited_on.strftime('%Y-%m-%d %H:%M'), - '%s' | format( - comment.Editor.Username, comment.Editor.Username)) - | safe - }}) - - {% endif %} - {% if request.user.is_elevated() or pkgbase.Maintainer == request.user %} -
    -
    - - - - -
    -
    - Edit comment +{% set header_cls = "comment-header" %} +{% set article_cls = "article-content" %} +{% if comment.Deleter %} + {% set header_cls = "%s %s" | format(header_cls, "comment-deleted") %} + {% set article_cls = "%s %s" | format(article_cls, "comment-deleted") %} +{% endif %} -
    -
    - - - - - -
    -
    - {% endif %} -

    -
    -
    - {% if comment.RenderedComment %} - {{ comment.RenderedComment | safe }} - {% else %} - {{ comment.Comments }} - {% endif %} -
    -
    +{% if not comment.Deleter or request.user.has_credential("CRED_COMMENT_VIEW_DELETED", approved=[comment.Deleter]) %} +

    + {% set commented_at = comment.CommentTS | dt | as_timezone(timezone) %} + {% set view_account_info = 'View account information for %s' | tr | format(comment.User.Username) %} + {{ + "%s commented on %s" | tr | format( + ('%s' | format( + comment.User.Username, + view_account_info, + comment.User.Username + )) if request.user.is_authenticated() else + (comment.User.Username), + '%s' | format( + comment.ID, + commented_at.strftime("%Y-%m-%d %H:%M") + ) + ) + | safe + }} + {% if comment.Editor %} + {% set edited_on = comment.EditedTS | dt | as_timezone(timezone) %} + + ({{ "edited on %s by %s" | tr + | format(edited_on.strftime('%Y-%m-%d %H:%M'), + '%s' | format( + comment.Editor.Username, comment.Editor.Username)) + | safe + }}) + + {% endif %} + {% if not comment.Deleter %} + {% if request.user.has_credential('CRED_COMMENT_DELETE', approved=[comment.User]) %} +
    +
    + +
    +
    + {% endif %} + + {% if request.user.has_credential('CRED_COMMENT_EDIT', approved=[comment.User]) %} + Edit comment + {% endif %} + + {% if request.user.has_credential("CRED_COMMENT_PIN", approved=[pkgbase.Maintainer]) %} +
    +
    + + + + + +
    +
    + {% endif %} + {% else %} + {% if request.user.has_credential("CRED_COMMENT_UNDELETE", approved=[comment.User]) %} +
    +
    + +
    +
    + {% endif %} + {% endif %} +

    +
    +
    + {% if comment.RenderedComment %} + {{ comment.RenderedComment | safe }} + {% else %} + {{ comment.Comments }} + {% endif %} +
    +
    +{% endif %} From 6644c42922ad4645104bef114e1685973ea1a92d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 30 Sep 2021 13:56:14 -0700 Subject: [PATCH 363/844] fix(FastAPI): AnonymousUser.has_credential also takes kwargs Signed-off-by: Kevin Morris --- aurweb/auth.py | 2 +- test/test_auth.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index 19c3a276..d1a9d9cb 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -66,7 +66,7 @@ class AnonymousUser: return False @staticmethod - def has_credential(credential): + def has_credential(credential, **kwargs): return False @staticmethod diff --git a/test/test_auth.py b/test/test_auth.py index ced64064..7aea17a0 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -117,3 +117,8 @@ def test_voted_for(): def test_notified(): user_ = AnonymousUser() assert not user_.notified(None) + + +def test_has_credential(): + user_ = AnonymousUser() + assert not user_.has_credential("FAKE_CREDENTIAL") From d3be30744ccfe6556f0299d08a1bf8bc63b2ae44 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 1 Oct 2021 15:44:24 -0700 Subject: [PATCH 364/844] add(FeatAPI): comment pytest.fixture Signed-off-by: Kevin Morris --- test/test_packages_routes.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 1bfa5fc0..93a7f524 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -125,6 +125,20 @@ def package(maintainer: User) -> Package: yield package +@pytest.fixture +def comment(user: User, package: Package) -> PackageComment: + pkgbase = package.PackageBase + now = int(datetime.utcnow().timestamp()) + with db.begin(): + comment = db.create(PackageComment, + User=user, + PackageBase=pkgbase, + Comments="Test comment.", + RenderedComment=str(), + CommentTS=now) + yield comment + + @pytest.fixture def packages(maintainer: User) -> List[Package]: """ Yield 55 packages named pkg_0 .. pkg_54. """ From 40cd1b9029cc50e41c21d69e502947996862b7b4 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 30 Sep 2021 13:58:37 -0700 Subject: [PATCH 365/844] feat(FastAPI): add /pkgbase/{name}/comments/{id}/delete (post) Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 25 ++++++++++++++++++++- test/test_packages_routes.py | 43 +++++++++++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index d5c99e8d..3a5ca047 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -11,7 +11,7 @@ import aurweb.models.package_comment import aurweb.models.package_keyword import aurweb.packages.util -from aurweb import db +from aurweb import db, l10n from aurweb.auth import auth_required from aurweb.models.license import License from aurweb.models.package import Package @@ -298,3 +298,26 @@ async def pkgbase_comment_post( # Redirect to the pkgbase page anchored to the updated comment. return RedirectResponse(f"/pkgbase/{pkgbase.Name}#comment-{db_comment.ID}", status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.post("/pkgbase/{name}/comments/{id}/delete") +@auth_required(True) +async def pkgbase_comment_delete(request: Request, name: str, id: int): + pkgbase = get_pkg_or_base(name, PackageBase) + comment = get_pkgbase_comment(pkgbase, id) + + authorized = request.user.has_credential("CRED_COMMENT_DELETE", + [comment.User]) + if not authorized: + _ = l10n.get_translator_for_request(request) + raise HTTPException( + status_code=int(HTTPStatus.UNAUTHORIZED), + detail=_("You are not allowed to delete this comment.")) + + now = int(datetime.utcnow().timestamp()) + with db.begin(): + comment.Deleter = request.user + comment.DelTS = now + + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 93a7f524..eb3da41a 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -995,7 +995,7 @@ def test_pkgbase_comments_missing_comment(client: TestClient, maintainer: User, assert resp.status_code == int(HTTPStatus.EXPECTATION_FAILED) -def test_pkgbase_comments(client: TestClient, maintainer: User, +def test_pkgbase_comments(client: TestClient, maintainer: User, user: User, package: Package): cookies = {"AURSID": maintainer.login(Request(), "testPassword")} pkgbasename = package.PackageBase.Name @@ -1077,3 +1077,44 @@ def test_pkgbase_comments(client: TestClient, maintainer: User, data = resp.json() assert "form" in data + + +def test_pkgbase_comment_delete(client: TestClient, + user: User, + package: Package, + comment: PackageComment): + # Test the unauthorized case of comment deletion. + cookies = {"AURSID": user.login(Request(), "testPassword")} + pkgbasename = package.PackageBase.Name + endpoint = f"/pkgbase/{pkgbasename}/comments/{comment.ID}/delete" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + expected = f"/pkgbase/{pkgbasename}" + assert resp.headers.get("location") == expected + + +def test_pkgbase_comment_delete_unauthorized(client: TestClient, + maintainer: User, + package: Package, + comment: PackageComment): + # Test the unauthorized case of comment deletion. + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + pkgbasename = package.PackageBase.Name + endpoint = f"/pkgbase/{pkgbasename}/comments/{comment.ID}/delete" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.UNAUTHORIZED) + + +def test_pkgbase_comment_delete_not_found(client: TestClient, + maintainer: User, + package: Package): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + comment_id = 12345 # Non-existing comment. + pkgbasename = package.PackageBase.Name + endpoint = f"/pkgbase/{pkgbasename}/comments/{comment_id}/delete" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.NOT_FOUND) From bb45ae7ac3f8da424c51051981bc6af91742e70e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 30 Sep 2021 19:48:25 -0700 Subject: [PATCH 366/844] feat(FastAPI): add /pkgbase/{name}/comments/{id}/undelete (post) Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 22 ++++++++++++++++++++++ test/test_packages_routes.py | 25 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 3a5ca047..92fc9361 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -321,3 +321,25 @@ async def pkgbase_comment_delete(request: Request, name: str, id: int): return RedirectResponse(f"/pkgbase/{name}", status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.post("/pkgbase/{name}/comments/{id}/undelete") +@auth_required(True) +async def pkgbase_comment_undelete(request: Request, name: str, id: int): + pkgbase = get_pkg_or_base(name, PackageBase) + comment = get_pkgbase_comment(pkgbase, id) + + has_cred = request.user.has_credential("CRED_COMMENT_UNDELETE", + approved=[comment.User]) + if not has_cred: + _ = l10n.get_translator_for_request(request) + raise HTTPException( + status_code=int(HTTPStatus.UNAUTHORIZED), + detail=_("You are not allowed to undelete this comment.")) + + with db.begin(): + comment.Deleter = None + comment.DelTS = None + + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index eb3da41a..47b5ed81 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1080,6 +1080,7 @@ def test_pkgbase_comments(client: TestClient, maintainer: User, user: User, def test_pkgbase_comment_delete(client: TestClient, + maintainer: User, user: User, package: Package, comment: PackageComment): @@ -1094,6 +1095,18 @@ def test_pkgbase_comment_delete(client: TestClient, expected = f"/pkgbase/{pkgbasename}" assert resp.headers.get("location") == expected + # Test the unauthorized case of comment undeletion. + maint_cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + endpoint = f"/pkgbase/{pkgbasename}/comments/{comment.ID}/undelete" + with client as request: + resp = request.post(endpoint, cookies=maint_cookies) + assert resp.status_code == int(HTTPStatus.UNAUTHORIZED) + + # And move on to undeleting it. + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + def test_pkgbase_comment_delete_unauthorized(client: TestClient, maintainer: User, @@ -1118,3 +1131,15 @@ def test_pkgbase_comment_delete_not_found(client: TestClient, with client as request: resp = request.post(endpoint, cookies=cookies) assert resp.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_pkgbase_comment_undelete_not_found(client: TestClient, + maintainer: User, + package: Package): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + comment_id = 12345 # Non-existing comment. + pkgbasename = package.PackageBase.Name + endpoint = f"/pkgbase/{pkgbasename}/comments/{comment_id}/undelete" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.NOT_FOUND) From 0895dd07ee621d24007a30b4f3d076d7b55c23f0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 1 Oct 2021 15:47:16 -0700 Subject: [PATCH 367/844] feat(FastAPI): add /pkgbase/{name}/comments/{id}/pin (post) In addition, fix up some templates to display pinned comments, and include the unpin form input for pinned comments, which is not yet implemented. Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 26 ++++++++++++++ templates/packages/show.html | 4 +-- templates/partials/packages/comment.html | 42 +++++++++++++++++------ templates/partials/packages/comments.html | 16 ++++++++- test/test_packages_routes.py | 26 ++++++++++++++ 5 files changed, 100 insertions(+), 14 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 92fc9361..681cde4f 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -131,6 +131,10 @@ async def make_single_context(request: Request, context["comments"] = pkgbase.comments.order_by( PackageComment.CommentTS.desc() ) + context["pinned_comments"] = pkgbase.comments.filter( + PackageComment.PinnedTS != 0 + ).order_by(PackageComment.CommentTS.desc()) + context["is_maintainer"] = (request.user.is_authenticated() and request.user.ID == pkgbase.MaintainerUID) context["notified"] = request.user.notified(pkgbase) @@ -343,3 +347,25 @@ async def pkgbase_comment_undelete(request: Request, name: str, id: int): return RedirectResponse(f"/pkgbase/{name}", status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.post("/pkgbase/{name}/comments/{id}/pin") +@auth_required(True) +async def pkgbase_comment_pin(request: Request, name: str, id: int): + pkgbase = get_pkg_or_base(name, PackageBase) + comment = get_pkgbase_comment(pkgbase, id) + + has_cred = request.user.has_credential("CRED_COMMENT_PIN", + approved=[pkgbase.Maintainer]) + if not has_cred: + _ = l10n.get_translator_for_request(request) + raise HTTPException( + status_code=int(HTTPStatus.UNAUTHORIZED), + detail=_("You are not allowed to pin this comment.")) + + now = int(datetime.utcnow().timestamp()) + with db.begin(): + comment.PinnedTS = now + + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/templates/packages/show.html b/templates/packages/show.html index 0bf8d9fd..ba531fc8 100644 --- a/templates/packages/show.html +++ b/templates/packages/show.html @@ -18,7 +18,5 @@ {% set pkgname = result.Name %} {% set pkgbase_id = result.ID %} - {% if comments.count() %} - {% include "partials/packages/comments.html" %} - {% endif %} + {% include "partials/packages/comments.html" %} {% endblock %} diff --git a/templates/partials/packages/comment.html b/templates/partials/packages/comment.html index 6af5cd9e..97f11723 100644 --- a/templates/partials/packages/comment.html +++ b/templates/partials/packages/comment.html @@ -49,16 +49,38 @@ Edit comment {% endif %} - {% if request.user.has_credential("CRED_COMMENT_PIN", approved=[pkgbase.Maintainer]) %} -
    -
    - - - - - -
    -
    + {% if request.user.has_credential("CRED_COMMENT_PIN", approved=[pkgbase.Maintainer]) %} + {% if comment.PinnedTS %} +
    +
    + +
    +
    + {% else %} +
    +
    + +
    +
    + {% endif %} {% endif %} {% else %} {% if request.user.has_credential("CRED_COMMENT_UNDELETE", approved=[comment.User]) %} diff --git a/templates/partials/packages/comments.html b/templates/partials/packages/comments.html index 7c8a32e5..56b5ab03 100644 --- a/templates/partials/packages/comments.html +++ b/templates/partials/packages/comments.html @@ -12,7 +12,21 @@
    {% endif %} -{% if comments %} +{% if pinned_comments.count() %} +
    +
    +

    + {{ "Pinned Comments" | tr }} + +

    +
    + {% for comment in pinned_comments.all() %} + {% include "partials/packages/comment.html" %} + {% endfor %} +
    +{% endif %} + +{% if comments.count() %}

    diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 47b5ed81..6bf4b975 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1143,3 +1143,29 @@ def test_pkgbase_comment_undelete_not_found(client: TestClient, with client as request: resp = request.post(endpoint, cookies=cookies) assert resp.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_pkgbase_comment_pin(client: TestClient, + maintainer: User, + package: Package, + comment: PackageComment): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + comment_id = comment.ID + pkgbasename = package.PackageBase.Name + endpoint = f"/pkgbase/{pkgbasename}/comments/{comment_id}/pin" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + +def test_pkgbase_comment_pin_unauthorized(client: TestClient, + user: User, + package: Package, + comment: PackageComment): + cookies = {"AURSID": user.login(Request(), "testPassword")} + comment_id = comment.ID + pkgbasename = package.PackageBase.Name + endpoint = f"/pkgbase/{pkgbasename}/comments/{comment_id}/pin" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.UNAUTHORIZED) From 2efd254974fd2db0253ebc4aec725a08ba525d67 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 1 Oct 2021 17:51:39 -0700 Subject: [PATCH 368/844] feat(FastAPI): add /pkgbase/{name}/comments/{id}/unpin (post) Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 21 +++++++++++++++++++++ test/test_packages_routes.py | 27 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 681cde4f..5ae19d07 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -369,3 +369,24 @@ async def pkgbase_comment_pin(request: Request, name: str, id: int): return RedirectResponse(f"/pkgbase/{name}", status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.post("/pkgbase/{name}/comments/{id}/unpin") +@auth_required(True) +async def pkgbase_comment_unpin(request: Request, name: str, id: int): + pkgbase = get_pkg_or_base(name, PackageBase) + comment = get_pkgbase_comment(pkgbase, id) + + has_cred = request.user.has_credential("CRED_COMMENT_PIN", + approved=[pkgbase.Maintainer]) + if not has_cred: + _ = l10n.get_translator_for_request(request) + raise HTTPException( + status_code=int(HTTPStatus.UNAUTHORIZED), + detail=_("You are not allowed to unpin this comment.")) + + with db.begin(): + comment.PinnedTS = 0 + + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 6bf4b975..1c7d5d3e 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1152,11 +1152,25 @@ def test_pkgbase_comment_pin(client: TestClient, cookies = {"AURSID": maintainer.login(Request(), "testPassword")} comment_id = comment.ID pkgbasename = package.PackageBase.Name + + # Pin the comment. endpoint = f"/pkgbase/{pkgbasename}/comments/{comment_id}/pin" with client as request: resp = request.post(endpoint, cookies=cookies) assert resp.status_code == int(HTTPStatus.SEE_OTHER) + # Assert that PinnedTS got set. + assert comment.PinnedTS > 0 + + # Unpin the comment we just pinned. + endpoint = f"/pkgbase/{pkgbasename}/comments/{comment_id}/unpin" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + # Let's assert that PinnedTS was unset. + assert comment.PinnedTS == 0 + def test_pkgbase_comment_pin_unauthorized(client: TestClient, user: User, @@ -1169,3 +1183,16 @@ def test_pkgbase_comment_pin_unauthorized(client: TestClient, with client as request: resp = request.post(endpoint, cookies=cookies) assert resp.status_code == int(HTTPStatus.UNAUTHORIZED) + + +def test_pkgbase_comment_unpin_unauthorized(client: TestClient, + user: User, + package: Package, + comment: PackageComment): + cookies = {"AURSID": user.login(Request(), "testPassword")} + comment_id = comment.ID + pkgbasename = package.PackageBase.Name + endpoint = f"/pkgbase/{pkgbasename}/comments/{comment_id}/unpin" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.UNAUTHORIZED) From 986fa9ee305ed113172f7f214d451a7af071ecc2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 26 Sep 2021 20:26:24 -0700 Subject: [PATCH 369/844] feat(PHP): add aurweb Prometheus metrics Along with this initial requests metric implementation, we also now serve the `/metrics` route, which grabs request metrics out of cache and renders them properly for Prometheus. **NOTE** Metrics are only enabled when the aurweb system admin has enabled caching by configuring `options.cache` correctly in `$AUR_CONFIG`. Otherwise, an error is logged about no cache being configured. New dependencies have been added which require the use of `composer`. See `INSTALL` for the dependency section in regards to composer dependencies and how to install them properly for aurweb. Metrics are in the following forms: aurweb_http_requests_count(method="GET",route="/some_route") aurweb_api_requests_count(method="GET",route="/rpc",type="search") This should allow us to search through the requests for specific routes and queries. Signed-off-by: Kevin Morris --- INSTALL | 8 ++- web/html/index.php | 29 ++++++++ web/html/metrics.php | 16 +++++ web/lib/metricfuncs.inc.php | 129 ++++++++++++++++++++++++++++++++++++ web/lib/routing.inc.php | 3 +- 5 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 web/html/metrics.php create mode 100644 web/lib/metricfuncs.inc.php diff --git a/INSTALL b/INSTALL index 9bcd0759..b161edd2 100644 --- a/INSTALL +++ b/INSTALL @@ -49,9 +49,15 @@ read the instructions below. # pacman -S python-mysql-connector python-pygit2 python-srcinfo python-sqlalchemy \ python-bleach python-markdown python-alembic python-jinja \ - python-itsdangerous python-authlib python-httpx hypercorn + python-itsdangerous python-authlib python-httpx hypercorn \ + composer # python3 setup.py install +4a) Install `composer` dependencies while inside of aurweb's root: + + $ cd /path/to/aurweb + /path/to/aurweb $ composer require promphp/prometheus_client_php + 5) Create a new MySQL database and a user and import the aurweb SQL schema: $ python -m aurweb.initdb diff --git a/web/html/index.php b/web/html/index.php index e57e7708..82a44c55 100644 --- a/web/html/index.php +++ b/web/html/index.php @@ -3,10 +3,39 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); include_once("pkgfuncs.inc.php"); +include_once("cachefuncs.inc.php"); +include_once("metricfuncs.inc.php"); $path = $_SERVER['PATH_INFO']; $tokens = explode('/', $path); +$query_string = $_SERVER['QUERY_STRING']; + +// If no options.cache is configured, we no-op metric storage operations. +$is_cached = defined('EXTENSION_LOADED_APC') || defined('EXTENSION_LOADED_MEMCACHE'); +if ($is_cached) { + $method = $_SERVER['REQUEST_METHOD']; + // We'll always add +1 to our total request count to this $path, + // unless this path == /metrics. + if ($path !== "/metrics") + add_metric("http_requests_count", $method, $path); + + // Extract $type out of $query_string, if we can. + $type = null; + $query = array(); + if ($query_string) + parse_str($query_string, $query); + $type = $query['type']; + + // Only store RPC metrics for valid types. + $good_types = [ + "info", "multiinfo", "search", "msearch", + "suggest", "suggest-pkgbase", "get-comment-form" + ]; + if ($path === "/rpc" && in_array($type, $good_types)) + add_metric("api_requests_count", $method, $path, $type); +} + if (config_get_bool('options', 'enable-maintenance') && (empty($tokens[1]) || ($tokens[1] != "css" && $tokens[1] != "images"))) { if (!in_array($_SERVER['REMOTE_ADDR'], explode(" ", config_get('options', 'maintenance-exceptions')))) { header("HTTP/1.0 503 Service Unavailable"); diff --git a/web/html/metrics.php b/web/html/metrics.php new file mode 100644 index 00000000..dfa860ed --- /dev/null +++ b/web/html/metrics.php @@ -0,0 +1,16 @@ + diff --git a/web/lib/metricfuncs.inc.php b/web/lib/metricfuncs.inc.php new file mode 100644 index 00000000..acfc30d7 --- /dev/null +++ b/web/lib/metricfuncs.inc.php @@ -0,0 +1,129 @@ +, 'query_string': }. + $metrics = get_cache_value("prometheus_metrics"); + $metrics = $metrics ? json_decode($metrics) : array(); + + $key = "$path:$type"; + + // If the current request $path isn't yet in $metrics create + // a new assoc array for it and push it into $metrics. + if (!in_array($key, $metrics)) { + $data = array( + 'anchor' => $anchor, + 'method' => $method, + 'path' => $path, + 'type' => $type + ); + array_push($metrics, json_encode($data)); + } + + // Cache-wise, we also store the count values of each route + // through the "prometheus:" key. Grab the cache value + // representing the current $path we're on (defaulted to 1). + $count = get_cache_value("prometheus:$key"); + $count = $count ? $count + 1 : 1; + + $labels = ["method", "route"]; + if ($type) + array_push($labels, "type"); + + $gauge = $registry->getOrRegisterGauge( + 'aurweb', + $anchor, + 'A metric count for the aurweb platform.', + $labels + ); + + $label_values = [$data['method'], $data['path']]; + if ($type) + array_push($label_values, $type); + + $gauge->set($count, $label_values); + + // Update cache values. + set_cache_value("prometheus:$key", $count, 0); + set_cache_value("prometheus_metrics", json_encode($metrics), 0); + +} + +function render_metrics() { + if (!defined('EXTENSION_LOADED_APC') && !defined('EXTENSION_LOADED_MEMCACHE')) { + error_log("The /metrics route requires a valid 'options.cache' " + . "configuration; no cache is configured."); + return http_response_code(417); // EXPECTATION_FAILED + } + + global $registry; + + // First, we grab the set of metrics we're interested in in the + // form of a cached JSON list, if we can. + $metrics = get_cache_value("prometheus_metrics"); + if (!$metrics) + $metrics = array(); + else + $metrics = json_decode($metrics); + + // Now, we walk through each of those list values one by one, + // which happen to be JSON-serialized associative arrays, + // and process each metric via its associative array's contents: + // The route path and the query string. + // See web/html/index.php for the creation of such metrics. + foreach ($metrics as $metric) { + $data = json_decode($metric, true); + + $anchor = $data['anchor']; + $path = $data['path']; + $type = $data['type']; + $key = "$path:$type"; + + $labels = ["method", "route"]; + if ($type) + array_push($labels, "type"); + + $count = get_cache_value("prometheus:$key"); + $gauge = $registry->getOrRegisterGauge( + 'aurweb', + $anchor, + 'A metric count for the aurweb platform.', + $labels + ); + + $label_values = [$data['method'], $data['path']]; + if ($type) + array_push($label_values, $type); + + $gauge->set($count, $label_values); + } + + // Construct the results from RenderTextFormat renderer and + // registry's samples. + $renderer = new RenderTextFormat(); + $result = $renderer->render($registry->getMetricFamilySamples()); + + // Output the results with the right content type header. + http_response_code(200); // OK + header('Content-Type: ' . RenderTextFormat::MIME_TYPE); + echo $result; +} + +?> diff --git a/web/lib/routing.inc.php b/web/lib/routing.inc.php index 73c667d2..0f452f22 100644 --- a/web/lib/routing.inc.php +++ b/web/lib/routing.inc.php @@ -19,7 +19,8 @@ $ROUTES = array( '/rss' => 'rss.php', '/tos' => 'tos.php', '/tu' => 'tu.php', - '/addvote' => 'addvote.php', + '/addvote' => 'addvote.php', + '/metrics' => 'metrics.php' // Prometheus Metrics ); $PKG_PATH = '/packages'; From 4d191b51f9d5b9a4d365c2b2f806c257f0a720e3 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 16 Sep 2021 19:42:09 -0700 Subject: [PATCH 370/844] feat(FastAPI): add /pkgbase/{name}/comaintainers (get, post) Changes from PHP: - Form action now points to `/pkgbase/{name}/comaintainers`. - When an error occurs, users are sent back to `/pkgbase/{name}/comaintainers` with an error at the top of the page. (PHP used to send people to /pkgbase/, which ended up at a blank search page). Closes: https://gitlab.archlinux.org/archlinux/aurweb/-/issues/51 Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 145 +++++++++++++++++++++++ templates/partials/packages/actions.html | 4 +- templates/pkgbase/comaintainers.html | 40 +++++++ test/test_packages_routes.py | 108 +++++++++++++++++ 4 files changed, 295 insertions(+), 2 deletions(-) create mode 100644 templates/pkgbase/comaintainers.html diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 5ae19d07..385d91db 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -16,6 +16,7 @@ from aurweb.auth import auth_required from aurweb.models.license import License from aurweb.models.package import Package from aurweb.models.package_base import PackageBase +from aurweb.models.package_comaintainer import PackageComaintainer from aurweb.models.package_comment import PackageComment from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_license import PackageLicense @@ -25,8 +26,10 @@ from aurweb.models.package_request import PackageRequest from aurweb.models.package_source import PackageSource from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import CONFLICTS_ID +from aurweb.models.user import User from aurweb.packages.search import PackageSearch from aurweb.packages.util import get_pkg_or_base, get_pkgbase_comment, query_notified, query_voted +from aurweb.scripts import notify from aurweb.scripts.rendercomment import update_comment_render from aurweb.templates import make_context, render_raw_template, render_template @@ -390,3 +393,145 @@ async def pkgbase_comment_unpin(request: Request, name: str, id: int): return RedirectResponse(f"/pkgbase/{name}", status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.get("/pkgbase/{name}/comaintainers") +@auth_required(True) +async def package_base_comaintainers(request: Request, name: str) -> Response: + # Get the PackageBase. + pkgbase = get_pkg_or_base(name, PackageBase) + + # Unauthorized users (Non-TU/Dev and not the pkgbase maintainer) + # get redirected to the package base's page. + has_creds = request.user.has_credential("CRED_PKGBASE_EDIT_COMAINTAINERS", + approved=[pkgbase.Maintainer]) + if not has_creds: + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) + + # Add our base information. + context = make_context(request, "Manage Co-maintainers") + context["pkgbase"] = pkgbase + + context["comaintainers"] = [ + c.User.Username for c in pkgbase.comaintainers + ] + + return render_template(request, "pkgbase/comaintainers.html", context) + + +def remove_users(pkgbase, usernames): + conn = db.ConnectionExecutor(db.get_engine().raw_connection()) + notifications = [] + with db.begin(): + for username in usernames: + # We know that the users we passed here are in the DB. + # No need to check for their existence. + comaintainer = pkgbase.comaintainers.join(User).filter( + User.Username == username + ).first() + notifications.append( + notify.ComaintainerRemoveNotification( + conn, comaintainer.User.ID, pkgbase.ID + ) + ) + db.session.delete(comaintainer) + + # Send out notifications if need be. + for notify_ in notifications: + notify_.send() + + +@router.post("/pkgbase/{name}/comaintainers") +@auth_required(True) +async def package_base_comaintainers_post( + request: Request, name: str, + users: str = Form(default=str())) -> Response: + # Get the PackageBase. + pkgbase = get_pkg_or_base(name, PackageBase) + + # Unauthorized users (Non-TU/Dev and not the pkgbase maintainer) + # get redirected to the package base's page. + has_creds = request.user.has_credential("CRED_PKGBASE_EDIT_COMAINTAINERS", + approved=[pkgbase.Maintainer]) + if not has_creds: + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) + + users = set(users.split("\n")) + users.remove(str()) # Remove any empty strings from the set. + records = {c.User.Username for c in pkgbase.comaintainers} + + remove_users(pkgbase, records.difference(users)) + + # Default priority (lowest value; most preferred). + priority = 1 + + # Get the highest priority in the comaintainer set. + last_priority = pkgbase.comaintainers.order_by( + PackageComaintainer.Priority.desc() + ).limit(1).first() + + # If that record exists, we use a priority which is 1 higher. + # TODO: This needs to ensure that it wraps around and preserves + # ordering in the case where we hit the max number allowed by + # the Priority column type. + if last_priority: + priority = last_priority.Priority + 1 + + def add_users(usernames): + """ Add users as comaintainers to pkgbase. + + :param usernames: An iterable of username strings + :return: None on success, an error string on failure. """ + nonlocal request, pkgbase, priority + + # First, perform a check against all usernames given; for each + # username, add its related User object to memo. + _ = l10n.get_translator_for_request(request) + memo = {} + for username in usernames: + user = db.query(User).filter(User.Username == username).first() + if not user: + return _("Invalid user name: %s") % username + memo[username] = user + + # Alright, now that we got past the check, add them all to the DB. + conn = db.ConnectionExecutor(db.get_engine().raw_connection()) + notifications = [] + with db.begin(): + for username in usernames: + user = memo.get(username) + if pkgbase.Maintainer == user: + # Already a maintainer. Move along. + continue + + # If we get here, our user model object is in the memo. + comaintainer = db.create( + PackageComaintainer, + PackageBase=pkgbase, + User=user, + Priority=priority) + priority += 1 + + notifications.append( + notify.ComaintainerAddNotification( + conn, comaintainer.User.ID, pkgbase.ID) + ) + + # Send out notifications. + for notify_ in notifications: + notify_.send() + + error = add_users(users.difference(records)) + if error: + context = make_context(request, "Manage Co-maintainers") + context["pkgbase"] = pkgbase + context["comaintainers"] = [ + c.User.Username for c in pkgbase.comaintainers + ] + context["errors"] = [error] + return render_template(request, "pkgbase/comaintainers.html", context) + + return RedirectResponse(f"/pkgbase/{pkgbase.Name}", + status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/templates/partials/packages/actions.html b/templates/partials/packages/actions.html index d552f2dd..6c30153c 100644 --- a/templates/partials/packages/actions.html +++ b/templates/partials/packages/actions.html @@ -117,9 +117,9 @@ {% endif %} - {% if is_maintainer %} + {% if request.user.has_credential('CRED_PKGBASE_EDIT_COMAINTAINERS', approved=[pkgbase.Maintainer]) %}
  • - + {{ "Manage Co-Maintainers" | tr }}
  • diff --git a/templates/pkgbase/comaintainers.html b/templates/pkgbase/comaintainers.html new file mode 100644 index 00000000..06e8b9d7 --- /dev/null +++ b/templates/pkgbase/comaintainers.html @@ -0,0 +1,40 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} + {% if errors %} +
      + {% for error in errors %} +
    • {{ error | tr }}
    • + {% endfor %} +
    + {% endif %} + +
    +

    {{ "Manage Co-maintainers" | tr }}:

    +

    + {{ + "Use this form to add co-maintainers for %s%s%s " + "(one user name per line):" + | tr | format("", pkgbase.Name, "") + | safe + }} +

    + +
    +
    +

    + + +

    + +

    + +

    +
    +
    + +
    +{% endblock %} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 1c7d5d3e..f1c20067 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1196,3 +1196,111 @@ def test_pkgbase_comment_unpin_unauthorized(client: TestClient, with client as request: resp = request.post(endpoint, cookies=cookies) assert resp.status_code == int(HTTPStatus.UNAUTHORIZED) + + +def test_pkgbase_comaintainers_not_found(client: TestClient, maintainer: User): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + endpoint = "/pkgbase/fake/comaintainers" + with client as request: + resp = request.get(endpoint, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_pkgbase_comaintainers_post_not_found(client: TestClient, + maintainer: User): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + endpoint = "/pkgbase/fake/comaintainers" + with client as request: + resp = request.post(endpoint, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_pkgbase_comaintainers_unauthorized(client: TestClient, user: User, + package: Package): + pkgbase = package.PackageBase + endpoint = f"/pkgbase/{pkgbase.Name}/comaintainers" + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.get(endpoint, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert resp.headers.get("location") == f"/pkgbase/{pkgbase.Name}" + + +def test_pkgbase_comaintainers_post_unauthorized(client: TestClient, + user: User, + package: Package): + pkgbase = package.PackageBase + endpoint = f"/pkgbase/{pkgbase.Name}/comaintainers" + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert resp.headers.get("location") == f"/pkgbase/{pkgbase.Name}" + + +def test_pkgbase_comaintainers_post_invalid_user(client: TestClient, + maintainer: User, + package: Package): + pkgbase = package.PackageBase + endpoint = f"/pkgbase/{pkgbase.Name}/comaintainers" + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, data={ + "users": "\nfake\n" + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + error = root.xpath('//ul[@class="errorlist"]/li')[0] + assert error.text.strip() == "Invalid user name: fake" + + +def test_pkgbase_comaintainers(client: TestClient, user: User, + maintainer: User, package: Package): + pkgbase = package.PackageBase + endpoint = f"/pkgbase/{pkgbase.Name}/comaintainers" + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + + # Start off by adding user as a comaintainer to package. + # The maintainer username given should be ignored. + with client as request: + resp = request.post(endpoint, data={ + "users": f"\n{user.Username}\n{maintainer.Username}\n" + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert resp.headers.get("location") == f"/pkgbase/{pkgbase.Name}" + + # Do it again to exercise the last_priority bump path. + with client as request: + resp = request.post(endpoint, data={ + "users": f"\n{user.Username}\n{maintainer.Username}\n" + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert resp.headers.get("location") == f"/pkgbase/{pkgbase.Name}" + + # Now that we've added a comaintainer to the pkgbase, + # let's perform a GET request to make sure that the backend produces + # the user we added in the users textarea. + with client as request: + resp = request.get(endpoint, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + users = root.xpath('//textarea[@id="id_users"]')[0] + assert users.text.strip() == user.Username + + # Finish off by removing all the comaintainers. + with client as request: + resp = request.post(endpoint, data={ + "users": str() + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert resp.headers.get("location") == f"/pkgbase/{pkgbase.Name}" + + with client as request: + resp = request.get(endpoint, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + users = root.xpath('//textarea[@id="id_users"]')[0] + assert users is not None and users.text is None From c164abe256e2c1ee71be1ab3815b365fcb3f80de Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 1 Sep 2021 13:55:41 -0700 Subject: [PATCH 371/844] feat(FastAPI): add Requests navigation item Along with this, created a new test suite at test/test_html.py, which has the responsibility of testing various HTML things that are not suitable for another test suite. Signed-off-by: Kevin Morris --- templates/partials/archdev-navbar.html | 5 ++ test/test_html.py | 99 ++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 test/test_html.py diff --git a/templates/partials/archdev-navbar.html b/templates/partials/archdev-navbar.html index 13459e1a..9a3ba780 100644 --- a/templates/partials/archdev-navbar.html +++ b/templates/partials/archdev-navbar.html @@ -7,6 +7,11 @@ {% endif %}
  • {% trans %}Packages{% endtrans %}
  • {% if request.user.is_authenticated() %} +
  • + + {% trans %}Requests{% endtrans %} + +
  • {% if request.user.is_trusted_user() or request.user.is_developer() %}
  • diff --git a/test/test_html.py b/test/test_html.py new file mode 100644 index 00000000..562d6a63 --- /dev/null +++ b/test/test_html.py @@ -0,0 +1,99 @@ +""" A test suite used to test HTML renders in different cases. """ +from http import HTTPStatus + +import pytest + +from fastapi.testclient import TestClient + +from aurweb import asgi, db +from aurweb.models.account_type import TRUSTED_USER_ID, USER_ID, AccountType +from aurweb.models.user import User +from aurweb.testing import setup_test_db +from aurweb.testing.html import parse_root +from aurweb.testing.requests import Request + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db(User.__tablename__) + + +@pytest.fixture +def client() -> TestClient: + yield TestClient(app=asgi.app) + + +@pytest.fixture +def user() -> User: + user_type = db.query(AccountType, AccountType.ID == USER_ID).first() + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + Passwd="testPassword", AccountType=user_type) + yield user + + +@pytest.fixture +def trusted_user(user: User) -> User: + tu_type = db.query(AccountType, + AccountType.ID == TRUSTED_USER_ID).first() + with db.begin(): + user.AccountType = tu_type + yield user + + +def test_archdev_navbar(client: TestClient): + expected = [ + "AUR Home", + "Packages", + "Register", + "Login" + ] + with client as request: + resp = request.get("/") + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + items = root.xpath('//div[@id="archdev-navbar"]/ul/li/a') + for i, item in enumerate(items): + assert item.text.strip() == expected[i] + + +def test_archdev_navbar_authenticated(client: TestClient, user: User): + expected = [ + "Dashboard", + "Packages", + "Requests", + "My Account", + "Logout" + ] + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.get("/", cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + items = root.xpath('//div[@id="archdev-navbar"]/ul/li/a') + for i, item in enumerate(items): + assert item.text.strip() == expected[i] + + +def test_archdev_navbar_authenticated_tu(client: TestClient, + trusted_user: User): + expected = [ + "Dashboard", + "Packages", + "Requests", + "Accounts", + "My Account", + "Trusted User", + "Logout" + ] + cookies = {"AURSID": trusted_user.login(Request(), "testPassword")} + with client as request: + resp = request.get("/", cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + items = root.xpath('//div[@id="archdev-navbar"]/ul/li/a') + for i, item in enumerate(items): + assert item.text.strip() == expected[i] From 99482f9962de96de0d5b248f0ee99cdd15a6a740 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 10 Sep 2021 13:28:11 -0700 Subject: [PATCH 372/844] feat(FastAPI): added /requests (get) route Introduces `aurweb.defaults` and `aurweb.filters`. `aurweb.filters` is a location developers can put their additional Jinja2 filters and/or functions. We should slowly move all of our filters over here, where it makes sense. `aurweb.defaults` is a new module which hosts some default constants and utility functions, starting with offsets (O) and per page values (PP). As far as the new GET /requests is concerned, we match up here to PHP's implementation, with some minor improvements: Improvements: * PP on this page is now configurable: 50 (default), 100, or 250. * Example: `https://localhost:8444/requests?PP=250` Modifications: * The pagination is a bit different, but serves the exact same purpose. * "Last" no longer goes to an empty page. * Closes: https://gitlab.archlinux.org/archlinux/aurweb/-/issues/14 Signed-off-by: Kevin Morris --- aurweb/auth.py | 15 +++++ aurweb/defaults.py | 18 ++++++ aurweb/filters.py | 14 ++++- aurweb/routers/packages.py | 40 ++++++++++-- aurweb/templates.py | 15 +++++ setup.cfg | 19 +++--- templates/requests.html | 115 +++++++++++++++++++++++++++++++++++ test/test_defaults.py | 14 +++++ test/test_packages_routes.py | 84 ++++++++++++++++++++++++- test/test_templates.py | 16 ++++- test/test_util.py | 8 ++- 11 files changed, 341 insertions(+), 17 deletions(-) create mode 100644 aurweb/defaults.py create mode 100644 templates/requests.html create mode 100644 test/test_defaults.py diff --git a/aurweb/auth.py b/aurweb/auth.py index d1a9d9cb..21d31081 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -31,8 +31,23 @@ class StubQuery: class AnonymousUser: + """ A stubbed User class used when an unauthenticated User + makes a request against FastAPI. """ # Stub attributes used to mimic a real user. ID = 0 + + class AccountType: + """ A stubbed AccountType static class. In here, we use an ID + and AccountType which do not exist in our constant records. + All records primary keys (AccountType.ID) should be non-zero, + so using a zero here means that we'll never match against a + real AccountType. """ + ID = 0 + AccountType = "Anonymous" + + # AccountTypeID == AccountType.ID; assign a stubbed column. + AccountTypeID = AccountType.ID + LangPreference = aurweb.config.get("options", "default_lang") Timezone = aurweb.config.get("options", "default_timezone") diff --git a/aurweb/defaults.py b/aurweb/defaults.py new file mode 100644 index 00000000..c2568d05 --- /dev/null +++ b/aurweb/defaults.py @@ -0,0 +1,18 @@ +""" Constant default values centralized in one place. """ + +# Default [O]ffset +O = 0 + +# Default [P]er [P]age +PP = 50 + +# A whitelist of valid PP values +PP_WHITELIST = {50, 100, 250} + + +def fallback_pp(per_page: int) -> int: + """ If `per_page` is a valid value in PP_WHITELIST, return it. + Otherwise, return defaults.PP. """ + if per_page not in PP_WHITELIST: + return PP + return per_page diff --git a/aurweb/filters.py b/aurweb/filters.py index bb56c656..f9f56b5d 100644 --- a/aurweb/filters.py +++ b/aurweb/filters.py @@ -4,8 +4,8 @@ import paginate from jinja2 import pass_context -from aurweb import util -from aurweb.templates import register_filter +from aurweb import config, util +from aurweb.templates import register_filter, register_function @register_filter("pager_nav") @@ -48,3 +48,13 @@ def pager_nav(context: Dict[str, Any], symbol_previous="‹ Previous", symbol_next="Next ›", symbol_last="Last »") + + +@register_function("config_getint") +def config_getint(section: str, key: str) -> int: + return config.getint(section, key) + + +@register_function("round") +def do_round(f: float) -> int: + return round(f) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 385d91db..5751a3ee 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -2,17 +2,18 @@ from datetime import datetime from http import HTTPStatus from typing import Any, Dict -from fastapi import APIRouter, Form, HTTPException, Request, Response +from fastapi import APIRouter, Form, HTTPException, Query, Request, Response from fastapi.responses import JSONResponse, RedirectResponse -from sqlalchemy import and_ +from sqlalchemy import and_, case import aurweb.filters import aurweb.models.package_comment import aurweb.models.package_keyword import aurweb.packages.util -from aurweb import db, l10n -from aurweb.auth import auth_required +from aurweb import db, defaults, l10n +from aurweb.auth import account_type_required, auth_required +from aurweb.models.account_type import DEVELOPER, TRUSTED_USER, TRUSTED_USER_AND_DEV from aurweb.models.license import License from aurweb.models.package import Package from aurweb.models.package_base import PackageBase @@ -22,10 +23,11 @@ from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_license import PackageLicense from aurweb.models.package_notification import PackageNotification from aurweb.models.package_relation import PackageRelation -from aurweb.models.package_request import PackageRequest +from aurweb.models.package_request import PENDING_ID, PackageRequest from aurweb.models.package_source import PackageSource from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import CONFLICTS_ID +from aurweb.models.request_type import RequestType from aurweb.models.user import User from aurweb.packages.search import PackageSearch from aurweb.packages.util import get_pkg_or_base, get_pkgbase_comment, query_notified, query_voted @@ -535,3 +537,31 @@ async def package_base_comaintainers_post( return RedirectResponse(f"/pkgbase/{pkgbase.Name}", status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.get("/requests") +@account_type_required({TRUSTED_USER, DEVELOPER, TRUSTED_USER_AND_DEV}) +@auth_required(True, redirect="/") +async def requests(request: Request, + O: int = Query(default=defaults.O), + PP: int = Query(default=defaults.PP)): + context = make_context(request, "Requests") + + context["q"] = dict(request.query_params) + context["O"] = O + context["PP"] = PP + + # A PackageRequest query, with left inner joined User and RequestType. + query = db.query(PackageRequest).join( + User, PackageRequest.UsersID == User.ID + ).join(RequestType) + + context["total"] = query.count() + context["results"] = query.order_by( + # Order primarily by the Status column being PENDING_ID, + # and secondarily by RequestTS; both in descending order. + case([(PackageRequest.Status == PENDING_ID, 1)], else_=0).desc(), + PackageRequest.RequestTS.desc() + ).limit(PP).offset(O).all() + + return render_template(request, "requests.html", context) diff --git a/aurweb/templates.py b/aurweb/templates.py index ef020bdf..2301cfe2 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -71,6 +71,20 @@ def register_filter(name: str) -> Callable: return decorator +def register_function(name: str) -> Callable: + """ A decorator that can be used to register a function. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + if name in _env.globals: + raise KeyError(f"Jinja already has a function named '{name}'") + _env.globals[name] = wrapper + return wrapper + return decorator + + def make_context(request: Request, title: str, next: str = None): """ Create a context for a jinja2 TemplateResponse. """ @@ -83,6 +97,7 @@ def make_context(request: Request, title: str, next: str = None): "timezones": time.SUPPORTED_TIMEZONES, "title": title, "now": datetime.now(tz=zoneinfo.ZoneInfo(timezone)), + "utcnow": int(datetime.utcnow().timestamp()), "config": aurweb.config, "next": next if next else request.url.path } diff --git a/setup.cfg b/setup.cfg index 4f2bdf7d..cec1bcf5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,17 +6,22 @@ ignore = E741, W503 max-line-length = 127 max-complexity = 10 -# aurweb/routers/accounts.py # Ignore some unavoidable flake8 warnings; we know this is against -# pycodestyle, but some of the existing codebase uses `I` variables, +# PEP8, but some of the existing codebase uses `I` variables, # so specifically silence warnings about it in pre-defined files. +# # In E741, the 'I', 'O', 'l' are ambiguous variable names. # Our current implementation uses these variables through HTTP # and the FastAPI form specification wants them named as such. -# In C901's case, our process_account_form function is way too -# complex for PEP (too many if statements). However, we need to -# process these anyways, and making it any more complex would -# just add confusion to the implementation. +# +# With {W503,W504}, PEP8 does not want us to break lines before +# or after a binary operator. We have many scripts that already +# do this, so we're ignoring it here. +ignore = E741, W503, W504 + +# aurweb/routers/accounts.py +# Ignore over-reaching complexity. +# TODO: This should actually be addressed so we do not ignore C901. # # test/test_ssh_pub_key.py # E501 is detected due to our >127 width test constant. Ignore it. @@ -24,7 +29,7 @@ max-complexity = 10 # Anything like this should be questioned. # per-file-ignores = - aurweb/routers/accounts.py:E741,C901 + aurweb/routers/accounts.py:C901 test/test_ssh_pub_key.py:E501 aurweb/routers/packages.py:E741 diff --git a/templates/requests.html b/templates/requests.html new file mode 100644 index 00000000..a9017e2f --- /dev/null +++ b/templates/requests.html @@ -0,0 +1,115 @@ +{% extends "partials/layout.html" %} + +{% set singular = "%d package request found." %} +{% set plural = "%d package requests found." %} + +{% block pageContent %} +
    + {% if not total %} +

    {{ "No requests matched your search criteria." | tr }}

    + {% else %} + {% include "partials/widgets/pager.html" %} + + + + + + + + + + + + + {% for result in results %} + + + {# Type #} + + {# Comments #} + + + {% set idle_time = config_getint("options", "request_idle_time") %} + {% set time_delta = (utcnow - result.RequestTS) | int %} + + {% set due = result.Status == 0 and time_delta > idle_time %} + + + + {% endfor %} + +
    {{ "Package" | tr }}{{ "Type" | tr }}{{ "Comments" | tr }}{{ "Filed by" | tr }}{{ "Date" | tr }}{{ "Status" | tr }}
    + {# Package #} + + {{ result.PackageBaseName }} + + + {{ result.RequestType.name_display() }} + {# If the RequestType is a merge and request.MergeBaseName is valid... #} + {% if result.RequestType.ID == 3 and result.MergeBaseName %} + ({{ result.MergeBaseName }}) + {% endif %} + {{ result.Comments }} + {# Filed by #} + + {{ result.User.Username }} + + + {# Date #} + {% set date = result.RequestTS | dt | as_timezone(timezone) %} + {{ date.strftime("%Y-%m-%d %H:%M") }} + + {# Status #} + {% if result.Status == 0 %} + {% set temp_q = { "next": "/requests" } %} + + {% if result.RequestType.ID == 1 %} + {% set action = "delete" %} + {% elif result.RequestType.ID == 2 %} + {% set action = "disown" %} + {% elif result.RequestType.ID == 3 %} + {% set action = "merge" %} + {# Add the 'via' url query parameter. #} + {% set temp_q = temp_q | extend_query( + ["via", result.ID], + ["into", result.MergeBaseName] + ) %} + {% endif %} + + {% if request.user.is_elevated() %} + {% if result.RequestType.ID == 2 and not due %} + {% set time_left = idle_time - time_delta %} + {% if time_left > 48 * 3600 %} + {% set n = round(time_left / (24 * 3600)) %} + {% set time_left_fmt = (n | tn("~%d day left", "~%d days left") | format(n)) %} + {% elif time_left > 3600 %} + {% set n = round(time_left / 3600) %} + {% set time_left_fmt = (n | tn("~%d hour left", "~%d hours left") | format(n)) %} + {% else %} + {% set time_left_fmt = ("<1 hour left" | tr) %} + {% endif %} + {{ "Locked" | tr }} + ({{ time_left_fmt }}) + {% else %} + {# Only elevated users (TU or Dev) are allowed to accept requests. #} + + {{ "Accept" | tr }} + + {% endif %} +
    + {% endif %} + + {{ "Close" | tr }} + + {% else %} + {{ result.status_display() }} + {% endif %} +
    + {% include "partials/widgets/pager.html" %} + {% endif %} +
    +{% endblock %} diff --git a/test/test_defaults.py b/test/test_defaults.py new file mode 100644 index 00000000..4803fb5a --- /dev/null +++ b/test/test_defaults.py @@ -0,0 +1,14 @@ +from aurweb import defaults + + +def test_fallback_pp(): + assert defaults.fallback_pp(75) == defaults.PP + assert defaults.fallback_pp(100) == 100 + + +def test_pp(): + assert defaults.PP == 50 + + +def test_o(): + assert defaults.O == 0 diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index f1c20067..a25fcb7e 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -8,7 +8,7 @@ import pytest from fastapi.testclient import TestClient -from aurweb import asgi, db +from aurweb import asgi, db, defaults from aurweb.models.account_type import USER_ID, AccountType from aurweb.models.dependency_type import DependencyType from aurweb.models.official_provider import OfficialProvider @@ -74,6 +74,7 @@ def setup(): PackageNotification.__tablename__, PackageComaintainer.__tablename__, PackageComment.__tablename__, + PackageRequest.__tablename__, OfficialProvider.__tablename__ ) @@ -108,6 +109,18 @@ def maintainer() -> User: yield maintainer +@pytest.fixture +def tu_user(): + tu_type = db.query(AccountType, + AccountType.AccountType == "Trusted User").first() + with db.begin(): + tu_user = db.create(User, Username="test_tu", + Email="test_tu@example.org", + RealName="Test TU", Passwd="testPassword", + AccountType=tu_type) + yield tu_user + + @pytest.fixture def package(maintainer: User) -> Package: """ Yield a Package created by user. """ @@ -160,6 +173,25 @@ def packages(maintainer: User) -> List[Package]: yield packages_ +@pytest.fixture +def requests(user: User, packages: List[Package]) -> List[PackageRequest]: + pkgreqs = [] + deletion_type = db.query(RequestType).filter( + RequestType.ID == DELETION_ID + ).first() + with db.begin(): + for i in range(55): + pkgreq = db.create(PackageRequest, + RequestType=deletion_type, + User=user, + PackageBase=packages[i].PackageBase, + PackageBaseName=packages[i].Name, + Comments=f"Deletion request for pkg_{i}", + ClosureComment=str()) + pkgreqs.append(pkgreq) + yield pkgreqs + + def test_package_not_found(client: TestClient): with client as request: resp = request.get("/packages/not_found") @@ -1304,3 +1336,53 @@ def test_pkgbase_comaintainers(client: TestClient, user: User, root = parse_root(resp.text) users = root.xpath('//textarea[@id="id_users"]')[0] assert users is not None and users.text is None + + +def test_requests_unauthorized(client: TestClient, + maintainer: User, + tu_user: User, + packages: List[Package], + requests: List[PackageRequest]): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + with client as request: + resp = request.get("/requests", cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + +def test_requests(client: TestClient, + maintainer: User, + tu_user: User, + packages: List[Package], + requests: List[PackageRequest]): + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + resp = request.get("/requests", params={ + # Pass in url query parameters O, SeB and SB to exercise + # their paths inside of the pager_nav used in this request. + "O": 0, # Page 1 + "SeB": "nd", + "SB": "n" + }, cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + assert "Next ›" in resp.text + assert "Last »" in resp.text + + root = parse_root(resp.text) + # We have 55 requests, our defaults.PP is 50, so expect we have 50 rows. + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == defaults.PP + + # Request page 2 of the requests page. + with client as request: + resp = request.get("/requests", params={ + "O": 50 # Page 2 + }, cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + assert "‹ Previous" in resp.text + assert "« First" in resp.text + + root = parse_root(resp.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 5 # There are five records left on the second page. diff --git a/test/test_templates.py b/test/test_templates.py index b6aa2055..86fbf611 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -1,6 +1,6 @@ import pytest -from aurweb.templates import register_filter +from aurweb.templates import register_filter, register_function @register_filter("func") @@ -8,6 +8,11 @@ def func(): pass +@register_function("function") +def function(): + pass + + def test_register_filter_exists_key_error(): """ Most instances of register_filter are tested through module imports or template renders, so we only test failures here. """ @@ -15,3 +20,12 @@ def test_register_filter_exists_key_error(): @register_filter("func") def some_func(): pass + + +def test_register_function_exists_key_error(): + """ Most instances of register_filter are tested through module + imports or template renders, so we only test failures here. """ + with pytest.raises(KeyError): + @register_function("function") + def some_func(): + pass diff --git a/test/test_util.py b/test/test_util.py index 0cc45409..99b77a78 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,7 +1,7 @@ from datetime import datetime from zoneinfo import ZoneInfo -from aurweb import util +from aurweb import filters, util def test_timestamp_to_datetime(): @@ -34,3 +34,9 @@ def test_to_qs(): query = {"a": "b", "c": [1, 2, 3]} qs = util.to_qs(query) assert qs == "a=b&c=1&c=2&c=3" + + +def test_round(): + assert filters.do_round(1.3) == 1 + assert filters.do_round(1.5) == 2 + assert filters.do_round(2.0) == 2 From 1cf9420997bc6788b4ae3264d8e019fd0ec13d56 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 12 Sep 2021 20:05:49 -0700 Subject: [PATCH 373/844] feat(FastAPI): allow reporters to cancel their own requests (1/2) This change required a slight modification of how we handle the Requests page. It is now available to all users. This commit provides 1/2 of the implementation which actually satisfies this feature. 2/2 will contain the actual implementation of closures of requests, which will also allow users who created the request to decide to close it. Issue: https://gitlab.archlinux.org/archlinux/aurweb/-/issues/20 Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 9 ++++++--- test/test_packages_routes.py | 28 +++++++++++++++++++++------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 5751a3ee..2b350478 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -12,8 +12,7 @@ import aurweb.models.package_keyword import aurweb.packages.util from aurweb import db, defaults, l10n -from aurweb.auth import account_type_required, auth_required -from aurweb.models.account_type import DEVELOPER, TRUSTED_USER, TRUSTED_USER_AND_DEV +from aurweb.auth import auth_required from aurweb.models.license import License from aurweb.models.package import Package from aurweb.models.package_base import PackageBase @@ -540,7 +539,6 @@ async def package_base_comaintainers_post( @router.get("/requests") -@account_type_required({TRUSTED_USER, DEVELOPER, TRUSTED_USER_AND_DEV}) @auth_required(True, redirect="/") async def requests(request: Request, O: int = Query(default=defaults.O), @@ -556,6 +554,11 @@ async def requests(request: Request, User, PackageRequest.UsersID == User.ID ).join(RequestType) + # If the request user is not elevated (TU or Dev), then + # filter PackageRequests which are owned by the request user. + if not request.user.is_elevated(): + query = query.filter(PackageRequest.UsersID == request.user.ID) + context["total"] = query.count() context["results"] = query.order_by( # Order primarily by the Status column being PENDING_ID, diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index a25fcb7e..9867ce42 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1338,14 +1338,9 @@ def test_pkgbase_comaintainers(client: TestClient, user: User, assert users is not None and users.text is None -def test_requests_unauthorized(client: TestClient, - maintainer: User, - tu_user: User, - packages: List[Package], - requests: List[PackageRequest]): - cookies = {"AURSID": maintainer.login(Request(), "testPassword")} +def test_requests_unauthorized(client: TestClient): with client as request: - resp = request.get("/requests", cookies=cookies, allow_redirects=False) + resp = request.get("/requests", allow_redirects=False) assert resp.status_code == int(HTTPStatus.SEE_OTHER) @@ -1386,3 +1381,22 @@ def test_requests(client: TestClient, root = parse_root(resp.text) rows = root.xpath('//table[@class="results"]/tbody/tr') assert len(rows) == 5 # There are five records left on the second page. + + +def test_requests_selfmade(client: TestClient, user: User, + requests: List[PackageRequest]): + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.get("/requests", cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + # As the user who creates all of the requests, we should see all of them. + # However, we are not allowed to accept any of them ourselves. + root = parse_root(resp.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == defaults.PP + + # Our first and only link in the last row should be "Close". + for row in rows: + last_row = row.xpath('./td')[-1].xpath('./a')[0] + assert last_row.text.strip() == "Close" From ad8369395e323d99bd4b3cae430269ac8dd19491 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 12 Sep 2021 21:51:20 -0700 Subject: [PATCH 374/844] feat(FastAPI): add /pkgbase/{name}/request (get) This change brings in the package base request form for new submissions. Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 14 ++++ templates/pkgbase/request.html | 87 ++++++++++++++++++++++++ test/test_packages_routes.py | 20 ++++++ web/html/js/typeahead-pkgbase-request.js | 36 ++++++++++ 4 files changed, 157 insertions(+) create mode 100644 templates/pkgbase/request.html create mode 100644 web/html/js/typeahead-pkgbase-request.js diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 2b350478..9c9a41e3 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -568,3 +568,17 @@ async def requests(request: Request, ).limit(PP).offset(O).all() return render_template(request, "requests.html", context) + + +@router.get("/pkgbase/{name}/request") +@auth_required(True) +async def package_request(request: Request, name: str): + context = make_context(request, "Submit Request") + + pkgbase = db.query(PackageBase).filter(PackageBase.Name == name).first() + + if not pkgbase: + raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) + + context["pkgbase"] = pkgbase + return render_template(request, "pkgbase/request.html", context) diff --git a/templates/pkgbase/request.html b/templates/pkgbase/request.html new file mode 100644 index 00000000..66d69f07 --- /dev/null +++ b/templates/pkgbase/request.html @@ -0,0 +1,87 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} +
    +

    {{ "Submit Request" | tr }}: {{ pkgbase.Name }}

    +

    + {{ "Use this form to file a request against package base " + "%s%s%s which includes the following packages:" + | tr | format("", pkgbase.Name, "") | safe }} +

    +
      + {% for package in pkgbase.packages %} +
    • {{ package.Name }}
    • + {% endfor %} +
    + + {# Request form #} +
    +
    +

    + + +

    + + {# Javascript included for HTML-changing triggers depending + on the selected type (above). #} + + + + +

    + + +

    + +

    + {{ + "By submitting a deletion request, you ask a Trusted " + "User to delete the package base. This type of " + "request should be used for duplicates, software " + "abandoned by upstream, as well as illegal and " + "irreparably broken packages." | tr + }} +

    + + + + + +

    + +

    + +
    +
    + +
    +{% endblock %} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 9867ce42..8704d702 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1400,3 +1400,23 @@ def test_requests_selfmade(client: TestClient, user: User, for row in rows: last_row = row.xpath('./td')[-1].xpath('./a')[0] assert last_row.text.strip() == "Close" + + +def test_pkgbase_request_not_found(client: TestClient, user: User): + pkgbase_name = "fake" + endpoint = f"/pkgbase/{pkgbase_name}/request" + + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.get(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_pkgbase_request(client: TestClient, user: User, package: Package): + pkgbase = package.PackageBase + endpoint = f"/pkgbase/{pkgbase.Name}/request" + + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.get(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) diff --git a/web/html/js/typeahead-pkgbase-request.js b/web/html/js/typeahead-pkgbase-request.js new file mode 100644 index 00000000..e012d55f --- /dev/null +++ b/web/html/js/typeahead-pkgbase-request.js @@ -0,0 +1,36 @@ +function showHideMergeSection() { + const elem = document.getElementById('id_type'); + const merge_section = document.getElementById('merge_section'); + if (elem.value == 'merge') { + merge_section.style.display = ''; + } else { + merge_section.style.display = 'none'; + } +} + +function showHideRequestHints() { + document.getElementById('deletion_hint').style.display = 'none'; + document.getElementById('merge_hint').style.display = 'none'; + document.getElementById('orphan_hint').style.display = 'none'; + + const elem = document.getElementById('id_type'); + document.getElementById(elem.value + '_hint').style.display = ''; +} + +document.addEventListener('DOMContentLoaded', function() { + showHideMergeSection(); + showHideRequestHints(); + + const input = document.getElementById('id_merge_into'); + const form = document.getElementById('request-form'); + const type = "suggest-pkgbase"; + + typeahead.init(type, input, form, false); +}); + +// Bind the change event here, otherwise we have to inline javascript, +// which angers CSP (Content Security Policy). +document.getElementById("id_type").addEventListener("change", function() { + showHideMergeSection(); + showHideRequestHints(); +}); From 1c031638c65588ef5c219adffdaf1a7b695d0d02 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 13 Sep 2021 17:26:25 -0700 Subject: [PATCH 375/844] feat(FastAPI): add /pkgbase/{name}/request (post) This change implements the FastAPI version of the /pkgbase/{name}/request form's action. Changes from PHP: - Additional errors are now displayed for the **merge_into** field, which are only displayed when the Merge type is selected. - If the **merge_into** field is empty, a new error is displayed: 'The "Merge into" field must not be empty.' - If the **merge_into** field is given the name of a package base which does not exist, a new error is displayed: "The package base you want to merge into does not exist." - If the **merge_into** field is given the name of the package base that a request is being created for, a new error is displayed: "You cannot merge a package base into itself." - When an error is encountered, users are now brought back to the request form which they submitted and an error is displayed at the top of the page. - If an invalid type is provided, users are returned to a BAD_REQUEST status rendering of the request form. Signed-off-by: Kevin Morris --- aurweb/models/request_type.py | 6 ++ aurweb/routers/packages.py | 69 ++++++++++++++ templates/pkgbase/request.html | 12 ++- test/test_packages_routes.py | 161 +++++++++++++++++++++++++++++++++ 4 files changed, 247 insertions(+), 1 deletion(-) diff --git a/aurweb/models/request_type.py b/aurweb/models/request_type.py index a26dcf9a..48ace3a3 100644 --- a/aurweb/models/request_type.py +++ b/aurweb/models/request_type.py @@ -20,6 +20,12 @@ class RequestType(Base): name = self.Name return name[0].upper() + name[1:] + def title(self) -> str: + return self.name_display() + + def __getitem__(self, n: int) -> str: + return self.Name[n] + DELETION_ID = db.query(RequestType, RequestType.Name == DELETION).first().ID ORPHAN_ID = db.query(RequestType, RequestType.Name == ORPHAN).first().ID diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 9c9a41e3..231f953b 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -582,3 +582,72 @@ async def package_request(request: Request, name: str): context["pkgbase"] = pkgbase return render_template(request, "pkgbase/request.html", context) + + +@router.post("/pkgbase/{name}/request") +@auth_required(True) +async def pkgbase_request_post(request: Request, name: str, + type: str = Form(...), + merge_into: str = Form(default=None), + comments: str = Form(default=str())): + pkgbase = get_pkg_or_base(name, PackageBase) + + # Create our render context. + context = make_context(request, "Submit Request") + context["pkgbase"] = pkgbase + if type not in {"deletion", "merge", "orphan"}: + # In the case that someone crafted a POST request with an invalid + # type, just return them to the request form with BAD_REQUEST status. + return render_template(request, "pkgbase/request.html", context, + status_code=HTTPStatus.BAD_REQUEST) + + if not comments: + context["errors"] = ["The comment field must not be empty."] + return render_template(request, "pkgbase/request.html", context) + + if type == "merge": + # Perform merge-related checks. + if not merge_into: + # TODO: This error needs to be translated. + context["errors"] = ['The "Merge into" field must not be empty.'] + return render_template(request, "pkgbase/request.html", context) + + target = db.query(PackageBase).filter( + PackageBase.Name == merge_into + ).first() + if not target: + # TODO: This error needs to be translated. + context["errors"] = [ + "The package base you want to merge into does not exist." + ] + return render_template(request, "pkgbase/request.html", context) + + if target.ID == pkgbase.ID: + # TODO: This error needs to be translated. + context["errors"] = [ + "You cannot merge a package base into itself." + ] + return render_template(request, "pkgbase/request.html", context) + + # All good. Create a new PackageRequest based on the given type. + now = int(datetime.utcnow().timestamp()) + reqtype = db.query(RequestType, RequestType.Name == type).first() + conn = db.ConnectionExecutor(db.get_engine().raw_connection()) + notify_ = None + with db.begin(): + pkgreq = db.create(PackageRequest, RequestType=reqtype, RequestTS=now, + PackageBase=pkgbase, PackageBaseName=pkgbase.Name, + MergeBaseName=merge_into, User=request.user, + Comments=comments, ClosureComment=str()) + + # Prepare notification object. + notify_ = notify.RequestOpenNotification( + conn, request.user.ID, pkgreq.ID, reqtype, + pkgreq.PackageBase.ID, merge_into=merge_into or None) + + # Send the notification now that we're out of the DB scope. + notify_.send() + + # Redirect the submitting user to /packages. + return RedirectResponse("/packages", + status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/templates/pkgbase/request.html b/templates/pkgbase/request.html index 66d69f07..bb9b5aba 100644 --- a/templates/pkgbase/request.html +++ b/templates/pkgbase/request.html @@ -1,8 +1,17 @@ {% extends "partials/layout.html" %} {% block pageContent %} + {% if errors %} +
      + {% for error in errors %} +
    • {{ error | tr }}
    • + {% endfor %} +
    + {% endif %} +

    {{ "Submit Request" | tr }}: {{ pkgbase.Name }}

    +

    {{ "Use this form to file a request against package base " "%s%s%s which includes the following packages:" @@ -15,7 +24,8 @@ {# Request form #} -

    +

    diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 8704d702..5353d3bf 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1420,3 +1420,164 @@ def test_pkgbase_request(client: TestClient, user: User, package: Package): with client as request: resp = request.get(endpoint, cookies=cookies) assert resp.status_code == int(HTTPStatus.OK) + + +def test_pkgbase_request_post_deletion(client: TestClient, user: User, + package: Package): + endpoint = f"/pkgbase/{package.PackageBase.Name}/request" + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, data={ + "type": "deletion", + "comments": "We want to delete this." + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + pkgreq = db.query(PackageRequest).filter( + PackageRequest.PackageBaseID == package.PackageBase.ID + ).first() + assert pkgreq is not None + assert pkgreq.RequestType.Name == "deletion" + assert pkgreq.PackageBaseName == package.PackageBase.Name + assert pkgreq.Comments == "We want to delete this." + + +def test_pkgbase_request_post_orphan(client: TestClient, user: User, + package: Package): + endpoint = f"/pkgbase/{package.PackageBase.Name}/request" + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, data={ + "type": "orphan", + "comments": "We want to disown this." + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + pkgreq = db.query(PackageRequest).filter( + PackageRequest.PackageBaseID == package.PackageBase.ID + ).first() + assert pkgreq is not None + assert pkgreq.RequestType.Name == "orphan" + assert pkgreq.PackageBaseName == package.PackageBase.Name + assert pkgreq.Comments == "We want to disown this." + + +def test_pkgbase_request_post_merge(client: TestClient, user: User, + package: Package): + with db.begin(): + pkgbase2 = db.create(PackageBase, Name="new-pkgbase", + Submitter=user, Maintainer=user, Packager=user) + target = db.create(Package, PackageBase=pkgbase2, + Name=pkgbase2.Name, Version="1.0.0") + + endpoint = f"/pkgbase/{package.PackageBase.Name}/request" + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, data={ + "type": "merge", + "merge_into": target.PackageBase.Name, + "comments": "We want to merge this." + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + pkgreq = db.query(PackageRequest).filter( + PackageRequest.PackageBaseID == package.PackageBase.ID + ).first() + assert pkgreq is not None + assert pkgreq.RequestType.Name == "merge" + assert pkgreq.PackageBaseName == package.PackageBase.Name + assert pkgreq.MergeBaseName == target.PackageBase.Name + assert pkgreq.Comments == "We want to merge this." + + +def test_pkgbase_request_post_not_found(client: TestClient, user: User): + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post("/pkgbase/fake/request", data={ + "type": "fake" + }, cookies=cookies) + assert resp.status_code == int(HTTPStatus.NOT_FOUND) + + +def test_pkgbase_request_post_invalid_type(client: TestClient, + user: User, + package: Package): + endpoint = f"/pkgbase/{package.PackageBase.Name}/request" + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, data={"type": "fake"}, cookies=cookies) + assert resp.status_code == int(HTTPStatus.BAD_REQUEST) + + +def test_pkgbase_request_post_no_comment_error(client: TestClient, + user: User, + package: Package): + endpoint = f"/pkgbase/{package.PackageBase.Name}/request" + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, data={ + "type": "deletion", + "comments": "" # An empty comment field causes an error. + }, cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + error = root.xpath('//ul[@class="errorlist"]/li')[0] + expected = "The comment field must not be empty." + assert error.text.strip() == expected + + +def test_pkgbase_request_post_merge_not_found_error(client: TestClient, + user: User, + package: Package): + endpoint = f"/pkgbase/{package.PackageBase.Name}/request" + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, data={ + "type": "merge", + "merge_into": "fake", # There is no PackageBase.Name "fake" + "comments": "We want to merge this." + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + error = root.xpath('//ul[@class="errorlist"]/li')[0] + expected = "The package base you want to merge into does not exist." + assert error.text.strip() == expected + + +def test_pkgbase_request_post_merge_no_merge_into_error(client: TestClient, + user: User, + package: Package): + endpoint = f"/pkgbase/{package.PackageBase.Name}/request" + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, data={ + "type": "merge", + "merge_into": "", # There is no PackageBase.Name "fake" + "comments": "We want to merge this." + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + error = root.xpath('//ul[@class="errorlist"]/li')[0] + expected = 'The "Merge into" field must not be empty.' + assert error.text.strip() == expected + + +def test_pkgbase_request_post_merge_self_error(client: TestClient, user: User, + package: Package): + endpoint = f"/pkgbase/{package.PackageBase.Name}/request" + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, data={ + "type": "merge", + "merge_into": package.PackageBase.Name, + "comments": "We want to merge this." + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + error = root.xpath('//ul[@class="errorlist"]/li')[0] + expected = "You cannot merge a package base into itself." + assert error.text.strip() == expected From f6141ff1778e8d1376a0db3b92e7a7d7fa2f9097 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 14 Sep 2021 21:37:35 -0700 Subject: [PATCH 376/844] feat(FastAPI): add /requests/{id}/close (get, post) Changes from PHP: - If a user submits a POST request with an invalid reason, they are returned back to the closure form with a BAD_REQUEST status. - Now, users which created a PackageRequest have the ability to close their own. - Form action has been changed to `/requests/{id}/close`. Closes https://gitlab.archlinux.org/archlinux/aurweb/-/issues/20 Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 52 ++++++++++++++++++++- templates/requests/close.html | 60 ++++++++++++++++++++++++ test/test_packages_routes.py | 86 ++++++++++++++++++++++++++++++++++- 3 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 templates/requests/close.html diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 231f953b..a3effb36 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -22,7 +22,7 @@ from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_license import PackageLicense from aurweb.models.package_notification import PackageNotification from aurweb.models.package_relation import PackageRelation -from aurweb.models.package_request import PENDING_ID, PackageRequest +from aurweb.models.package_request import ACCEPTED_ID, PENDING_ID, REJECTED_ID, PackageRequest from aurweb.models.package_source import PackageSource from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import CONFLICTS_ID @@ -651,3 +651,53 @@ async def pkgbase_request_post(request: Request, name: str, # Redirect the submitting user to /packages. return RedirectResponse("/packages", status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.get("/requests/{id}/close") +@auth_required(True) +async def requests_close(request: Request, id: int): + pkgreq = db.query(PackageRequest).filter(PackageRequest.ID == id).first() + if not request.user.is_elevated() and request.user != pkgreq.User: + # Request user doesn't have permission here: redirect to '/'. + return RedirectResponse("/", status_code=int(HTTPStatus.SEE_OTHER)) + + context = make_context(request, "Close Request") + context["pkgreq"] = pkgreq + return render_template(request, "requests/close.html", context) + + +@router.post("/requests/{id}/close") +@auth_required(True) +async def requests_close_post(request: Request, id: int, + reason: int = Form(default=0), + comments: str = Form(default=str())): + pkgreq = db.query(PackageRequest).filter(PackageRequest.ID == id).first() + if not request.user.is_elevated() and request.user != pkgreq.User: + # Request user doesn't have permission here: redirect to '/'. + return RedirectResponse("/", status_code=int(HTTPStatus.SEE_OTHER)) + + context = make_context(request, "Close Request") + context["pkgreq"] = pkgreq + + if reason not in {ACCEPTED_ID, REJECTED_ID}: + # If the provided reason is not valid, send the user back to + # the closure form with a BAD_REQUEST status. + return render_template(request, "requests/close.html", context, + status_code=HTTPStatus.BAD_REQUEST) + + if not request.user.is_elevated(): + # If we're closing the request as the user who created it, + # the reason should just be a REJECTION. + reason = REJECTED_ID + + with db.begin(): + pkgreq.Closer = request.user + pkgreq.Status = reason + pkgreq.ClosureComment = comments + + conn = db.ConnectionExecutor(db.get_engine().raw_connection()) + notify_ = notify.RequestCloseNotification( + conn, request.user.ID, pkgreq.ID, pkgreq.status_display()) + notify_.send() + + return RedirectResponse("/requests", status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/templates/requests/close.html b/templates/requests/close.html new file mode 100644 index 00000000..7862064a --- /dev/null +++ b/templates/requests/close.html @@ -0,0 +1,60 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} +

    +

    {{ "Close Request" | tr }}: {{ pkgreq.PackageBaseName }}

    + +

    + {{ + "Use this form to close the request for package base %s%s%s." + | tr | format("", pkgreq.PackageBaseName, "") + | safe + }} +

    + +

    + {{ "Note" | tr }}: + {{ + "The comments field can be left empty. However, it is highly " + "recommended to add a comment when rejecting a request." + | tr + }} +

    + + +
    +

    + + +

    + +

    + + +

    + +

    + +

    + +
    + + +
    +{% endblock %} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 5353d3bf..5afe011a 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -20,7 +20,7 @@ from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_keyword import PackageKeyword from aurweb.models.package_notification import PackageNotification from aurweb.models.package_relation import PackageRelation -from aurweb.models.package_request import PackageRequest +from aurweb.models.package_request import ACCEPTED_ID, REJECTED_ID, PackageRequest from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import PROVIDES_ID, RelationType from aurweb.models.request_type import DELETION_ID, RequestType @@ -1581,3 +1581,87 @@ def test_pkgbase_request_post_merge_self_error(client: TestClient, user: User, error = root.xpath('//ul[@class="errorlist"]/li')[0] expected = "You cannot merge a package base into itself." assert error.text.strip() == expected + + +@pytest.fixture +def pkgreq(user: User, package: Package) -> PackageRequest: + reqtype = db.query(RequestType).filter( + RequestType.ID == DELETION_ID + ).first() + with db.begin(): + pkgreq = db.create(PackageRequest, + RequestType=reqtype, + User=user, + PackageBase=package.PackageBase, + PackageBaseName=package.PackageBase.Name, + Comments=str(), + ClosureComment=str()) + yield pkgreq + + +def test_requests_close(client: TestClient, user: User, + pkgreq: PackageRequest): + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.get(f"/requests/{pkgreq.ID}/close", cookies=cookies, + allow_redirects=False) + assert resp.status_code == int(HTTPStatus.OK) + + +def test_requests_close_unauthorized(client: TestClient, maintainer: User, + pkgreq: PackageRequest): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + with client as request: + resp = request.get(f"/requests/{pkgreq.ID}/close", cookies=cookies, + allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert resp.headers.get("location") == "/" + + +def test_requests_close_post_invalid_reason(client: TestClient, user: User, + pkgreq: PackageRequest): + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(f"/requests/{pkgreq.ID}/close", data={ + "reason": 0 + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.BAD_REQUEST) + + +def test_requests_close_post_unauthorized(client: TestClient, maintainer: User, + pkgreq: PackageRequest): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + with client as request: + resp = request.post(f"/requests/{pkgreq.ID}/close", data={ + "reason": ACCEPTED_ID + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert resp.headers.get("location") == "/" + + +def test_requests_close_post(client: TestClient, user: User, + pkgreq: PackageRequest): + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(f"/requests/{pkgreq.ID}/close", data={ + "reason": REJECTED_ID + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + assert pkgreq.Status == REJECTED_ID + assert pkgreq.Closer == user + assert pkgreq.ClosureComment == str() + + +def test_requests_close_post_rejected(client: TestClient, user: User, + pkgreq: PackageRequest): + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(f"/requests/{pkgreq.ID}/close", data={ + "reason": REJECTED_ID + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + assert pkgreq.Status == REJECTED_ID + assert pkgreq.Closer == user + assert pkgreq.ClosureComment == str() From b5f8e69b8aaefc093eabb3163eb0dd6445682a8b Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 3 Oct 2021 10:22:34 -0700 Subject: [PATCH 377/844] feat(FastAPI): use SQLAlchemy's scoped_session Closes #113 Signed-off-by: Kevin Morris --- aurweb/db.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aurweb/db.py b/aurweb/db.py index ea6b6918..2b934300 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -3,6 +3,7 @@ import math import re from sqlalchemy import event +from sqlalchemy.orm import scoped_session import aurweb.config import aurweb.util @@ -167,7 +168,8 @@ def get_engine(echo: bool = False): connect_args=connect_args, echo=echo) - Session = sessionmaker(autocommit=True, autoflush=False, bind=engine) + Session = scoped_session( + sessionmaker(autocommit=True, autoflush=False, bind=engine)) session = Session() if db_backend == "sqlite": From 7bfc2bf9b44ba13526b20160531aea208f694c89 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 3 Oct 2021 15:11:42 -0700 Subject: [PATCH 378/844] fix(FastAPI): Improve sqlite testing speed This commit adds a new Arch dependency: `libeatmydata`, which provides the `eatmydata` executable that stubs out fsync() operations. We use `eatmydata` to run our sharness and pytests in Docker now. With `autocommit=True`, required by SQLAlchemy to keep the session up to date with external DB modifications, many fsync calls are used in the SQLite backend; especially because we're wiping and creating records in every DB-bound test. **Before:** - mysql: 1m42s (elapsed during pytest run) - sqlite: 3m06s (elapsed during pytest run) **After:** - mysql: 1m40s (elapsed during pytest run) - sqlite: 1m50s (elapsed during pytest run) Shout out to @klausenbusk, who suggested this as a possible fix, and it was. Thanks, Kristian! Closes #120 Signed-off-by: Kevin Morris --- docker/scripts/install-deps.sh | 2 +- docker/scripts/run-pytests.sh | 2 +- docker/scripts/run-sharness.sh | 2 +- test/README.md | 10 ++++++++++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh index f8881d05..fc068b06 100755 --- a/docker/scripts/install-deps.sh +++ b/docker/scripts/install-deps.sh @@ -8,7 +8,7 @@ pacman -Syu --noconfirm --noprogressbar \ --cachedir .pkg-cache git gpgme nginx redis openssh \ mariadb mariadb-libs cgit uwsgi uwsgi-plugin-cgi \ php php-fpm memcached php-memcached python-pip pyalpm \ - python-srcinfo curl + python-srcinfo curl libeatmydata # https://python-poetry.org/docs/ Installation section. curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - diff --git a/docker/scripts/run-pytests.sh b/docker/scripts/run-pytests.sh index c6baa939..ef8a2318 100755 --- a/docker/scripts/run-pytests.sh +++ b/docker/scripts/run-pytests.sh @@ -27,7 +27,7 @@ python -m aurweb.initdb 2>/dev/null || \ (echo "Error: aurweb.initdb failed; already initialized?" && /bin/true) # Run pytest with optional targets in front of it. -make -C test "${PARAMS[@]}" pytest +eatmydata -- make -C test "${PARAMS[@]}" pytest # By default, report coverage and move it into cache. if [ $COVERAGE -eq 1 ]; then diff --git a/docker/scripts/run-sharness.sh b/docker/scripts/run-sharness.sh index 8e928b3f..fe16751c 100755 --- a/docker/scripts/run-sharness.sh +++ b/docker/scripts/run-sharness.sh @@ -4,4 +4,4 @@ set -eou pipefail # Initialize the new database; ignore errors. python -m aurweb.initdb 2>/dev/null || /bin/true -make -C test sh +eatmydata -- make -C test sh diff --git a/test/README.md b/test/README.md index ef8a08f4..13fb0a0c 100644 --- a/test/README.md +++ b/test/README.md @@ -31,6 +31,10 @@ For all the test to run, the following Arch packages should be installed: - postfix - openssh +Optional (faster testing) + +- libeatmydata + Test Configuration ------------------ @@ -115,6 +119,12 @@ To run `pytest` Python test suites: $ make -C test pytest +**Note:** For SQLite tests, users may want to use `eatmydata` +to improve speed: + + $ eatmydata -- make -C test sh + $ eatmydata -- make -C test pytest + To produce coverage reports related to Python when running tests manually, use the following method: From 08068e0a5c70ef8d5ef94c20952d9aa15ba6c8dc Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Mon, 4 Oct 2021 13:30:25 -0400 Subject: [PATCH 379/844] fix(FastAPI): use configured letter case for SSH fingerprints Currently, the config parser converts all keys to lowercase which is inconsistent with the old PHP behavior. This has been fixed and relevant fingerprint-getting functions have been simplified without changes in behavior. Signed-off-by: Steven Guikal --- aurweb/config.py | 8 ++++---- aurweb/util.py | 8 +------- test/test_homepage.py | 4 +++- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/aurweb/config.py b/aurweb/config.py index 52fadda2..aa111f15 100644 --- a/aurweb/config.py +++ b/aurweb/config.py @@ -17,6 +17,7 @@ def _get_parser(): defaults = os.environ.get('AUR_CONFIG_DEFAULTS', path + '.defaults') _parser = configparser.RawConfigParser() + _parser.optionxform = lambda option: option if os.path.isfile(defaults): with open(defaults) as f: _parser.read_file(f) @@ -48,7 +49,6 @@ def getint(section, option, fallback=None): return _get_parser().getint(section, option, fallback=fallback) -def get_section(section_name): - for section in _get_parser().sections(): - if section == section_name: - return _get_parser()[section] +def get_section(section): + if section in _get_parser().sections(): + return _get_parser()[section] diff --git a/aurweb/util.py b/aurweb/util.py index f9181811..08e6d7c6 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -166,10 +166,4 @@ def add_samesite_fields(response: Response, value: str): def get_ssh_fingerprints(): - fingerprints = {} - fingerprint_section = aurweb.config.get_section("fingerprints") - - if fingerprint_section: - fingerprints = {key: fingerprint_section[key] for key in fingerprint_section.keys()} - - return fingerprints + return aurweb.config.get_section("fingerprints") or {} diff --git a/test/test_homepage.py b/test/test_homepage.py index fef3532d..5c678b71 100644 --- a/test/test_homepage.py +++ b/test/test_homepage.py @@ -96,7 +96,9 @@ def test_homepage_ssh_fingerprints(get_ssh_fingerprints_mock): with client as request: response = request.get("/") - assert list(fingerprints.values())[0] in response.content.decode() + for key, value in fingerprints.items(): + assert key in response.content.decode() + assert value in response.content.decode() assert 'The following SSH fingerprints are used for the AUR' in response.content.decode() From 5c179dc4d35b8c2bbacce61d0664fc7285d99e56 Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Mon, 4 Oct 2021 17:04:23 -0400 Subject: [PATCH 380/844] fix(FastAPI): use consistent ordering on dashboard and request page Signed-off-by: Steven Guikal --- aurweb/routers/html.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index 3d44cf87..6e7697e4 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -6,7 +6,7 @@ from http import HTTPStatus from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse -from sqlalchemy import and_, or_ +from sqlalchemy import and_, case, or_ import aurweb.config import aurweb.models.package_request @@ -17,7 +17,7 @@ from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.package_comaintainer import PackageComaintainer -from aurweb.models.package_request import PackageRequest +from aurweb.models.package_request import PENDING_ID, PackageRequest from aurweb.models.user import User from aurweb.packages.util import query_notified, query_voted, updated_packages from aurweb.templates import make_context, render_template @@ -166,6 +166,11 @@ async def index(request: Request): # Package requests created by request.user. context["package_requests"] = request.user.package_requests.filter( PackageRequest.RequestTS >= start + ).order_by( + # Order primarily by the Status column being PENDING_ID, + # and secondarily by RequestTS; both in descending order. + case([(PackageRequest.Status == PENDING_ID, 1)], else_=0).desc(), + PackageRequest.RequestTS.desc() ).limit(50).all() # Packages that the request user maintains or comaintains. From 9af76a73a331b889815e42866e3df4bbe8ddc5d0 Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Mon, 4 Oct 2021 16:32:10 -0400 Subject: [PATCH 381/844] fix(FastAPI): include MergeBaseName in merge request type This was done on the dedicated requests page, but missed on the dashboard. Signed-off-by: Steven Guikal --- templates/partials/packages/requests.html | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/templates/partials/packages/requests.html b/templates/partials/packages/requests.html index 5239ca72..5188a476 100644 --- a/templates/partials/packages/requests.html +++ b/templates/partials/packages/requests.html @@ -20,7 +20,13 @@ {{ request.PackageBase.Name }}
    - {{ request.RequestType.name_display() | tr }} + + {{ request.RequestType.name_display() | tr }} + {# If the RequestType is a merge and request.MergeBaseName is valid... #} + {% if request.RequestType.ID == 3 and request.MergeBaseName %} + ({{ request.MergeBaseName }}) + {% endif %} + {{ request.Comments }} Date: Mon, 4 Oct 2021 17:37:25 -0400 Subject: [PATCH 382/844] fix(FastAPI): add missing translation filter for request type Signed-off-by: Steven Guikal --- templates/requests.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/requests.html b/templates/requests.html index a9017e2f..74ea6416 100644 --- a/templates/requests.html +++ b/templates/requests.html @@ -31,7 +31,7 @@ {# Type #} - {{ result.RequestType.name_display() }} + {{ result.RequestType.name_display() | tr }} {# If the RequestType is a merge and request.MergeBaseName is valid... #} {% if result.RequestType.ID == 3 and result.MergeBaseName %} ({{ result.MergeBaseName }}) From 1956be0f469e5870d957a8dec2fa48100a247273 Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Tue, 5 Oct 2021 14:00:12 -0400 Subject: [PATCH 383/844] fix(FastAPI): prefill login fields with entered data --- aurweb/routers/auth.py | 16 ++++++++-------- templates/login.html | 9 +++++++-- test/test_auth_routes.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index 8f37fe27..a985281e 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -9,14 +9,14 @@ import aurweb.config from aurweb import util from aurweb.auth import auth_required from aurweb.models.user import User -from aurweb.templates import make_context, render_template +from aurweb.templates import make_variable_context, render_template router = APIRouter() -def login_template(request: Request, next: str, errors: list = None): +async def login_template(request: Request, next: str, errors: list = None): """ Provide login-specific template context to render_template. """ - context = make_context(request, "Login", next) + context = await make_variable_context(request, "Login", next) context["errors"] = errors context["url_base"] = f"{request.url.scheme}://{request.url.netloc}" return render_template(request, "login.html", context) @@ -25,7 +25,7 @@ def login_template(request: Request, next: str, errors: list = None): @router.get("/login", response_class=HTMLResponse) @auth_required(False) async def login_get(request: Request, next: str = "/"): - return login_template(request, next) + return await login_template(request, next) @router.post("/login", response_class=HTMLResponse) @@ -39,8 +39,8 @@ async def login_post(request: Request, user = session.query(User).filter(User.Username == user).first() if not user: - return login_template(request, next, - errors=["Bad username or password."]) + return await login_template(request, next, + errors=["Bad username or password."]) cookie_timeout = 0 @@ -50,8 +50,8 @@ async def login_post(request: Request, sid = user.login(request, passwd, cookie_timeout) if not sid: - return login_template(request, next, - errors=["Bad username or password."]) + return await login_template(request, next, + errors=["Bad username or password."]) login_timeout = aurweb.config.getint("options", "login_timeout") diff --git a/templates/login.html b/templates/login.html index da7bd722..3c4f945f 100644 --- a/templates/login.html +++ b/templates/login.html @@ -45,7 +45,8 @@ + maxlength="254" autofocus="autofocus" + value="{{ user or '' }}">

    @@ -57,7 +58,11 @@

    - + diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index 1d8f9cbe..313f9927 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -160,6 +160,11 @@ def test_login_missing_username(): response = request.post("/login", data=post_data) assert "AURSID" not in response.cookies + # Make sure password isn't prefilled and remember_me isn't checked. + content = response.content.decode() + assert post_data["passwd"] not in content + assert "checked" not in content + def test_login_remember_me(): post_data = { @@ -188,6 +193,26 @@ def test_login_remember_me(): assert _session.LastUpdateTS < expected_ts + 5 +def test_login_incorrect_password_remember_me(): + post_data = { + "user": "test", + "passwd": "badPassword", + "next": "/", + "remember_me": "on" + } + + with client as request: + response = request.post("/login", data=post_data) + assert "AURSID" not in response.cookies + + # Make sure username is prefilled, password isn't prefilled, and remember_me + # is checked. + content = response.content.decode() + assert post_data["user"] in content + assert post_data["passwd"] not in content + assert "checked" in content + + def test_login_missing_password(): post_data = { "user": "test", @@ -198,6 +223,11 @@ def test_login_missing_password(): response = request.post("/login", data=post_data) assert "AURSID" not in response.cookies + # Make sure username is prefilled and remember_me isn't checked. + content = response.content.decode() + assert post_data["user"] in content + assert "checked" not in content + def test_login_incorrect_password(): post_data = { @@ -209,3 +239,10 @@ def test_login_incorrect_password(): with client as request: response = request.post("/login", data=post_data) assert "AURSID" not in response.cookies + + # Make sure username is prefilled, password isn't prefilled and remember_me + # isn't checked. + content = response.content.decode() + assert post_data["user"] in content + assert post_data["passwd"] not in content + assert "checked" not in content From 1bce53bbb7343fc861f253e97be171403bb930f4 Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Tue, 5 Oct 2021 14:36:46 -0400 Subject: [PATCH 384/844] fix(FastAPI): mark user and passwd as required fields --- templates/login.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/login.html b/templates/login.html index 3c4f945f..45fc1645 100644 --- a/templates/login.html +++ b/templates/login.html @@ -46,7 +46,7 @@ + required="required" value="{{ user or '' }}">

    @@ -54,7 +54,7 @@ {% trans %}Password{% endtrans %}: + size="30" required="required">

    From a54a09f61d0495789b22724c5f83bf883af83b45 Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Tue, 5 Oct 2021 12:38:31 -0400 Subject: [PATCH 385/844] fix(FastAPI): fix padding on email inputs Signed-off-by: Steven Guikal --- web/html/css/archweb.css | 1 + 1 file changed, 1 insertion(+) diff --git a/web/html/css/archweb.css b/web/html/css/archweb.css index b935d7db..45b9bff0 100644 --- a/web/html/css/archweb.css +++ b/web/html/css/archweb.css @@ -329,6 +329,7 @@ label { input[type=text], input[type=password], +input[type=email], textarea { padding: 0.10em; } From 889c5b1e21788a1f3126dfa121b6253ea9497501 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 6 Oct 2021 22:08:17 -0700 Subject: [PATCH 386/844] fix(FastAPI): pkgbase actions template Display Delete, Merge and Disown actions based on user credentials. Signed-off-by: Kevin Morris --- templates/partials/packages/actions.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/templates/partials/packages/actions.html b/templates/partials/packages/actions.html index 6c30153c..a54d4c90 100644 --- a/templates/partials/packages/actions.html +++ b/templates/partials/packages/actions.html @@ -142,17 +142,21 @@ {% endif %}

  • - {% if is_maintainer %} + {% if request.user.has_credential("CRED_PKGBASE_DELETE") %}
  • {{ "Delete Package" | tr }}
  • + {% endif %} + {% if request.user.has_credential("CRED_PKGBASE_MERGE") %}
  • {{ "Merge Package" | tr }}
  • + {% endif %} + {% if request.user.has_credential("CRED_PKGBASE_DISOWN", approved=[pkgbase.Maintainer]) %}
  • Date: Wed, 6 Oct 2021 22:29:53 -0700 Subject: [PATCH 387/844] feat(FastAPI): add CRED_PKGBASE_MERGE Signed-off-by: Kevin Morris --- aurweb/auth.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aurweb/auth.py b/aurweb/auth.py index 21d31081..fb062eab 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -268,6 +268,7 @@ CRED_PKGREQ_LIST = 18 CRED_TU_ADD_VOTE = 19 CRED_TU_LIST_VOTES = 20 CRED_TU_VOTE = 21 +CRED_PKGBASE_MERGE = 29 def has_any(user, *account_types): @@ -321,6 +322,7 @@ cred_filters = { CRED_TU_LIST_VOTES: trusted_user, CRED_TU_VOTE: trusted_user, CRED_ACCOUNT_EDIT_DEV: developer, + CRED_PKGBASE_MERGE: trusted_user_or_dev, } From e5299b5ed4c9c9041217f196f5721d8b49bfbf00 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 6 Oct 2021 23:17:08 -0700 Subject: [PATCH 388/844] fix(FastAPI): pkgbase/package tests Signed-off-by: Kevin Morris --- test/test_packages_routes.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 5afe011a..4118744a 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -335,6 +335,30 @@ def test_package_authenticated_maintainer(client: TestClient, resp = request.get(package_endpoint(package), cookies=cookies) assert resp.status_code == int(HTTPStatus.OK) + expected = [ + "View PKGBUILD", + "View Changes", + "Download snapshot", + "Search wiki", + "Flag package out-of-date", + "Vote for this package", + "Enable notifications", + "Manage Co-Maintainers", + "Submit Request", + "Disown Package" + ] + for expected_text in expected: + assert expected_text in resp.text + + +def test_package_authenticated_tu(client: TestClient, + tu_user: User, + package: Package): + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + resp = request.get(package_endpoint(package), cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + expected = [ "View PKGBUILD", "View Changes", From 75c49e4f8ada4cae1c5c5fd02ddd3ce73e7ac06a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 7 Oct 2021 00:03:24 -0700 Subject: [PATCH 389/844] feat(FastAPI): support {named} fmt in auth_required redirect Signed-off-by: Kevin Morris --- aurweb/auth.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index fb062eab..d44d4ded 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -1,4 +1,5 @@ import functools +import re from datetime import datetime from http import HTTPStatus @@ -121,6 +122,7 @@ class BasicAuthBackend(AuthenticationBackend): def auth_required(is_required: bool = True, + login: bool = False, redirect: str = "/", template: tuple = None, status_code: HTTPStatus = HTTPStatus.UNAUTHORIZED): @@ -162,8 +164,16 @@ def auth_required(is_required: bool = True, async def wrapper(request, *args, **kwargs): if request.user.is_authenticated() != is_required: url = "/" + if redirect: - url = redirect + path_params_expr = re.compile(r'\{(\w+)\}') + match = re.findall(path_params_expr, redirect) + args = {k: request.path_params.get(k) for k in match} + url = redirect.format(**args) + + if login: + url = "/login?" + util.urlencode({"next": url}) + if template: # template=("template.html", # ["Some Title", "someFormatted {}"], From 8bc1fab74df9f14a47ad1923a718633702ae82eb Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 7 Oct 2021 00:26:57 -0700 Subject: [PATCH 390/844] change(FastAPI): automate request login requirement Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 2 +- templates/partials/packages/actions.html | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index a3effb36..539f9526 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -571,7 +571,7 @@ async def requests(request: Request, @router.get("/pkgbase/{name}/request") -@auth_required(True) +@auth_required(True, login=True, redirect="/pkgbase/{name}") async def package_request(request: Request, name: str): context = make_context(request, "Submit Request") diff --git a/templates/partials/packages/actions.html b/templates/partials/packages/actions.html index a54d4c90..7355420c 100644 --- a/templates/partials/packages/actions.html +++ b/templates/partials/packages/actions.html @@ -132,15 +132,9 @@
  • {% endif %}
  • - {% if not request.user.is_authenticated() %} - - {{ "Submit Request" | tr }} - - {% else %} {{ "Submit Request" | tr }} - {% endif %}
  • {% if request.user.has_credential("CRED_PKGBASE_DELETE") %}
  • From dc11a88ed35f9cfc8c253e23f5814867448f3ac0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 7 Oct 2021 00:39:25 -0700 Subject: [PATCH 391/844] change(FastAPI): depend on auth_required redirect for pkgbase actions Signed-off-by: Kevin Morris --- templates/partials/packages/actions.html | 133 +++++++++-------------- 1 file changed, 51 insertions(+), 82 deletions(-) diff --git a/templates/partials/packages/actions.html b/templates/partials/packages/actions.html index 7355420c..f1863663 100644 --- a/templates/partials/packages/actions.html +++ b/templates/partials/packages/actions.html @@ -23,99 +23,68 @@ {{ "Search wiki" | tr }}
  • - {% if not request.user.is_authenticated() %} - {% if not out_of_date %} + {% if not out_of_date %}
  • {{ "Flag package out-of-date" | tr }}
  • - {% else %} -
  • - - {% set ood_ts = result.OutOfDateTS | dt | as_timezone(timezone) %} - {{ - "Flagged out-of-date (%s)" - | tr | format(ood_ts.strftime("%Y-%m-%d")) - }} - -
  • - {% endif %} -
  • - - {{ "Vote for this package" | tr }} - -
  • -
  • - - {{ "Enable notifications" | tr }} - -
  • {% else %} - {% if not out_of_date %} -
  • - - {{ "Flag package out-of-date" | tr }} - -
  • - {% else %} -
  • - - {% set ood_ts = result.OutOfDateTS | dt | as_timezone(timezone) %} - {{ - "Flagged out-of-date (%s)" - | tr | format(ood_ts.strftime("%Y-%m-%d")) - }} - -
  • -
  • - - + + {% set ood_ts = result.OutOfDateTS | dt | as_timezone(timezone) %} + {{ + "Flagged out-of-date (%s)" + | tr | format(ood_ts.strftime("%Y-%m-%d")) + }} + +
  • +
  • + + + +
  • + {% endif %} +
  • + {% if not voted %} +
    + +
    + {% else %} +
    + +
    + {% endif %} +
  • +
  • + {% if notified %} +
    +
    -
  • - {% endif %} -
  • - {% if not voted %} -
    + {% else %} + + name="do_Notify" + value="{{ 'Enable notifications' | tr }}" + />
    - {% else %} -
    - -
    - {% endif %} -
  • -
  • - {% if notified %} -
    - -
    - {% else %} -
    - -
    - {% endif %} -
  • - - {% endif %} + {% endif %} + {% if request.user.has_credential('CRED_PKGBASE_EDIT_COMAINTAINERS', approved=[pkgbase.Maintainer]) %}
  • From a756691d08408b2098557ebc6a50cf205ffe0084 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 7 Oct 2021 10:00:46 -0700 Subject: [PATCH 392/844] change(FastAPI): user_developer_or_trusted_user always True Signed-off-by: Kevin Morris --- aurweb/auth.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index fb062eab..9f56f90f 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -276,8 +276,7 @@ def has_any(user, *account_types): def user_developer_or_trusted_user(user): - return has_any(user, "User", "Trusted User", "Developer", - "Trusted User & Developer") + return True def trusted_user(user): From 2e6f8cb9f40869198bd6aaba34597467a5951476 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 7 Oct 2021 09:43:47 -0700 Subject: [PATCH 393/844] change(FastAPI): @auth_required login kwarg defaulted to True We pretty much want @auth_required to send users to login if we enforce auth requirements but don't otherwise specify a way to deal with it. Signed-off-by: Kevin Morris --- aurweb/auth.py | 3 ++- aurweb/routers/accounts.py | 20 ++++++++++---------- aurweb/routers/packages.py | 28 ++++++++++++++-------------- aurweb/routers/trusted_user.py | 10 +++++----- test/test_trusted_user_routes.py | 6 ++++-- 5 files changed, 35 insertions(+), 32 deletions(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index d44d4ded..fb52fade 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -122,7 +122,7 @@ class BasicAuthBackend(AuthenticationBackend): def auth_required(is_required: bool = True, - login: bool = False, + login: bool = True, redirect: str = "/", template: tuple = None, status_code: HTTPStatus = HTTPStatus.UNAUTHORIZED): @@ -152,6 +152,7 @@ def auth_required(is_required: bool = True, applying any format operations. :param is_required: A boolean indicating whether the function requires auth + :param login: Redirect to `/login`, passing `next=` :param redirect: Path to redirect to if is_required isn't True :param template: A three-element template tuple: (path, title_iterable, variable_iterable) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 3c799938..fc1c5242 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -30,14 +30,14 @@ logger = logging.getLogger(__name__) @router.get("/passreset", response_class=HTMLResponse) -@auth_required(False) +@auth_required(False, login=False) async def passreset(request: Request): context = await make_variable_context(request, "Password Reset") return render_template(request, "passreset.html", context) @router.post("/passreset", response_class=HTMLResponse) -@auth_required(False) +@auth_required(False, login=False) async def passreset_post(request: Request, user: str = Form(...), resetkey: str = Form(default=None), @@ -315,7 +315,7 @@ def make_account_form_context(context: dict, @router.get("/register", response_class=HTMLResponse) -@auth_required(False) +@auth_required(False, login=False) async def account_register(request: Request, U: str = Form(default=str()), # Username E: str = Form(default=str()), # Email @@ -341,7 +341,7 @@ async def account_register(request: Request, @router.post("/register", response_class=HTMLResponse) -@auth_required(False) +@auth_required(False, login=False) async def account_register_post(request: Request, U: str = Form(default=str()), # Username E: str = Form(default=str()), # Email @@ -432,7 +432,7 @@ def cannot_edit(request, user): @router.get("/account/{username}/edit", response_class=HTMLResponse) -@auth_required(True) +@auth_required(True, redirect="/account/{username}") async def account_edit(request: Request, username: str): user = db.query(User, User.Username == username).first() @@ -448,7 +448,7 @@ async def account_edit(request: Request, @router.post("/account/{username}/edit", response_class=HTMLResponse) -@auth_required(True) +@auth_required(True, redirect="/account/{username}") async def account_edit_post(request: Request, username: str, U: str = Form(default=str()), # Username @@ -594,7 +594,7 @@ async def account(request: Request, username: str): @router.get("/accounts/") -@auth_required(True) +@auth_required(True, redirect="/accounts/") @account_type_required({TRUSTED_USER, DEVELOPER, TRUSTED_USER_AND_DEV}) async def accounts(request: Request): context = make_context(request, "Accounts") @@ -602,7 +602,7 @@ async def accounts(request: Request): @router.post("/accounts/") -@auth_required(True) +@auth_required(True, redirect="/accounts/") @account_type_required({TRUSTED_USER, DEVELOPER, TRUSTED_USER_AND_DEV}) async def accounts_post(request: Request, O: int = Form(default=0), # Offset @@ -688,7 +688,7 @@ def render_terms_of_service(request: Request, @router.get("/tos") -@auth_required(True, redirect="/") +@auth_required(True, redirect="/tos") async def terms_of_service(request: Request): # Query the database for terms that were previously accepted, # but now have a bumped Revision that needs to be accepted. @@ -709,7 +709,7 @@ async def terms_of_service(request: Request): @router.post("/tos") -@auth_required(True, redirect="/") +@auth_required(True, redirect="/tos") async def terms_of_service_post(request: Request, accept: bool = Form(default=False)): # Query the database for terms that were previously accepted, diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 539f9526..ee6d71ba 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -222,7 +222,7 @@ async def package_base_voters(request: Request, name: str) -> Response: @router.post("/pkgbase/{name}/comments") -@auth_required(True) +@auth_required(True, redirect="/pkgbase/{name}/comments") async def pkgbase_comments_post( request: Request, name: str, comment: str = Form(default=str()), @@ -254,7 +254,7 @@ async def pkgbase_comments_post( @router.get("/pkgbase/{name}/comments/{id}/form") -@auth_required(True) +@auth_required(True, login=False) async def pkgbase_comment_form(request: Request, name: str, id: int): """ Produce a comment form for comment {id}. """ pkgbase = get_pkg_or_base(name, PackageBase) @@ -274,7 +274,7 @@ async def pkgbase_comment_form(request: Request, name: str, id: int): @router.post("/pkgbase/{name}/comments/{id}") -@auth_required(True) +@auth_required(True, redirect="/pkgbase/{name}/comments/{id}") async def pkgbase_comment_post( request: Request, name: str, id: int, comment: str = Form(default=str()), @@ -309,7 +309,7 @@ async def pkgbase_comment_post( @router.post("/pkgbase/{name}/comments/{id}/delete") -@auth_required(True) +@auth_required(True, redirect="/pkgbase/{name}/comments/{id}/delete") async def pkgbase_comment_delete(request: Request, name: str, id: int): pkgbase = get_pkg_or_base(name, PackageBase) comment = get_pkgbase_comment(pkgbase, id) @@ -332,7 +332,7 @@ async def pkgbase_comment_delete(request: Request, name: str, id: int): @router.post("/pkgbase/{name}/comments/{id}/undelete") -@auth_required(True) +@auth_required(True, redirect="/pkgbase/{name}/comments/{id}/undelete") async def pkgbase_comment_undelete(request: Request, name: str, id: int): pkgbase = get_pkg_or_base(name, PackageBase) comment = get_pkgbase_comment(pkgbase, id) @@ -354,7 +354,7 @@ async def pkgbase_comment_undelete(request: Request, name: str, id: int): @router.post("/pkgbase/{name}/comments/{id}/pin") -@auth_required(True) +@auth_required(True, redirect="/pkgbase/{name}/comments/{id}/pin") async def pkgbase_comment_pin(request: Request, name: str, id: int): pkgbase = get_pkg_or_base(name, PackageBase) comment = get_pkgbase_comment(pkgbase, id) @@ -376,7 +376,7 @@ async def pkgbase_comment_pin(request: Request, name: str, id: int): @router.post("/pkgbase/{name}/comments/{id}/unpin") -@auth_required(True) +@auth_required(True, redirect="/pkgbase/{name}/comments/{id}/unpin") async def pkgbase_comment_unpin(request: Request, name: str, id: int): pkgbase = get_pkg_or_base(name, PackageBase) comment = get_pkgbase_comment(pkgbase, id) @@ -397,7 +397,7 @@ async def pkgbase_comment_unpin(request: Request, name: str, id: int): @router.get("/pkgbase/{name}/comaintainers") -@auth_required(True) +@auth_required(True, redirect="/pkgbase/{name}/comaintainers") async def package_base_comaintainers(request: Request, name: str) -> Response: # Get the PackageBase. pkgbase = get_pkg_or_base(name, PackageBase) @@ -444,7 +444,7 @@ def remove_users(pkgbase, usernames): @router.post("/pkgbase/{name}/comaintainers") -@auth_required(True) +@auth_required(True, redirect="/pkgbase/{name}/comaintainers") async def package_base_comaintainers_post( request: Request, name: str, users: str = Form(default=str())) -> Response: @@ -539,7 +539,7 @@ async def package_base_comaintainers_post( @router.get("/requests") -@auth_required(True, redirect="/") +@auth_required(True, redirect="/requests") async def requests(request: Request, O: int = Query(default=defaults.O), PP: int = Query(default=defaults.PP)): @@ -571,7 +571,7 @@ async def requests(request: Request, @router.get("/pkgbase/{name}/request") -@auth_required(True, login=True, redirect="/pkgbase/{name}") +@auth_required(True, redirect="/pkgbase/{name}") async def package_request(request: Request, name: str): context = make_context(request, "Submit Request") @@ -585,7 +585,7 @@ async def package_request(request: Request, name: str): @router.post("/pkgbase/{name}/request") -@auth_required(True) +@auth_required(True, redirect="/pkgbase/{name}/request") async def pkgbase_request_post(request: Request, name: str, type: str = Form(...), merge_into: str = Form(default=None), @@ -654,7 +654,7 @@ async def pkgbase_request_post(request: Request, name: str, @router.get("/requests/{id}/close") -@auth_required(True) +@auth_required(True, redirect="/requests/{id}/close") async def requests_close(request: Request, id: int): pkgreq = db.query(PackageRequest).filter(PackageRequest.ID == id).first() if not request.user.is_elevated() and request.user != pkgreq.User: @@ -667,7 +667,7 @@ async def requests_close(request: Request, id: int): @router.post("/requests/{id}/close") -@auth_required(True) +@auth_required(True, redirect="/requests/{id}/close") async def requests_close_post(request: Request, id: int, reason: int = Form(default=0), comments: str = Form(default=str())): diff --git a/aurweb/routers/trusted_user.py b/aurweb/routers/trusted_user.py index a977b31a..b897a635 100644 --- a/aurweb/routers/trusted_user.py +++ b/aurweb/routers/trusted_user.py @@ -45,7 +45,7 @@ ADDVOTE_SPECIFICS = { @router.get("/tu") -@auth_required(True, redirect="/") +@auth_required(True, redirect="/tu") @account_type_required(REQUIRED_TYPES) async def trusted_user(request: Request, coff: int = 0, # current offset @@ -149,7 +149,7 @@ def render_proposal(request: Request, @router.get("/tu/{proposal}") -@auth_required(True, redirect="/") +@auth_required(True, redirect="/tu/{proposal}") @account_type_required(REQUIRED_TYPES) async def trusted_user_proposal(request: Request, proposal: int): context = await make_variable_context(request, "Trusted User") @@ -175,7 +175,7 @@ async def trusted_user_proposal(request: Request, proposal: int): @router.post("/tu/{proposal}") -@auth_required(True, redirect="/") +@auth_required(True, redirect="/tu/{proposal}") @account_type_required(REQUIRED_TYPES) async def trusted_user_proposal_post(request: Request, proposal: int, @@ -223,7 +223,7 @@ async def trusted_user_proposal_post(request: Request, @router.get("/addvote") -@auth_required(True) +@auth_required(True, redirect="/addvote") @account_type_required({"Trusted User", "Trusted User & Developer"}) async def trusted_user_addvote(request: Request, user: str = str(), @@ -243,7 +243,7 @@ async def trusted_user_addvote(request: Request, @router.post("/addvote") -@auth_required(True) +@auth_required(True, redirect="/addvote") @account_type_required({TRUSTED_USER, TRUSTED_USER_AND_DEV}) async def trusted_user_addvote_post(request: Request, user: str = Form(default=str()), diff --git a/test/test_trusted_user_routes.py b/test/test_trusted_user_routes.py index 67181db3..0579247e 100644 --- a/test/test_trusted_user_routes.py +++ b/test/test_trusted_user_routes.py @@ -9,7 +9,7 @@ import pytest from fastapi.testclient import TestClient -from aurweb import db +from aurweb import db, util from aurweb.models.account_type import AccountType from aurweb.models.tu_vote import TUVote from aurweb.models.tu_voteinfo import TUVoteInfo @@ -128,7 +128,9 @@ def test_tu_index_guest(client): with client as request: response = request.get("/tu", allow_redirects=False) assert response.status_code == int(HTTPStatus.SEE_OTHER) - assert response.headers.get("location") == "/" + + params = util.urlencode({"next": "/tu"}) + assert response.headers.get("location") == f"/login?{params}" def test_tu_index_unauthorized(client, user): From 8eadb4251da6029cb4edb528b222eebb0d3b821c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 5 Oct 2021 16:04:19 -0700 Subject: [PATCH 394/844] feat(FastAPI): add /pkgbase/{name}/[un]flag (post) Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 32 ++++++++++++++++++++++++++++++++ test/test_packages_routes.py | 31 +++++++++++++++++++++++++++++++ test/test_user.py | 1 - 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index ee6d71ba..8790327f 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -701,3 +701,35 @@ async def requests_close_post(request: Request, id: int, notify_.send() return RedirectResponse("/requests", status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.post("/pkgbase/{name}/flag") +@auth_required(True, redirect="/pkgbase/{name}") +async def pkgbase_flag(request: Request, name: str): + pkgbase = get_pkg_or_base(name, PackageBase) + + has_cred = request.user.has_credential("CRED_PKGBASE_FLAG") + if has_cred and not pkgbase.Flagger: + now = int(datetime.utcnow().timestamp()) + with db.begin(): + pkgbase.OutOfDateTS = now + pkgbase.Flagger = request.user + + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.post("/pkgbase/{name}/unflag") +@auth_required(True, redirect="/pkgbase/{name}") +async def pkgbase_unflag(request: Request, name: str): + pkgbase = get_pkg_or_base(name, PackageBase) + + has_cred = request.user.has_credential( + "CRED_PKGBASE_UNFLAG", approved=[pkgbase.Flagger]) + if has_cred: + with db.begin(): + pkgbase.OutOfDateTS = None + pkgbase.Flagger = None + + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 4118744a..db36d1a9 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1689,3 +1689,34 @@ def test_requests_close_post_rejected(client: TestClient, user: User, assert pkgreq.Status == REJECTED_ID assert pkgreq.Closer == user assert pkgreq.ClosureComment == str() + + +def test_pkgbase_flag(client: TestClient, user: User, maintainer: User, + package: Package): + pkgbase = package.PackageBase + + # We shouldn't have flagged the package yet; assert so. + assert pkgbase.Flagger is None + + # Flag it. + cookies = {"AURSID": user.login(Request(), "testPassword")} + endpoint = f"/pkgbase/{pkgbase.Name}/flag" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert pkgbase.Flagger == user + + # Now, test that the 'maintainer' user can't unflag it, because they + # didn't flag it to begin with. + maint_cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + endpoint = f"/pkgbase/{pkgbase.Name}/unflag" + with client as request: + resp = request.post(endpoint, cookies=maint_cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert pkgbase.Flagger == user + + # Now, unflag it for real. + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert pkgbase.Flagger is None diff --git a/test/test_user.py b/test/test_user.py index 43cbf58a..771611d8 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -166,7 +166,6 @@ def test_user_minimum_passwd_length(): def test_user_has_credential(): - assert user.has_credential("CRED_PKGBASE_FLAG") assert not user.has_credential("CRED_ACCOUNT_CHANGE_TYPE") From 0dfff2bcb24d8915f5fd317c79f5e750f0897e5a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 5 Oct 2021 21:13:51 -0700 Subject: [PATCH 395/844] feat(FastAPI): add /pkgbase/{name}/[un]notify (post) Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 36 ++++++++++++++++++++++++++++++++++++ test/test_packages_routes.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 8790327f..ced9ea3d 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -733,3 +733,39 @@ async def pkgbase_unflag(request: Request, name: str): return RedirectResponse(f"/pkgbase/{name}", status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.post("/pkgbase/{name}/notify") +@auth_required(True, redirect="/pkgbase/{name}") +async def pkgbase_notify(request: Request, name: str): + pkgbase = get_pkg_or_base(name, PackageBase) + + notif = db.query(pkgbase.notifications.filter( + PackageNotification.UserID == request.user.ID + ).exists()).scalar() + has_cred = request.user.has_credential("CRED_PKGBASE_NOTIFY") + if has_cred and not notif: + with db.begin(): + db.create(PackageNotification, + PackageBase=pkgbase, + User=request.user) + + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.post("/pkgbase/{name}/unnotify") +@auth_required(True, redirect="/pkgbase/{name}") +async def pkgbase_unnotify(request: Request, name: str): + pkgbase = get_pkg_or_base(name, PackageBase) + + notif = pkgbase.notifications.filter( + PackageNotification.UserID == request.user.ID + ).first() + has_cred = request.user.has_credential("CRED_PKGBASE_NOTIFY") + if has_cred and notif: + with db.begin(): + db.session.delete(notif) + + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index db36d1a9..1d7fe17c 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1720,3 +1720,36 @@ def test_pkgbase_flag(client: TestClient, user: User, maintainer: User, resp = request.post(endpoint, cookies=cookies) assert resp.status_code == int(HTTPStatus.SEE_OTHER) assert pkgbase.Flagger is None + + +def test_pkgbase_notify(client: TestClient, user: User, package: Package): + pkgbase = package.PackageBase + + # We have no notif record yet; assert that. + notif = pkgbase.notifications.filter( + PackageNotification.UserID == user.ID + ).first() + assert notif is None + + # Enable notifications. + cookies = {"AURSID": user.login(Request(), "testPassword")} + endpoint = f"/pkgbase/{pkgbase.Name}/notify" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + notif = pkgbase.notifications.filter( + PackageNotification.UserID == user.ID + ).first() + assert notif is not None + + # Disable notifications. + endpoint = f"/pkgbase/{pkgbase.Name}/unnotify" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + notif = pkgbase.notifications.filter( + PackageNotification.UserID == user.ID + ).first() + assert notif is None From 0a02df363a80e7571c9a0bbb9829b84f18cf7f4c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 5 Oct 2021 21:32:12 -0700 Subject: [PATCH 396/844] feat(FastAPI): add /pkgbase/{name}/[un]vote (post) Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 38 ++++++++++++++++++++++++++++++++++++ test/test_packages_routes.py | 27 +++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index ced9ea3d..8bfb680e 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -769,3 +769,41 @@ async def pkgbase_unnotify(request: Request, name: str): return RedirectResponse(f"/pkgbase/{name}", status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.post("/pkgbase/{name}/vote") +@auth_required(True, redirect="/pkgbase/{name}") +async def pkgbase_vote(request: Request, name: str): + pkgbase = get_pkg_or_base(name, PackageBase) + + vote = pkgbase.package_votes.filter( + PackageVote.UsersID == request.user.ID + ).first() + has_cred = request.user.has_credential("CRED_PKGBASE_VOTE") + if has_cred and not vote: + now = int(datetime.utcnow().timestamp()) + with db.begin(): + db.create(PackageVote, + User=request.user, + PackageBase=pkgbase, + VoteTS=now) + + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.post("/pkgbase/{name}/unvote") +@auth_required(True, redirect="/pkgbase/{name}") +async def pkgbase_unvote(request: Request, name: str): + pkgbase = get_pkg_or_base(name, PackageBase) + + vote = pkgbase.package_votes.filter( + PackageVote.UsersID == request.user.ID + ).first() + has_cred = request.user.has_credential("CRED_PKGBASE_VOTE") + if has_cred and vote: + with db.begin(): + db.session.delete(vote) + + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 1d7fe17c..a03c5920 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1753,3 +1753,30 @@ def test_pkgbase_notify(client: TestClient, user: User, package: Package): PackageNotification.UserID == user.ID ).first() assert notif is None + + +def test_pkgbase_vote(client: TestClient, user: User, package: Package): + pkgbase = package.PackageBase + + # We haven't voted yet. + vote = pkgbase.package_votes.filter(PackageVote.UsersID == user.ID).first() + assert vote is None + + # Vote for the package. + cookies = {"AURSID": user.login(Request(), "testPassword")} + endpoint = f"/pkgbase/{pkgbase.Name}/vote" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + vote = pkgbase.package_votes.filter(PackageVote.UsersID == user.ID).first() + assert vote is not None + + # Remove vote. + endpoint = f"/pkgbase/{pkgbase.Name}/unvote" + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + vote = pkgbase.package_votes.filter(PackageVote.UsersID == user.ID).first() + assert vote is None From 16d516c221112d36ead6ce36b5beb6a54015c8a2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 6 Oct 2021 22:07:20 -0700 Subject: [PATCH 397/844] feat(FastAPI): add /pkgbase/{name}/disown (get, post) Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 63 +++++++++++++++++++++++ templates/packages/disown.html | 55 ++++++++++++++++++++ templates/partials/packages/actions.html | 10 ++-- test/test_packages_routes.py | 65 ++++++++++++++++++++++++ 4 files changed, 186 insertions(+), 7 deletions(-) create mode 100644 templates/packages/disown.html diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 8bfb680e..af1ebe46 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -807,3 +807,66 @@ async def pkgbase_unvote(request: Request, name: str): return RedirectResponse(f"/pkgbase/{name}", status_code=int(HTTPStatus.SEE_OTHER)) + + +def disown_pkgbase(pkgbase: PackageBase, disowner: User): + conn = db.ConnectionExecutor(db.get_engine().raw_connection()) + notif = notify.DisownNotification(conn, disowner.ID, pkgbase.ID) + + if disowner != pkgbase.Maintainer: + with db.begin(): + pkgbase.Maintainer = None + else: + co = pkgbase.comaintainers.order_by( + PackageComaintainer.Priority.asc() + ).limit(1).first() + + if co: + with db.begin(): + pkgbase.Maintainer = co.User + db.session.delete(co) + else: + pkgbase.Maintainer = None + + notif.send() + + +@router.get("/pkgbase/{name}/disown") +@auth_required(True, redirect="/pkgbase/{name}") +async def pkgbase_disown_get(request: Request, name: str): + pkgbase = get_pkg_or_base(name, PackageBase) + + has_cred = request.user.has_credential("CRED_PKGBASE_DISOWN", + approved=[pkgbase.Maintainer]) + if not has_cred: + return RedirectResponse(f"/pkgbase/{name}", + int(HTTPStatus.SEE_OTHER)) + + context = make_context(request, "Disown Package") + context["pkgbase"] = pkgbase + return render_template(request, "packages/disown.html", context) + + +@router.post("/pkgbase/{name}/disown") +@auth_required(True, redirect="/pkgbase/{name}") +async def pkgbase_disown_post(request: Request, name: str, + confirm: bool = Form(default=False)): + pkgbase = get_pkg_or_base(name, PackageBase) + + has_cred = request.user.has_credential("CRED_PKGBASE_DISOWN", + approved=[pkgbase.Maintainer]) + if not has_cred: + return RedirectResponse(f"/pkgbase/{name}", + int(HTTPStatus.SEE_OTHER)) + + if not confirm: + context = make_context(request, "Disown Package") + context["pkgbase"] = pkgbase + context["errors"] = [("The selected packages have not been disowned, " + "check the confirmation checkbox.")] + return render_template(request, "packages/disown.html", context, + status_code=int(HTTPStatus.EXPECTATION_FAILED)) + + disown_pkgbase(pkgbase, request.user) + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/templates/packages/disown.html b/templates/packages/disown.html new file mode 100644 index 00000000..8d5a8574 --- /dev/null +++ b/templates/packages/disown.html @@ -0,0 +1,55 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} + + {% if errors %} +
      + {% for error in errors %} +
    • {{ error | tr }}
    • + {% endfor %} +
    + {% endif %} + +
    +

    {{ "Disown Package" | tr }}: {{ pkgbase.Name }}

    + +

    + {{ + "Use this form to disown the package base %s%s%s which " + "includes the following packages: " + | tr | format("", pkgbase.Name, "") | safe + }} +

    + +
      + {% for package in pkgbase.packages.all() %} +
    • {{ package.Name }}
    • + {% endfor %} +
    + +

    + {{ + "By selecting the checkbox, you confirm that you want to " + "disown the package." | tr + }} +

    + +
    +
    +

    + +

    +

    + +

    +
    +
    + +
    +{% endblock %} diff --git a/templates/partials/packages/actions.html b/templates/partials/packages/actions.html index f1863663..2b26144e 100644 --- a/templates/partials/packages/actions.html +++ b/templates/partials/packages/actions.html @@ -121,13 +121,9 @@ {% endif %} {% if request.user.has_credential("CRED_PKGBASE_DISOWN", approved=[pkgbase.Maintainer]) %}
  • -
    - -
    + + {{ "Disown Package" | tr }} +
  • {% endif %} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index a03c5920..c9622431 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1780,3 +1780,68 @@ def test_pkgbase_vote(client: TestClient, user: User, package: Package): vote = pkgbase.package_votes.filter(PackageVote.UsersID == user.ID).first() assert vote is None + + +def test_pkgbase_disown_as_tu(client: TestClient, tu_user: User, + package: Package): + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + pkgbase = package.PackageBase + endpoint = f"/pkgbase/{pkgbase.Name}/disown" + + # But we do here. + with client as request: + resp = request.post(endpoint, data={"confirm": True}, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + +def test_pkgbase_disown_as_sole_maintainer(client: TestClient, + maintainer: User, + package: Package): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + pkgbase = package.PackageBase + endpoint = f"/pkgbase/{pkgbase.Name}/disown" + + # But we do here. + with client as request: + resp = request.post(endpoint, data={"confirm": True}, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + +def test_pkgbase_disown(client: TestClient, user: User, maintainer: User, + package: Package): + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + user_cookies = {"AURSID": user.login(Request(), "testPassword")} + pkgbase = package.PackageBase + endpoint = f"/pkgbase/{pkgbase.Name}/disown" + + with db.begin(): + db.create(PackageComaintainer, + User=user, + PackageBase=pkgbase, + Priority=1) + + # GET as a normal user, which is rejected for lack of credentials. + with client as request: + resp = request.get(endpoint, cookies=user_cookies, + allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + # GET as the maintainer. + with client as request: + resp = request.get(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + # POST as a normal user, which is rejected for lack of credentials. + with client as request: + resp = request.post(endpoint, cookies=user_cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + # POST as the maintainer without "confirm". + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.EXPECTATION_FAILED) + + # POST as the maintainer with "confirm". + with client as request: + resp = request.post(endpoint, data={"confirm": True}, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) From c8d01cc5e8083a6586ae61a6c3371d7ed2428f6a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 19 Sep 2021 19:27:29 -0700 Subject: [PATCH 398/844] feat(FastAPI): add aurweb.util.apply_all(iterable, fn) A helper which allows us to apply a specific function to each item in an iterable. Signed-off-by: Kevin Morris --- aurweb/util.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/aurweb/util.py b/aurweb/util.py index 08e6d7c6..44f711f1 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -7,7 +7,7 @@ import secrets import string from datetime import datetime -from typing import Any, Dict +from typing import Any, Callable, Dict, Iterable from urllib.parse import urlencode, urlparse from zoneinfo import ZoneInfo @@ -167,3 +167,8 @@ def add_samesite_fields(response: Response, value: str): def get_ssh_fingerprints(): return aurweb.config.get_section("fingerprints") or {} + + +def apply_all(iterable: Iterable, fn: Callable): + for item in iterable: + fn(item) From ed68fa2b57f7f4cf916fd2e40312e1f64da2c71e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 19 Sep 2021 19:27:03 -0700 Subject: [PATCH 399/844] feat(FastAPI): add aurweb.db.delete_all(iterable) Signed-off-by: Kevin Morris --- aurweb/db.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/aurweb/db.py b/aurweb/db.py index 2b934300..c1e80751 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -2,6 +2,8 @@ import functools import math import re +from typing import Iterable + from sqlalchemy import event from sqlalchemy.orm import scoped_session @@ -71,6 +73,12 @@ def delete(model, *args, **kwargs): session.delete(record) +def delete_all(iterable: Iterable): + with begin(): + for obj in iterable: + session.delete(obj) + + def rollback(): session.rollback() From 0ddc969bdcd07fe0181e05a8e2abdb2e23301212 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 6 Oct 2021 15:33:23 -0700 Subject: [PATCH 400/844] feat(FastAPI-dev): add package_delete helper Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 76 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index af1ebe46..40322785 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -1,6 +1,6 @@ from datetime import datetime from http import HTTPStatus -from typing import Any, Dict +from typing import Any, Dict, List from fastapi import APIRouter, Form, HTTPException, Query, Request, Response from fastapi.responses import JSONResponse, RedirectResponse @@ -11,7 +11,7 @@ import aurweb.models.package_comment import aurweb.models.package_keyword import aurweb.packages.util -from aurweb import db, defaults, l10n +from aurweb import db, defaults, l10n, util from aurweb.auth import auth_required from aurweb.models.license import License from aurweb.models.package import Package @@ -26,7 +26,7 @@ from aurweb.models.package_request import ACCEPTED_ID, PENDING_ID, REJECTED_ID, from aurweb.models.package_source import PackageSource from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import CONFLICTS_ID -from aurweb.models.request_type import RequestType +from aurweb.models.request_type import DELETION_ID, RequestType from aurweb.models.user import User from aurweb.packages.search import PackageSearch from aurweb.packages.util import get_pkg_or_base, get_pkgbase_comment, query_notified, query_voted @@ -116,6 +116,76 @@ async def packages(request: Request) -> Response: return await packages_get(request, context) +def create_request_if_missing(requests: List[PackageRequest], + reqtype: RequestType, + user: User, + package: Package): + now = int(datetime.utcnow().timestamp()) + pkgreq = db.query(PackageRequest).filter( + PackageRequest.PackageBaseName == package.PackageBase.Name + ).first() + if not pkgreq: + # No PackageRequest existed. Create one. + comments = "Automatically generated by aurweb." + closure_comment = "Deleted by aurweb." + pkgreq = db.create(PackageRequest, + RequestType=reqtype, + PackageBase=package.PackageBase, + PackageBaseName=package.PackageBase.Name, + User=user, + Status=ACCEPTED_ID, + Comments=comments, + ClosureComment=closure_comment, + ClosedTS=now, + Closer=user) + requests.append(pkgreq) + + +def delete_package(deleter: User, + package: Package): + notifications = [] + requests = [] + bases_to_delete = [] + + conn = db.ConnectionExecutor(db.get_engine().raw_connection()) + # In all cases, though, just delete the Package in question. + if package.PackageBase.packages.count() == 1: + reqtype = db.query(RequestType).filter( + RequestType.ID == DELETION_ID + ).first() + + with db.begin(): + create_request_if_missing( + requests, reqtype, deleter, package) + + bases_to_delete.append(package.PackageBase) + + # Prepare DeleteNotification. + notifications.append( + notify.DeleteNotification(conn, deleter.ID, package.PackageBase.ID) + ) + + # For each PackageRequest created, mock up an open and close notification. + basename = package.PackageBase.Name + for pkgreq in requests: + notifications.append( + notify.RequestOpenNotification( + conn, deleter.ID, pkgreq.ID, reqtype.Name, + pkgreq.PackageBase.ID, merge_into=basename or None) + ) + notifications.append( + notify.RequestCloseNotification( + conn, deleter.ID, pkgreq.ID, pkgreq.status_display()) + ) + + # Perform all the deletions. + db.delete_all([package]) + db.delete_all(bases_to_delete) + + # Send out all the notifications. + util.apply_all(notifications, lambda n: n.send()) + + async def make_single_context(request: Request, pkgbase: PackageBase) -> Dict[str, Any]: """ Make a basic context for package or pkgbase. From 4e7d2295da657ece2dc0fd1422ecd0e75e4facb7 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 6 Oct 2021 20:17:58 -0700 Subject: [PATCH 401/844] fix(FastAPI): add package-related missing backref cascades Signed-off-by: Kevin Morris --- aurweb/models/package_comment.py | 3 ++- aurweb/models/package_dependency.py | 3 ++- aurweb/models/package_relation.py | 3 ++- aurweb/models/package_source.py | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/aurweb/models/package_comment.py b/aurweb/models/package_comment.py index c52ee270..92ae8911 100644 --- a/aurweb/models/package_comment.py +++ b/aurweb/models/package_comment.py @@ -17,7 +17,8 @@ class PackageComment(Base): Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), nullable=False) PackageBase = relationship( - "PackageBase", backref=backref("comments", lazy="dynamic"), + "PackageBase", backref=backref("comments", lazy="dynamic", + cascade="all,delete"), foreign_keys=[PackageBaseID]) UsersID = Column(Integer, ForeignKey("Users.ID", ondelete="SET NULL")) diff --git a/aurweb/models/package_dependency.py b/aurweb/models/package_dependency.py index 9ce0b019..fb66c6f2 100644 --- a/aurweb/models/package_dependency.py +++ b/aurweb/models/package_dependency.py @@ -15,7 +15,8 @@ class PackageDependency(Base): Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), nullable=False) Package = relationship( - "Package", backref=backref("package_dependencies", lazy="dynamic"), + "Package", backref=backref("package_dependencies", lazy="dynamic", + cascade="all,delete"), foreign_keys=[PackageID]) DepTypeID = Column( diff --git a/aurweb/models/package_relation.py b/aurweb/models/package_relation.py index 1e6c146c..d4921859 100644 --- a/aurweb/models/package_relation.py +++ b/aurweb/models/package_relation.py @@ -16,7 +16,8 @@ class PackageRelation(Base): Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), nullable=False) Package = relationship( - "Package", backref=backref("package_relations", lazy="dynamic"), + "Package", backref=backref("package_relations", lazy="dynamic", + cascade="all,delete"), foreign_keys=[PackageID]) RelTypeID = Column( diff --git a/aurweb/models/package_source.py b/aurweb/models/package_source.py index 4ffa23df..f016bee0 100644 --- a/aurweb/models/package_source.py +++ b/aurweb/models/package_source.py @@ -13,7 +13,8 @@ class PackageSource(Base): PackageID = Column(Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), nullable=False) Package = relationship( - "Package", backref=backref("package_sources", lazy="dynamic"), + "Package", backref=backref("package_sources", lazy="dynamic", + cascade="all,delete"), foreign_keys=[PackageID]) __mapper_args__ = {"primary_key": [PackageID]} From d38abd783250d13f93294944f3897a38e1209d31 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 6 Oct 2021 13:54:14 -0700 Subject: [PATCH 402/844] feat(FastAPI): add /pkgbase/{name}/delete (get, post) In addition, we've had to add cascade arguments to backref so sqlalchemy treats the relationships as proper cascades. Furthermore, our pkgbase actions template was not rendering actions properly based on TU credentials. Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 37 ++++++++++++++++++++++ templates/packages/delete.html | 56 ++++++++++++++++++++++++++++++++++ test/test_packages_routes.py | 46 ++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 templates/packages/delete.html diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 40322785..4426d0be 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -940,3 +940,40 @@ async def pkgbase_disown_post(request: Request, name: str, disown_pkgbase(pkgbase, request.user) return RedirectResponse(f"/pkgbase/{name}", status_code=int(HTTPStatus.SEE_OTHER)) + + +@router.get("/pkgbase/{name}/delete") +@auth_required(True) +async def pkgbase_delete_get(request: Request, name: str): + if not request.user.has_credential("CRED_PKGBASE_DELETE"): + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) + + context = make_context(request, "Package Deletion") + context["pkgbase"] = get_pkg_or_base(name, PackageBase) + return render_template(request, "packages/delete.html", context) + + +@router.post("/pkgbase/{name}/delete") +@auth_required(True) +async def pkgbase_delete_post(request: Request, name: str, + confirm: bool = Form(default=False)): + pkgbase = get_pkg_or_base(name, PackageBase) + + if not request.user.has_credential("CRED_PKGBASE_DELETE"): + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) + + if not confirm: + context = make_context(request, "Package Deletion") + context["pkgbase"] = pkgbase + context["errors"] = [("The selected packages have not been deleted, " + "check the confirmation checkbox.")] + return render_template(request, "packages/delete.html", context, + status_code=int(HTTPStatus.EXPECTATION_FAILED)) + + packages = pkgbase.packages.all() + for package in packages: + delete_package(request.user, package) + + return RedirectResponse("/packages", status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/templates/packages/delete.html b/templates/packages/delete.html new file mode 100644 index 00000000..6e882d05 --- /dev/null +++ b/templates/packages/delete.html @@ -0,0 +1,56 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} + + {% if errors %} +
      + {% for error in errors %} +
    • {{ error | tr }}
    • + {% endfor %} +
    + {% endif %} + +
    +

    {{ "Delete Package" | tr }}: {{ pkgbase.Name }}

    + +

    + {{ + "Use this form to delete the package base %s%s%s and " + "the following packages from the AUR: " + | tr | format("", pkgbase.Name, "") | safe + }} +

    + +
      + {% for package in pkgbase.packages.all() %} +
    • {{ package.Name }}
    • + {% endfor %} +
    + +

    + {{ + "Deletion of a package is permanent. " + "Select the checkbox to confirm action." | tr + }} +

    + +
    +
    +

    + +

    + +

    + +

    +
    +
    + +
    +{% endblock %} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index c9622431..1f258497 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1845,3 +1845,49 @@ def test_pkgbase_disown(client: TestClient, user: User, maintainer: User, with client as request: resp = request.post(endpoint, data={"confirm": True}, cookies=cookies) assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + +def test_pkgbase_delete_unauthorized(client: TestClient, user: User, + package: Package): + pkgbase = package.PackageBase + cookies = {"AURSID": user.login(Request(), "testPassword")} + endpoint = f"/pkgbase/{pkgbase.Name}/delete" + + # Test GET. + with client as request: + resp = request.get(endpoint, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert resp.headers.get("location") == f"/pkgbase/{pkgbase.Name}" + + # Test POST. + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert resp.headers.get("location") == f"/pkgbase/{pkgbase.Name}" + + +def test_pkgbase_delete(client: TestClient, tu_user: User, package: Package): + pkgbase = package.PackageBase + + # Test that the GET request works. + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + endpoint = f"/pkgbase/{pkgbase.Name}/delete" + with client as request: + resp = request.get(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + # Test that POST works and denies us because we haven't confirmed. + with client as request: + resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.EXPECTATION_FAILED) + + # Test that we can actually delete the pkgbase. + with client as request: + resp = request.post(endpoint, data={"confirm": True}, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + # Let's assert that the package base record got removed. + record = db.query(PackageBase).filter( + PackageBase.Name == pkgbase.Name + ).first() + assert record is None From 01fb42c5d97fe930b63ef8b71d4b2a2b62f32a5e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 7 Oct 2021 22:44:54 -0700 Subject: [PATCH 403/844] fix(scripts.popupdate): use forced-utc timestamp Additionally, clean up some controversial PEP-8 warnings by removing the '+' string concatenation. Signed-off-by: Kevin Morris --- aurweb/scripts/popupdate.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/aurweb/scripts/popupdate.py b/aurweb/scripts/popupdate.py index b1e70403..96155eef 100755 --- a/aurweb/scripts/popupdate.py +++ b/aurweb/scripts/popupdate.py @@ -1,21 +1,21 @@ #!/usr/bin/env python3 -import time +from datetime import datetime import aurweb.db def main(): conn = aurweb.db.Connection() - conn.execute("UPDATE PackageBases SET NumVotes = (" + - "SELECT COUNT(*) FROM PackageVotes " + - "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") + conn.execute(("UPDATE PackageBases SET NumVotes = (" + "SELECT COUNT(*) FROM PackageVotes " + "WHERE PackageVotes.PackageBaseID = PackageBases.ID)")) - now = int(time.time()) - conn.execute("UPDATE PackageBases SET Popularity = (" + - "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " + - "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + - "PackageBases.ID AND NOT VoteTS IS NULL)", [now]) + now = int(datetime.utcnow().timestamp()) + conn.execute(("UPDATE PackageBases SET Popularity = (" + "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " + "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + "PackageBases.ID AND NOT VoteTS IS NULL)"), [now]) conn.commit() conn.close() From 63498f5edde58d231eba34359ac231ea217a34d9 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 7 Oct 2021 22:48:31 -0700 Subject: [PATCH 404/844] fix(FastAPI): use popupdate when [un]voting The `aurweb.scripts.popupdate` script is used to maintain the NumVotes and Popularity field. We could do the NumVotes change more simply; however, since this is already a long-term implementation, we're going to use it until we move scripts over to ORM. Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 10 +++++++- aurweb/scripts/popupdate.py | 45 ++++++++++++++++++++++++++++-------- test/test_packages_routes.py | 2 ++ 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 4426d0be..f806f054 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -30,7 +30,7 @@ from aurweb.models.request_type import DELETION_ID, RequestType from aurweb.models.user import User from aurweb.packages.search import PackageSearch from aurweb.packages.util import get_pkg_or_base, get_pkgbase_comment, query_notified, query_voted -from aurweb.scripts import notify +from aurweb.scripts import notify, popupdate from aurweb.scripts.rendercomment import update_comment_render from aurweb.templates import make_context, render_raw_template, render_template @@ -858,6 +858,10 @@ async def pkgbase_vote(request: Request, name: str): PackageBase=pkgbase, VoteTS=now) + # Update NumVotes/Popularity. + conn = db.ConnectionExecutor(db.get_engine().raw_connection()) + popupdate.run_single(conn, pkgbase) + return RedirectResponse(f"/pkgbase/{name}", status_code=int(HTTPStatus.SEE_OTHER)) @@ -875,6 +879,10 @@ async def pkgbase_unvote(request: Request, name: str): with db.begin(): db.session.delete(vote) + # Update NumVotes/Popularity. + conn = db.ConnectionExecutor(db.get_engine().raw_connection()) + popupdate.run_single(conn, pkgbase) + return RedirectResponse(f"/pkgbase/{name}", status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/aurweb/scripts/popupdate.py b/aurweb/scripts/popupdate.py index 96155eef..fa82208d 100755 --- a/aurweb/scripts/popupdate.py +++ b/aurweb/scripts/popupdate.py @@ -5,17 +5,44 @@ from datetime import datetime import aurweb.db -def main(): - conn = aurweb.db.Connection() - conn.execute(("UPDATE PackageBases SET NumVotes = (" - "SELECT COUNT(*) FROM PackageVotes " - "WHERE PackageVotes.PackageBaseID = PackageBases.ID)")) +def run_single(conn, pkgbase): + """ A single popupdate. The given pkgbase instance will be + refreshed after the database update is done. + + NOTE: This function is compatible only with aurweb FastAPI. + + :param conn: db.Connection[Executor] + :param pkgbase: Instance of db.PackageBase + """ + + conn.execute("UPDATE PackageBases SET NumVotes = (" + "SELECT COUNT(*) FROM PackageVotes " + "WHERE PackageVotes.PackageBaseID = PackageBases.ID) " + "WHERE PackageBases.ID = ?", [pkgbase.ID]) now = int(datetime.utcnow().timestamp()) - conn.execute(("UPDATE PackageBases SET Popularity = (" - "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " - "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " - "PackageBases.ID AND NOT VoteTS IS NULL)"), [now]) + conn.execute("UPDATE PackageBases SET Popularity = (" + "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " + "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + "PackageBases.ID AND NOT VoteTS IS NULL) WHERE " + "PackageBases.ID = ?", [now, pkgbase.ID]) + + conn.commit() + conn.close() + aurweb.db.session.refresh(pkgbase) + + +def main(): + conn = aurweb.db.Connection() + conn.execute("UPDATE PackageBases SET NumVotes = (" + "SELECT COUNT(*) FROM PackageVotes " + "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") + + now = int(datetime.utcnow().timestamp()) + conn.execute("UPDATE PackageBases SET Popularity = (" + "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " + "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + "PackageBases.ID AND NOT VoteTS IS NULL)", [now]) conn.commit() conn.close() diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 1f258497..7b9c520c 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1771,6 +1771,7 @@ def test_pkgbase_vote(client: TestClient, user: User, package: Package): vote = pkgbase.package_votes.filter(PackageVote.UsersID == user.ID).first() assert vote is not None + assert pkgbase.NumVotes == 1 # Remove vote. endpoint = f"/pkgbase/{pkgbase.Name}/unvote" @@ -1780,6 +1781,7 @@ def test_pkgbase_vote(client: TestClient, user: User, package: Package): vote = pkgbase.package_votes.filter(PackageVote.UsersID == user.ID).first() assert vote is None + assert pkgbase.NumVotes == 0 def test_pkgbase_disown_as_tu(client: TestClient, tu_user: User, From 305d07797371087b5e04bb49827d619c793a2d63 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 7 Oct 2021 22:01:04 -0700 Subject: [PATCH 405/844] feat(FastAPI): add /pkgbase/{name}/adopt (post) Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 17 ++++++++++++ templates/partials/packages/actions.html | 19 +++++++++++--- test/test_packages_routes.py | 33 ++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index f806f054..b623ca10 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -950,6 +950,23 @@ async def pkgbase_disown_post(request: Request, name: str, status_code=int(HTTPStatus.SEE_OTHER)) +@router.post("/pkgbase/{name}/adopt") +@auth_required(True) +async def pkgbase_adopt_post(request: Request, name: str): + pkgbase = get_pkg_or_base(name, PackageBase) + + has_cred = request.user.has_credential("CRED_PKGBASE_ADOPT") + if has_cred or not pkgbase.Maintainer: + # If the user has credentials, they'll adopt the package regardless + # of maintainership. Otherwise, we'll promote the user to maintainer + # if no maintainer currently exists. + with db.begin(): + pkgbase.Maintainer = request.user + + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) + + @router.get("/pkgbase/{name}/delete") @auth_required(True) async def pkgbase_delete_get(request: Request, name: str): diff --git a/templates/partials/packages/actions.html b/templates/partials/packages/actions.html index 2b26144e..dd83c84d 100644 --- a/templates/partials/packages/actions.html +++ b/templates/partials/packages/actions.html @@ -119,12 +119,23 @@ {% endif %} - {% if request.user.has_credential("CRED_PKGBASE_DISOWN", approved=[pkgbase.Maintainer]) %} + {% if not result.Maintainer %}
  • - - {{ "Disown Package" | tr }} - +
    + +
  • + {% else %} + {% if request.user.has_credential("CRED_PKGBASE_DISOWN", approved=[pkgbase.Maintainer]) %} +
  • + + {{ "Disown Package" | tr }} + +
  • + {% endif %} {% endif %}

    diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 7b9c520c..86949996 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1849,6 +1849,39 @@ def test_pkgbase_disown(client: TestClient, user: User, maintainer: User, assert resp.status_code == int(HTTPStatus.SEE_OTHER) +def test_pkgbase_adopt(client: TestClient, user: User, tu_user: User, + maintainer: User, package: Package): + # Unset the maintainer as if package is orphaned. + with db.begin(): + package.PackageBase.Maintainer = None + + pkgbasename = package.PackageBase.Name + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + endpoint = f"/pkgbase/{pkgbasename}/adopt" + + # Adopt the package base. + with client as request: + resp = request.post(endpoint, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert package.PackageBase.Maintainer == maintainer + + # Try to adopt it when it already has a maintainer; nothing changes. + user_cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, cookies=user_cookies, + allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert package.PackageBase.Maintainer == maintainer + + # Steal the package as a TU. + tu_cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, cookies=tu_cookies, + allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert package.PackageBase.Maintainer == tu_user + + def test_pkgbase_delete_unauthorized(client: TestClient, user: User, package: Package): pkgbase = package.PackageBase From 5bbc94f2ef333bbe5d33ee1893067e2864de5eb1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 9 Oct 2021 18:41:32 -0700 Subject: [PATCH 406/844] fix(FastAPI): add /pkgbase/{name}/flag (get) This was missed in the [un]flag (post) commit. Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 28 +++++++++++++++++- templates/packages/flag.html | 57 ++++++++++++++++++++++++++++++++++++ test/test_packages_routes.py | 20 ++++++++++++- 3 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 templates/packages/flag.html diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index b623ca10..8f4a7e1f 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -773,17 +773,42 @@ async def requests_close_post(request: Request, id: int, return RedirectResponse("/requests", status_code=int(HTTPStatus.SEE_OTHER)) +@router.get("/pkgbase/{name}/flag") +@auth_required(True, redirect="/pkgbase/{name}") +async def pkgbase_flag_get(request: Request, name: str): + pkgbase = get_pkg_or_base(name, PackageBase) + + has_cred = request.user.has_credential("CRED_PKGBASE_FLAG") + if not has_cred or pkgbase.Flagger is not None: + return RedirectResponse(f"/pkgbase/{name}", + status_code=int(HTTPStatus.SEE_OTHER)) + + context = make_context(request, "Flag Package Out-Of-Date") + context["pkgbase"] = pkgbase + return render_template(request, "packages/flag.html", context) + + @router.post("/pkgbase/{name}/flag") @auth_required(True, redirect="/pkgbase/{name}") -async def pkgbase_flag(request: Request, name: str): +async def pkgbase_flag_post(request: Request, name: str, + comments: str = Form(default=str())): pkgbase = get_pkg_or_base(name, PackageBase) + if not comments: + context = make_context(request, "Flag Package Out-Of-Date") + context["pkgbase"] = pkgbase + context["errors"] = ["The selected packages have not been flagged, " + "please enter a comment."] + return render_template(request, "packages/flag.html", context, + status_code=int(HTTPStatus.BAD_REQUEST)) + has_cred = request.user.has_credential("CRED_PKGBASE_FLAG") if has_cred and not pkgbase.Flagger: now = int(datetime.utcnow().timestamp()) with db.begin(): pkgbase.OutOfDateTS = now pkgbase.Flagger = request.user + pkgbase.FlaggerComment = comments return RedirectResponse(f"/pkgbase/{name}", status_code=int(HTTPStatus.SEE_OTHER)) @@ -800,6 +825,7 @@ async def pkgbase_unflag(request: Request, name: str): with db.begin(): pkgbase.OutOfDateTS = None pkgbase.Flagger = None + pkgbase.FlaggerComment = str() return RedirectResponse(f"/pkgbase/{name}", status_code=int(HTTPStatus.SEE_OTHER)) diff --git a/templates/packages/flag.html b/templates/packages/flag.html new file mode 100644 index 00000000..4e133acb --- /dev/null +++ b/templates/packages/flag.html @@ -0,0 +1,57 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} +
    +

    {{ "Flag Package Out-Of-Date" | tr }}: {{ pkgbase.Name }}

    + +

    + {{ + "Use this form to flag the package base %s%s%s and " + "the following packages out-of-date: " + | tr | format("", pkgbase.Name, "") | safe + }} +

    + +
      + {% for package in pkgbase.packages.all() %} +
    • {{ package.Name }}
    • + {% endfor %} +
    + +

    + {{ + "Please do %snot%s use this form to report bugs. " + "Use the package comments instead." + | tr | format("", "") | safe + }} + {{ + "Enter details on why the package is out-of-date below, " + "preferably including links to the release announcement " + "or the new release tarball." | tr + }} +

    + + {% if errors %} +
      + {% for error in errors %} +
    • {{ error | tr }}
    • + {% endfor %} +
    + {% endif %} + +
    +
    +

    + + +

    +

    + +

    +
    +
    +
    +{% endblock %} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 86949996..12d7e33e 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1698,13 +1698,31 @@ def test_pkgbase_flag(client: TestClient, user: User, maintainer: User, # We shouldn't have flagged the package yet; assert so. assert pkgbase.Flagger is None - # Flag it. cookies = {"AURSID": user.login(Request(), "testPassword")} endpoint = f"/pkgbase/{pkgbase.Name}/flag" + + # Get the flag page. + with client as request: + resp = request.get(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + # Try to flag it without a comment. with client as request: resp = request.post(endpoint, cookies=cookies) + assert resp.status_code == int(HTTPStatus.BAD_REQUEST) + + # Flag it with a valid comment. + with client as request: + resp = request.post(endpoint, {"comments": "Test"}, cookies=cookies) assert resp.status_code == int(HTTPStatus.SEE_OTHER) assert pkgbase.Flagger == user + assert pkgbase.FlaggerComment == "Test" + + # Now try to perform a get; we should be redirected because + # it's already flagged. + with client as request: + resp = request.get(endpoint, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) # Now, test that the 'maintainer' user can't unflag it, because they # didn't flag it to begin with. From d9ab65cb6f2e0f0985f2463c352acc52621e1026 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 9 Oct 2021 20:46:52 -0700 Subject: [PATCH 407/844] add Feedback.md GitLab issue template Signed-off-by: Kevin Morris --- .gitlab/issue_templates/Feedback.md | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .gitlab/issue_templates/Feedback.md diff --git a/.gitlab/issue_templates/Feedback.md b/.gitlab/issue_templates/Feedback.md new file mode 100644 index 00000000..e32120aa --- /dev/null +++ b/.gitlab/issue_templates/Feedback.md @@ -0,0 +1,56 @@ +**NOTE:** This issue template is only applicable to FastAPI implementations +in the code-base, which only exists within the `pu` branch. If you wish to +file an issue for the current PHP implementation of aurweb, please file a +standard issue prefixed with `[Bug]` or `[Feature]`. + + +**Checklist** + +- [ ] I have prefixed the issue title with `[Feedback]` along with a message + pointing to the route or feature tested. + - Example: `[Feedback] /packages/{name}` +- [ ] I have completed the [Changes](#changes) section. +- [ ] I have completed the [Bugs](#bugs) section. +- [ ] I have completed the [Improvements](#improvements) section. +- [ ] I have completed the [Summary](#summary) section. + +### Changes + +Please describe changes in user experience when compared to the PHP +implementation. This section can actually hold a lot of info if you +are up for it -- changes in routes, HTML rendering, back-end behavior, +etc. + +If you cannot see any changes from your standpoint, include a short +statement about that fact. + +### Bugs + +Please describe any bugs you've experienced while testing the route +pertaining to this issue. A "perfect" bug report would include your +specific experience, what you expected to occur, and what happened +otherwise. If you can, please include output of `docker-compose logs fastapi` +with your report; especially if any unintended exceptions occurred. + +### Improvements + +If you've experienced improvements in the route when compared to PHP, +please do include those here. We'd like to know if users are noticing +these improvements and how they feel about them. + +There are multiple routes with no improvements. For these, just include +a short sentence about the fact that you've experienced none. + +### Summary + +First: If you've gotten here and completed the [Changes](#changes), +[Bugs](#bugs), and [Improvements](#improvements) sections, we'd like +to thank you very much for your contribution and willingness to test. +We are not a company, and we are not a large team; any bit of assistance +here helps the project astronomically and moves us closer toward a +new release. + +That being said: please include an overall summary of your experience +and how you felt about the current implementation which you're testing +in comparison with PHP (current aur.archlinux.org, or https://localhost:8443 +through docker). From 34c96ed81b017b1b8273e14d98d69a69f9029e37 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 9 Oct 2021 20:46:52 -0700 Subject: [PATCH 408/844] add Feedback.md GitLab issue template Signed-off-by: Kevin Morris --- .gitlab/issue_templates/Feedback.md | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .gitlab/issue_templates/Feedback.md diff --git a/.gitlab/issue_templates/Feedback.md b/.gitlab/issue_templates/Feedback.md new file mode 100644 index 00000000..e32120aa --- /dev/null +++ b/.gitlab/issue_templates/Feedback.md @@ -0,0 +1,56 @@ +**NOTE:** This issue template is only applicable to FastAPI implementations +in the code-base, which only exists within the `pu` branch. If you wish to +file an issue for the current PHP implementation of aurweb, please file a +standard issue prefixed with `[Bug]` or `[Feature]`. + + +**Checklist** + +- [ ] I have prefixed the issue title with `[Feedback]` along with a message + pointing to the route or feature tested. + - Example: `[Feedback] /packages/{name}` +- [ ] I have completed the [Changes](#changes) section. +- [ ] I have completed the [Bugs](#bugs) section. +- [ ] I have completed the [Improvements](#improvements) section. +- [ ] I have completed the [Summary](#summary) section. + +### Changes + +Please describe changes in user experience when compared to the PHP +implementation. This section can actually hold a lot of info if you +are up for it -- changes in routes, HTML rendering, back-end behavior, +etc. + +If you cannot see any changes from your standpoint, include a short +statement about that fact. + +### Bugs + +Please describe any bugs you've experienced while testing the route +pertaining to this issue. A "perfect" bug report would include your +specific experience, what you expected to occur, and what happened +otherwise. If you can, please include output of `docker-compose logs fastapi` +with your report; especially if any unintended exceptions occurred. + +### Improvements + +If you've experienced improvements in the route when compared to PHP, +please do include those here. We'd like to know if users are noticing +these improvements and how they feel about them. + +There are multiple routes with no improvements. For these, just include +a short sentence about the fact that you've experienced none. + +### Summary + +First: If you've gotten here and completed the [Changes](#changes), +[Bugs](#bugs), and [Improvements](#improvements) sections, we'd like +to thank you very much for your contribution and willingness to test. +We are not a company, and we are not a large team; any bit of assistance +here helps the project astronomically and moves us closer toward a +new release. + +That being said: please include an overall summary of your experience +and how you felt about the current implementation which you're testing +in comparison with PHP (current aur.archlinux.org, or https://localhost:8443 +through docker). From 27fbda5e7ba21a43c64ca0324c24f42a484196c0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 9 Oct 2021 22:00:18 -0700 Subject: [PATCH 409/844] feat(FastAPI): add get_(errors|successes) testing HTML helpers These functions will allow us to more easily check errors or success messages when testing routes. Signed-off-by: Kevin Morris --- aurweb/testing/html.py | 11 +++++++++++ test/test_html.py | 22 +++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/aurweb/testing/html.py b/aurweb/testing/html.py index d5f0c256..f01aaf3d 100644 --- a/aurweb/testing/html.py +++ b/aurweb/testing/html.py @@ -1,4 +1,5 @@ from io import StringIO +from typing import List from lxml import etree @@ -12,3 +13,13 @@ def parse_root(html: str) -> etree.Element: :return: etree.Element """ return etree.parse(StringIO(html), parser) + + +def get_errors(content: str) -> List[etree._Element]: + root = parse_root(content) + return root.xpath('//ul[@class="errorlist"]/li') + + +def get_successes(content: str) -> List[etree._Element]: + root = parse_root(content) + return root.xpath('//ul[@class="success"]/li') diff --git a/test/test_html.py b/test/test_html.py index 562d6a63..2018840b 100644 --- a/test/test_html.py +++ b/test/test_html.py @@ -9,7 +9,7 @@ from aurweb import asgi, db from aurweb.models.account_type import TRUSTED_USER_ID, USER_ID, AccountType from aurweb.models.user import User from aurweb.testing import setup_test_db -from aurweb.testing.html import parse_root +from aurweb.testing.html import get_errors, get_successes, parse_root from aurweb.testing.requests import Request @@ -97,3 +97,23 @@ def test_archdev_navbar_authenticated_tu(client: TestClient, items = root.xpath('//div[@id="archdev-navbar"]/ul/li/a') for i, item in enumerate(items): assert item.text.strip() == expected[i] + + +def test_get_errors(): + html = """ +
      +
    • Test
    • +
    +""" + errors = get_errors(html) + assert errors[0].text.strip() == "Test" + + +def test_get_successes(): + html = """ +
      +
    • Test
    • +
    +""" + successes = get_successes(html) + assert successes[0].text.strip() == "Test" From 4525a11d923f3669e46204626b7c1115927d4703 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 10 Oct 2021 00:59:08 -0700 Subject: [PATCH 410/844] fix(FastAPI): change a deep copy instead of original This was updating offsets and causing unintended behavior. We should be a bit more functional anyway. Signed-off-by: Kevin Morris --- aurweb/util.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/aurweb/util.py b/aurweb/util.py index 44f711f1..61ed5cfb 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -1,4 +1,5 @@ import base64 +import copy import logging import math import random @@ -127,9 +128,10 @@ def as_timezone(dt: datetime, timezone: str): def extend_query(query: Dict[str, Any], *additions) -> Dict[str, Any]: """ Add additional key value pairs to query. """ + q = copy.copy(query) for k, v in list(additions): - query[k] = v - return query + q[k] = v + return q def to_qs(query: Dict[str, Any]) -> str: From 68383b79e24b645c0079627149187c06b3dda734 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 11 Oct 2021 14:13:29 -0700 Subject: [PATCH 411/844] add Feature.md GitLab issue template Signed-off-by: Kevin Morris --- .gitlab/issue_templates/Feature.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .gitlab/issue_templates/Feature.md diff --git a/.gitlab/issue_templates/Feature.md b/.gitlab/issue_templates/Feature.md new file mode 100644 index 00000000..5b1524b1 --- /dev/null +++ b/.gitlab/issue_templates/Feature.md @@ -0,0 +1,30 @@ +- [ ] I have summed up the feature in concise words in the [Summary](#summary) section. +- [ ] I have completely described the feature in the [Description](#description) section. +- [ ] I have completed the [Blockers](#blockers) section. + +### Summary + +Fill this section out with a concise wording about the feature being +requested. + +Example: _A new `Tyrant` account type for users_. + +### Description + +Describe your feature in full detail. + +Example: _The `Tyrant` account type should be used to allow a user to be +tyrannical. When a user is a `Tyrant`, they should be able to assassinate +users due to not complying with their laws. Laws can be configured by updating +the Tyrant laws page at https://aur.archlinux.org/account/{username}/laws. +More specifics about laws._ + +### Blockers + +Include any blockers in a list. If there are no blockers, this section +should be omitted from the issue. + +Example: + +- [Feature] Do not allow users to be Tyrants + - \<(issue|merge_request)_link\> From 3d971bfc8d70c48f69090b7ec0b0b1899178c03d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 11 Oct 2021 14:48:00 -0700 Subject: [PATCH 412/844] add Bug.md GitLab issue template Signed-off-by: Kevin Morris --- .gitlab/issue_templates/Bug.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .gitlab/issue_templates/Bug.md diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md new file mode 100644 index 00000000..d84a5181 --- /dev/null +++ b/.gitlab/issue_templates/Bug.md @@ -0,0 +1,34 @@ +- [ ] I have described the bug in complete detail in the + [Description](#description) section. +- [ ] I have specified steps in the [Reproduction](#reproduction) section. +- [ ] I have included any logs related to the bug in the + [Logs](#logs) section. +- [ ] I have included the versions which are affected in the + [Version(s)](#versions) section. + +### Description + +Describe the bug in full detail. + +### Reproduction + +Describe a specific set of actions that can be used to reproduce +this bug. + +### Logs + +If you have any logs relevent to the bug, include them here in +quoted or code blocks. + +### Version(s) + +In this section, please include a list of versions you have found +to be affected by this program. This can either come in the form +of `major.minor.patch` (if it affects a release tarball), or a +commit hash if the bug does not directly affect a release version. + +All development is done without modifying version displays in +aurweb's HTML render output. If you're testing locally, use the +commit on which you are experiencing the bug. If you have found +a bug which exists on live aur.archlinux.org, include the version +located at the bottom of the webpage. From 748faca87d314e23557a1e20223db939d8b19192 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 12 Oct 2021 17:55:03 -0700 Subject: [PATCH 413/844] fix(FastAPI): translate some untranslated strings Affects: templates/partials/packages/search_actions.html Signed-off-by: Kevin Morris --- templates/partials/packages/search_actions.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/partials/packages/search_actions.html b/templates/partials/packages/search_actions.html index 2f5fe2e7..221189fb 100644 --- a/templates/partials/packages/search_actions.html +++ b/templates/partials/packages/search_actions.html @@ -18,8 +18,8 @@ - +

    From 22b3af61b568732861be17e9759be68715a709fb Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 13 Oct 2021 17:10:16 -0700 Subject: [PATCH 414/844] fix(PHP): sanitize and produce metrics at shutdown This change now requires that PHP routes do not return HTTP 404 to be considered for the /metrics population. Additionally, we make a small sanitization here to avoid trailing '/' characters, unless we're on the homepage route. Signed-off-by: Kevin Morris --- web/html/index.php | 24 ++--------------------- web/lib/metricfuncs.inc.php | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/web/html/index.php b/web/html/index.php index 82a44c55..99046930 100644 --- a/web/html/index.php +++ b/web/html/index.php @@ -13,28 +13,8 @@ $query_string = $_SERVER['QUERY_STRING']; // If no options.cache is configured, we no-op metric storage operations. $is_cached = defined('EXTENSION_LOADED_APC') || defined('EXTENSION_LOADED_MEMCACHE'); -if ($is_cached) { - $method = $_SERVER['REQUEST_METHOD']; - // We'll always add +1 to our total request count to this $path, - // unless this path == /metrics. - if ($path !== "/metrics") - add_metric("http_requests_count", $method, $path); - - // Extract $type out of $query_string, if we can. - $type = null; - $query = array(); - if ($query_string) - parse_str($query_string, $query); - $type = $query['type']; - - // Only store RPC metrics for valid types. - $good_types = [ - "info", "multiinfo", "search", "msearch", - "suggest", "suggest-pkgbase", "get-comment-form" - ]; - if ($path === "/rpc" && in_array($type, $good_types)) - add_metric("api_requests_count", $method, $path, $type); -} +if ($is_cached) + register_shutdown_function('update_metrics'); if (config_get_bool('options', 'enable-maintenance') && (empty($tokens[1]) || ($tokens[1] != "css" && $tokens[1] != "images"))) { if (!in_array($_SERVER['REMOTE_ADDR'], explode(" ", config_get('options', 'maintenance-exceptions')))) { diff --git a/web/lib/metricfuncs.inc.php b/web/lib/metricfuncs.inc.php index acfc30d7..7ebb59be 100644 --- a/web/lib/metricfuncs.inc.php +++ b/web/lib/metricfuncs.inc.php @@ -13,6 +13,44 @@ use \Prometheus\RenderTextFormat; // and will start again at 0 if it's restarted. $registry = new CollectorRegistry(new InMemory()); +function update_metrics() { + // With no code given to http_response_code, it gets the current + // response code set (via http_response_code or header). + if(http_response_code() == 404) + return; + + $path = $_SERVER['PATH_INFO']; + $method = $_SERVER['REQUEST_METHOD']; + $query_string = $_SERVER['QUERY_STRING']; + + // If $path is at least 1 character, strip / off the end. + // This turns $paths like '/packages/' into '/packages'. + if (strlen($path) > 1) + $path = rtrim($path, "/"); + + // We'll always add +1 to our total request count to this $path, + // unless this path == /metrics. + if ($path !== "/metrics") + add_metric("http_requests_count", $method, $path); + + // Extract $type out of $query_string, if we can. + $type = null; + $query = array(); + if ($query_string) + parse_str($query_string, $query); + + if (array_key_exists("type", $query)) + $type = $query["type"]; + + // Only store RPC metrics for valid types. + $good_types = [ + "info", "multiinfo", "search", "msearch", + "suggest", "suggest-pkgbase", "get-comment-form" + ]; + if ($path === "/rpc" && in_array($type, $good_types)) + add_metric("api_requests_count", $method, $path, $type); +} + function add_metric($anchor, $method, $path, $type = null) { global $registry; From 5bfc1e9094e2a67bad534f7380b1b6b20fe13ef9 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 15 Oct 2021 13:18:58 -0700 Subject: [PATCH 415/844] Revert "fix(PHP): sanitize and produce metrics at shutdown" This reverts commit 22b3af61b568732861be17e9759be68715a709fb. --- web/html/index.php | 24 +++++++++++++++++++++-- web/lib/metricfuncs.inc.php | 38 ------------------------------------- 2 files changed, 22 insertions(+), 40 deletions(-) diff --git a/web/html/index.php b/web/html/index.php index 99046930..82a44c55 100644 --- a/web/html/index.php +++ b/web/html/index.php @@ -13,8 +13,28 @@ $query_string = $_SERVER['QUERY_STRING']; // If no options.cache is configured, we no-op metric storage operations. $is_cached = defined('EXTENSION_LOADED_APC') || defined('EXTENSION_LOADED_MEMCACHE'); -if ($is_cached) - register_shutdown_function('update_metrics'); +if ($is_cached) { + $method = $_SERVER['REQUEST_METHOD']; + // We'll always add +1 to our total request count to this $path, + // unless this path == /metrics. + if ($path !== "/metrics") + add_metric("http_requests_count", $method, $path); + + // Extract $type out of $query_string, if we can. + $type = null; + $query = array(); + if ($query_string) + parse_str($query_string, $query); + $type = $query['type']; + + // Only store RPC metrics for valid types. + $good_types = [ + "info", "multiinfo", "search", "msearch", + "suggest", "suggest-pkgbase", "get-comment-form" + ]; + if ($path === "/rpc" && in_array($type, $good_types)) + add_metric("api_requests_count", $method, $path, $type); +} if (config_get_bool('options', 'enable-maintenance') && (empty($tokens[1]) || ($tokens[1] != "css" && $tokens[1] != "images"))) { if (!in_array($_SERVER['REMOTE_ADDR'], explode(" ", config_get('options', 'maintenance-exceptions')))) { diff --git a/web/lib/metricfuncs.inc.php b/web/lib/metricfuncs.inc.php index 7ebb59be..acfc30d7 100644 --- a/web/lib/metricfuncs.inc.php +++ b/web/lib/metricfuncs.inc.php @@ -13,44 +13,6 @@ use \Prometheus\RenderTextFormat; // and will start again at 0 if it's restarted. $registry = new CollectorRegistry(new InMemory()); -function update_metrics() { - // With no code given to http_response_code, it gets the current - // response code set (via http_response_code or header). - if(http_response_code() == 404) - return; - - $path = $_SERVER['PATH_INFO']; - $method = $_SERVER['REQUEST_METHOD']; - $query_string = $_SERVER['QUERY_STRING']; - - // If $path is at least 1 character, strip / off the end. - // This turns $paths like '/packages/' into '/packages'. - if (strlen($path) > 1) - $path = rtrim($path, "/"); - - // We'll always add +1 to our total request count to this $path, - // unless this path == /metrics. - if ($path !== "/metrics") - add_metric("http_requests_count", $method, $path); - - // Extract $type out of $query_string, if we can. - $type = null; - $query = array(); - if ($query_string) - parse_str($query_string, $query); - - if (array_key_exists("type", $query)) - $type = $query["type"]; - - // Only store RPC metrics for valid types. - $good_types = [ - "info", "multiinfo", "search", "msearch", - "suggest", "suggest-pkgbase", "get-comment-form" - ]; - if ($path === "/rpc" && in_array($type, $good_types)) - add_metric("api_requests_count", $method, $path, $type); -} - function add_metric($anchor, $method, $path, $type = null) { global $registry; From 040bb0d7f4d43af66126abdc677fcea05afa058a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 15 Oct 2021 13:19:07 -0700 Subject: [PATCH 416/844] Revert "feat(PHP): add aurweb Prometheus metrics" This reverts commit 986fa9ee305ed113172f7f214d451a7af071ecc2. --- INSTALL | 8 +-- web/html/index.php | 29 -------- web/html/metrics.php | 16 ----- web/lib/metricfuncs.inc.php | 129 ------------------------------------ web/lib/routing.inc.php | 3 +- 5 files changed, 2 insertions(+), 183 deletions(-) delete mode 100644 web/html/metrics.php delete mode 100644 web/lib/metricfuncs.inc.php diff --git a/INSTALL b/INSTALL index b161edd2..9bcd0759 100644 --- a/INSTALL +++ b/INSTALL @@ -49,15 +49,9 @@ read the instructions below. # pacman -S python-mysql-connector python-pygit2 python-srcinfo python-sqlalchemy \ python-bleach python-markdown python-alembic python-jinja \ - python-itsdangerous python-authlib python-httpx hypercorn \ - composer + python-itsdangerous python-authlib python-httpx hypercorn # python3 setup.py install -4a) Install `composer` dependencies while inside of aurweb's root: - - $ cd /path/to/aurweb - /path/to/aurweb $ composer require promphp/prometheus_client_php - 5) Create a new MySQL database and a user and import the aurweb SQL schema: $ python -m aurweb.initdb diff --git a/web/html/index.php b/web/html/index.php index 82a44c55..e57e7708 100644 --- a/web/html/index.php +++ b/web/html/index.php @@ -3,39 +3,10 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); include_once("pkgfuncs.inc.php"); -include_once("cachefuncs.inc.php"); -include_once("metricfuncs.inc.php"); $path = $_SERVER['PATH_INFO']; $tokens = explode('/', $path); -$query_string = $_SERVER['QUERY_STRING']; - -// If no options.cache is configured, we no-op metric storage operations. -$is_cached = defined('EXTENSION_LOADED_APC') || defined('EXTENSION_LOADED_MEMCACHE'); -if ($is_cached) { - $method = $_SERVER['REQUEST_METHOD']; - // We'll always add +1 to our total request count to this $path, - // unless this path == /metrics. - if ($path !== "/metrics") - add_metric("http_requests_count", $method, $path); - - // Extract $type out of $query_string, if we can. - $type = null; - $query = array(); - if ($query_string) - parse_str($query_string, $query); - $type = $query['type']; - - // Only store RPC metrics for valid types. - $good_types = [ - "info", "multiinfo", "search", "msearch", - "suggest", "suggest-pkgbase", "get-comment-form" - ]; - if ($path === "/rpc" && in_array($type, $good_types)) - add_metric("api_requests_count", $method, $path, $type); -} - if (config_get_bool('options', 'enable-maintenance') && (empty($tokens[1]) || ($tokens[1] != "css" && $tokens[1] != "images"))) { if (!in_array($_SERVER['REMOTE_ADDR'], explode(" ", config_get('options', 'maintenance-exceptions')))) { header("HTTP/1.0 503 Service Unavailable"); diff --git a/web/html/metrics.php b/web/html/metrics.php deleted file mode 100644 index dfa860ed..00000000 --- a/web/html/metrics.php +++ /dev/null @@ -1,16 +0,0 @@ - diff --git a/web/lib/metricfuncs.inc.php b/web/lib/metricfuncs.inc.php deleted file mode 100644 index acfc30d7..00000000 --- a/web/lib/metricfuncs.inc.php +++ /dev/null @@ -1,129 +0,0 @@ -, 'query_string': }. - $metrics = get_cache_value("prometheus_metrics"); - $metrics = $metrics ? json_decode($metrics) : array(); - - $key = "$path:$type"; - - // If the current request $path isn't yet in $metrics create - // a new assoc array for it and push it into $metrics. - if (!in_array($key, $metrics)) { - $data = array( - 'anchor' => $anchor, - 'method' => $method, - 'path' => $path, - 'type' => $type - ); - array_push($metrics, json_encode($data)); - } - - // Cache-wise, we also store the count values of each route - // through the "prometheus:" key. Grab the cache value - // representing the current $path we're on (defaulted to 1). - $count = get_cache_value("prometheus:$key"); - $count = $count ? $count + 1 : 1; - - $labels = ["method", "route"]; - if ($type) - array_push($labels, "type"); - - $gauge = $registry->getOrRegisterGauge( - 'aurweb', - $anchor, - 'A metric count for the aurweb platform.', - $labels - ); - - $label_values = [$data['method'], $data['path']]; - if ($type) - array_push($label_values, $type); - - $gauge->set($count, $label_values); - - // Update cache values. - set_cache_value("prometheus:$key", $count, 0); - set_cache_value("prometheus_metrics", json_encode($metrics), 0); - -} - -function render_metrics() { - if (!defined('EXTENSION_LOADED_APC') && !defined('EXTENSION_LOADED_MEMCACHE')) { - error_log("The /metrics route requires a valid 'options.cache' " - . "configuration; no cache is configured."); - return http_response_code(417); // EXPECTATION_FAILED - } - - global $registry; - - // First, we grab the set of metrics we're interested in in the - // form of a cached JSON list, if we can. - $metrics = get_cache_value("prometheus_metrics"); - if (!$metrics) - $metrics = array(); - else - $metrics = json_decode($metrics); - - // Now, we walk through each of those list values one by one, - // which happen to be JSON-serialized associative arrays, - // and process each metric via its associative array's contents: - // The route path and the query string. - // See web/html/index.php for the creation of such metrics. - foreach ($metrics as $metric) { - $data = json_decode($metric, true); - - $anchor = $data['anchor']; - $path = $data['path']; - $type = $data['type']; - $key = "$path:$type"; - - $labels = ["method", "route"]; - if ($type) - array_push($labels, "type"); - - $count = get_cache_value("prometheus:$key"); - $gauge = $registry->getOrRegisterGauge( - 'aurweb', - $anchor, - 'A metric count for the aurweb platform.', - $labels - ); - - $label_values = [$data['method'], $data['path']]; - if ($type) - array_push($label_values, $type); - - $gauge->set($count, $label_values); - } - - // Construct the results from RenderTextFormat renderer and - // registry's samples. - $renderer = new RenderTextFormat(); - $result = $renderer->render($registry->getMetricFamilySamples()); - - // Output the results with the right content type header. - http_response_code(200); // OK - header('Content-Type: ' . RenderTextFormat::MIME_TYPE); - echo $result; -} - -?> diff --git a/web/lib/routing.inc.php b/web/lib/routing.inc.php index 0f452f22..73c667d2 100644 --- a/web/lib/routing.inc.php +++ b/web/lib/routing.inc.php @@ -19,8 +19,7 @@ $ROUTES = array( '/rss' => 'rss.php', '/tos' => 'tos.php', '/tu' => 'tu.php', - '/addvote' => 'addvote.php', - '/metrics' => 'metrics.php' // Prometheus Metrics + '/addvote' => 'addvote.php', ); $PKG_PATH = '/packages'; From dd420f8c4148a7696554499fd99eb8c5c726a994 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 11 Oct 2021 14:13:29 -0700 Subject: [PATCH 417/844] add Feature.md GitLab issue template Signed-off-by: Kevin Morris --- .gitlab/issue_templates/Feature.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .gitlab/issue_templates/Feature.md diff --git a/.gitlab/issue_templates/Feature.md b/.gitlab/issue_templates/Feature.md new file mode 100644 index 00000000..5b1524b1 --- /dev/null +++ b/.gitlab/issue_templates/Feature.md @@ -0,0 +1,30 @@ +- [ ] I have summed up the feature in concise words in the [Summary](#summary) section. +- [ ] I have completely described the feature in the [Description](#description) section. +- [ ] I have completed the [Blockers](#blockers) section. + +### Summary + +Fill this section out with a concise wording about the feature being +requested. + +Example: _A new `Tyrant` account type for users_. + +### Description + +Describe your feature in full detail. + +Example: _The `Tyrant` account type should be used to allow a user to be +tyrannical. When a user is a `Tyrant`, they should be able to assassinate +users due to not complying with their laws. Laws can be configured by updating +the Tyrant laws page at https://aur.archlinux.org/account/{username}/laws. +More specifics about laws._ + +### Blockers + +Include any blockers in a list. If there are no blockers, this section +should be omitted from the issue. + +Example: + +- [Feature] Do not allow users to be Tyrants + - \<(issue|merge_request)_link\> From 81c9312606458395c90984be3f4a758636f93c9f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 11 Oct 2021 14:48:00 -0700 Subject: [PATCH 418/844] add Bug.md GitLab issue template Signed-off-by: Kevin Morris --- .gitlab/issue_templates/Bug.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .gitlab/issue_templates/Bug.md diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md new file mode 100644 index 00000000..d84a5181 --- /dev/null +++ b/.gitlab/issue_templates/Bug.md @@ -0,0 +1,34 @@ +- [ ] I have described the bug in complete detail in the + [Description](#description) section. +- [ ] I have specified steps in the [Reproduction](#reproduction) section. +- [ ] I have included any logs related to the bug in the + [Logs](#logs) section. +- [ ] I have included the versions which are affected in the + [Version(s)](#versions) section. + +### Description + +Describe the bug in full detail. + +### Reproduction + +Describe a specific set of actions that can be used to reproduce +this bug. + +### Logs + +If you have any logs relevent to the bug, include them here in +quoted or code blocks. + +### Version(s) + +In this section, please include a list of versions you have found +to be affected by this program. This can either come in the form +of `major.minor.patch` (if it affects a release tarball), or a +commit hash if the bug does not directly affect a release version. + +All development is done without modifying version displays in +aurweb's HTML render output. If you're testing locally, use the +commit on which you are experiencing the bug. If you have found +a bug which exists on live aur.archlinux.org, include the version +located at the bottom of the webpage. From 71b3f781f799f8c9d1d8b3e39682972b89d6c9c2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 15 Oct 2021 15:11:45 -0700 Subject: [PATCH 419/844] fix(FastAPI): maintainers are allowed to unflag their packages Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 2 +- test/test_packages_routes.py | 30 ++++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 8f4a7e1f..15d0591c 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -820,7 +820,7 @@ async def pkgbase_unflag(request: Request, name: str): pkgbase = get_pkg_or_base(name, PackageBase) has_cred = request.user.has_credential( - "CRED_PKGBASE_UNFLAG", approved=[pkgbase.Flagger]) + "CRED_PKGBASE_UNFLAG", approved=[pkgbase.Flagger, pkgbase.Maintainer]) if has_cred: with db.begin(): pkgbase.OutOfDateTS = None diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 12d7e33e..e2811a46 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1713,7 +1713,9 @@ def test_pkgbase_flag(client: TestClient, user: User, maintainer: User, # Flag it with a valid comment. with client as request: - resp = request.post(endpoint, {"comments": "Test"}, cookies=cookies) + resp = request.post(endpoint, data={ + "comments": "Test" + }, cookies=cookies) assert resp.status_code == int(HTTPStatus.SEE_OTHER) assert pkgbase.Flagger == user assert pkgbase.FlaggerComment == "Test" @@ -1724,14 +1726,34 @@ def test_pkgbase_flag(client: TestClient, user: User, maintainer: User, resp = request.get(endpoint, cookies=cookies, allow_redirects=False) assert resp.status_code == int(HTTPStatus.SEE_OTHER) - # Now, test that the 'maintainer' user can't unflag it, because they + with db.begin(): + user2 = db.create(User, Username="test2", + Email="test2@example.org", + Passwd="testPassword", + AccountType=user.AccountType) + + # Now, test that the 'user2' user can't unflag it, because they # didn't flag it to begin with. - maint_cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + user2_cookies = {"AURSID": user2.login(Request(), "testPassword")} endpoint = f"/pkgbase/{pkgbase.Name}/unflag" + with client as request: + resp = request.post(endpoint, cookies=user2_cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + assert pkgbase.Flagger == user + + # Now, test that the 'maintainer' user can. + maint_cookies = {"AURSID": maintainer.login(Request(), "testPassword")} with client as request: resp = request.post(endpoint, cookies=maint_cookies) assert resp.status_code == int(HTTPStatus.SEE_OTHER) - assert pkgbase.Flagger == user + assert pkgbase.Flagger is None + + # Flag it again. + with client as request: + resp = request.post(f"/pkgbase/{pkgbase.Name}/flag", data={ + "comments": "Test" + }, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) # Now, unflag it for real. with client as request: From 2d46811c45a14b89f0c9251ed25b53c8a7f0e775 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 15 Oct 2021 16:15:53 -0700 Subject: [PATCH 420/844] fix(FastAPI): display VCS note when flagging a VCS package Closes: #131 Signed-off-by: Kevin Morris --- po/aurweb.pot | 9 +++++++++ templates/packages/flag.html | 14 ++++++++++++++ test/test_packages_routes.py | 21 +++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/po/aurweb.pot b/po/aurweb.pot index aeed9f02..1f7e8784 100644 --- a/po/aurweb.pot +++ b/po/aurweb.pot @@ -568,6 +568,15 @@ msgstr "" msgid "Flag Package Out-Of-Date" msgstr "" +#: templates/packages/flag.html +msgid "This seems to be a VCS package. Please do %snot%s flag " +"it out-of-date if the package version in the AUR does " +"not match the most recent commit. Flagging this package " +"should only be done if the sources moved or changes in " +"the PKGBUILD are required because of recent upstream " +"changes." +msgstr "" + #: html/pkgflag.php #, php-format msgid "" diff --git a/templates/packages/flag.html b/templates/packages/flag.html index 4e133acb..0335cf18 100644 --- a/templates/packages/flag.html +++ b/templates/packages/flag.html @@ -18,6 +18,20 @@ {% endfor %} + {% if pkgbase.Name.endswith(('-cvs', '-svn', '-git', '-hg', '-bzr', '-darcs')) %} +

    + {# TODO: This error is not yet translated. #} + {{ + "This seems to be a VCS package. Please do %snot%s flag " + "it out-of-date if the package version in the AUR does " + "not match the most recent commit. Flagging this package " + "should only be done if the sources moved or changes in " + "the PKGBUILD are required because of recent upstream " + "changes." | tr | format("", "") | safe + }} +

    + {% endif %} +

    {{ "Please do %snot%s use this form to report bugs. " diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index e2811a46..7eb4e532 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1762,6 +1762,27 @@ def test_pkgbase_flag(client: TestClient, user: User, maintainer: User, assert pkgbase.Flagger is None +def test_pkgbase_flag_vcs(client: TestClient, user: User, package: Package): + # Morph our package fixture into a VCS package (-git). + with db.begin(): + package.PackageBase.Name += "-git" + package.Name += "-git" + + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.get(f"/pkgbase/{package.PackageBase.Name}/flag", + cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + expected = ("This seems to be a VCS package. Please do " + "not flag it out-of-date if the package " + "version in the AUR does not match the most recent commit. " + "Flagging this package should only be done if the sources " + "moved or changes in the PKGBUILD are required because of " + "recent upstream changes.") + assert expected in resp.text + + def test_pkgbase_notify(client: TestClient, user: User, package: Package): pkgbase = package.PackageBase From 8040ef5a9c53048598eb6d0b356923db00467b7e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 15 Oct 2021 19:02:53 -0700 Subject: [PATCH 421/844] fix(FastAPI): use pkgbase in package actions Previously, `result` was being used which was directly set to `pkgbase` before rendering the actions.html partial. It didn't make much sense. This commit cleans things up a bit. Signed-off-by: Kevin Morris --- templates/packages/show.html | 5 ++-- templates/partials/packages/actions.html | 36 ++++++++++++------------ 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/templates/packages/show.html b/templates/packages/show.html index ba531fc8..fbc9c0ea 100644 --- a/templates/packages/show.html +++ b/templates/packages/show.html @@ -5,7 +5,6 @@

    {{ 'Package Details' | tr }}: {{ package.Name }} {{ package.Version }}

    - {% set result = pkgbase %} {% include "partials/packages/actions.html" %} {% set show_package_details = True %} @@ -16,7 +15,7 @@
    - {% set pkgname = result.Name %} - {% set pkgbase_id = result.ID %} + {% set pkgname = package.Name %} + {% set pkgbase_id = pkgbase.ID %} {% include "partials/packages/comments.html" %} {% endblock %} diff --git a/templates/partials/packages/actions.html b/templates/partials/packages/actions.html index dd83c84d..81536a3d 100644 --- a/templates/partials/packages/actions.html +++ b/templates/partials/packages/actions.html @@ -1,38 +1,38 @@ diff --git a/templates/partials/account/comment.html b/templates/partials/account/comment.html new file mode 100644 index 00000000..bc167cf7 --- /dev/null +++ b/templates/partials/account/comment.html @@ -0,0 +1,40 @@ +{% set header_cls = "comment-header" %} +{% if comment.Deleter %} + {% set header_cls = "%s %s" | format(header_cls, "comment-deleted") %} +{% endif %} + +{% if not comment.Deleter or request.user.has_credential("CRED_COMMENT_VIEW_DELETED", approved=[comment.Deleter]) %} + + {% set commented_at = comment.CommentTS | dt | as_timezone(timezone) %} +

    + {{ + "Commented on package %s%s%s on %s%s%s" | tr + | format( + '' | format(comment.PackageBase.Name), + comment.PackageBase.Name, + "", + '' | format( + username, + comment.ID + ), + commented_at.strftime("%Y-%m-%d %H:%M"), + "" + ) | safe + }} + {% if comment.Editor %} + {% set edited_on = comment.EditedTS | dt | as_timezone(timezone) %} + + ({{ "edited on %s by %s" | tr + | format(edited_on.strftime('%Y-%m-%d %H:%M'), + '%s' | format( + comment.Editor.Username, comment.Editor.Username)) + | safe + }}) + + {% endif %} + + {% include "partials/comment_actions.html" %} +

    + + {% include "partials/comment_content.html" %} +{% endif %} From 7f4c011dc3d4db377c7676bde50b67db9b937c72 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 29 Oct 2021 20:26:57 -0700 Subject: [PATCH 497/844] fix(fastapi): sanitize PP/O parameters for package search This definitely leaked through in more areas. We'll need to reuse this new utility function in a few other routes in upcoming commits. Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 7 +++++-- aurweb/util.py | 18 ++++++++++++++++-- test/test_packages_routes.py | 10 +++------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index b0da3bf9..14b91221 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -30,8 +30,11 @@ async def packages_get(request: Request, context: Dict[str, Any], context["q"] = dict(request.query_params) # Per page and offset. - per_page = context["PP"] = int(request.query_params.get("PP", 50)) - offset = context["O"] = int(request.query_params.get("O", 0)) + offset, per_page = util.sanitize_params( + request.query_params.get("O", defaults.O), + request.query_params.get("PP", defaults.PP)) + context["O"] = offset + context["PP"] = per_page # Query search by. search_by = context["SeB"] = request.query_params.get("SeB", "nd") diff --git a/aurweb/util.py b/aurweb/util.py index dd7491d3..88142cbc 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -7,7 +7,7 @@ import secrets import string from datetime import datetime -from typing import Any, Callable, Dict, Iterable +from typing import Any, Callable, Dict, Iterable, Tuple from urllib.parse import urlencode, urlparse from zoneinfo import ZoneInfo @@ -18,7 +18,7 @@ from jinja2 import pass_context import aurweb.config -from aurweb import logging +from aurweb import defaults, logging logger = logging.get_logger(__name__) @@ -155,3 +155,17 @@ def get_ssh_fingerprints(): def apply_all(iterable: Iterable, fn: Callable): for item in iterable: fn(item) + + +def sanitize_params(offset: str, per_page: str) -> Tuple[int, int]: + try: + offset = int(offset) + except ValueError: + offset = defaults.O + + try: + per_page = int(per_page) + except ValueError: + per_page = defaults.PP + + return (offset, per_page) diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index b4a582e3..2ef3f3d8 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -486,15 +486,11 @@ def test_pkgbase(client: TestClient, package: Package): def test_packages(client: TestClient, packages: List[Package]): - """ Test the / packages route with defaults. - - Defaults: - 50 results per page - offset of 0 - """ with client as request: response = request.get("/packages", params={ - "SeB": "X" # "X" isn't valid, defaults to "nd" + "SeB": "X", # "X" isn't valid, defaults to "nd" + "PP": "1 or 1", + "O": "0 or 0" }) assert response.status_code == int(HTTPStatus.OK) From 01e27fa34719b8e68def295c575bfb1f9b0ed362 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 29 Oct 2021 20:29:56 -0700 Subject: [PATCH 498/844] fix(fastapi): sanitize /requests params Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 14b91221..27125b60 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -635,6 +635,8 @@ async def requests(request: Request, context = make_context(request, "Requests") context["q"] = dict(request.query_params) + + O, PP = util.sanitize_params(O, PP) context["O"] = O context["PP"] = PP From 9464de108f39e3fe633083ddf3c7a3526426fd98 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 29 Oct 2021 21:37:52 -0700 Subject: [PATCH 499/844] feat(fastapi): add /pkgbase/{name}/comments/{id}/edit (get) This is needed so that users can edit comments when they don't have Javascript being used in their browser. Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 15 +++++++++ templates/packages/comments/edit.html | 44 +++++++++++++++++++++++++++ test/test_packages_routes.py | 7 +++++ 3 files changed, 66 insertions(+) create mode 100644 templates/packages/comments/edit.html diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 27125b60..c574ec18 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -384,6 +384,21 @@ async def pkgbase_comment_post( status_code=HTTPStatus.SEE_OTHER) +@router.get("/pkgbase/{name}/comments/{id}/edit") +@auth_required(True, redirect="/pkgbase/{name}/comments/{id}/edit") +async def pkgbase_comment_edit(request: Request, name: str, id: int, + next: str = Form(default=None)): + pkgbase = get_pkg_or_base(name, models.PackageBase) + comment = get_pkgbase_comment(pkgbase, id) + + if not next: + next = f"/pkgbase/{name}" + + context = await make_variable_context(request, "Edit comment", next=next) + context["comment"] = comment + return render_template(request, "packages/comments/edit.html", context) + + @router.post("/pkgbase/{name}/comments/{id}/delete") @auth_required(True, redirect="/pkgbase/{name}/comments/{id}/delete") async def pkgbase_comment_delete(request: Request, name: str, id: int, diff --git a/templates/packages/comments/edit.html b/templates/packages/comments/edit.html new file mode 100644 index 00000000..f938287e --- /dev/null +++ b/templates/packages/comments/edit.html @@ -0,0 +1,44 @@ +{% extends "partials/layout.html" %} + +{% block pageContent %} +
    +

    {{ "Edit comment for: %s" | tr | format(comment.PackageBase.Name) }}

    + + +
    +
    + +
    + +

    + {{ + "Git commit identifiers referencing commits in " + "the AUR package repository and URLs are converted " + "to links automatically." | tr + }} + {{ + "%sMarkdown syntax%s is partiaully supported." + | tr | format( + '', + "" + ) | safe + }} +

    + +

    + +

    + +

    + +

    + +
    + + +
    +{% endblock %} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 2ef3f3d8..207be379 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -1084,6 +1084,13 @@ def test_pkgbase_comments(client: TestClient, maintainer: User, user: User, assert len(bodies) == 1 assert bodies[0].text.strip() == "Test comment." + comment_id = headers[0].attrib["id"].split("-")[-1] + + # Test the non-javascript version of comment editing by + # visiting the /pkgbase/{name}/comments/{id}/edit route. + with client as request: + resp = request.get(f"{endpoint}/{comment_id}/edit", cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) # Clear up the PackageNotification. This doubles as testing # that the notification was created and clears it up so we can From b3b31394e840b2372134fb23f9217e9c40ef242b Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 29 Oct 2021 22:59:40 -0700 Subject: [PATCH 500/844] fix(rpc): simplify json generation complexity This simply decouples depends and relations population into their own helper functions. Signed-off-by: Kevin Morris --- aurweb/rpc.py | 60 ++++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 84bae53c..e92f9c70 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import List +from typing import Any, Dict, List from sqlalchemy import and_ @@ -95,6 +95,36 @@ class RPC: raise RPCError( f"Request type '{self.type}' is not yet implemented.") + def _update_json_depends(self, package: models.Package, + data: Dict[str, Any]): + # Walk through all related PackageDependencies and produce + # the appropriate dict entries. + depends = package.package_dependencies + for dep in depends: + if dep.DepTypeID in DEP_TYPES: + key = DEP_TYPES.get(dep.DepTypeID) + + display = dep.DepName + if dep.DepCondition: + display += dep.DepCondition + + data[key].append(display) + + def _update_json_relations(self, package: models.Package, + data: Dict[str, Any]): + # Walk through all related PackageRelations and produce + # the appropriate dict entries. + relations = package.package_relations + for rel in relations: + if rel.RelTypeID in REL_TYPES: + key = REL_TYPES.get(rel.RelTypeID) + + display = rel.RelName + if rel.RelCondition: + display += rel.RelCondition + + data[key].append(display) + def _get_json_data(self, package: models.Package): """ Produce dictionary data of one Package that can be JSON-serialized. @@ -137,32 +167,8 @@ class RPC: # We do have a maintainer: set the Maintainer key. data["Maintainer"] = package.PackageBase.Maintainer.Username - # Walk through all related PackageDependencies and produce - # the appropriate dict entries. - if depends := package.package_dependencies: - for dep in depends: - if dep.DepTypeID in DEP_TYPES: - key = DEP_TYPES.get(dep.DepTypeID) - - display = dep.DepName - if dep.DepCondition: - display += dep.DepCondition - - data[key].append(display) - - # Walk through all related PackageRelations and produce - # the appropriate dict entries. - if relations := package.package_relations: - for rel in relations: - if rel.RelTypeID in REL_TYPES: - key = REL_TYPES.get(rel.RelTypeID) - - display = rel.RelName - if rel.RelCondition: - display += rel.RelCondition - - data[key].append(display) - + self._update_json_depends(package, data) + self._update_json_relations(package, data) return data def _handle_multiinfo_type(self, args: List[str] = []): From 0af6a2c32f04f1c0f8b98e1be9fa45983eec9bd5 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 29 Oct 2021 23:47:47 -0700 Subject: [PATCH 501/844] fix(docker): fix COMMIT_HASH variable check The previous method was super bad. Even if a variable was declared, if it was empty, we would run into a false-positive. Additionally, the previous method did not allow us to not specify the COMMIT_HASH variable; which is problematic for development environments. Signed-off-by: Kevin Morris --- docker/fastapi-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/fastapi-entrypoint.sh b/docker/fastapi-entrypoint.sh index ec9eb5c1..58fafe56 100755 --- a/docker/fastapi-entrypoint.sh +++ b/docker/fastapi-entrypoint.sh @@ -11,7 +11,7 @@ sed -ri "s;^(aur_location) = .+;\1 = ${AURWEB_FASTAPI_PREFIX};" conf/config sed -ri 's/^(cache) = .+/\1 = redis/' conf/config sed -ri 's|^(redis_address) = .+|\1 = redis://redis|' conf/config -if [ "$COMMIT_HASH" ]; then +if [ ! -z ${COMMIT_HASH+x} ]; then sed -ri "s/^;?(commit_hash) =.*$/\1 = $COMMIT_HASH/" conf/config fi From 6d376fed1576e036d8a4ceb687493e647f3d8c0d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 29 Oct 2021 23:10:20 -0700 Subject: [PATCH 502/844] feat(rpc): add ETag header with md5 hash content The ETag header can be used for client-side caching. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag Signed-off-by: Kevin Morris --- aurweb/routers/rpc.py | 25 +++++++++++++++++++++++-- aurweb/rpc.py | 6 ++---- test/test_rpc.py | 8 ++++++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/aurweb/routers/rpc.py b/aurweb/routers/rpc.py index 0616326b..0c52404c 100644 --- a/aurweb/routers/rpc.py +++ b/aurweb/routers/rpc.py @@ -1,8 +1,12 @@ +import hashlib + from http import HTTPStatus from typing import List, Optional from urllib.parse import unquote -from fastapi import APIRouter, Query, Request +import orjson + +from fastapi import APIRouter, Query, Request, Response from fastapi.responses import JSONResponse from aurweb.ratelimit import check_ratelimit @@ -74,4 +78,21 @@ async def rpc(request: Request, # Prepare list of arguments for input. If 'arg' was given, it'll # be a list with one element. arguments = parse_args(request) - return JSONResponse(rpc.handle(arguments)) + data = rpc.handle(arguments) + + # Serialize `data` into JSON in a sorted fashion. This way, our + # ETag header produced below will never end up changed. + output = orjson.dumps(data, option=orjson.OPT_SORT_KEYS) + + # Produce an md5 hash based on `output`. + md5 = hashlib.md5() + md5.update(output) + etag = md5.hexdigest() + + # Finally, return our JSONResponse with the ETag header. + # The ETag header expects quotes to surround any identifier. + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag + return Response(output.decode(), headers={ + "Content-Type": "application/json", + "ETag": f'"{etag}"' + }) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index e92f9c70..87700a2f 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -99,8 +99,7 @@ class RPC: data: Dict[str, Any]): # Walk through all related PackageDependencies and produce # the appropriate dict entries. - depends = package.package_dependencies - for dep in depends: + for dep in package.package_dependencies: if dep.DepTypeID in DEP_TYPES: key = DEP_TYPES.get(dep.DepTypeID) @@ -114,8 +113,7 @@ class RPC: data: Dict[str, Any]): # Walk through all related PackageRelations and produce # the appropriate dict entries. - relations = package.package_relations - for rel in relations: + for rel in package.package_relations: if rel.RelTypeID in REL_TYPES: key = REL_TYPES.get(rel.RelTypeID) diff --git a/test/test_rpc.py b/test/test_rpc.py index 9400ee06..00703c23 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -488,3 +488,11 @@ def test_rpc_ratelimit(getint: mock.MagicMock, pipeline: Pipeline): # The new first request should be good. response = make_request("/rpc?v=5&type=suggest-pkgbase&arg=big") assert response.status_code == int(HTTPStatus.OK) + + +def test_rpc_etag(): + response1 = make_request("/rpc?v=5&type=suggest-pkgbase&arg=big") + response2 = make_request("/rpc?v=5&type=suggest-pkgbase&arg=big") + assert response1.headers.get("ETag") is not None + assert response1.headers.get("ETag") != str() + assert response1.headers.get("ETag") == response2.headers.get("ETag") From 9d6dbaf0ecfaf576d0c966ca0c93ba68dbd07544 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 30 Oct 2021 00:36:21 -0700 Subject: [PATCH 503/844] feat(rpc): add suggest type handler Signed-off-by: Kevin Morris --- aurweb/rpc.py | 8 ++++++++ test/test_rpc.py | 24 +++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 87700a2f..5c9df1a7 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -175,6 +175,14 @@ class RPC: models.Package.Name.in_(args)) return [self._get_json_data(pkg) for pkg in packages] + def _handle_suggest_type(self, args: List[str] = []): + arg = args[0] + packages = db.query(models.Package).join(models.PackageBase).filter( + and_(models.PackageBase.PackagerUID.isnot(None), + models.Package.Name.like(f"%{arg}%")) + ).order_by(models.Package.Name.asc()).limit(20) + return [pkg.Name for pkg in packages] + def _handle_suggest_pkgbase_type(self, args: List[str] = []): records = db.query(models.PackageBase).filter( and_(models.PackageBase.PackagerUID.isnot(None), diff --git a/test/test_rpc.py b/test/test_rpc.py index 00703c23..71c7397f 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -98,6 +98,17 @@ def setup(): Maintainer=user1, Packager=user1) + pkgbase4 = create(PackageBase, Name="fugly-chungus", + Maintainer=user1, + Packager=user1) + + desc = "A Package belonging to a PackageBase with another name." + create(Package, + PackageBase=pkgbase4, + Name="other-pkg", + Description=desc, + URL="https://example.com") + create(Package, PackageBase=pkgbase3, Name=pkgbase3.Name, @@ -451,8 +462,19 @@ def test_rpc_suggest_pkgbase(): assert data == ["chungy-chungus"] +def test_rpc_suggest(): + response = make_request("/rpc?v=5&type=suggest&arg=other") + data = response.json() + assert data == ["other-pkg"] + + # Test non-existent Package. + response = make_request("/rpc?v=5&type=suggest&arg=nonexistent") + data = response.json() + assert data == [] + + def test_rpc_unimplemented_types(): - unimplemented = ["search", "msearch", "suggest"] + unimplemented = ["search", "msearch"] for type in unimplemented: response = make_request(f"/rpc?v=5&type={type}&arg=big") data = response.json() From c28f1695edb6d94c038363648c99768e23d7fcf5 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 30 Oct 2021 16:22:54 -0700 Subject: [PATCH 504/844] fix(fastapi): support `by` maintainer search with no keywords In this case, package search should return orphaned packages. Signed-off-by: Kevin Morris --- aurweb/packages/search.py | 10 +++++++--- test/test_packages_routes.py | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/aurweb/packages/search.py b/aurweb/packages/search.py index e4729d89..0319a2ba 100644 --- a/aurweb/packages/search.py +++ b/aurweb/packages/search.py @@ -90,9 +90,13 @@ class PackageSearch: return self def _search_by_maintainer(self, keywords: str) -> orm.Query: - self.query = self.query.join( - models.User, models.User.ID == models.PackageBase.MaintainerUID - ).filter(models.User.Username == keywords) + if keywords: + self.query = self.query.join( + models.User, models.User.ID == models.PackageBase.MaintainerUID + ).filter(models.User.Username == keywords) + else: + self.query = self.query.filter( + models.PackageBase.MaintainerUID.is_(None)) return self def _search_by_comaintainer(self, keywords: str) -> orm.Query: diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 207be379..c4d9ab1c 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -623,13 +623,36 @@ def test_packages_search_by_keywords(client: TestClient, def test_packages_search_by_maintainer(client: TestClient, maintainer: User, package: Package): + # We should expect that searching by `package`'s maintainer + # returns `package` in the results. with client as request: response = request.get("/packages", params={ "SeB": "m", "K": maintainer.Username }) assert response.status_code == int(HTTPStatus.OK) + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 + # Search again by maintainer with no keywords given. + # This kind of search returns all orphans instead. + # In this first case, there are no orphan packages; assert that. + with client as request: + response = request.get("/packages", params={"SeB": "m"}) + assert response.status_code == int(HTTPStatus.OK) + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 0 + + # Orphan `package`. + with db.begin(): + package.PackageBase.Maintainer = None + + # This time, we should get `package` returned, since it's now an orphan. + with client as request: + response = request.get("/packages", params={"SeB": "m"}) + assert response.status_code == int(HTTPStatus.OK) root = parse_root(response.text) rows = root.xpath('//table[@class="results"]/tbody/tr') assert len(rows) == 1 From af2f3694e7fa59f06ebe1af22ac6592b513ef42f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 30 Oct 2021 16:39:20 -0700 Subject: [PATCH 505/844] feat(rpc): add search type handler This commit introduces a PackageSearch-derivative class: `RPCSearch`. This derivative modifies callback behavior of PackageSearch to suit RPC searches, including [make|check|opt]depends `by` types. Signed-off-by: Kevin Morris --- aurweb/defaults.py | 3 + aurweb/packages/search.py | 118 ++++++++++++++++++++++++++++++++------ aurweb/routers/rpc.py | 12 ++-- aurweb/rpc.py | 75 +++++++++++++++++++----- test/test_rpc.py | 74 +++++++++++++++++++++++- 5 files changed, 245 insertions(+), 37 deletions(-) diff --git a/aurweb/defaults.py b/aurweb/defaults.py index c2568d05..51072e8f 100644 --- a/aurweb/defaults.py +++ b/aurweb/defaults.py @@ -9,6 +9,9 @@ PP = 50 # A whitelist of valid PP values PP_WHITELIST = {50, 100, 250} +# Default `by` parameter for RPC search. +RPC_SEARCH_BY = "name-desc" + def fallback_pp(per_page: int) -> int: """ If `per_page` is a valid value in PP_WHITELIST, return it. diff --git a/aurweb/packages/search.py b/aurweb/packages/search.py index 0319a2ba..a14fe19b 100644 --- a/aurweb/packages/search.py +++ b/aurweb/packages/search.py @@ -1,6 +1,7 @@ from sqlalchemy import and_, case, or_, orm -from aurweb import config, db, models +from aurweb import config, db, models, util +from aurweb.models.dependency_type import CHECKDEPENDS_ID, DEPENDS_ID, MAKEDEPENDS_ID, OPTDEPENDS_ID DEFAULT_MAX_RESULTS = 2500 @@ -11,24 +12,25 @@ class PackageSearch: # A constant mapping of short to full name sort orderings. FULL_SORT_ORDER = {"d": "desc", "a": "asc"} - def __init__(self, user: models.User): - """ Construct an instance of PackageSearch. - - This constructors performs several steps during initialization: - 1. Setup self.query: an ORM query of Package joined by PackageBase. - """ + def __init__(self, user: models.User = None): self.user = user - self.query = db.query(models.Package).join(models.PackageBase).join( - models.PackageVote, - and_(models.PackageVote.PackageBaseID == models.PackageBase.ID, - models.PackageVote.UsersID == self.user.ID), - isouter=True - ).join( - models.PackageNotification, - and_(models.PackageNotification.PackageBaseID == models.PackageBase.ID, - models.PackageNotification.UserID == self.user.ID), - isouter=True - ) + self.query = db.query(models.Package).join(models.PackageBase) + + if self.user: + PackageVote = models.PackageVote + join_vote_on = and_( + PackageVote.PackageBaseID == models.PackageBase.ID, + PackageVote.UsersID == self.user.ID) + + PackageNotification = models.PackageNotification + join_notif_on = and_( + PackageNotification.PackageBaseID == models.PackageBase.ID, + PackageNotification.UserID == self.user.ID) + + self.query = self.query.join( + models.PackageVote, join_vote_on, isouter=True + ).join(models.PackageNotification, join_notif_on, isouter=True) + self.ordering = "d" # Setup SeB (Search By) callbacks. @@ -198,3 +200,83 @@ class PackageSearch: # Return the query to the user. return self.query + + +class RPCSearch(PackageSearch): + """ A PackageSearch-derived RPC package search query builder. + + With RPC search, we need a subset of PackageSearch's handlers, + with a few additional handlers added. So, within the RPCSearch + constructor, we pop unneeded keys out of inherited self.search_by_cb + and add a few more keys to it, namely: depends, makedepends, + optdepends and checkdepends. + + Additionally, some logic within the inherited PackageSearch.search_by + method is not needed, so it is overridden in this class without + sanitization done for the PackageSearch `by` argument. + """ + + keys_removed = ("b", "N", "B", "k", "c", "M", "s") + + def __init__(self) -> "RPCSearch": + super().__init__() + + # Fix-up inherited search_by_cb to reflect RPC-specific by params. + # We keep: "nd", "n" and "m". We also overlay four new by params + # on top: "depends", "makedepends", "optdepends" and "checkdepends". + util.apply_all(RPCSearch.keys_removed, + lambda k: self.search_by_cb.pop(k)) + self.search_by_cb.update({ + "depends": self._search_by_depends, + "makedepends": self._search_by_makedepends, + "optdepends": self._search_by_optdepends, + "checkdepends": self._search_by_checkdepends + }) + + def _join_depends(self, dep_type_id: int) -> orm.Query: + """ Join Package with PackageDependency and filter results + based on `dep_type_id`. + + :param dep_type_id: DependencyType ID + :returns: PackageDependency-joined orm.Query + """ + self.query = self.query.join(models.PackageDependency).filter( + models.PackageDependency.DepTypeID == dep_type_id) + return self.query + + def _search_by_depends(self, keywords: str) -> "RPCSearch": + self.query = self._join_depends(DEPENDS_ID).filter( + models.PackageDependency.DepName == keywords) + return self + + def _search_by_makedepends(self, keywords: str) -> "RPCSearch": + self.query = self._join_depends(MAKEDEPENDS_ID).filter( + models.PackageDependency.DepName == keywords) + return self + + def _search_by_optdepends(self, keywords: str) -> "RPCSearch": + self.query = self._join_depends(OPTDEPENDS_ID).filter( + models.PackageDependency.DepName == keywords) + return self + + def _search_by_checkdepends(self, keywords: str) -> "RPCSearch": + self.query = self._join_depends(CHECKDEPENDS_ID).filter( + models.PackageDependency.DepName == keywords) + return self + + def search_by(self, by: str, keywords: str) -> "RPCSearch": + """ Override inherited search_by. In this override, we reduce the + scope of what we handle within this function. We do not set `by` + to a default of "nd" in the RPC, as the RPC returns an error when + incorrect `by` fields are specified. + + :param by: RPC `by` argument + :param keywords: RPC `arg` argument + :returns: self + """ + callback = self.search_by_cb.get(by) + result = callback(keywords) + return result + + def results(self) -> orm.Query: + return self.query diff --git a/aurweb/routers/rpc.py b/aurweb/routers/rpc.py index 0c52404c..6d3dce54 100644 --- a/aurweb/routers/rpc.py +++ b/aurweb/routers/rpc.py @@ -9,6 +9,7 @@ import orjson from fastapi import APIRouter, Query, Request, Response from fastapi.responses import JSONResponse +from aurweb import defaults from aurweb.ratelimit import check_ratelimit from aurweb.rpc import RPC @@ -62,10 +63,11 @@ def parse_args(request: Request): @router.get("/rpc") async def rpc(request: Request, - v: Optional[int] = Query(None), - type: Optional[str] = Query(None), - arg: Optional[str] = Query(None), - args: Optional[List[str]] = Query(None, alias="arg[]")): + v: Optional[int] = Query(default=None), + type: Optional[str] = Query(default=None), + by: Optional[str] = Query(default=defaults.RPC_SEARCH_BY), + arg: Optional[str] = Query(default=None), + args: Optional[List[str]] = Query(default=[], alias="arg[]")): # Create a handle to our RPC class. rpc = RPC(version=v, type=type) @@ -78,7 +80,7 @@ async def rpc(request: Request, # Prepare list of arguments for input. If 'arg' was given, it'll # be a list with one element. arguments = parse_args(request) - data = rpc.handle(arguments) + data = rpc.handle(by=by, args=arguments) # Serialize `data` into JSON in a sorted fashion. This way, our # ETag header produced below will never end up changed. diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 5c9df1a7..009b1440 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -5,8 +5,9 @@ from sqlalchemy import and_ import aurweb.config as config -from aurweb import db, models, util +from aurweb import db, defaults, models, util from aurweb.models import dependency_type, relation_type +from aurweb.packages.search import RPCSearch # Define dependency type mappings from ID to RPC-compatible keys. DEP_TYPES = { @@ -60,8 +61,16 @@ class RPC: "suggest", "suggest-pkgbase" } - # A mapping of aliases. - ALIASES = {"info": "multiinfo"} + # A mapping of type aliases. + TYPE_ALIASES = {"info": "multiinfo"} + + EXPOSED_BYS = { + "name-desc", "name", "maintainer", + "depends", "makedepends", "optdepends", "checkdepends" + } + + # A mapping of by aliases. + BY_ALIASES = {"name-desc": "nd", "name": "n", "maintainer": "m"} def __init__(self, version: int = 0, type: str = None): self.version = version @@ -76,14 +85,17 @@ class RPC: "error": message } - def _verify_inputs(self, args: List[str] = []): + def _verify_inputs(self, by: str = [], args: List[str] = []): if self.version is None: raise RPCError("Please specify an API version.") if self.version not in RPC.EXPOSED_VERSIONS: raise RPCError("Invalid version specified.") - if self.type is None or not len(args): + if by not in RPC.EXPOSED_BYS: + raise RPCError("Incorrect by field specified.") + + if self.type is None: raise RPCError("No request type/data specified.") if self.type not in RPC.EXPOSED_TYPES: @@ -95,6 +107,10 @@ class RPC: raise RPCError( f"Request type '{self.type}' is not yet implemented.") + def _enforce_args(self, args: List[str]): + if not args: + raise RPCError("No request type/data specified.") + def _update_json_depends(self, package: models.Package, data: Dict[str, Any]): # Walk through all related PackageDependencies and produce @@ -169,13 +185,36 @@ class RPC: self._update_json_relations(package, data) return data - def _handle_multiinfo_type(self, args: List[str] = []): + def _handle_multiinfo_type(self, args: List[str] = [], **kwargs): + self._enforce_args(args) args = set(args) packages = db.query(models.Package).filter( models.Package.Name.in_(args)) return [self._get_json_data(pkg) for pkg in packages] - def _handle_suggest_type(self, args: List[str] = []): + def _handle_search_type(self, by: str = defaults.RPC_SEARCH_BY, + args: List[str] = []): + # If `by` isn't maintainer and we don't have any args, raise an error. + # In maintainer's case, return all orphans if there are no args, + # so we need args to pass through to the handler without errors. + if by != "m" and not len(args): + raise RPCError("No request type/data specified.") + + arg = args[0] + if len(arg) < 2: + raise RPCError("Query arg too small.") + + search = RPCSearch() + search.search_by(by, arg) + + max_results = config.getint("options", "max_rpc_results") + results = search.results().limit(max_results) + return [self._get_json_data(pkg) for pkg in results] + + def _handle_suggest_type(self, args: List[str] = [], **kwargs): + if not args: + return [] + arg = args[0] packages = db.query(models.Package).join(models.PackageBase).filter( and_(models.PackageBase.PackagerUID.isnot(None), @@ -183,14 +222,17 @@ class RPC: ).order_by(models.Package.Name.asc()).limit(20) return [pkg.Name for pkg in packages] - def _handle_suggest_pkgbase_type(self, args: List[str] = []): + def _handle_suggest_pkgbase_type(self, args: List[str] = [], **kwargs): + if not args: + return [] + records = db.query(models.PackageBase).filter( and_(models.PackageBase.PackagerUID.isnot(None), models.PackageBase.Name.like(f"%{args[0]}%")) ).order_by(models.PackageBase.Name.asc()).limit(20) return [record.Name for record in records] - def handle(self, args: List[str] = []): + def handle(self, by: str = defaults.RPC_SEARCH_BY, args: List[str] = []): """ Request entrypoint. A router should pass v, type and args to this function and expect an output dictionary to be returned. @@ -199,22 +241,29 @@ class RPC: :param args: Deciphered list of arguments based on arg/arg[] inputs """ # Convert type aliased types. - if self.type in RPC.ALIASES: - self.type = RPC.ALIASES.get(self.type) + if self.type in RPC.TYPE_ALIASES: + self.type = RPC.TYPE_ALIASES.get(self.type) # Prepare our output data dictionary with some basic keys. data = {"version": self.version, "type": self.type} # Run some verification on our given arguments. try: - self._verify_inputs(args) + self._verify_inputs(by=by, args=args) except RPCError as exc: return self.error(str(exc)) + # Convert by to its aliased value if it has one. + if by in RPC.BY_ALIASES: + by = RPC.BY_ALIASES.get(by) + # Get a handle to our callback and trap an RPCError with # an empty list of results based on callback's execution. callback = getattr(self, f"_handle_{self.type.replace('-', '_')}_type") - results = callback(args) + try: + results = callback(by=by, args=args) + except RPCError as exc: + return self.error(str(exc)) # These types are special: we produce a different kind of # successful JSON output: a list of results. diff --git a/test/test_rpc.py b/test/test_rpc.py index 71c7397f..38b81226 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -461,6 +461,11 @@ def test_rpc_suggest_pkgbase(): data = response.json() assert data == ["chungy-chungus"] + # Test no arg supplied. + response = make_request("/rpc?v=5&type=suggest-pkgbase") + data = response.json() + assert data == [] + def test_rpc_suggest(): response = make_request("/rpc?v=5&type=suggest&arg=other") @@ -472,9 +477,14 @@ def test_rpc_suggest(): data = response.json() assert data == [] + # Test no arg supplied. + response = make_request("/rpc?v=5&type=suggest") + data = response.json() + assert data == [] + def test_rpc_unimplemented_types(): - unimplemented = ["search", "msearch"] + unimplemented = ["msearch"] for type in unimplemented: response = make_request(f"/rpc?v=5&type={type}&arg=big") data = response.json() @@ -518,3 +528,65 @@ def test_rpc_etag(): assert response1.headers.get("ETag") is not None assert response1.headers.get("ETag") != str() assert response1.headers.get("ETag") == response2.headers.get("ETag") + + +def test_rpc_search_arg_too_small(): + response = make_request("/rpc?v=5&type=search&arg=b") + assert response.status_code == int(HTTPStatus.OK) + assert response.json().get("error") == "Query arg too small." + + +def test_rpc_search(): + response = make_request("/rpc?v=5&type=search&arg=big") + assert response.status_code == int(HTTPStatus.OK) + + data = response.json() + assert data.get("resultcount") == 1 + + result = data.get("results")[0] + assert result.get("Name") == "big-chungus" + + # No args on non-m by types return an error. + response = make_request("/rpc?v=5&type=search") + assert response.json().get("error") == "No request type/data specified." + + +def test_rpc_search_depends(): + response = make_request( + "/rpc?v=5&type=search&by=depends&arg=chungus-depends") + data = response.json() + assert data.get("resultcount") == 1 + result = data.get("results")[0] + assert result.get("Name") == "big-chungus" + + +def test_rpc_search_makedepends(): + response = make_request( + "/rpc?v=5&type=search&by=makedepends&arg=chungus-makedepends") + data = response.json() + assert data.get("resultcount") == 1 + result = data.get("results")[0] + assert result.get("Name") == "big-chungus" + + +def test_rpc_search_optdepends(): + response = make_request( + "/rpc?v=5&type=search&by=optdepends&arg=chungus-optdepends") + data = response.json() + assert data.get("resultcount") == 1 + result = data.get("results")[0] + assert result.get("Name") == "big-chungus" + + +def test_rpc_search_checkdepends(): + response = make_request( + "/rpc?v=5&type=search&by=checkdepends&arg=chungus-checkdepends") + data = response.json() + assert data.get("resultcount") == 1 + result = data.get("results")[0] + assert result.get("Name") == "big-chungus" + + +def test_rpc_incorrect_by(): + response = make_request("/rpc?v=5&type=search&by=fake&arg=big") + assert response.json().get("error") == "Incorrect by field specified." From 9fef8b06114e011cedab7c25fc6d44f9de99b2ab Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 30 Oct 2021 22:53:30 -0700 Subject: [PATCH 506/844] fix(rpc): fix search arg check When by == 'maintainer', we allow an unspecified keyword, resulting in a search of orphan packages. Fix our search check so that when no arg is given, it is set to an empty str(). We already check for valid args when type is not maintainer, so there's no need to worry about other args falling through. Signed-off-by: Kevin Morris --- aurweb/rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 009b1440..16985f37 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -200,8 +200,8 @@ class RPC: if by != "m" and not len(args): raise RPCError("No request type/data specified.") - arg = args[0] - if len(arg) < 2: + arg = args[0] if args else str() + if by != "m" and len(arg) < 2: raise RPCError("Query arg too small.") search = RPCSearch() From 05e6cfca62162ffa5ca7e524f08810bc4d0df42a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 30 Oct 2021 22:56:18 -0700 Subject: [PATCH 507/844] feat(rpc): add msearch type handler Signed-off-by: Kevin Morris --- aurweb/rpc.py | 9 +++------ test/test_rpc.py | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 16985f37..56f75391 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -101,12 +101,6 @@ class RPC: if self.type not in RPC.EXPOSED_TYPES: raise RPCError("Incorrect request type specified.") - try: - getattr(self, f"_handle_{self.type.replace('-', '_')}_type") - except AttributeError: - raise RPCError( - f"Request type '{self.type}' is not yet implemented.") - def _enforce_args(self, args: List[str]): if not args: raise RPCError("No request type/data specified.") @@ -211,6 +205,9 @@ class RPC: results = search.results().limit(max_results) return [self._get_json_data(pkg) for pkg in results] + def _handle_msearch_type(self, args: List[str] = [], **kwargs): + return self._handle_search_type(by="m", args=args) + def _handle_suggest_type(self, args: List[str] = [], **kwargs): if not args: return [] diff --git a/test/test_rpc.py b/test/test_rpc.py index 38b81226..0636c792 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -483,15 +483,6 @@ def test_rpc_suggest(): assert data == [] -def test_rpc_unimplemented_types(): - unimplemented = ["msearch"] - for type in unimplemented: - response = make_request(f"/rpc?v=5&type={type}&arg=big") - data = response.json() - expected = f"Request type '{type}' is not yet implemented." - assert data.get("error") == expected - - def mock_config_getint(section: str, key: str): if key == "request_limit": return 4 @@ -551,6 +542,35 @@ def test_rpc_search(): assert response.json().get("error") == "No request type/data specified." +def test_rpc_msearch(): + response = make_request("/rpc?v=5&type=msearch&arg=user1") + data = response.json() + + # user1 maintains 4 packages; assert that we got them all. + assert data.get("resultcount") == 4 + names = list(sorted(r.get("Name") for r in data.get("results"))) + expected_results = list(sorted([ + "big-chungus", + "chungy-chungus", + "gluggly-chungus", + "other-pkg" + ])) + assert names == expected_results + + # Search for a non-existent maintainer, giving us zero packages. + response = make_request("/rpc?v=5&type=msearch&arg=blah-blah") + data = response.json() + assert data.get("resultcount") == 0 + + # A missing arg still succeeds, but it returns all orphans. + # Just verify that we receive no error and the orphaned result. + response = make_request("/rpc?v=5&type=msearch") + data = response.json() + assert data.get("resultcount") == 1 + result = data.get("results")[0] + assert result.get("Name") == "woogly-chungus" + + def test_rpc_search_depends(): response = make_request( "/rpc?v=5&type=search&by=depends&arg=chungus-depends") From 12b4269ba8c1b4dbe9aa55b9e0541db4f0cdac77 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 31 Oct 2021 00:28:55 -0700 Subject: [PATCH 508/844] feat(rpc): support jsonp callbacks This change introduces alternate rendering of text/javascript JSONP-compatible callback content. The `examples/jsonp.html` HTML document can be used to test this functionality against a running aurweb server. Signed-off-by: Kevin Morris --- aurweb/routers/rpc.py | 26 +++++++++++---- examples/jsonp.html | 74 +++++++++++++++++++++++++++++++++++++++++++ test/test_rpc.py | 14 ++++++++ 3 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 examples/jsonp.html diff --git a/aurweb/routers/rpc.py b/aurweb/routers/rpc.py index 6d3dce54..175e5f0f 100644 --- a/aurweb/routers/rpc.py +++ b/aurweb/routers/rpc.py @@ -67,7 +67,8 @@ async def rpc(request: Request, type: Optional[str] = Query(default=None), by: Optional[str] = Query(default=defaults.RPC_SEARCH_BY), arg: Optional[str] = Query(default=None), - args: Optional[List[str]] = Query(default=[], alias="arg[]")): + args: Optional[List[str]] = Query(default=[], alias="arg[]"), + callback: Optional[str] = Query(default=None)): # Create a handle to our RPC class. rpc = RPC(version=v, type=type) @@ -84,17 +85,28 @@ async def rpc(request: Request, # Serialize `data` into JSON in a sorted fashion. This way, our # ETag header produced below will never end up changed. - output = orjson.dumps(data, option=orjson.OPT_SORT_KEYS) + content = orjson.dumps(data, option=orjson.OPT_SORT_KEYS) # Produce an md5 hash based on `output`. md5 = hashlib.md5() - md5.update(output) + md5.update(content) etag = md5.hexdigest() - # Finally, return our JSONResponse with the ETag header. + # If `callback` was provided, produce a text/javascript response + # valid for the jsonp callback. Otherwise, by default, return + # application/json containing `output`. + # Note: Being the API hot path, `content` is not defaulted to + # avoid copying the JSON string in the case callback is provided. + content_type = "application/json" + if callback: + print("callback called") + content_type = "text/javascript" + content = f"/**/{callback}({content.decode()})" + # The ETag header expects quotes to surround any identifier. # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag - return Response(output.decode(), headers={ - "Content-Type": "application/json", + headers = { + "Content-Type": content_type, "ETag": f'"{etag}"' - }) + } + return Response(content, headers=headers) diff --git a/examples/jsonp.html b/examples/jsonp.html new file mode 100644 index 00000000..d73ec91e --- /dev/null +++ b/examples/jsonp.html @@ -0,0 +1,74 @@ + + + + + + + + JSONP Callback Test + + + + + +
    +
    +

    + Searching with the following form uses a JSONP callback + to log data out to the javascript console. +

    + +
    + + +
    + +
    + + +
    + +
    + +
    +
    +
    + + diff --git a/test/test_rpc.py b/test/test_rpc.py index 0636c792..acf1ae26 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -1,3 +1,5 @@ +import re + from http import HTTPStatus from unittest import mock @@ -610,3 +612,15 @@ def test_rpc_search_checkdepends(): def test_rpc_incorrect_by(): response = make_request("/rpc?v=5&type=search&by=fake&arg=big") assert response.json().get("error") == "Incorrect by field specified." + + +def test_rpc_jsonp_callback(): + """ Test the callback parameter. + + For end-to-end verification, the `examples/jsonp.html` file can be + used to submit jsonp callback requests to the RPC. + """ + response = make_request( + "/rpc?v=5&type=search&arg=big&callback=jsonCallback") + assert response.headers.get("content-type") == "text/javascript" + assert re.search(r'^/\*\*/jsonCallback\(.*\)$', response.text) is not None From 2cc44e8f28275c5ccdefc65e6075bd0bec245026 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 31 Oct 2021 01:17:16 -0700 Subject: [PATCH 509/844] fix(rpc): perform regex match against callback name Since we're in the hot path, a constant re.compiled JSONP_EXPR is defined for checks against the callback. Additionally, reorganized `content_type` and `content` to avoid performing a DB query when we encounter a regex mismatch. Signed-off-by: Kevin Morris --- aurweb/routers/rpc.py | 29 ++++++++++++++++++----------- test/test_rpc.py | 6 ++++++ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/aurweb/routers/rpc.py b/aurweb/routers/rpc.py index 175e5f0f..6abd73d9 100644 --- a/aurweb/routers/rpc.py +++ b/aurweb/routers/rpc.py @@ -1,4 +1,5 @@ import hashlib +import re from http import HTTPStatus from typing import List, Optional @@ -61,6 +62,9 @@ def parse_args(request: Request): return args +JSONP_EXPR = re.compile(r'^[a-zA-Z0-9()_.]{1,128}$') + + @router.get("/rpc") async def rpc(request: Request, v: Optional[int] = Query(default=None), @@ -78,6 +82,16 @@ async def rpc(request: Request, return JSONResponse(rpc.error("Rate limit reached"), status_code=int(HTTPStatus.TOO_MANY_REQUESTS)) + # If `callback` was provided, produce a text/javascript response + # valid for the jsonp callback. Otherwise, by default, return + # application/json containing `output`. + content_type = "application/json" + if callback: + if not re.match(JSONP_EXPR, callback): + return rpc.error("Invalid callback name.") + + content_type = "text/javascript" + # Prepare list of arguments for input. If 'arg' was given, it'll # be a list with one element. arguments = parse_args(request) @@ -92,21 +106,14 @@ async def rpc(request: Request, md5.update(content) etag = md5.hexdigest() - # If `callback` was provided, produce a text/javascript response - # valid for the jsonp callback. Otherwise, by default, return - # application/json containing `output`. - # Note: Being the API hot path, `content` is not defaulted to - # avoid copying the JSON string in the case callback is provided. - content_type = "application/json" - if callback: - print("callback called") - content_type = "text/javascript" - content = f"/**/{callback}({content.decode()})" - # The ETag header expects quotes to surround any identifier. # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag headers = { "Content-Type": content_type, "ETag": f'"{etag}"' } + + if callback: + content = f"/**/{callback}({content.decode()})" + return Response(content, headers=headers) diff --git a/test/test_rpc.py b/test/test_rpc.py index acf1ae26..f4ce6de8 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -624,3 +624,9 @@ def test_rpc_jsonp_callback(): "/rpc?v=5&type=search&arg=big&callback=jsonCallback") assert response.headers.get("content-type") == "text/javascript" assert re.search(r'^/\*\*/jsonCallback\(.*\)$', response.text) is not None + + # Test an invalid callback name; we get an application/json error. + response = make_request( + "/rpc?v=5&type=search&arg=big&callback=jsonCallback!") + assert response.headers.get("content-type") == "application/json" + assert response.json().get("error") == "Invalid callback name." From 61f3cb938ce601900aba70a1e73bf454689fa156 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 31 Oct 2021 01:22:54 -0700 Subject: [PATCH 510/844] feat(rpc): support the If-None-Match request header If the If-None-Match header is supplied with a previously obtained ETag from the same query, a 304 Not Modified is returned with no content. This allows clients to completely leverage the ETag header. Signed-off-by: Kevin Morris --- aurweb/routers/rpc.py | 5 +++++ test/test_rpc.py | 12 ++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/aurweb/routers/rpc.py b/aurweb/routers/rpc.py index 6abd73d9..66376067 100644 --- a/aurweb/routers/rpc.py +++ b/aurweb/routers/rpc.py @@ -113,6 +113,11 @@ async def rpc(request: Request, "ETag": f'"{etag}"' } + if_none_match = request.headers.get("If-None-Match", str()) + if if_none_match and if_none_match.strip("\t\n\r\" ") == etag: + return Response(headers=headers, + status_code=int(HTTPStatus.NOT_MODIFIED)) + if callback: content = f"/**/{callback}({content.decode()})" diff --git a/test/test_rpc.py b/test/test_rpc.py index f4ce6de8..055baa33 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -1,6 +1,7 @@ import re from http import HTTPStatus +from typing import Dict from unittest import mock import orjson @@ -28,9 +29,9 @@ from aurweb.redis import redis_connection from aurweb.testing import setup_test_db -def make_request(path): +def make_request(path, headers: Dict[str, str] = {}): with TestClient(app) as request: - return request.get(path) + return request.get(path, headers=headers) @pytest.fixture(autouse=True) @@ -539,6 +540,13 @@ def test_rpc_search(): result = data.get("results")[0] assert result.get("Name") == "big-chungus" + # Test the If-None-Match headers. + etag = response.headers.get("ETag").strip('"') + headers = {"If-None-Match": etag} + response = make_request("/rpc?v=5&type=search&arg=big", headers=headers) + assert response.status_code == int(HTTPStatus.NOT_MODIFIED) + assert response.content == b'' + # No args on non-m by types return an error. response = make_request("/rpc?v=5&type=search") assert response.json().get("error") == "No request type/data specified." From b7475a5bd0607200afae508729c53b95c2b40c6c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 31 Oct 2021 04:11:42 -0700 Subject: [PATCH 511/844] fix(rpc): fix performance of suggest[-pkgbase] We were selecting the entire record; we should just select the Name column as done in this commit. Signed-off-by: Kevin Morris --- aurweb/rpc.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 56f75391..ca838050 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -213,7 +213,9 @@ class RPC: return [] arg = args[0] - packages = db.query(models.Package).join(models.PackageBase).filter( + packages = db.query(models.Package.Name).join( + models.PackageBase + ).filter( and_(models.PackageBase.PackagerUID.isnot(None), models.Package.Name.like(f"%{arg}%")) ).order_by(models.Package.Name.asc()).limit(20) @@ -223,11 +225,11 @@ class RPC: if not args: return [] - records = db.query(models.PackageBase).filter( + packages = db.query(models.PackageBase.Name).filter( and_(models.PackageBase.PackagerUID.isnot(None), models.PackageBase.Name.like(f"%{args[0]}%")) ).order_by(models.PackageBase.Name.asc()).limit(20) - return [record.Name for record in records] + return [pkg.Name for pkg in packages] def handle(self, by: str = defaults.RPC_SEARCH_BY, args: List[str] = []): """ Request entrypoint. A router should pass v, type and args From cef69b634233b54642c54350d89b6831605f2e81 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 31 Oct 2021 15:47:39 -0700 Subject: [PATCH 512/844] fix(gitlab-ci): prune dangling images and build cache Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 156f0abf..e18df8ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,6 +65,7 @@ deploy: # Set secure login config for aurweb. - sed -ri "s/^(disable_http_login).*$/\1 = 1/" conf/config.dev - docker-compose build + - docker system prune -f - docker-compose -f docker-compose.yml -f docker-compose.aur-dev.yml up -d environment: name: development From f26cd1e9941d2ac3ffbd852c504251ce2f0d3c40 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 31 Oct 2021 16:13:01 -0700 Subject: [PATCH 513/844] fix(gitlab-ci): add `docker` dep to deploy target Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e18df8ee..1590bf34 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ deploy: COMMIT_HASH: $CI_COMMIT_SHA GIT_DATA_DIR: git_data script: - - pacman -Syu --noconfirm docker-compose socat openssh + - pacman -Syu --noconfirm docker docker-compose socat openssh - chmod 600 ${SSH_KEY} - socat "UNIX-LISTEN:/tmp/docker.sock,reuseaddr,fork" EXEC:"ssh -o UserKnownHostsFile=${SSH_KNOWN_HOSTS} -Ti ${SSH_KEY} ${SSH_USER}@${SSH_HOST}" & - export DOCKER_HOST="unix:///tmp/docker.sock" From 451eec0c28113425c951861f70ba1fd55ecabf87 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 31 Oct 2021 15:45:41 -0700 Subject: [PATCH 514/844] fix(fastapi): remove info-specific fields from search results Signed-off-by: Kevin Morris --- aurweb/rpc.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index ca838050..4ab005af 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -162,7 +162,20 @@ class RPC: "Popularity": pop, "OutOfDate": package.PackageBase.OutOfDateTS, "FirstSubmitted": package.PackageBase.SubmittedTS, - "LastModified": package.PackageBase.ModifiedTS, + "LastModified": package.PackageBase.ModifiedTS + }) + + if package.PackageBase.Maintainer is not None: + # We do have a maintainer: set the Maintainer key. + data["Maintainer"] = package.PackageBase.Maintainer.Username + + return data + + def _get_info_json_data(self, package: models.Package): + data = self._get_json_data(package) + + # Add licenses and keywords to info output. + data.update({ "License": [ lic.License.Name for lic in package.package_licenses ], @@ -171,10 +184,6 @@ class RPC: ] }) - if package.PackageBase.Maintainer is not None: - # We do have a maintainer: set the Maintainer key. - data["Maintainer"] = package.PackageBase.Maintainer.Username - self._update_json_depends(package, data) self._update_json_relations(package, data) return data @@ -184,7 +193,7 @@ class RPC: args = set(args) packages = db.query(models.Package).filter( models.Package.Name.in_(args)) - return [self._get_json_data(pkg) for pkg in packages] + return [self._get_info_json_data(pkg) for pkg in packages] def _handle_search_type(self, by: str = defaults.RPC_SEARCH_BY, args: List[str] = []): From a82879210c9dd04c511a5521d0cd163971db9838 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 31 Oct 2021 19:56:56 -0700 Subject: [PATCH 515/844] fix(poetry): add mysql-connector dep This is not used anymore in our FastAPI code, however, for back-compatibility with pre-FastAPI scripts, we need it. Signed-off-by: Kevin Morris --- poetry.lock | 13 ++++++++++++- pyproject.toml | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index eefec89b..4f41307d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -518,6 +518,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "mysql-connector" +version = "2.2.9" +description = "MySQL driver written in Python" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "mysqlclient" version = "2.0.3" @@ -962,7 +970,7 @@ h11 = ">=0.9.0,<1" [metadata] lock-version = "1.1" python-versions = ">=3.9,<3.10" -content-hash = "eb5ec82957f9fb964ca6f3852e353da51542982923dc6169658bd4bccfa78513" +content-hash = "6a45364297f5a6e88ee62240bb2eb1eaf3b41283b6d8f040ee67db02601f18e7" [metadata.files] aiofiles = [ @@ -1380,6 +1388,9 @@ mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] +mysql-connector = [ + {file = "mysql-connector-2.2.9.tar.gz", hash = "sha256:1733e6ce52a049243de3264f1fbc22a852cb35458c4ad739ba88189285efdf32"}, +] mysqlclient = [ {file = "mysqlclient-2.0.3-cp36-cp36m-win_amd64.whl", hash = "sha256:3381ca1a4f37ff1155fcfde20836b46416d66531add8843f6aa6d968982731c3"}, {file = "mysqlclient-2.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0ac0dd759c4ca02c35a9fedc24bc982cf75171651e8187c2495ec957a87dfff7"}, diff --git a/pyproject.toml b/pyproject.toml index 12812bc8..2f327318 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ SQLAlchemy = "^1.4.26" uvicorn = "^0.15.0" gunicorn = "^20.1.0" Hypercorn = "^0.11.2" +mysql-connector = "^2.2.9" [tool.poetry.dev-dependencies] flake8 = "^4.0.1" From cc45290ec274530d66d77bad5692576876b35446 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 1 Nov 2021 11:41:20 -0700 Subject: [PATCH 516/844] feat(poetry): add prometheus-fastapi-instrumentator Signed-off-by: Kevin Morris --- poetry.lock | 33 ++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 4f41307d..9f528d12 100644 --- a/poetry.lock +++ b/poetry.lock @@ -594,6 +594,29 @@ category = "main" optional = false python-versions = ">=3.6.1" +[[package]] +name = "prometheus-client" +version = "0.12.0" +description = "Python client for the Prometheus monitoring system." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +twisted = ["twisted"] + +[[package]] +name = "prometheus-fastapi-instrumentator" +version = "5.7.1" +description = "Instrument your FastAPI with Prometheus metrics" +category = "main" +optional = false +python-versions = ">=3.6.0,<4.0.0" + +[package.dependencies] +fastapi = ">=0.38.1,<1.0.0" +prometheus-client = ">=0.8.0,<1.0.0" + [[package]] name = "protobuf" version = "3.19.0" @@ -970,7 +993,7 @@ h11 = ">=0.9.0,<1" [metadata] lock-version = "1.1" python-versions = ">=3.9,<3.10" -content-hash = "6a45364297f5a6e88ee62240bb2eb1eaf3b41283b6d8f040ee67db02601f18e7" +content-hash = "569b0489389b884d269458f8e4252efcf3ebbbaa5fa77b6d09d7f0cdbda53362" [metadata.files] aiofiles = [ @@ -1440,6 +1463,14 @@ priority = [ {file = "priority-2.0.0-py3-none-any.whl", hash = "sha256:6f8eefce5f3ad59baf2c080a664037bb4725cd0a790d53d59ab4059288faf6aa"}, {file = "priority-2.0.0.tar.gz", hash = "sha256:c965d54f1b8d0d0b19479db3924c7c36cf672dbf2aec92d43fbdaf4492ba18c0"}, ] +prometheus-client = [ + {file = "prometheus_client-0.12.0-py2.py3-none-any.whl", hash = "sha256:317453ebabff0a1b02df7f708efbab21e3489e7072b61cb6957230dd004a0af0"}, + {file = "prometheus_client-0.12.0.tar.gz", hash = "sha256:1b12ba48cee33b9b0b9de64a1047cbd3c5f2d0ab6ebcead7ddda613a750ec3c5"}, +] +prometheus-fastapi-instrumentator = [ + {file = "prometheus-fastapi-instrumentator-5.7.1.tar.gz", hash = "sha256:5371f1b494e2b00017a02898d854119b4929025d1a203670b07b3f42dd0b5526"}, + {file = "prometheus_fastapi_instrumentator-5.7.1-py3-none-any.whl", hash = "sha256:da40ea0df14b0e95d584769747fba777522a8df6a8c47cec2edf798f1fff49b5"}, +] protobuf = [ {file = "protobuf-3.19.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:01a0645ef3acddfbc90237e1cdfae1086130fc7cb480b5874656193afd657083"}, {file = "protobuf-3.19.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d3861c9721a90ba83ee0936a9cfcc4fa1c4b4144ac9658fb6f6343b38558e9b4"}, diff --git a/pyproject.toml b/pyproject.toml index 2f327318..20855fa6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,7 @@ uvicorn = "^0.15.0" gunicorn = "^20.1.0" Hypercorn = "^0.11.2" mysql-connector = "^2.2.9" +prometheus-fastapi-instrumentator = "^5.7.1" [tool.poetry.dev-dependencies] flake8 = "^4.0.1" From f21765bfe467f225825ddeeca48c37dd5220d225 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 31 Oct 2021 02:16:50 -0700 Subject: [PATCH 517/844] feat(fastapi): add prometheus /metrics This commit provides custom metrics, so we can group requests into their route paths and not by the arguments given, e.g. /pkgbase/some-package -> /pkgbase/{name}. We also count RPC requests as `http_api_requests_total`, split by the RPC query "type" argument. - `http_api_requests_total` - Labels: ["type", "status"] - `http_requests_total` - Number of HTTP requests in total. - Labels: ["method", "path", "status"] Signed-off-by: Kevin Morris --- LICENSES/starlette_exporter | 201 ++++++++++++++++++++++++++++++++++++ aurweb/asgi.py | 8 ++ aurweb/prometheus.py | 101 ++++++++++++++++++ 3 files changed, 310 insertions(+) create mode 100644 LICENSES/starlette_exporter create mode 100644 aurweb/prometheus.py diff --git a/LICENSES/starlette_exporter b/LICENSES/starlette_exporter new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/LICENSES/starlette_exporter @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 8ebeef29..2ba2afd0 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -19,11 +19,18 @@ import aurweb.logging from aurweb.auth import BasicAuthBackend from aurweb.db import get_engine, query from aurweb.models import AcceptedTerm, Term +from aurweb.prometheus import http_api_requests_total, http_requests_total, instrumentator from aurweb.routers import accounts, auth, errors, html, packages, rpc, rss, sso, trusted_user # Setup the FastAPI app. app = FastAPI(exception_handlers=errors.exceptions) +# Instrument routes with the prometheus-fastapi-instrumentator +# library with custom collectors and expose /metrics. +instrumentator().add(http_api_requests_total()) +instrumentator().add(http_requests_total()) +instrumentator().instrument(app).expose(app) + @app.on_event("startup") async def app_startup(): @@ -67,6 +74,7 @@ async def app_startup(): app.include_router(rss.router) app.include_router(packages.router) app.include_router(rpc.router) + # Initialize the database engine and ORM. get_engine() diff --git a/aurweb/prometheus.py b/aurweb/prometheus.py new file mode 100644 index 00000000..0a3dd173 --- /dev/null +++ b/aurweb/prometheus.py @@ -0,0 +1,101 @@ +from typing import Any, Callable, Dict, List, Optional + +from prometheus_client import Counter +from prometheus_fastapi_instrumentator import Instrumentator +from prometheus_fastapi_instrumentator.metrics import Info +from starlette.routing import Match, Route + +from aurweb import logging + +logger = logging.get_logger(__name__) +_instrumentator = Instrumentator() + + +def instrumentator(): + return _instrumentator + + +# Taken from https://github.com/stephenhillier/starlette_exporter +# Their license is included in LICENSES/starlette_exporter. +# The code has been modified to remove child route checks +# (since we don't have any) and to stay within an 80-width limit. +def get_matching_route_path(scope: Dict[Any, Any], routes: List[Route], + route_name: Optional[str] = None) -> str: + """ + Find a matching route and return its original path string + + Will attempt to enter mounted routes and subrouters. + + Credit to https://github.com/elastic/apm-agent-python + + """ + for route in routes: + match, child_scope = route.matches(scope) + if match == Match.FULL: + route_name = route.path + + ''' + # This path exists in the original function's code, but we + # don't need it (currently), so it's been removed to avoid + # useless test coverage. + child_scope = {**scope, **child_scope} + if isinstance(route, Mount) and route.routes: + child_route_name = get_matching_route_path(child_scope, + route.routes, + route_name) + if child_route_name is None: + route_name = None + else: + route_name += child_route_name + ''' + + return route_name + elif match == Match.PARTIAL and route_name is None: + route_name = route.path + + +def http_requests_total() -> Callable[[Info], None]: + metric = Counter("http_requests_total", + "Number of HTTP requests.", + labelnames=("method", "path", "status")) + + def instrumentation(info: Info) -> None: + scope = info.request.scope + + # Taken from https://github.com/stephenhillier/starlette_exporter + # Their license is included at LICENSES/starlette_exporter. + # The code has been slightly modified: we no longer catch + # exceptions; we expect this collector to always succeed. + # Failures in this collector shall cause test failures. + if not (scope.get("endpoint", None) and scope.get("router", None)): + return None + + base_scope = { + "type": scope.get("type"), + "path": scope.get("root_path", "") + scope.get("path"), + "path_params": scope.get("path_params", {}), + "method": scope.get("method") + } + + method = scope.get("method") + path = get_matching_route_path(base_scope, scope.get("router").routes) + status = str(info.response.status_code)[:1] + "xx" + + metric.labels(method=method, path=path, status=status).inc() + + return instrumentation + + +def http_api_requests_total() -> Callable[[Info], None]: + metric = Counter( + "http_api_requests", + "Number of times an RPC API type has been requested.", + labelnames=("type", "status")) + + def instrumentation(info: Info) -> None: + if info.request.url.path.rstrip("/") == "/rpc": + type = info.request.query_params.get("type", "None") + status = str(info.response.status_code)[:1] + "xx" + metric.labels(type=type, status=status).inc() + + return instrumentation From 1be4ac2fde4f8d867fe476e52332f98ca18341f0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 1 Nov 2021 12:27:33 -0700 Subject: [PATCH 518/844] feat(docker): use PROMETHEUS_MULTIPROC_DIR Signed-off-by: Kevin Morris --- docker-compose.aur-dev.yml | 1 + docker-compose.yml | 1 + docker/fastapi-entrypoint.sh | 3 +++ 3 files changed, 5 insertions(+) diff --git a/docker-compose.aur-dev.yml b/docker-compose.aur-dev.yml index 3f574d42..1db306cc 100644 --- a/docker-compose.aur-dev.yml +++ b/docker-compose.aur-dev.yml @@ -53,6 +53,7 @@ services: - FASTAPI_WORKERS=${FASTAPI_WORKERS} - AURWEB_FASTAPI_PREFIX=${AURWEB_FASTAPI_PREFIX} - AURWEB_SSHD_PREFIX=${AURWEB_SSHD_PREFIX} + - PROMETHEUS_MULTIPROC_DIR=/tmp_prometheus volumes: - cache:/cache diff --git a/docker-compose.yml b/docker-compose.yml index 2b25c7d8..6c822e7c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -168,6 +168,7 @@ services: - FASTAPI_WORKERS=${FASTAPI_WORKERS} - AURWEB_FASTAPI_PREFIX=${AURWEB_FASTAPI_PREFIX} - AURWEB_SSHD_PREFIX=${AURWEB_SSHD_PREFIX} + - PROMETHEUS_MULTIPROC_DIR=/tmp_prometheus entrypoint: /docker/fastapi-entrypoint.sh command: /docker/scripts/run-fastapi.sh "${FASTAPI_BACKEND}" healthcheck: diff --git a/docker/fastapi-entrypoint.sh b/docker/fastapi-entrypoint.sh index 58fafe56..f4ceaafa 100755 --- a/docker/fastapi-entrypoint.sh +++ b/docker/fastapi-entrypoint.sh @@ -18,4 +18,7 @@ fi sed -ri "s|^(git_clone_uri_anon) = .+|\1 = ${AURWEB_FASTAPI_PREFIX}/%s.git|" conf/config.defaults sed -ri "s|^(git_clone_uri_priv) = .+|\1 = ${AURWEB_SSHD_PREFIX}/%s.git|" conf/config.defaults +rm -rf $PROMETHEUS_MULTIPROC_DIR +mkdir -p $PROMETHEUS_MULTIPROC_DIR + exec "$@" From dc397f6bd8d95efcebfa479d897cc00651d3bb20 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 1 Nov 2021 13:17:24 -0700 Subject: [PATCH 519/844] fix(fastapi): utilize PROMETHEUS_MULTIPROC_DIR in our own /metrics Signed-off-by: Kevin Morris --- aurweb/asgi.py | 9 ++++++++- aurweb/routers/html.py | 21 +++++++++++++++++++-- test/test_html.py | 7 +++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 2ba2afd0..16de771e 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -9,6 +9,7 @@ from urllib.parse import quote_plus from fastapi import FastAPI, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse from fastapi.staticfiles import StaticFiles +from prometheus_client import multiprocess from sqlalchemy import and_, or_ from starlette.middleware.authentication import AuthenticationMiddleware from starlette.middleware.sessions import SessionMiddleware @@ -29,7 +30,7 @@ app = FastAPI(exception_handlers=errors.exceptions) # library with custom collectors and expose /metrics. instrumentator().add(http_api_requests_total()) instrumentator().add(http_requests_total()) -instrumentator().instrument(app).expose(app) +instrumentator().instrument(app) @app.on_event("startup") @@ -79,6 +80,12 @@ async def app_startup(): get_engine() +def child_exit(server, worker): # pragma: no cover + """ This function is required for gunicorn customization + of prometheus multiprocessing. """ + multiprocess.mark_process_dead(worker.pid) + + @app.exception_handler(HTTPException) async def http_exception_handler(request, exc): """ diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index c749ca67..4cee5f99 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -1,11 +1,14 @@ """ AURWeb's primary routing module. Define all routes via @app.app.{get,post} decorators in some way; more complex routes should be defined in their own modules and imported here. """ +import os + from datetime import datetime from http import HTTPStatus -from fastapi import APIRouter, Form, HTTPException, Request +from fastapi import APIRouter, Form, HTTPException, Request, Response from fastapi.responses import HTMLResponse, RedirectResponse +from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, generate_latest, multiprocess from sqlalchemy import and_, case, or_ import aurweb.config @@ -203,7 +206,21 @@ async def index(request: Request): return render_template(request, "index.html", context) -# A route that returns a error 503. For testing purposes. +@router.get("/metrics") +async def metrics(request: Request): + registry = CollectorRegistry() + if os.environ.get("FASTAPI_BACKEND", "") == "gunicorn": # pragma: no cover + # This case only ever happens in production, when we are running + # gunicorn. We don't test with gunicorn, so we don't cover this path. + multiprocess.MultiProcessCollector(registry) + data = generate_latest(registry) + headers = { + "Content-Type": CONTENT_TYPE_LATEST, + "Content-Length": str(len(data)) + } + return Response(data, headers=headers) + + @router.get("/raisefivethree", response_class=HTMLResponse) async def raise_service_unavailable(request: Request): raise HTTPException(status_code=503) diff --git a/test/test_html.py b/test/test_html.py index 2018840b..8e7cb2d1 100644 --- a/test/test_html.py +++ b/test/test_html.py @@ -117,3 +117,10 @@ def test_get_successes(): """ successes = get_successes(html) assert successes[0].text.strip() == "Test" + + +def test_metrics(client: TestClient): + with client as request: + resp = request.get("/metrics") + assert resp.status_code == int(HTTPStatus.OK) + assert resp.headers.get("Content-Type").startswith("text/plain") From cdb854259af49ff759ce1481cd799612cd657bbc Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 1 Nov 2021 13:54:58 -0700 Subject: [PATCH 520/844] fix(docker): share FASTAPI_BACKEND with the server Signed-off-by: Kevin Morris --- docker-compose.yml | 1 + docker/scripts/run-fastapi.sh | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6c822e7c..5dffe5d3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -165,6 +165,7 @@ services: init: true environment: - AUR_CONFIG=conf/config + - FASTAPI_BACKEND=${FASTAPI_BACKEND} - FASTAPI_WORKERS=${FASTAPI_WORKERS} - AURWEB_FASTAPI_PREFIX=${AURWEB_FASTAPI_PREFIX} - AURWEB_SSHD_PREFIX=${AURWEB_SSHD_PREFIX} diff --git a/docker/scripts/run-fastapi.sh b/docker/scripts/run-fastapi.sh index 16ae3cb9..effc7fe4 100755 --- a/docker/scripts/run-fastapi.sh +++ b/docker/scripts/run-fastapi.sh @@ -14,10 +14,12 @@ fi # By default, set FASTAPI_WORKERS to 2. In production, this should # be configured by the deployer. -if [ -z "$FASTAPI_WORKERS" ]; then +if [ -z ${FASTAPI_WORKERS+x} ]; then FASTAPI_WORKERS=2 fi +export FASTAPI_BACKEND="$1" + echo "FASTAPI_BACKEND: $FASTAPI_BACKEND" echo "FASTAPI_WORKERS: $FASTAPI_WORKERS" From 9aa8decf403c618092b00db0f8b76c9917987b6b Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 1 Nov 2021 14:18:19 -0700 Subject: [PATCH 521/844] fix(fastapi): use metrics in cases where PROMETHEUS_MULTIPROC_DIR is defined Previously, we restricted this to gunicorn to get it working on aur-dev. This change makes it usable through any backend, and also no-op if PROMETHEUS_MULTIPROC_DIR is not defined. Signed-off-by: Kevin Morris --- aurweb/routers/html.py | 4 +--- docker-compose.yml | 3 +++ docker/scripts/run-pytests.sh | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index 4cee5f99..525fb626 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -209,9 +209,7 @@ async def index(request: Request): @router.get("/metrics") async def metrics(request: Request): registry = CollectorRegistry() - if os.environ.get("FASTAPI_BACKEND", "") == "gunicorn": # pragma: no cover - # This case only ever happens in production, when we are running - # gunicorn. We don't test with gunicorn, so we don't cover this path. + if os.environ.get("PROMETHEUS_MULTIPROC_DIR", None): # pragma: no cover multiprocess.MultiProcessCollector(registry) data = generate_latest(registry) headers = { diff --git a/docker-compose.yml b/docker-compose.yml index 5dffe5d3..225e5b9b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -241,6 +241,7 @@ services: environment: - AUR_CONFIG=conf/config - TEST_RECURSION_LIMIT=${TEST_RECURSION_LIMIT} + - PROMETHEUS_MULTIPROC_DIR=/tmp_prometheus entrypoint: /docker/test-mysql-entrypoint.sh command: /docker/scripts/run-pytests.sh clean stdin_open: true @@ -267,6 +268,7 @@ services: environment: - AUR_CONFIG=conf/config.sqlite - TEST_RECURSION_LIMIT=${TEST_RECURSION_LIMIT} + - PROMETHEUS_MULTIPROC_DIR=/tmp_prometheus entrypoint: /docker/test-sqlite-entrypoint.sh command: setup-sqlite.sh run-pytests.sh clean stdin_open: true @@ -289,6 +291,7 @@ services: environment: - AUR_CONFIG=conf/config - TEST_RECURSION_LIMIT=${TEST_RECURSION_LIMIT} + - PROMETHEUS_MULTIPROC_DIR=/tmp_prometheus entrypoint: /docker/tests-entrypoint.sh command: setup-sqlite.sh run-tests.sh stdin_open: true diff --git a/docker/scripts/run-pytests.sh b/docker/scripts/run-pytests.sh index d992bf06..ee546fb7 100755 --- a/docker/scripts/run-pytests.sh +++ b/docker/scripts/run-pytests.sh @@ -22,6 +22,9 @@ while [ $# -ne 0 ]; do esac done +rm -rf $PROMETHEUS_MULTIPROC_DIR +mkdir -p $PROMETHEUS_MULTIPROC_DIR + # Initialize the new database; ignore errors. python -m aurweb.initdb 2>/dev/null || \ (echo "Error: aurweb.initdb failed; already initialized?" && /bin/true) From 16e6fa2cdd002fecf6ad5e4251727315d7a3dfc8 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 1 Nov 2021 14:23:15 -0700 Subject: [PATCH 522/844] fix(fastapi): fix prometheus parsing of HTTPStatus This wasn't actually casting to int. We shouldn't be providing HTTPStatus.CONSTANTS directly anyway, but, in case we do, we now just convert the status to an int before converting it to a string. Signed-off-by: Kevin Morris --- aurweb/prometheus.py | 2 +- aurweb/routers/packages.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/prometheus.py b/aurweb/prometheus.py index 0a3dd173..a64f6b27 100644 --- a/aurweb/prometheus.py +++ b/aurweb/prometheus.py @@ -79,7 +79,7 @@ def http_requests_total() -> Callable[[Info], None]: method = scope.get("method") path = get_matching_route_path(base_scope, scope.get("router").routes) - status = str(info.response.status_code)[:1] + "xx" + status = str(int(info.response.status_code))[:1] + "xx" metric.labels(method=method, path=path, status=status).inc() diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index c574ec18..bcc0be56 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -269,7 +269,7 @@ async def package_base(request: Request, name: str) -> Response: # If this is not a split package, redirect to /packages/{name}. if pkgbase.packages.count() == 1: return RedirectResponse(f"/packages/{name}", - status_code=HTTPStatus.SEE_OTHER) + status_code=int(HTTPStatus.SEE_OTHER)) # Add our base information. context = await make_single_context(request, pkgbase) From e4a5b7fae968e1d1335b3c03a9b816b2d1b668ae Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 3 Nov 2021 05:39:28 -0700 Subject: [PATCH 523/844] fix(docker): use 3s intervals for all healthchecks This'll speed up the docker development and deployment processes significantly. Signed-off-by: Kevin Morris --- docker-compose.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 225e5b9b..038eb65b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,8 @@ services: command: /docker/scripts/run-memcached.sh healthcheck: test: "bash /docker/health/memcached.sh" - + interval: 3s + redis: image: aurweb:latest init: true @@ -45,6 +46,7 @@ services: command: /docker/scripts/run-redis.sh healthcheck: test: "bash /docker/health/redis.sh" + interval: 3s ports: - "16379:6379" @@ -62,6 +64,7 @@ services: - mariadb_data:/var/lib/mysql healthcheck: test: "bash /docker/health/mariadb.sh" + interval: 3s mariadb_init: image: aurweb:latest @@ -85,6 +88,7 @@ services: - "2222:2222" healthcheck: test: "bash /docker/health/sshd.sh" + interval: 3s depends_on: mariadb_init: condition: service_started @@ -100,6 +104,7 @@ services: command: /docker/scripts/run-smartgit.sh healthcheck: test: "bash /docker/health/smartgit.sh" + interval: 3s cgit-php: image: aurweb:latest @@ -111,6 +116,7 @@ services: command: /docker/scripts/run-cgit.sh 3000 healthcheck: test: "bash /docker/health/cgit.sh 3000" + interval: 3s depends_on: git: condition: service_healthy @@ -129,6 +135,7 @@ services: command: /docker/scripts/run-cgit.sh 3000 healthcheck: test: "bash /docker/health/cgit.sh 3000" + interval: 3s depends_on: git: condition: service_healthy @@ -148,6 +155,7 @@ services: command: /docker/scripts/run-php.sh healthcheck: test: "bash /docker/health/php.sh" + interval: 3s depends_on: ca: condition: service_started @@ -174,6 +182,7 @@ services: command: /docker/scripts/run-fastapi.sh "${FASTAPI_BACKEND}" healthcheck: test: "bash /docker/health/fastapi.sh ${FASTAPI_BACKEND}" + interval: 3s depends_on: ca: condition: service_started @@ -198,6 +207,7 @@ services: - "8444:8444" # FastAPI healthcheck: test: "bash /docker/health/nginx.sh" + interval: 3s depends_on: cgit-php: condition: service_healthy From 020409ef46452bafb0e1bf0d6a9537912059a6dd Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Mon, 1 Nov 2021 17:18:09 -0400 Subject: [PATCH 524/844] fix(FastAPI): prevent CSRF forging login requests Signed-off-by: Steven Guikal --- aurweb/routers/auth.py | 12 +++++++++++- po/aurweb.pot | 4 ++++ test/test_auth_routes.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index 4e6a416a..b8e83c7d 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -1,13 +1,14 @@ from datetime import datetime from http import HTTPStatus -from fastapi import APIRouter, Form, Request +from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse import aurweb.config from aurweb import cookies from aurweb.auth import auth_required +from aurweb.l10n import get_translator_for_request from aurweb.models import User from aurweb.templates import make_variable_context, render_template @@ -35,6 +36,15 @@ async def login_post(request: Request, user: str = Form(default=str()), passwd: str = Form(default=str()), remember_me: bool = Form(default=False)): + # TODO: Once the Origin header gets broader adoption, this code can be + # slightly simplified to use it. + login_path = aurweb.config.get("options", "aur_location") + "/login" + referer = request.headers.get("Referer") + if not referer or not referer.startswith(login_path): + _ = get_translator_for_request(request) + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, + detail=_("Bad Referer header.")) + from aurweb.db import session user = session.query(User).filter(User.Username == user).first() diff --git a/po/aurweb.pot b/po/aurweb.pot index 721f874e..dd93ca27 100644 --- a/po/aurweb.pot +++ b/po/aurweb.pot @@ -964,6 +964,10 @@ msgstr "" msgid "Package details could not be found." msgstr "" +#: aurweb/routers/auth.py +msgid "Bad Referer header." +msgstr "" + #: aurweb/routers/packages.py msgid "You did not select any packages to be notified about." msgstr "" diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index 313f9927..39afc6f9 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -18,6 +18,9 @@ from aurweb.testing import setup_test_db # Some test global constants. TEST_USERNAME = "test" TEST_EMAIL = "test@example.org" +TEST_REFERER = { + "referer": aurweb.config.get("options", "aur_location") + "/login", +} # Global mutables. user = client = None @@ -39,6 +42,10 @@ def setup(): client = TestClient(app) + # Necessary for forged login CSRF protection on the login route. Set here + # instead of only on the necessary requests for convenience. + client.headers.update(TEST_REFERER) + def test_login_logout(): post_data = { @@ -92,6 +99,10 @@ def test_secure_login(mock): # Create a local TestClient here since we mocked configuration. client = TestClient(app) + # Necessary for forged login CSRF protection on the login route. Set here + # instead of only on the necessary requests for convenience. + client.headers.update(TEST_REFERER) + # Data used for our upcoming http post request. post_data = { "user": user.Username, @@ -246,3 +257,26 @@ def test_login_incorrect_password(): assert post_data["user"] in content assert post_data["passwd"] not in content assert "checked" not in content + + +def test_login_bad_referer(): + post_data = { + "user": "test", + "passwd": "testPassword", + "next": "/", + } + + # Create new TestClient without a Referer header. + client = TestClient(app) + + with client as request: + response = request.post("/login", data=post_data) + assert "AURSID" not in response.cookies + + BAD_REFERER = { + "referer": aurweb.config.get("options", "aur_location") + ".mal.local", + } + with client as request: + response = request.post("/login", data=post_data, headers=BAD_REFERER) + assert response.status_code == int(HTTPStatus.BAD_REQUEST) + assert "AURSID" not in response.cookies From 69773a5b58ba86bf43a4c4240e5db4842c5dfbe0 Mon Sep 17 00:00:00 2001 From: Kristian Klausen Date: Fri, 15 Oct 2021 20:14:31 +0200 Subject: [PATCH 525/844] feat(PHP): Add packages dump file with more metadata --- aurweb/scripts/mkpkglists.py | 10 ++++++++++ conf/config.defaults | 1 + test/setup.sh | 1 + web/html/index.php | 1 + 4 files changed, 13 insertions(+) diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index 6724141a..c73cc3be 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -2,11 +2,13 @@ import datetime import gzip +import json import aurweb.config import aurweb.db packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') +packagesmetafile = aurweb.config.get('mkpkglists', 'packagesmetafile') pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') userfile = aurweb.config.get('mkpkglists', 'userfile') @@ -27,6 +29,14 @@ def main(): "WHERE PackageBases.PackagerUID IS NOT NULL") f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + with gzip.open(packagesmetafile, "wt") as f: + cur = conn.execute("SELECT * FROM Packages") + json.dump({ + "warning": "This is a experimental! It can be removed or modified without warning!", + "columns": [d[0] for d in cur.description], + "data": cur.fetchall() + }, f) + with gzip.open(pkgbasefile, "w") as f: f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) cur = conn.execute("SELECT Name FROM PackageBases " + diff --git a/conf/config.defaults b/conf/config.defaults index b7bc0368..36ea02ef 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -92,5 +92,6 @@ server = ftp://mirrors.kernel.org/archlinux/%s/os/x86_64 [mkpkglists] packagesfile = /srv/http/aurweb/web/html/packages.gz +packagesmetafile = /srv/http/aurweb/web/html/packages-meta-v1.json.gz pkgbasefile = /srv/http/aurweb/web/html/pkgbase.gz userfile = /srv/http/aurweb/web/html/users.gz diff --git a/test/setup.sh b/test/setup.sh index 764d4518..24bb5f48 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -61,6 +61,7 @@ server = file://$(pwd)/remote/ [mkpkglists] packagesfile = packages.gz +packagesmetafile = packages-meta-v1.json.gz pkgbasefile = pkgbase.gz userfile = users.gz EOF diff --git a/web/html/index.php b/web/html/index.php index e57e7708..3163c3e8 100644 --- a/web/html/index.php +++ b/web/html/index.php @@ -189,6 +189,7 @@ if (!empty($tokens[1]) && '/' . $tokens[1] == get_pkg_route()) { readfile("./$path"); break; case "/packages.gz": + case "/packages-teapot.json.gz": case "/pkgbase.gz": case "/users.gz": header("Content-Type: text/plain"); From 51fb24ab730f3b09d78e200f020b01974dc9e457 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 31 Oct 2021 16:52:30 -0700 Subject: [PATCH 526/844] fix(mkpkglists): improve package meta archive The SQL logic in this file for package metadata now exactly reflects RPC's search logic, without searching for specific packages. Two command line arguments are available: --extended | Include License, Keywords, Groups, relations and dependencies. When --extended is passed, the script will create a packages-meta-ext-v1.json.gz, configured via packagesmetaextfile. Archive JSON is in the following format: line-separated package objects enclosed in a list: [ {...}, {...}, {...} ] Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 2 +- INSTALL | 3 +- aurweb/scripts/mkpkglists.py | 273 ++++++++++++++++++++++++++++++++--- conf/config.defaults | 1 + test/setup.sh | 2 + web/html/index.php | 3 +- 6 files changed, 258 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aff18a83..ce374082 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,7 +12,7 @@ before_script: python-pygit2 python-srcinfo python-bleach python-markdown python-sqlalchemy python-alembic python-pytest python-werkzeug python-pytest-tap python-fastapi hypercorn nginx python-authlib - python-itsdangerous python-httpx + python-itsdangerous python-httpx python-orjson test: script: diff --git a/INSTALL b/INSTALL index 9bcd0759..dc9cc51f 100644 --- a/INSTALL +++ b/INSTALL @@ -49,7 +49,8 @@ read the instructions below. # pacman -S python-mysql-connector python-pygit2 python-srcinfo python-sqlalchemy \ python-bleach python-markdown python-alembic python-jinja \ - python-itsdangerous python-authlib python-httpx hypercorn + python-itsdangerous python-authlib python-httpx hypercorn \ + python-orjson # python3 setup.py install 5) Create a new MySQL database and a user and import the aurweb SQL schema: diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index c73cc3be..f2095a20 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -1,16 +1,192 @@ #!/usr/bin/env python3 +""" +Produces package, package base and user archives for the AUR +database. + +Archives: + + packages.gz | A line-separated list of package names + packages-meta-v1.json | A type=search RPC-formatted JSON dataset + packages-meta-ext-v1.json | An --extended archive + pkgbase.gz | A line-separated list of package base names + users.gz | A line-separated list of user names + +This script takes an optional argument: --extended. Based +on the following, right-hand side fields are added to each item. + + --extended | License, Keywords, Groups, relations and dependencies + +""" import datetime import gzip -import json +import os +import sys + +from collections import defaultdict +from decimal import Decimal +from typing import Tuple + +import orjson import aurweb.config import aurweb.db + +def state_path(archive: str) -> str: + # A hard-coded /tmp state directory. + # TODO: Use Redis cache to store this state after we merge + # FastAPI into master and removed PHP from the tree. + return os.path.join("/tmp", os.path.basename(archive) + ".state") + + packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') packagesmetafile = aurweb.config.get('mkpkglists', 'packagesmetafile') +packagesmetaextfile = aurweb.config.get('mkpkglists', 'packagesmetaextfile') +packages_state = state_path(packagesfile) + pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') +pkgbases_state = state_path(pkgbasefile) + userfile = aurweb.config.get('mkpkglists', 'userfile') +users_state = state_path(userfile) + + +def should_update(state: str, tablename: str) -> Tuple[bool, int]: + if aurweb.config.get("database", "backend") != "mysql": + return (False, 0) + + db_name = aurweb.config.get("database", "name") + conn = aurweb.db.Connection() + cur = conn.execute("SELECT auto_increment FROM information_schema.tables " + "WHERE table_schema = ? AND table_name = ?", + (db_name, tablename,)) + update_time = cur.fetchone()[0] + + saved_update_time = 0 + if os.path.exists(state): + with open(state) as f: + saved_update_time = int(f.read().strip()) + + return (saved_update_time == update_time, update_time) + + +def update_state(state: str, update_time: int) -> None: + with open(state, "w") as f: + f.write(str(update_time)) + + +TYPE_MAP = { + "depends": "Depends", + "makedepends": "MakeDepends", + "checkdepends": "CheckDepends", + "optdepends": "OptDepends", + "conflicts": "Conflicts", + "provides": "Provides", + "replaces": "Replaces", +} + + +def get_extended_dict(query: str): + """ + Produce data in the form in a single bulk SQL query: + + { + : { + "Depends": [...], + "Conflicts": [...], + "License": [...] + } + } + + The caller can then use this data to populate a dataset of packages. + + output = produce_base_output_data() + data = get_extended_dict(query) + for i in range(len(output)): + package_id = output[i].get("ID") + output[i].update(data.get(package_id)) + """ + + conn = aurweb.db.Connection() + + cursor = conn.execute(query) + + data = defaultdict(lambda: defaultdict(list)) + + for result in cursor.fetchall(): + + pkgid = result[0] + key = TYPE_MAP.get(result[1]) + output = result[2] + if result[3]: + output += result[3] + + # In all cases, we have at least an empty License list. + if "License" not in data[pkgid]: + data[pkgid]["License"] = [] + + # In all cases, we have at least an empty Keywords list. + if "Keywords" not in data[pkgid]: + data[pkgid]["Keywords"] = [] + + data[pkgid][key].append(output) + + conn.close() + return data + + +def get_extended_fields(): + # Returns: [ID, Type, Name, Cond] + query = """ + SELECT PackageDepends.PackageID AS ID, DependencyTypes.Name AS Type, + PackageDepends.DepName AS Name, PackageDepends.DepCondition AS Cond + FROM PackageDepends + LEFT JOIN DependencyTypes + ON DependencyTypes.ID = PackageDepends.DepTypeID + UNION SELECT PackageRelations.PackageID AS ID, RelationTypes.Name AS Type, + PackageRelations.RelName AS Name, + PackageRelations.RelCondition AS Cond + FROM PackageRelations + LEFT JOIN RelationTypes + ON RelationTypes.ID = PackageRelations.RelTypeID + UNION SELECT PackageGroups.PackageID AS ID, 'Groups' AS Type, + Groups.Name, '' AS Cond + FROM Groups + INNER JOIN PackageGroups ON PackageGroups.GroupID = Groups.ID + UNION SELECT PackageLicenses.PackageID AS ID, 'License' AS Type, + Licenses.Name, '' as Cond + FROM Licenses + INNER JOIN PackageLicenses ON PackageLicenses.LicenseID = Licenses.ID + UNION SELECT Packages.ID AS ID, 'Keywords' AS Type, + PackageKeywords.Keyword AS Name, '' as Cond + FROM PackageKeywords + INNER JOIN Packages ON Packages.PackageBaseID = PackageKeywords.PackageBaseID + """ + return get_extended_dict(query) + + +EXTENDED_FIELD_HANDLERS = { + "--extended": get_extended_fields +} + + +def is_decimal(column): + """ Check if an SQL column is of decimal.Decimal type. """ + if isinstance(column, Decimal): + return float(column) + return column + + +def write_archive(archive: str, output: list): + with gzip.open(archive, "wb") as f: + f.write(b"[\n") + for i, item in enumerate(output): + f.write(orjson.dumps(item)) + if i < len(output) - 1: + f.write(b",") + f.write(b"\n") + f.write(b"]") def main(): @@ -21,32 +197,83 @@ def main(): pkgbaselist_header = "# AUR package base list, generated on " + datestr userlist_header = "# AUR user name list, generated on " + datestr - with gzip.open(packagesfile, "w") as f: - f.write(bytes(pkglist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT Packages.Name FROM Packages " + - "INNER JOIN PackageBases " + - "ON PackageBases.ID = Packages.PackageBaseID " + + updated, update_time = should_update(packages_state, "Packages") + if not updated: + print("Updating Packages...") + + # Query columns; copied from RPC. + columns = ("Packages.ID, Packages.Name, " + "PackageBases.ID AS PackageBaseID, " + "PackageBases.Name AS PackageBase, " + "Version, Description, URL, NumVotes, " + "Popularity, OutOfDateTS AS OutOfDate, " + "Users.UserName AS Maintainer, " + "SubmittedTS AS FirstSubmitted, " + "ModifiedTS AS LastModified") + + # Perform query. + cur = conn.execute(f"SELECT {columns} FROM Packages " + "LEFT JOIN PackageBases " + "ON PackageBases.ID = Packages.PackageBaseID " + "LEFT JOIN Users " + "ON PackageBases.MaintainerUID = Users.ID " "WHERE PackageBases.PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - with gzip.open(packagesmetafile, "wt") as f: - cur = conn.execute("SELECT * FROM Packages") - json.dump({ - "warning": "This is a experimental! It can be removed or modified without warning!", - "columns": [d[0] for d in cur.description], - "data": cur.fetchall() - }, f) + # Produce packages-meta-v1.json.gz + output = list() + snapshot_uri = aurweb.config.get("options", "snapshot_uri") + for result in cur.fetchall(): + item = { + column[0]: is_decimal(result[i]) + for i, column in enumerate(cur.description) + } + item["URLPath"] = snapshot_uri % item.get("Name") + output.append(item) - with gzip.open(pkgbasefile, "w") as f: - f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT Name FROM PackageBases " + - "WHERE PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + write_archive(packagesmetafile, output) - with gzip.open(userfile, "w") as f: - f.write(bytes(userlist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT UserName FROM Users") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + # Produce packages-meta-ext-v1.json.gz + if len(sys.argv) > 1 and sys.argv[1] in EXTENDED_FIELD_HANDLERS: + f = EXTENDED_FIELD_HANDLERS.get(sys.argv[1]) + data = f() + + default_ = {"Groups": [], "License": [], "Keywords": []} + for i in range(len(output)): + data_ = data.get(output[i].get("ID"), default_) + output[i].update(data_) + + write_archive(packagesmetaextfile, output) + + # Produce packages.gz + with gzip.open(packagesfile, "wb") as f: + f.write(bytes(pkglist_header + "\n", "UTF-8")) + f.writelines([ + bytes(x.get("Name") + "\n", "UTF-8") + for x in output + ]) + + update_state(packages_state, update_time) + + updated, update_time = should_update(pkgbases_state, "PackageBases") + if not updated: + print("Updating PackageBases...") + # Produce pkgbase.gz + with gzip.open(pkgbasefile, "w") as f: + f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT Name FROM PackageBases " + + "WHERE PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + update_state(pkgbases_state, update_time) + + updated, update_time = should_update(users_state, "Users") + if not updated: + print("Updating Users...") + # Produce users.gz + with gzip.open(userfile, "w") as f: + f.write(bytes(userlist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT UserName FROM Users") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + update_state(users_state, update_time) conn.close() diff --git a/conf/config.defaults b/conf/config.defaults index 36ea02ef..a04f21bc 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -93,5 +93,6 @@ server = ftp://mirrors.kernel.org/archlinux/%s/os/x86_64 [mkpkglists] packagesfile = /srv/http/aurweb/web/html/packages.gz packagesmetafile = /srv/http/aurweb/web/html/packages-meta-v1.json.gz +packagesmetaextfile = /srv/http/aurweb/web/html/packages-meta-ext-v1.json.gz pkgbasefile = /srv/http/aurweb/web/html/pkgbase.gz userfile = /srv/http/aurweb/web/html/users.gz diff --git a/test/setup.sh b/test/setup.sh index 24bb5f48..f74cd1b7 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -31,6 +31,7 @@ enable-maintenance = 0 maintenance-exceptions = 127.0.0.1 commit_uri = https://aur.archlinux.org/cgit/aur.git/log/?h=%s&id=%s localedir = $TOPLEVEL/web/locale/ +snapshot_uri = /cgit/aur.git/snapshot/%s.tar.gz [notifications] notify-cmd = $NOTIFY @@ -62,6 +63,7 @@ server = file://$(pwd)/remote/ [mkpkglists] packagesfile = packages.gz packagesmetafile = packages-meta-v1.json.gz +packagesmetaextfile = packages-meta-ext-v1.json.gz pkgbasefile = pkgbase.gz userfile = users.gz EOF diff --git a/web/html/index.php b/web/html/index.php index 3163c3e8..dc435162 100644 --- a/web/html/index.php +++ b/web/html/index.php @@ -189,7 +189,8 @@ if (!empty($tokens[1]) && '/' . $tokens[1] == get_pkg_route()) { readfile("./$path"); break; case "/packages.gz": - case "/packages-teapot.json.gz": + case "/packages-meta-v1.json.gz": + case "/packages-meta-ext-v1.json.gz": case "/pkgbase.gz": case "/users.gz": header("Content-Type: text/plain"); From cdca8bd2953f3c3aa3a1b4cedb89c3b7b9fd4ddb Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 6 Nov 2021 16:23:08 -0700 Subject: [PATCH 527/844] feat(mkpkglists): added metadata archives Two new archives are available: - packages-meta-v1.json.gz - RPC search formatted data for all packages - ~2.1MB at the time of writing. - packages-meta-ext-v1.json.gz (via --extended) - RPC multiinfo formatted data for all packages. - ~9.8MB at the time of writing. New dependencies are required for this update: - `python-orjson` All archives served out by aur.archlinux.org distribute the Last-Modified header and support the If-Modified-Since header, which should be populated with Last-Modified's value. These should be used by clients to avoid redownloading the archive when unnecessary. Additionally, the new meta archives contain a format suitable for streaming the data as the file is retrieved. It is still in JSON format, however, users can parse package objects line by line after the first '[' found in the file, until the last ']'; both contained on their own lines. Note: This commit is a documentation change and commit body. Signed-off-by: Kevin Morris --- doc/maintenance.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/maintenance.txt b/doc/maintenance.txt index d6094545..2c5c9faf 100644 --- a/doc/maintenance.txt +++ b/doc/maintenance.txt @@ -70,7 +70,8 @@ computations and clean up the database: * aurweb-pkgmaint automatically removes empty repositories that were created within the last 24 hours but never populated. -* aurweb-mkpkglists generates the package list files. +* aurweb-mkpkglists generates the package list files; it takes an optional + --extended flag, which additionally produces multiinfo metadata. * aurweb-usermaint removes the last login IP address of all users that did not login within the past seven days. @@ -79,7 +80,7 @@ These scripts can be installed by running `python3 setup.py install` and are usually scheduled using Cron. The current setup is: ---- -*/5 * * * * aurweb-mkpkglists +*/5 * * * * aurweb-mkpkglists [--extended] 1 */2 * * * aurweb-popupdate 2 */2 * * * aurweb-aurblup 3 */2 * * * aurweb-pkgmaint From 9f1f39995740e04d44309c7aed69a9b15d26dac0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 6 Nov 2021 17:13:16 -0700 Subject: [PATCH 528/844] fix(mkpkglists): remove caching We really need caching for this; however, our current caching method will cause the script to bypass changes to columns if they have nothing to do with IDs. Signed-off-by: Kevin Morris --- aurweb/scripts/mkpkglists.py | 159 ++++++++++++----------------------- 1 file changed, 54 insertions(+), 105 deletions(-) diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index f2095a20..2566a146 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -20,60 +20,23 @@ on the following, right-hand side fields are added to each item. import datetime import gzip -import os import sys from collections import defaultdict from decimal import Decimal -from typing import Tuple import orjson import aurweb.config import aurweb.db - -def state_path(archive: str) -> str: - # A hard-coded /tmp state directory. - # TODO: Use Redis cache to store this state after we merge - # FastAPI into master and removed PHP from the tree. - return os.path.join("/tmp", os.path.basename(archive) + ".state") - - packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') packagesmetafile = aurweb.config.get('mkpkglists', 'packagesmetafile') packagesmetaextfile = aurweb.config.get('mkpkglists', 'packagesmetaextfile') -packages_state = state_path(packagesfile) pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') -pkgbases_state = state_path(pkgbasefile) userfile = aurweb.config.get('mkpkglists', 'userfile') -users_state = state_path(userfile) - - -def should_update(state: str, tablename: str) -> Tuple[bool, int]: - if aurweb.config.get("database", "backend") != "mysql": - return (False, 0) - - db_name = aurweb.config.get("database", "name") - conn = aurweb.db.Connection() - cur = conn.execute("SELECT auto_increment FROM information_schema.tables " - "WHERE table_schema = ? AND table_name = ?", - (db_name, tablename,)) - update_time = cur.fetchone()[0] - - saved_update_time = 0 - if os.path.exists(state): - with open(state) as f: - saved_update_time = int(f.read().strip()) - - return (saved_update_time == update_time, update_time) - - -def update_state(state: str, update_time: int) -> None: - with open(state, "w") as f: - f.write(str(update_time)) TYPE_MAP = { @@ -197,83 +160,69 @@ def main(): pkgbaselist_header = "# AUR package base list, generated on " + datestr userlist_header = "# AUR user name list, generated on " + datestr - updated, update_time = should_update(packages_state, "Packages") - if not updated: - print("Updating Packages...") + # Query columns; copied from RPC. + columns = ("Packages.ID, Packages.Name, " + "PackageBases.ID AS PackageBaseID, " + "PackageBases.Name AS PackageBase, " + "Version, Description, URL, NumVotes, " + "Popularity, OutOfDateTS AS OutOfDate, " + "Users.UserName AS Maintainer, " + "SubmittedTS AS FirstSubmitted, " + "ModifiedTS AS LastModified") - # Query columns; copied from RPC. - columns = ("Packages.ID, Packages.Name, " - "PackageBases.ID AS PackageBaseID, " - "PackageBases.Name AS PackageBase, " - "Version, Description, URL, NumVotes, " - "Popularity, OutOfDateTS AS OutOfDate, " - "Users.UserName AS Maintainer, " - "SubmittedTS AS FirstSubmitted, " - "ModifiedTS AS LastModified") + # Perform query. + cur = conn.execute(f"SELECT {columns} FROM Packages " + "LEFT JOIN PackageBases " + "ON PackageBases.ID = Packages.PackageBaseID " + "LEFT JOIN Users " + "ON PackageBases.MaintainerUID = Users.ID " + "WHERE PackageBases.PackagerUID IS NOT NULL") - # Perform query. - cur = conn.execute(f"SELECT {columns} FROM Packages " - "LEFT JOIN PackageBases " - "ON PackageBases.ID = Packages.PackageBaseID " - "LEFT JOIN Users " - "ON PackageBases.MaintainerUID = Users.ID " - "WHERE PackageBases.PackagerUID IS NOT NULL") + # Produce packages-meta-v1.json.gz + output = list() + snapshot_uri = aurweb.config.get("options", "snapshot_uri") + for result in cur.fetchall(): + item = { + column[0]: is_decimal(result[i]) + for i, column in enumerate(cur.description) + } + item["URLPath"] = snapshot_uri % item.get("Name") + output.append(item) - # Produce packages-meta-v1.json.gz - output = list() - snapshot_uri = aurweb.config.get("options", "snapshot_uri") - for result in cur.fetchall(): - item = { - column[0]: is_decimal(result[i]) - for i, column in enumerate(cur.description) - } - item["URLPath"] = snapshot_uri % item.get("Name") - output.append(item) + write_archive(packagesmetafile, output) - write_archive(packagesmetafile, output) + # Produce packages-meta-ext-v1.json.gz + if len(sys.argv) > 1 and sys.argv[1] in EXTENDED_FIELD_HANDLERS: + f = EXTENDED_FIELD_HANDLERS.get(sys.argv[1]) + data = f() - # Produce packages-meta-ext-v1.json.gz - if len(sys.argv) > 1 and sys.argv[1] in EXTENDED_FIELD_HANDLERS: - f = EXTENDED_FIELD_HANDLERS.get(sys.argv[1]) - data = f() + default_ = {"Groups": [], "License": [], "Keywords": []} + for i in range(len(output)): + data_ = data.get(output[i].get("ID"), default_) + output[i].update(data_) - default_ = {"Groups": [], "License": [], "Keywords": []} - for i in range(len(output)): - data_ = data.get(output[i].get("ID"), default_) - output[i].update(data_) + write_archive(packagesmetaextfile, output) - write_archive(packagesmetaextfile, output) + # Produce packages.gz + with gzip.open(packagesfile, "wb") as f: + f.write(bytes(pkglist_header + "\n", "UTF-8")) + f.writelines([ + bytes(x.get("Name") + "\n", "UTF-8") + for x in output + ]) - # Produce packages.gz - with gzip.open(packagesfile, "wb") as f: - f.write(bytes(pkglist_header + "\n", "UTF-8")) - f.writelines([ - bytes(x.get("Name") + "\n", "UTF-8") - for x in output - ]) + # Produce pkgbase.gz + with gzip.open(pkgbasefile, "w") as f: + f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT Name FROM PackageBases " + + "WHERE PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - update_state(packages_state, update_time) - - updated, update_time = should_update(pkgbases_state, "PackageBases") - if not updated: - print("Updating PackageBases...") - # Produce pkgbase.gz - with gzip.open(pkgbasefile, "w") as f: - f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT Name FROM PackageBases " + - "WHERE PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - update_state(pkgbases_state, update_time) - - updated, update_time = should_update(users_state, "Users") - if not updated: - print("Updating Users...") - # Produce users.gz - with gzip.open(userfile, "w") as f: - f.write(bytes(userlist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT UserName FROM Users") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - update_state(users_state, update_time) + # Produce users.gz + with gzip.open(userfile, "w") as f: + f.write(bytes(userlist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT UserName FROM Users") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) conn.close() From 446a082352bf2bb8d7398afbdbe9dfad42f21fed Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 7 Nov 2021 17:26:05 -0800 Subject: [PATCH 529/844] change(fastapi): refactor database ORM model definitions We don't want to depend on the database to load up data about the models we define. We now leverage the existing `aurweb.schema` module for table definitions and set __table_args__["autoload"] to False. Signed-off-by: Kevin Morris --- aurweb/models/accepted_term.py | 16 +++----- aurweb/models/account_type.py | 56 +++++++++++---------------- aurweb/models/api_rate_limit.py | 10 ++--- aurweb/models/ban.py | 10 ++--- aurweb/models/declarative.py | 6 +-- aurweb/models/dependency_type.py | 27 +++++-------- aurweb/models/group.py | 10 ++--- aurweb/models/license.py | 10 ++--- aurweb/models/official_provider.py | 10 ++--- aurweb/models/package.py | 15 +++---- aurweb/models/package_base.py | 21 ++++------ aurweb/models/package_blacklist.py | 10 ++--- aurweb/models/package_comaintainer.py | 20 ++++------ aurweb/models/package_comment.py | 26 ++++--------- aurweb/models/package_dependency.py | 22 ++++------- aurweb/models/package_group.py | 18 ++++----- aurweb/models/package_keyword.py | 19 ++++----- aurweb/models/package_license.py | 20 ++++------ aurweb/models/package_notification.py | 20 ++++------ aurweb/models/package_relation.py | 22 ++++------- aurweb/models/package_request.py | 25 ++++-------- aurweb/models/package_source.py | 14 +++---- aurweb/models/package_vote.py | 20 ++++------ aurweb/models/relation_type.py | 24 ++++-------- aurweb/models/request_type.py | 21 ++++------ aurweb/models/session.py | 13 +++---- aurweb/models/ssh_pub_key.py | 19 +++------ aurweb/models/term.py | 10 ++--- aurweb/models/tu_vote.py | 18 ++++----- aurweb/models/tu_voteinfo.py | 15 +++---- aurweb/models/user.py | 21 ++++------ 31 files changed, 212 insertions(+), 356 deletions(-) diff --git a/aurweb/models/accepted_term.py b/aurweb/models/accepted_term.py index b4dbb410..0f9b187e 100644 --- a/aurweb/models/accepted_term.py +++ b/aurweb/models/accepted_term.py @@ -1,28 +1,24 @@ -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.term import Term as _Term from aurweb.models.user import User as _User class AcceptedTerm(Base): - __tablename__ = "AcceptedTerms" + __table__ = schema.AcceptedTerms + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.TermsID]} - UsersID = Column(Integer, ForeignKey("Users.ID", ondelete="CASCADE"), - nullable=False) User = relationship( _User, backref=backref("accepted_terms", lazy="dynamic"), - foreign_keys=[UsersID]) + foreign_keys=[__table__.c.UsersID]) - TermsID = Column(Integer, ForeignKey("Terms.ID", ondelete="CASCADE"), - nullable=False) Term = relationship( _Term, backref=backref("accepted_terms", lazy="dynamic"), - foreign_keys=[TermsID]) - - __mapper_args__ = {"primary_key": [TermsID]} + foreign_keys=[__table__.c.TermsID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/account_type.py b/aurweb/models/account_type.py index 7aa7733c..a849df02 100644 --- a/aurweb/models/account_type.py +++ b/aurweb/models/account_type.py @@ -1,6 +1,4 @@ -from sqlalchemy import Column, Integer - -from aurweb import db +from aurweb import schema from aurweb.models.declarative import Base USER = "User" @@ -8,37 +6,10 @@ TRUSTED_USER = "Trusted User" DEVELOPER = "Developer" TRUSTED_USER_AND_DEV = "Trusted User & Developer" - -class AccountType(Base): - """ An ORM model of a single AccountTypes record. """ - __tablename__ = "AccountTypes" - - ID = Column(Integer, primary_key=True) - - __mapper_args__ = {"primary_key": [ID]} - - def __init__(self, **kwargs): - self.AccountType = kwargs.pop("AccountType") - - def __str__(self): - return str(self.AccountType) - - def __repr__(self): - return "" % ( - self.ID, str(self)) - - -# Fetch account type IDs from the database for constants. -_account_types = db.query(AccountType) -USER_ID = _account_types.filter( - AccountType.AccountType == USER).first().ID -TRUSTED_USER_ID = _account_types.filter( - AccountType.AccountType == TRUSTED_USER).first().ID -DEVELOPER_ID = _account_types.filter( - AccountType.AccountType == DEVELOPER).first().ID -TRUSTED_USER_AND_DEV_ID = _account_types.filter( - AccountType.AccountType == TRUSTED_USER_AND_DEV).first().ID -_account_types = None # Get rid of the query handle. +USER_ID = 1 +TRUSTED_USER_ID = 2 +DEVELOPER_ID = 3 +TRUSTED_USER_AND_DEV_ID = 4 # Map string constants to integer constants. ACCOUNT_TYPE_ID = { @@ -50,3 +21,20 @@ ACCOUNT_TYPE_ID = { # Reversed ACCOUNT_TYPE_ID mapping. ACCOUNT_TYPE_NAME = {v: k for k, v in ACCOUNT_TYPE_ID.items()} + + +class AccountType(Base): + """ An ORM model of a single AccountTypes record. """ + __table__ = schema.AccountTypes + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} + + def __init__(self, **kwargs): + self.AccountType = kwargs.pop("AccountType") + + def __str__(self): + return str(self.AccountType) + + def __repr__(self): + return "" % ( + self.ID, str(self)) diff --git a/aurweb/models/api_rate_limit.py b/aurweb/models/api_rate_limit.py index f8641896..19b656df 100644 --- a/aurweb/models/api_rate_limit.py +++ b/aurweb/models/api_rate_limit.py @@ -1,15 +1,13 @@ -from sqlalchemy import Column, String from sqlalchemy.exc import IntegrityError +from aurweb import schema from aurweb.models.declarative import Base class ApiRateLimit(Base): - __tablename__ = "ApiRateLimit" - - IP = Column(String(45), primary_key=True, unique=True, default=str()) - - __mapper_args__ = {"primary_key": [IP]} + __table__ = schema.ApiRateLimit + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.IP]} def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/ban.py b/aurweb/models/ban.py index e10087b0..a70be7b9 100644 --- a/aurweb/models/ban.py +++ b/aurweb/models/ban.py @@ -1,15 +1,13 @@ from fastapi import Request -from sqlalchemy import Column, String +from aurweb import schema from aurweb.models.declarative import Base class Ban(Base): - __tablename__ = "Bans" - - IPAddress = Column(String(45), primary_key=True) - - __mapper_args__ = {"primary_key": [IPAddress]} + __table__ = schema.Bans + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.IPAddress]} def __init__(self, **kwargs): self.IPAddress = kwargs.get("IPAddress") diff --git a/aurweb/models/declarative.py b/aurweb/models/declarative.py index 96ee1829..20ddd20c 100644 --- a/aurweb/models/declarative.py +++ b/aurweb/models/declarative.py @@ -2,8 +2,6 @@ import json from sqlalchemy.ext.declarative import declarative_base -import aurweb.db - from aurweb import util @@ -25,12 +23,10 @@ Base = declarative_base() # Setup __table_args__ applicable to every table. Base.__table_args__ = { - "autoload": True, - "autoload_with": aurweb.db.get_engine(), + "autoload": False, "extend_existing": True } - # Setup Base.as_dict and Base.json. # # With this, declarative models can use .as_dict() or .json() diff --git a/aurweb/models/dependency_type.py b/aurweb/models/dependency_type.py index 3b5fafcc..98418802 100644 --- a/aurweb/models/dependency_type.py +++ b/aurweb/models/dependency_type.py @@ -1,6 +1,4 @@ -from sqlalchemy import Column, Integer - -from aurweb import db +from aurweb import schema from aurweb.models.declarative import Base DEPENDS = "depends" @@ -8,23 +6,16 @@ MAKEDEPENDS = "makedepends" CHECKDEPENDS = "checkdepends" OPTDEPENDS = "optdepends" +DEPENDS_ID = 1 +MAKEDEPENDS_ID = 2 +CHECKDEPENDS_ID = 3 +OPTDEPENDS_ID = 4 + class DependencyType(Base): - __tablename__ = "DependencyTypes" - - ID = Column(Integer, primary_key=True) - - __mapper_args__ = {"primary_key": [ID]} + __table__ = schema.DependencyTypes + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} def __init__(self, Name: str = None): self.Name = Name - - -DEPENDS_ID = db.query(DependencyType).filter( - DependencyType.Name == DEPENDS).first().ID -MAKEDEPENDS_ID = db.query(DependencyType).filter( - DependencyType.Name == MAKEDEPENDS).first().ID -CHECKDEPENDS_ID = db.query(DependencyType).filter( - DependencyType.Name == CHECKDEPENDS).first().ID -OPTDEPENDS_ID = db.query(DependencyType).filter( - DependencyType.Name == OPTDEPENDS).first().ID diff --git a/aurweb/models/group.py b/aurweb/models/group.py index 5493bb7f..0275ed94 100644 --- a/aurweb/models/group.py +++ b/aurweb/models/group.py @@ -1,15 +1,13 @@ -from sqlalchemy import Column, Integer from sqlalchemy.exc import IntegrityError +from aurweb import schema from aurweb.models.declarative import Base class Group(Base): - __tablename__ = "Groups" - - ID = Column(Integer, primary_key=True) - - __mapper_args__ = {"primary_key": [ID]} + __table__ = schema.Groups + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/license.py b/aurweb/models/license.py index fa863379..86aeaa86 100644 --- a/aurweb/models/license.py +++ b/aurweb/models/license.py @@ -1,15 +1,13 @@ -from sqlalchemy import Column, Integer from sqlalchemy.exc import IntegrityError +from aurweb import schema from aurweb.models.declarative import Base class License(Base): - __tablename__ = "Licenses" - - ID = Column(Integer, primary_key=True) - - __mapper_args__ = {"primary_key": [ID]} + __table__ = schema.Licenses + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/official_provider.py b/aurweb/models/official_provider.py index a273dd06..a8282ff1 100644 --- a/aurweb/models/official_provider.py +++ b/aurweb/models/official_provider.py @@ -1,6 +1,6 @@ -from sqlalchemy import Column, Integer from sqlalchemy.exc import IntegrityError +from aurweb import schema from aurweb.models.declarative import Base # TODO: Fix this! Official packages aren't from aur.archlinux.org... @@ -8,11 +8,9 @@ OFFICIAL_BASE = "https://aur.archlinux.org" class OfficialProvider(Base): - __tablename__ = "OfficialProviders" - - ID = Column(Integer, primary_key=True) - - __mapper_args__ = {"primary_key": [ID]} + __table__ = schema.OfficialProviders + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/package.py b/aurweb/models/package.py index ef119f3c..8f82dadd 100644 --- a/aurweb/models/package.py +++ b/aurweb/models/package.py @@ -1,24 +1,19 @@ -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.package_base import PackageBase as _PackageBase class Package(Base): - __tablename__ = "Packages" + __table__ = schema.Packages + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} - ID = Column(Integer, primary_key=True) - - PackageBaseID = Column( - Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), - nullable=False) PackageBase = relationship( _PackageBase, backref=backref("packages", lazy="dynamic"), - foreign_keys=[PackageBaseID]) - - __mapper_args__ = {"primary_key": [ID]} + foreign_keys=[__table__.c.PackageBaseID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/package_base.py b/aurweb/models/package_base.py index e6f28050..8c88b7b5 100644 --- a/aurweb/models/package_base.py +++ b/aurweb/models/package_base.py @@ -1,38 +1,33 @@ from datetime import datetime -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.user import User as _User class PackageBase(Base): - __tablename__ = "PackageBases" + __table__ = schema.PackageBases + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} - FlaggerUID = Column(Integer, - ForeignKey("Users.ID", ondelete="SET NULL")) Flagger = relationship( _User, backref=backref("flagged_bases", lazy="dynamic"), - foreign_keys=[FlaggerUID]) + foreign_keys=[__table__.c.FlaggerUID]) - SubmitterUID = Column(Integer, - ForeignKey("Users.ID", ondelete="SET NULL")) Submitter = relationship( _User, backref=backref("submitted_bases", lazy="dynamic"), - foreign_keys=[SubmitterUID]) + foreign_keys=[__table__.c.SubmitterUID]) - MaintainerUID = Column(Integer, - ForeignKey("Users.ID", ondelete="SET NULL")) Maintainer = relationship( _User, backref=backref("maintained_bases", lazy="dynamic"), - foreign_keys=[MaintainerUID]) + foreign_keys=[__table__.c.MaintainerUID]) - PackagerUID = Column(Integer, ForeignKey("Users.ID", ondelete="SET NULL")) Packager = relationship( _User, backref=backref("package_bases", lazy="dynamic"), - foreign_keys=[PackagerUID]) + foreign_keys=[__table__.c.PackagerUID]) # A set used to check for floatable values. TO_FLOAT = {"Popularity"} diff --git a/aurweb/models/package_blacklist.py b/aurweb/models/package_blacklist.py index 4ba3f308..0f8f0cee 100644 --- a/aurweb/models/package_blacklist.py +++ b/aurweb/models/package_blacklist.py @@ -1,15 +1,13 @@ -from sqlalchemy import Column, Integer from sqlalchemy.exc import IntegrityError +from aurweb import schema from aurweb.models.declarative import Base class PackageBlacklist(Base): - __tablename__ = "PackageBlacklist" - - ID = Column(Integer, primary_key=True) - - __mapper_args__ = {"primary_key": [ID]} + __table__ = schema.PackageBlacklist + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/package_comaintainer.py b/aurweb/models/package_comaintainer.py index 2f77782c..7641fb43 100644 --- a/aurweb/models/package_comaintainer.py +++ b/aurweb/models/package_comaintainer.py @@ -1,30 +1,26 @@ -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.package_base import PackageBase as _PackageBase from aurweb.models.user import User as _User class PackageComaintainer(Base): - __tablename__ = "PackageComaintainers" + __table__ = schema.PackageComaintainers + __tablename__ = __table__.name + __mapper_args__ = { + "primary_key": [__table__.c.UsersID, __table__.c.PackageBaseID] + } - UsersID = Column( - Integer, ForeignKey("Users.ID", ondelete="CASCADE"), - nullable=False) User = relationship( _User, backref=backref("comaintained", lazy="dynamic"), - foreign_keys=[UsersID]) + foreign_keys=[__table__.c.UsersID]) - PackageBaseID = Column( - Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), - nullable=False) PackageBase = relationship( _PackageBase, backref=backref("comaintainers", lazy="dynamic"), - foreign_keys=[PackageBaseID]) - - __mapper_args__ = {"primary_key": [UsersID, PackageBaseID]} + foreign_keys=[__table__.c.PackageBaseID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/package_comment.py b/aurweb/models/package_comment.py index a511df9b..2a529c9c 100644 --- a/aurweb/models/package_comment.py +++ b/aurweb/models/package_comment.py @@ -1,43 +1,33 @@ -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.package_base import PackageBase as _PackageBase from aurweb.models.user import User as _User class PackageComment(Base): - __tablename__ = "PackageComments" + __table__ = schema.PackageComments + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} - ID = Column(Integer, primary_key=True) - - PackageBaseID = Column( - Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), - nullable=False) PackageBase = relationship( _PackageBase, backref=backref("comments", lazy="dynamic", cascade="all, delete"), - foreign_keys=[PackageBaseID]) + foreign_keys=[__table__.c.PackageBaseID]) - UsersID = Column(Integer, ForeignKey("Users.ID", ondelete="SET NULL")) User = relationship( _User, backref=backref("package_comments", lazy="dynamic"), - foreign_keys=[UsersID]) + foreign_keys=[__table__.c.UsersID]) - EditedUsersID = Column( - Integer, ForeignKey("Users.ID", ondelete="SET NULL")) Editor = relationship( _User, backref=backref("edited_comments", lazy="dynamic"), - foreign_keys=[EditedUsersID]) + foreign_keys=[__table__.c.EditedUsersID]) - DelUsersID = Column( - Integer, ForeignKey("Users.ID", ondelete="SET NULL")) Deleter = relationship( _User, backref=backref("deleted_comments", lazy="dynamic"), - foreign_keys=[DelUsersID]) - - __mapper_args__ = {"primary_key": [ID]} + foreign_keys=[__table__.c.DelUsersID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/package_dependency.py b/aurweb/models/package_dependency.py index 3f4e2baa..edaa6538 100644 --- a/aurweb/models/package_dependency.py +++ b/aurweb/models/package_dependency.py @@ -1,34 +1,28 @@ -from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.dependency_type import DependencyType as _DependencyType from aurweb.models.package import Package as _Package class PackageDependency(Base): - __tablename__ = "PackageDepends" + __table__ = schema.PackageDepends + __tablename__ = __table__.name + __mapper_args__ = { + "primary_key": [__table__.c.PackageID, __table__.c.DepName] + } - PackageID = Column( - Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), - nullable=False) Package = relationship( _Package, backref=backref("package_dependencies", lazy="dynamic", cascade="all, delete"), - foreign_keys=[PackageID]) + foreign_keys=[__table__.c.PackageID]) - DepTypeID = Column( - Integer, ForeignKey("DependencyTypes.ID", ondelete="NO ACTION"), - nullable=False) DependencyType = relationship( _DependencyType, backref=backref("package_dependencies", lazy="dynamic"), - foreign_keys=[DepTypeID]) - - DepName = Column(String(255), nullable=False) - - __mapper_args__ = {"primary_key": [PackageID, DepName]} + foreign_keys=[__table__.c.DepTypeID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/package_group.py b/aurweb/models/package_group.py index c1d1e4f8..3b6db37d 100644 --- a/aurweb/models/package_group.py +++ b/aurweb/models/package_group.py @@ -1,28 +1,26 @@ -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.group import Group as _Group from aurweb.models.package import Package as _Package class PackageGroup(Base): - __tablename__ = "PackageGroups" + __table__ = schema.PackageGroups + __tablename__ = __table__.name + __mapper_args__ = { + "primary_key": [__table__.c.PackageID, __table__.c.GroupID] + } - PackageID = Column(Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), - primary_key=True, nullable=True) Package = relationship( _Package, backref=backref("package_groups", lazy="dynamic"), - foreign_keys=[PackageID]) + foreign_keys=[__table__.c.PackageID]) - GroupID = Column(Integer, ForeignKey("Groups.ID", ondelete="CASCADE"), - primary_key=True, nullable=True) Group = relationship( _Group, backref=backref("package_groups", lazy="dynamic"), - foreign_keys=[GroupID]) - - __mapper_args__ = {"primary_key": [PackageID, GroupID]} + foreign_keys=[__table__.c.GroupID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/package_keyword.py b/aurweb/models/package_keyword.py index 25bd340b..581aafdc 100644 --- a/aurweb/models/package_keyword.py +++ b/aurweb/models/package_keyword.py @@ -1,27 +1,22 @@ -from sqlalchemy import Column, ForeignKey, Integer, String, text from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.package_base import PackageBase as _PackageBase class PackageKeyword(Base): - __tablename__ = "PackageKeywords" + __table__ = schema.PackageKeywords + __tablename__ = __table__.name + __mapper_args__ = { + "primary_key": [__table__.c.PackageBaseID, __table__.c.Keyword] + } - PackageBaseID = Column( - Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), - primary_key=True, nullable=True) PackageBase = relationship( _PackageBase, backref=backref("keywords", lazy="dynamic", cascade="all, delete"), - foreign_keys=[PackageBaseID]) - - Keyword = Column( - String(255), primary_key=True, nullable=False, - server_default=text("''")) - - __mapper_args__ = {"primary_key": [PackageBaseID, Keyword]} + foreign_keys=[__table__.c.PackageBaseID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/package_license.py b/aurweb/models/package_license.py index db12a7c3..43dd0339 100644 --- a/aurweb/models/package_license.py +++ b/aurweb/models/package_license.py @@ -1,32 +1,28 @@ -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.license import License as _License from aurweb.models.package import Package as _Package class PackageLicense(Base): - __tablename__ = "PackageLicenses" + __table__ = schema.PackageLicenses + __tablename__ = __table__.name + __mapper_args__ = { + "primary_key": [__table__.c.PackageID, __table__.c.LicenseID] + } - PackageID = Column( - Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), - primary_key=True, nullable=True) Package = relationship( _Package, backref=backref("package_licenses", lazy="dynamic", cascade="all, delete"), - foreign_keys=[PackageID]) + foreign_keys=[__table__.c.PackageID]) - LicenseID = Column( - Integer, ForeignKey("Licenses.ID", ondelete="CASCADE"), - primary_key=True, nullable=True) License = relationship( _License, backref=backref("package_licenses", lazy="dynamic", cascade="all, delete"), - foreign_keys=[LicenseID]) - - __mapper_args__ = {"primary_key": [PackageID, LicenseID]} + foreign_keys=[__table__.c.LicenseID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/package_notification.py b/aurweb/models/package_notification.py index 221067e1..97dbe38f 100644 --- a/aurweb/models/package_notification.py +++ b/aurweb/models/package_notification.py @@ -1,31 +1,27 @@ -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.package_base import PackageBase as _PackageBase from aurweb.models.user import User as _User class PackageNotification(Base): - __tablename__ = "PackageNotifications" + __table__ = schema.PackageNotifications + __tablename__ = __table__.name + __mapper_args__ = { + "primary_key": [__table__.c.UserID, __table__.c.PackageBaseID] + } - UserID = Column( - Integer, ForeignKey("Users.ID", ondelete="CASCADE"), - nullable=False) User = relationship( _User, backref=backref("notifications", lazy="dynamic"), - foreign_keys=[UserID]) + foreign_keys=[__table__.c.UserID]) - PackageBaseID = Column( - Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), - nullable=False) PackageBase = relationship( _PackageBase, backref=backref("notifications", lazy="dynamic"), - foreign_keys=[PackageBaseID]) - - __mapper_args__ = {"primary_key": [UserID, PackageBaseID]} + foreign_keys=[__table__.c.PackageBaseID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/package_relation.py b/aurweb/models/package_relation.py index e79a90d6..eb6caa84 100644 --- a/aurweb/models/package_relation.py +++ b/aurweb/models/package_relation.py @@ -1,33 +1,27 @@ -from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.package import Package as _Package from aurweb.models.relation_type import RelationType as _RelationType class PackageRelation(Base): - __tablename__ = "PackageRelations" + __table__ = schema.PackageRelations + __tablename__ = __table__.name + __mapper_args__ = { + "primary_key": [__table__.c.PackageID, __table__.c.RelName] + } - PackageID = Column( - Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), - nullable=False) Package = relationship( _Package, backref=backref("package_relations", lazy="dynamic", cascade="all, delete"), - foreign_keys=[PackageID]) + foreign_keys=[__table__.c.PackageID]) - RelTypeID = Column( - Integer, ForeignKey("RelationTypes.ID", ondelete="CASCADE"), - nullable=False) RelationType = relationship( _RelationType, backref=backref("package_relations", lazy="dynamic"), - foreign_keys=[RelTypeID]) - - RelName = Column(String(255), unique=True) - - __mapper_args__ = {"primary_key": [PackageID, RelName]} + foreign_keys=[__table__.c.RelTypeID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/package_request.py b/aurweb/models/package_request.py index f600566c..9669ec46 100644 --- a/aurweb/models/package_request.py +++ b/aurweb/models/package_request.py @@ -1,7 +1,7 @@ -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.package_base import PackageBase as _PackageBase from aurweb.models.request_type import RequestType as _RequestType @@ -20,34 +20,25 @@ REJECTED_ID = 3 class PackageRequest(Base): - __tablename__ = "PackageRequests" + __table__ = schema.PackageRequests + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} - ID = Column(Integer, primary_key=True) - - ReqTypeID = Column( - Integer, ForeignKey("RequestTypes.ID", ondelete="NO ACTION"), - nullable=False) RequestType = relationship( _RequestType, backref=backref("package_requests", lazy="dynamic"), - foreign_keys=[ReqTypeID]) + foreign_keys=[__table__.c.ReqTypeID]) - UsersID = Column(Integer, ForeignKey("Users.ID", ondelete="SET NULL")) User = relationship( _User, backref=backref("package_requests", lazy="dynamic"), - foreign_keys=[UsersID]) + foreign_keys=[__table__.c.UsersID]) - PackageBaseID = Column( - Integer, ForeignKey("PackageBases.ID", ondelete="SET NULL")) PackageBase = relationship( _PackageBase, backref=backref("requests", lazy="dynamic"), - foreign_keys=[PackageBaseID]) + foreign_keys=[__table__.c.PackageBaseID]) - ClosedUID = Column(Integer, ForeignKey("Users.ID", ondelete="SET NULL")) Closer = relationship( _User, backref=backref("closed_requests", lazy="dynamic"), - foreign_keys=[ClosedUID]) - - __mapper_args__ = {"primary_key": [ID]} + foreign_keys=[__table__.c.ClosedUID]) STATUS_DISPLAY = { PENDING_ID: PENDING, diff --git a/aurweb/models/package_source.py b/aurweb/models/package_source.py index db983272..59046bbd 100644 --- a/aurweb/models/package_source.py +++ b/aurweb/models/package_source.py @@ -1,22 +1,22 @@ -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.package import Package as _Package class PackageSource(Base): - __tablename__ = "PackageSources" + __table__ = schema.PackageSources + __tablename__ = __table__.name + __mapper_args__ = { + "primary_key": [__table__.c.PackageID] + } - PackageID = Column(Integer, ForeignKey("Packages.ID", ondelete="CASCADE"), - nullable=False) Package = relationship( _Package, backref=backref("package_sources", lazy="dynamic", cascade="all, delete"), - foreign_keys=[PackageID]) - - __mapper_args__ = {"primary_key": [PackageID]} + foreign_keys=[__table__.c.PackageID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/package_vote.py b/aurweb/models/package_vote.py index 2d70be16..7221d527 100644 --- a/aurweb/models/package_vote.py +++ b/aurweb/models/package_vote.py @@ -1,30 +1,26 @@ -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.package_base import PackageBase as _PackageBase from aurweb.models.user import User as _User class PackageVote(Base): - __tablename__ = "PackageVotes" + __table__ = schema.PackageVotes + __tablename__ = __table__.name + __mapper_args__ = { + "primary_key": [__table__.c.UsersID, __table__.c.PackageBaseID] + } - UsersID = Column( - Integer, ForeignKey("Users.ID", ondelete="CASCADE"), - nullable=False) User = relationship( _User, backref=backref("package_votes", lazy="dynamic"), - foreign_keys=[UsersID]) + foreign_keys=[__table__.c.UsersID]) - PackageBaseID = Column( - Integer, ForeignKey("PackageBases.ID", ondelete="CASCADE"), - nullable=False) PackageBase = relationship( _PackageBase, backref=backref("package_votes", lazy="dynamic"), - foreign_keys=[PackageBaseID]) - - __mapper_args__ = {"primary_key": [UsersID, PackageBaseID]} + foreign_keys=[__table__.c.PackageBaseID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/relation_type.py b/aurweb/models/relation_type.py index 71b6adbb..b52c91ec 100644 --- a/aurweb/models/relation_type.py +++ b/aurweb/models/relation_type.py @@ -1,27 +1,19 @@ -from sqlalchemy import Column, Integer - -from aurweb import db +from aurweb import schema from aurweb.models.declarative import Base CONFLICTS = "conflicts" PROVIDES = "provides" REPLACES = "replaces" +CONFLICTS_ID = 1 +PROVIDES_ID = 2 +REPLACES_ID = 3 + class RelationType(Base): - __tablename__ = "RelationTypes" - - ID = Column(Integer, primary_key=True) - - __mapper_args__ = {"primary_key": [ID]} + __table__ = schema.RelationTypes + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} def __init__(self, Name: str = None): self.Name = Name - - -CONFLICTS_ID = db.query(RelationType).filter( - RelationType.Name == CONFLICTS).first().ID -PROVIDES_ID = db.query(RelationType).filter( - RelationType.Name == PROVIDES).first().ID -REPLACES_ID = db.query(RelationType).filter( - RelationType.Name == REPLACES).first().ID diff --git a/aurweb/models/request_type.py b/aurweb/models/request_type.py index 4578464c..cabab3d2 100644 --- a/aurweb/models/request_type.py +++ b/aurweb/models/request_type.py @@ -1,25 +1,20 @@ -from sqlalchemy import Column, Integer - -from aurweb import db +from aurweb import schema from aurweb.models.declarative import Base DELETION = "deletion" ORPHAN = "orphan" MERGE = "merge" +DELETION_ID = 1 +ORPHAN_ID = 2 +MERGE_ID = 3 + class RequestType(Base): - __tablename__ = "RequestTypes" - - ID = Column(Integer, primary_key=True) - - __mapper_args__ = {"primary_key": [ID]} + __table__ = schema.RequestTypes + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} def name_display(self) -> str: """ Return the Name column with its first char capitalized. """ return self.Name.title() - - -DELETION_ID = db.query(RequestType, RequestType.Name == DELETION).first().ID -ORPHAN_ID = db.query(RequestType, RequestType.Name == ORPHAN).first().ID -MERGE_ID = db.query(RequestType, RequestType.Name == MERGE).first().ID diff --git a/aurweb/models/session.py b/aurweb/models/session.py index a4034678..96f88d85 100644 --- a/aurweb/models/session.py +++ b/aurweb/models/session.py @@ -1,23 +1,20 @@ -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.db import make_random_value, query from aurweb.models.declarative import Base from aurweb.models.user import User as _User class Session(Base): - __tablename__ = "Sessions" + __table__ = schema.Sessions + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.UsersID]} - UsersID = Column( - Integer, ForeignKey("Users.ID", ondelete="CASCADE"), - nullable=False) User = relationship( _User, backref=backref("session", uselist=False), - foreign_keys=[UsersID]) - - __mapper_args__ = {"primary_key": [UsersID]} + foreign_keys=[__table__.c.UsersID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/ssh_pub_key.py b/aurweb/models/ssh_pub_key.py index 268a585b..789be629 100644 --- a/aurweb/models/ssh_pub_key.py +++ b/aurweb/models/ssh_pub_key.py @@ -3,30 +3,23 @@ import tempfile from subprocess import PIPE, Popen -from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base class SSHPubKey(Base): - __tablename__ = "SSHPubKeys" + __table__ = schema.SSHPubKeys + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.Fingerprint]} - UserID = Column( - Integer, ForeignKey("Users.ID", ondelete="CASCADE"), - nullable=False) User = relationship( "User", backref=backref("ssh_pub_key", uselist=False), - foreign_keys=[UserID]) - - Fingerprint = Column(String(44), primary_key=True) - - __mapper_args__ = {"primary_key": Fingerprint} + foreign_keys=[__table__.c.UserID]) def __init__(self, **kwargs): - self.UserID = kwargs.get("UserID") - self.Fingerprint = kwargs.get("Fingerprint") - self.PubKey = kwargs.get("PubKey") + super().__init__(**kwargs) def get_fingerprint(pubkey): diff --git a/aurweb/models/term.py b/aurweb/models/term.py index 0985cd76..59534bbc 100644 --- a/aurweb/models/term.py +++ b/aurweb/models/term.py @@ -1,15 +1,13 @@ -from sqlalchemy import Column, Integer from sqlalchemy.exc import IntegrityError +from aurweb import schema from aurweb.models.declarative import Base class Term(Base): - __tablename__ = "Terms" - - ID = Column(Integer, primary_key=True) - - __mapper_args__ = {"primary_key": [ID]} + __table__ = schema.Terms + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/tu_vote.py b/aurweb/models/tu_vote.py index 634c041e..efb23b19 100644 --- a/aurweb/models/tu_vote.py +++ b/aurweb/models/tu_vote.py @@ -1,28 +1,26 @@ -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.tu_voteinfo import TUVoteInfo as _TUVoteInfo from aurweb.models.user import User as _User class TUVote(Base): - __tablename__ = "TU_Votes" + __table__ = schema.TU_Votes + __tablename__ = __table__.name + __mapper_args__ = { + "primary_key": [__table__.c.VoteID, __table__.c.UserID] + } - VoteID = Column(Integer, ForeignKey("TU_VoteInfo.ID", ondelete="CASCADE"), - nullable=False) VoteInfo = relationship( _TUVoteInfo, backref=backref("tu_votes", lazy="dynamic"), - foreign_keys=[VoteID]) + foreign_keys=[__table__.c.VoteID]) - UserID = Column(Integer, ForeignKey("Users.ID", ondelete="CASCADE"), - nullable=False) User = relationship( _User, backref=backref("tu_votes", lazy="dynamic"), - foreign_keys=[UserID]) - - __mapper_args__ = {"primary_key": [VoteID, UserID]} + foreign_keys=[__table__.c.UserID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/tu_voteinfo.py b/aurweb/models/tu_voteinfo.py index da43b097..35675ccc 100644 --- a/aurweb/models/tu_voteinfo.py +++ b/aurweb/models/tu_voteinfo.py @@ -2,27 +2,22 @@ import typing from datetime import datetime -from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship +from aurweb import schema from aurweb.models.declarative import Base from aurweb.models.user import User as _User class TUVoteInfo(Base): - __tablename__ = "TU_VoteInfo" + __table__ = schema.TU_VoteInfo + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} - ID = Column(Integer, primary_key=True) - - SubmitterID = Column( - Integer, ForeignKey("Users.ID", ondelete="CASCADE"), - nullable=False) Submitter = relationship( _User, backref=backref("tu_voteinfo_set", lazy="dynamic"), - foreign_keys=[SubmitterID]) - - __mapper_args__ = {"primary_key": [ID]} + foreign_keys=[__table__.c.SubmitterID]) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/aurweb/models/user.py b/aurweb/models/user.py index e4223144..8db34c38 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -5,14 +5,14 @@ from datetime import datetime import bcrypt from fastapi import Request -from sqlalchemy import Column, ForeignKey, Integer, String, or_, text +from sqlalchemy import or_ from sqlalchemy.orm import backref, relationship import aurweb.config import aurweb.models.account_type import aurweb.schema -from aurweb import db +from aurweb import db, schema from aurweb.models.account_type import AccountType as _AccountType from aurweb.models.ban import is_banned from aurweb.models.declarative import Base @@ -22,23 +22,16 @@ SALT_ROUNDS_DEFAULT = 12 class User(Base): """ An ORM model of a single Users record. """ - __tablename__ = "Users" + __table__ = schema.Users + __tablename__ = __table__.name + __mapper_args__ = {"primary_key": [__table__.c.ID]} - ID = Column(Integer, primary_key=True) - - AccountTypeID = Column( - Integer, ForeignKey("AccountTypes.ID", ondelete="NO ACTION"), - nullable=False, server_default=text("1")) AccountType = relationship( _AccountType, backref=backref("users", lazy="dynamic"), - foreign_keys=[AccountTypeID], + foreign_keys=[__table__.c.AccountTypeID], uselist=False) - Passwd = Column(String(255), default=str()) - - __mapper_args__ = {"primary_key": [ID]} - # High-level variables used to track authentication (not in DB). authenticated = False nonce = None @@ -49,7 +42,7 @@ class User(Base): SALT_ROUNDS_DEFAULT) def __init__(self, Passwd: str = str(), **kwargs): - super().__init__(**kwargs) + super().__init__(**kwargs, Passwd=str()) # Run this again in the constructor in case we rehashed config. self.salt_rounds = aurweb.config.getint("options", "salt_rounds", From 3517862ecdaf665693bbfb6e06fe547249d34005 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 8 Nov 2021 18:46:21 -0800 Subject: [PATCH 530/844] change(poetry): use kevr@upgrade-starlette-0.17.0 as fastapi source Starlette 0.16.0 has a pretty bad bug in terms of logging which has been fixed in the 0.17.0 release. That being said, FastAPI has not yet merged a request at https://github.com/tiangolo/fastapi/pull/4145 which resolves this dependency resolution so we can use the updated starlette package. kevr has forked the pull request in question and we are using it for now in our poetry dependencies to get ahead of the game. When FastAPI upstream is updated to support 0.17.0, we'll need to switch this back to using upstream's source. Signed-off-by: Kevin Morris --- poetry.lock | 317 +++++++++++++++++++++++++++---------------------- pyproject.toml | 2 +- 2 files changed, 177 insertions(+), 142 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9f528d12..37e2f8f9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -165,7 +165,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "coverage" -version = "6.0.2" +version = "6.1.1" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -213,12 +213,15 @@ trio = ["trio (>=0.14.0)", "sniffio (>=1.1)"] [[package]] name = "dunamai" -version = "1.6.0" +version = "1.7.0" description = "Dynamic version generation" category = "main" optional = false python-versions = ">=3.5,<4.0" +[package.dependencies] +packaging = ">=20.9" + [[package]] name = "email-validator" version = "1.1.3" @@ -256,10 +259,11 @@ description = "FastAPI framework, high performance, easy to learn, fast to code, category = "main" optional = false python-versions = ">=3.6.1" +develop = false [package.dependencies] pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0" -starlette = "0.16.0" +starlette = "0.17.0" [package.extras] all = ["requests (>=2.24.0,<3.0.0)", "jinja2 (>=2.11.2,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "itsdangerous (>=1.1.0,<3.0.0)", "pyyaml (>=5.3.1,<6.0.0)", "ujson (>=4.0.1,<5.0.0)", "orjson (>=3.2.1,<4.0.0)", "email_validator (>=1.1.1,<2.0.0)", "uvicorn[standard] (>=0.12.0,<0.16.0)"] @@ -267,6 +271,12 @@ dev = ["python-jose[cryptography] (>=3.3.0,<4.0.0)", "passlib[bcrypt] (>=1.7.2,< doc = ["mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=7.1.9,<8.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "typer-cli (>=0.0.12,<0.0.13)", "pyyaml (>=5.3.1,<6.0.0)"] test = ["pytest (>=6.2.4,<7.0.0)", "pytest-cov (>=2.12.0,<4.0.0)", "mypy (==0.910)", "flake8 (>=3.8.3,<4.0.0)", "black (==21.9b0)", "isort (>=5.0.6,<6.0.0)", "requests (>=2.24.0,<3.0.0)", "httpx (>=0.14.0,<0.19.0)", "email_validator (>=1.1.1,<2.0.0)", "sqlalchemy (>=1.3.18,<1.5.0)", "peewee (>=3.13.3,<4.0.0)", "databases[sqlite] (>=0.3.2,<0.6.0)", "orjson (>=3.2.1,<4.0.0)", "ujson (>=4.0.1,<5.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "flask (>=1.1.2,<3.0.0)", "anyio[trio] (>=3.2.1,<4.0.0)", "types-ujson (==0.1.1)", "types-orjson (==3.6.0)", "types-dataclasses (==0.1.7)"] +[package.source] +type = "git" +url = "https://github.com/kevr/fastapi.git" +reference = "upgrade-starlette-0.17.0" +resolved_reference = "5d2d79e6bafd86564c318b7f99153132cd6ca466" + [[package]] name = "feedgen" version = "0.9.0" @@ -428,7 +438,7 @@ python-versions = "*" [[package]] name = "isort" -version = "5.9.3" +version = "5.10.0" description = "A Python utility / library to sort Python imports." category = "dev" optional = false @@ -464,7 +474,7 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "lxml" -version = "4.6.3" +version = "4.6.4" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." category = "main" optional = false @@ -544,14 +554,14 @@ python-versions = ">=3.7" [[package]] name = "packaging" -version = "21.0" +version = "21.2" description = "Core utilities for Python packages" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -pyparsing = ">=2.0.2" +pyparsing = ">=2.0.2,<3" [[package]] name = "paginate" @@ -619,7 +629,7 @@ prometheus-client = ">=0.8.0,<1.0.0" [[package]] name = "protobuf" -version = "3.19.0" +version = "3.19.1" description = "Protocol Buffers" category = "main" optional = false @@ -627,11 +637,11 @@ python-versions = ">=3.5" [[package]] name = "py" -version = "1.10.0" +version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pycodestyle" @@ -643,7 +653,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pycparser" -version = "2.20" +version = "2.21" description = "C parser in Python" category = "main" optional = false @@ -743,7 +753,7 @@ testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtuale [[package]] name = "pytest-tap" -version = "3.2" +version = "3.3" description = "Test Anything Protocol (TAP) reporting plugin for pytest" category = "dev" optional = false @@ -876,7 +886,7 @@ sqlcipher = ["sqlcipher3-binary"] [[package]] name = "starlette" -version = "0.16.0" +version = "0.17.0" description = "The little ASGI library that shines." category = "main" optional = false @@ -886,7 +896,7 @@ python-versions = ">=3.6" anyio = ">=3.0.0,<4" [package.extras] -full = ["itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests", "graphene"] +full = ["itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"] [[package]] name = "tap.py" @@ -909,7 +919,7 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "tomli" -version = "1.2.1" +version = "1.2.2" description = "A lil' TOML parser" category = "dev" optional = false @@ -993,7 +1003,7 @@ h11 = ">=0.9.0,<1" [metadata] lock-version = "1.1" python-versions = ">=3.9,<3.10" -content-hash = "569b0489389b884d269458f8e4252efcf3ebbbaa5fa77b6d09d7f0cdbda53362" +content-hash = "356b37d545d78b8aa1e1939f42522207bcf79526abe8193308c5a2955897d6fd" [metadata.files] aiofiles = [ @@ -1106,39 +1116,55 @@ colorama = [ {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] coverage = [ - {file = "coverage-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1549e1d08ce38259de2bc3e9a0d5f3642ff4a8f500ffc1b2df73fd621a6cdfc0"}, - {file = "coverage-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcae10fccb27ca2a5f456bf64d84110a5a74144be3136a5e598f9d9fb48c0caa"}, - {file = "coverage-6.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:53a294dc53cfb39c74758edaa6305193fb4258a30b1f6af24b360a6c8bd0ffa7"}, - {file = "coverage-6.0.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8251b37be1f2cd9c0e5ccd9ae0380909c24d2a5ed2162a41fcdbafaf59a85ebd"}, - {file = "coverage-6.0.2-cp310-cp310-win32.whl", hash = "sha256:db42baa892cba723326284490283a68d4de516bfb5aaba369b4e3b2787a778b7"}, - {file = "coverage-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:bbffde2a68398682623d9dd8c0ca3f46fda074709b26fcf08ae7a4c431a6ab2d"}, - {file = "coverage-6.0.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:60e51a3dd55540bec686d7fff61b05048ca31e804c1f32cbb44533e6372d9cc3"}, - {file = "coverage-6.0.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a6a9409223a27d5ef3cca57dd7cd4dfcb64aadf2fad5c3b787830ac9223e01a"}, - {file = "coverage-6.0.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4b34ae4f51bbfa5f96b758b55a163d502be3dcb24f505d0227858c2b3f94f5b9"}, - {file = "coverage-6.0.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3bbda1b550e70fa6ac40533d3f23acd4f4e9cb4e6e77251ce77fdf41b3309fb2"}, - {file = "coverage-6.0.2-cp36-cp36m-win32.whl", hash = "sha256:4e28d2a195c533b58fc94a12826f4431726d8eb029ac21d874345f943530c122"}, - {file = "coverage-6.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:a82d79586a0a4f5fd1cf153e647464ced402938fbccb3ffc358c7babd4da1dd9"}, - {file = "coverage-6.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3be1206dc09fb6298de3fce70593e27436862331a85daee36270b6d0e1c251c4"}, - {file = "coverage-6.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9cd3828bbe1a40070c11fe16a51df733fd2f0cb0d745fb83b7b5c1f05967df7"}, - {file = "coverage-6.0.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d036dc1ed8e1388e995833c62325df3f996675779541f682677efc6af71e96cc"}, - {file = "coverage-6.0.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:04560539c19ec26995ecfb3d9307ff154fbb9a172cb57e3b3cfc4ced673103d1"}, - {file = "coverage-6.0.2-cp37-cp37m-win32.whl", hash = "sha256:e4fb7ced4d9dec77d6cf533acfbf8e1415fe799430366affb18d69ee8a3c6330"}, - {file = "coverage-6.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:77b1da5767ed2f44611bc9bc019bc93c03fa495728ec389759b6e9e5039ac6b1"}, - {file = "coverage-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:61b598cbdbaae22d9e34e3f675997194342f866bb1d781da5d0be54783dce1ff"}, - {file = "coverage-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36e9040a43d2017f2787b28d365a4bb33fcd792c7ff46a047a04094dc0e2a30d"}, - {file = "coverage-6.0.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9f1627e162e3864a596486774876415a7410021f4b67fd2d9efdf93ade681afc"}, - {file = "coverage-6.0.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e7a0b42db2a47ecb488cde14e0f6c7679a2c5a9f44814393b162ff6397fcdfbb"}, - {file = "coverage-6.0.2-cp38-cp38-win32.whl", hash = "sha256:a1b73c7c4d2a42b9d37dd43199c5711d91424ff3c6c22681bc132db4a4afec6f"}, - {file = "coverage-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:1db67c497688fd4ba85b373b37cc52c50d437fd7267520ecd77bddbd89ea22c9"}, - {file = "coverage-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f2f184bf38e74f152eed7f87e345b51f3ab0b703842f447c22efe35e59942c24"}, - {file = "coverage-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1cf1deb3d5544bd942356364a2fdc8959bad2b6cf6eb17f47d301ea34ae822"}, - {file = "coverage-6.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ad9b8c1206ae41d46ec7380b78ba735ebb77758a650643e841dd3894966c31d0"}, - {file = "coverage-6.0.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:381d773d896cc7f8ba4ff3b92dee4ed740fb88dfe33b6e42efc5e8ab6dfa1cfe"}, - {file = "coverage-6.0.2-cp39-cp39-win32.whl", hash = "sha256:424c44f65e8be58b54e2b0bd1515e434b940679624b1b72726147cfc6a9fc7ce"}, - {file = "coverage-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:abbff240f77347d17306d3201e14431519bf64495648ca5a49571f988f88dee9"}, - {file = "coverage-6.0.2-pp36-none-any.whl", hash = "sha256:7092eab374346121805fb637572483270324407bf150c30a3b161fc0c4ca5164"}, - {file = "coverage-6.0.2-pp37-none-any.whl", hash = "sha256:30922626ce6f7a5a30bdba984ad21021529d3d05a68b4f71ea3b16bda35b8895"}, - {file = "coverage-6.0.2.tar.gz", hash = "sha256:6807947a09510dc31fa86f43595bf3a14017cd60bf633cc746d52141bfa6b149"}, + {file = "coverage-6.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:42a1fb5dee3355df90b635906bb99126faa7936d87dfc97eacc5293397618cb7"}, + {file = "coverage-6.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a00284dbfb53b42e35c7dd99fc0e26ef89b4a34efff68078ed29d03ccb28402a"}, + {file = "coverage-6.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:51a441011a30d693e71dea198b2a6f53ba029afc39f8e2aeb5b77245c1b282ef"}, + {file = "coverage-6.1.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e76f017b6d4140a038c5ff12be1581183d7874e41f1c0af58ecf07748d36a336"}, + {file = "coverage-6.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7833c872718dc913f18e51ee97ea0dece61d9930893a58b20b3daf09bb1af6b6"}, + {file = "coverage-6.1.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8186b5a4730c896cbe1e4b645bdc524e62d874351ae50e1db7c3e9f5dc81dc26"}, + {file = "coverage-6.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bbca34dca5a2d60f81326d908d77313816fad23d11b6069031a3d6b8c97a54f9"}, + {file = "coverage-6.1.1-cp310-cp310-win32.whl", hash = "sha256:72bf437d54186d104388cbae73c9f2b0f8a3e11b6e8d7deb593bd14625c96026"}, + {file = "coverage-6.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:994ce5a7b3d20981b81d83618aa4882f955bfa573efdbef033d5632b58597ba9"}, + {file = "coverage-6.1.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ab6a0fe4c96f8058d41948ddf134420d3ef8c42d5508b5a341a440cce7a37a1d"}, + {file = "coverage-6.1.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10ab138b153e4cc408b43792cb7f518f9ee02f4ff55cd1ab67ad6fd7e9905c7e"}, + {file = "coverage-6.1.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7e083d32965d2eb6638a77e65b622be32a094fdc0250f28ce6039b0732fbcaa8"}, + {file = "coverage-6.1.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:359a32515e94e398a5c0fa057e5887a42e647a9502d8e41165cf5cb8d3d1ca67"}, + {file = "coverage-6.1.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:bf656cd74ff7b4ed7006cdb2a6728150aaad69c7242b42a2a532f77b63ea233f"}, + {file = "coverage-6.1.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:dc5023be1c2a8b0a0ab5e31389e62c28b2453eb31dd069f4b8d1a0f9814d951a"}, + {file = "coverage-6.1.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:557594a50bfe3fb0b1b57460f6789affe8850ad19c1acf2d14a3e12b2757d489"}, + {file = "coverage-6.1.1-cp36-cp36m-win32.whl", hash = "sha256:9eb0a1923354e0fdd1c8a6f53f5db2e6180d670e2b587914bf2e79fa8acfd003"}, + {file = "coverage-6.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:04a92a6cf9afd99f9979c61348ec79725a9f9342fb45e63c889e33c04610d97b"}, + {file = "coverage-6.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:479228e1b798d3c246ac89b09897ee706c51b3e5f8f8d778067f38db73ccc717"}, + {file = "coverage-6.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78287731e3601ea5ce9d6468c82d88a12ef8fe625d6b7bdec9b45d96c1ad6533"}, + {file = "coverage-6.1.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c95257aa2ccf75d3d91d772060538d5fea7f625e48157f8ca44594f94d41cb33"}, + {file = "coverage-6.1.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9ad5895938a894c368d49d8470fe9f519909e5ebc6b8f8ea5190bd0df6aa4271"}, + {file = "coverage-6.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:326d944aad0189603733d646e8d4a7d952f7145684da973c463ec2eefe1387c2"}, + {file = "coverage-6.1.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e7d5606b9240ed4def9cbdf35be4308047d11e858b9c88a6c26974758d6225ce"}, + {file = "coverage-6.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:572f917267f363101eec375c109c9c1118037c7cc98041440b5eabda3185ac7b"}, + {file = "coverage-6.1.1-cp37-cp37m-win32.whl", hash = "sha256:35cd2230e1ed76df7d0081a997f0fe705be1f7d8696264eb508076e0d0b5a685"}, + {file = "coverage-6.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:65ad3ff837c89a229d626b8004f0ee32110f9bfdb6a88b76a80df36ccc60d926"}, + {file = "coverage-6.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:977ce557d79577a3dd510844904d5d968bfef9489f512be65e2882e1c6eed7d8"}, + {file = "coverage-6.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62512c0ec5d307f56d86504c58eace11c1bc2afcdf44e3ff20de8ca427ca1d0e"}, + {file = "coverage-6.1.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2e5b9c17a56b8bf0c0a9477fcd30d357deb486e4e1b389ed154f608f18556c8a"}, + {file = "coverage-6.1.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:666c6b32b69e56221ad1551d377f718ed00e6167c7a1b9257f780b105a101271"}, + {file = "coverage-6.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:fb2fa2f6506c03c48ca42e3fe5a692d7470d290c047ee6de7c0f3e5fa7639ac9"}, + {file = "coverage-6.1.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f0f80e323a17af63eac6a9db0c9188c10f1fd815c3ab299727150cc0eb92c7a4"}, + {file = "coverage-6.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:738e823a746841248b56f0f3bd6abf3b73af191d1fd65e4c723b9c456216f0ad"}, + {file = "coverage-6.1.1-cp38-cp38-win32.whl", hash = "sha256:8605add58e6a960729aa40c0fd9a20a55909dd9b586d3e8104cc7f45869e4c6b"}, + {file = "coverage-6.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:6e994003e719458420e14ffb43c08f4c14990e20d9e077cb5cad7a3e419bbb54"}, + {file = "coverage-6.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e3c4f5211394cd0bf6874ac5d29684a495f9c374919833dcfff0bd6d37f96201"}, + {file = "coverage-6.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e14bceb1f3ae8a14374be2b2d7bc12a59226872285f91d66d301e5f41705d4d6"}, + {file = "coverage-6.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0147f7833c41927d84f5af9219d9b32f875c0689e5e74ac8ca3cb61e73a698f9"}, + {file = "coverage-6.1.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b1d0a1bce919de0dd8da5cff4e616b2d9e6ebf3bd1410ff645318c3dd615010a"}, + {file = "coverage-6.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ae6de0e41f44794e68d23644636544ed8003ce24845f213b24de097cbf44997f"}, + {file = "coverage-6.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2797ed7a7e883b9ab76e8e778bb4c859fc2037d6fd0644d8675e64d58d1653"}, + {file = "coverage-6.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c40966b683d92869b72ea3c11fd6b99a091fd30e12652727eca117273fc97366"}, + {file = "coverage-6.1.1-cp39-cp39-win32.whl", hash = "sha256:a11a2c019324fc111485e79d55907e7289e53d0031275a6c8daed30690bc50c0"}, + {file = "coverage-6.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d8b453764b9b26b0dd2afb83086a7c3f9379134e340288d2a52f8a91592394b"}, + {file = "coverage-6.1.1-pp36-none-any.whl", hash = "sha256:3b270c6b48d3ff5a35deb3648028ba2643ad8434b07836782b1139cf9c66313f"}, + {file = "coverage-6.1.1-pp37-none-any.whl", hash = "sha256:ffa8fee2b1b9e60b531c4c27cf528d6b5d5da46b1730db1f4d6eee56ff282e07"}, + {file = "coverage-6.1.1-pp38-none-any.whl", hash = "sha256:4cd919057636f63ab299ccb86ea0e78b87812400c76abab245ca385f17d19fb5"}, + {file = "coverage-6.1.1.tar.gz", hash = "sha256:b8e4f15b672c9156c1154249a9c5746e86ac9ae9edc3799ee3afebc323d9d9e0"}, ] cryptography = [ {file = "cryptography-35.0.0-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:d57e0cdc1b44b6cdf8af1d01807db06886f10177469312fbde8f44ccbb284bc9"}, @@ -1167,8 +1193,8 @@ dnspython = [ {file = "dnspython-2.1.0.zip", hash = "sha256:e4a87f0b573201a0f3727fa18a516b055fd1107e0e5477cded4a2de497df1dd4"}, ] dunamai = [ - {file = "dunamai-1.6.0-py3-none-any.whl", hash = "sha256:44a94a4edebb145bb6198a2f26de957b12b77d43b7c9c0646be814c60cf5d8df"}, - {file = "dunamai-1.6.0.tar.gz", hash = "sha256:6f1111f47e869ed58d44a7d37f112e3e7c761dce3c71f2c5464526928d7e9896"}, + {file = "dunamai-1.7.0-py3-none-any.whl", hash = "sha256:375e017eb014681e9c8f6e7f2c4c2065ef35832d429f8b70900bed24e8be83f8"}, + {file = "dunamai-1.7.0.tar.gz", hash = "sha256:6abfeb91768caea59d65a4989cec49472fa66ee04dcd6a5c9f92ebc019926a93"}, ] email-validator = [ {file = "email_validator-1.1.3-py2.py3-none-any.whl", hash = "sha256:5675c8ceb7106a37e40e2698a57c056756bf3f272cfa8682a4f87ebd95d8440b"}, @@ -1178,10 +1204,7 @@ fakeredis = [ {file = "fakeredis-1.6.1-py3-none-any.whl", hash = "sha256:5eb1516f1fe1813e9da8f6c482178fc067af09f53de587ae03887ef5d9d13024"}, {file = "fakeredis-1.6.1.tar.gz", hash = "sha256:0d06a9384fb79da9f2164ce96e34eb9d4e2ea46215070805ea6fd3c174590b47"}, ] -fastapi = [ - {file = "fastapi-0.70.0-py3-none-any.whl", hash = "sha256:a36d5f2fad931aa3575c07a3472c784e81f3e664e3bb5c8b9c88d0ec1104f59c"}, - {file = "fastapi-0.70.0.tar.gz", hash = "sha256:66da43cfe5185ea1df99552acffd201f1832c6b364e0f4136c0a99f933466ced"}, -] +fastapi = [] feedgen = [ {file = "feedgen-0.9.0.tar.gz", hash = "sha256:8e811bdbbed6570034950db23a4388453628a70e689a6e8303ccec430f5a804a"}, ] @@ -1282,8 +1305,8 @@ iniconfig = [ {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] isort = [ - {file = "isort-5.9.3-py3-none-any.whl", hash = "sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2"}, - {file = "isort-5.9.3.tar.gz", hash = "sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899"}, + {file = "isort-5.10.0-py3-none-any.whl", hash = "sha256:1a18ccace2ed8910bd9458b74a3ecbafd7b2f581301b0ab65cfdd4338272d76f"}, + {file = "isort-5.10.0.tar.gz", hash = "sha256:e52ff6d38012b131628cf0f26c51e7bd3a7c81592eefe3ac71411e692f1b9345"}, ] itsdangerous = [ {file = "itsdangerous-2.0.1-py3-none-any.whl", hash = "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c"}, @@ -1294,54 +1317,66 @@ jinja2 = [ {file = "Jinja2-3.0.2.tar.gz", hash = "sha256:827a0e32839ab1600d4eb1c4c33ec5a8edfbc5cb42dafa13b81f182f97784b45"}, ] lxml = [ - {file = "lxml-4.6.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:df7c53783a46febb0e70f6b05df2ba104610f2fb0d27023409734a3ecbb78fb2"}, - {file = "lxml-4.6.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1b7584d421d254ab86d4f0b13ec662a9014397678a7c4265a02a6d7c2b18a75f"}, - {file = "lxml-4.6.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:079f3ae844f38982d156efce585bc540c16a926d4436712cf4baee0cce487a3d"}, - {file = "lxml-4.6.3-cp27-cp27m-win32.whl", hash = "sha256:bc4313cbeb0e7a416a488d72f9680fffffc645f8a838bd2193809881c67dd106"}, - {file = "lxml-4.6.3-cp27-cp27m-win_amd64.whl", hash = "sha256:8157dadbb09a34a6bd95a50690595e1fa0af1a99445e2744110e3dca7831c4ee"}, - {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7728e05c35412ba36d3e9795ae8995e3c86958179c9770e65558ec3fdfd3724f"}, - {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:4bff24dfeea62f2e56f5bab929b4428ae6caba2d1eea0c2d6eb618e30a71e6d4"}, - {file = "lxml-4.6.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:64812391546a18896adaa86c77c59a4998f33c24788cadc35789e55b727a37f4"}, - {file = "lxml-4.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c1a40c06fd5ba37ad39caa0b3144eb3772e813b5fb5b084198a985431c2f1e8d"}, - {file = "lxml-4.6.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:74f7d8d439b18fa4c385f3f5dfd11144bb87c1da034a466c5b5577d23a1d9b51"}, - {file = "lxml-4.6.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f90ba11136bfdd25cae3951af8da2e95121c9b9b93727b1b896e3fa105b2f586"}, - {file = "lxml-4.6.3-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:4c61b3a0db43a1607d6264166b230438f85bfed02e8cff20c22e564d0faff354"}, - {file = "lxml-4.6.3-cp35-cp35m-manylinux2014_x86_64.whl", hash = "sha256:5c8c163396cc0df3fd151b927e74f6e4acd67160d6c33304e805b84293351d16"}, - {file = "lxml-4.6.3-cp35-cp35m-win32.whl", hash = "sha256:f2380a6376dfa090227b663f9678150ef27543483055cc327555fb592c5967e2"}, - {file = "lxml-4.6.3-cp35-cp35m-win_amd64.whl", hash = "sha256:c4f05c5a7c49d2fb70223d0d5bcfbe474cf928310ac9fa6a7c6dddc831d0b1d4"}, - {file = "lxml-4.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d2e35d7bf1c1ac8c538f88d26b396e73dd81440d59c1ef8522e1ea77b345ede4"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:289e9ca1a9287f08daaf796d96e06cb2bc2958891d7911ac7cae1c5f9e1e0ee3"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:bccbfc27563652de7dc9bdc595cb25e90b59c5f8e23e806ed0fd623755b6565d"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:d916d31fd85b2f78c76400d625076d9124de3e4bda8b016d25a050cc7d603f24"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:820628b7b3135403540202e60551e741f9b6d3304371712521be939470b454ec"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:c47ff7e0a36d4efac9fd692cfa33fbd0636674c102e9e8d9b26e1b93a94e7617"}, - {file = "lxml-4.6.3-cp36-cp36m-win32.whl", hash = "sha256:5a0a14e264069c03e46f926be0d8919f4105c1623d620e7ec0e612a2e9bf1c04"}, - {file = "lxml-4.6.3-cp36-cp36m-win_amd64.whl", hash = "sha256:92e821e43ad382332eade6812e298dc9701c75fe289f2a2d39c7960b43d1e92a"}, - {file = "lxml-4.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:efd7a09678fd8b53117f6bae4fa3825e0a22b03ef0a932e070c0bdbb3a35e654"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:efac139c3f0bf4f0939f9375af4b02c5ad83a622de52d6dfa8e438e8e01d0eb0"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:0fbcf5565ac01dff87cbfc0ff323515c823081c5777a9fc7703ff58388c258c3"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:36108c73739985979bf302006527cf8a20515ce444ba916281d1c43938b8bb96"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:122fba10466c7bd4178b07dba427aa516286b846b2cbd6f6169141917283aae2"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:cdaf11d2bd275bf391b5308f86731e5194a21af45fbaaaf1d9e8147b9160ea92"}, - {file = "lxml-4.6.3-cp37-cp37m-win32.whl", hash = "sha256:3439c71103ef0e904ea0a1901611863e51f50b5cd5e8654a151740fde5e1cade"}, - {file = "lxml-4.6.3-cp37-cp37m-win_amd64.whl", hash = "sha256:4289728b5e2000a4ad4ab8da6e1db2e093c63c08bdc0414799ee776a3f78da4b"}, - {file = "lxml-4.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b007cbb845b28db4fb8b6a5cdcbf65bacb16a8bd328b53cbc0698688a68e1caa"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:76fa7b1362d19f8fbd3e75fe2fb7c79359b0af8747e6f7141c338f0bee2f871a"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:26e761ab5b07adf5f555ee82fb4bfc35bf93750499c6c7614bd64d12aaa67927"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:e1cbd3f19a61e27e011e02f9600837b921ac661f0c40560eefb366e4e4fb275e"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:66e575c62792c3f9ca47cb8b6fab9e35bab91360c783d1606f758761810c9791"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:1b38116b6e628118dea5b2186ee6820ab138dbb1e24a13e478490c7db2f326ae"}, - {file = "lxml-4.6.3-cp38-cp38-win32.whl", hash = "sha256:89b8b22a5ff72d89d48d0e62abb14340d9e99fd637d046c27b8b257a01ffbe28"}, - {file = "lxml-4.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:2a9d50e69aac3ebee695424f7dbd7b8c6d6eb7de2a2eb6b0f6c7db6aa41e02b7"}, - {file = "lxml-4.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ce256aaa50f6cc9a649c51be3cd4ff142d67295bfc4f490c9134d0f9f6d58ef0"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:7610b8c31688f0b1be0ef882889817939490a36d0ee880ea562a4e1399c447a1"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f8380c03e45cf09f8557bdaa41e1fa7c81f3ae22828e1db470ab2a6c96d8bc23"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:3082c518be8e97324390614dacd041bb1358c882d77108ca1957ba47738d9d59"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:884ab9b29feaca361f7f88d811b1eea9bfca36cf3da27768d28ad45c3ee6f969"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:6f12e1427285008fd32a6025e38e977d44d6382cf28e7201ed10d6c1698d2a9a"}, - {file = "lxml-4.6.3-cp39-cp39-win32.whl", hash = "sha256:33bb934a044cf32157c12bfcfbb6649807da20aa92c062ef51903415c704704f"}, - {file = "lxml-4.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:542d454665a3e277f76954418124d67516c5f88e51a900365ed54a9806122b83"}, - {file = "lxml-4.6.3.tar.gz", hash = "sha256:39b78571b3b30645ac77b95f7c69d1bffc4cf8c3b157c435a34da72e78c82468"}, + {file = "lxml-4.6.4-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:bbf2dc330bd44bfc0254ab37677ec60f7c7ecea55ad8ba1b8b2ea7bf20c265f5"}, + {file = "lxml-4.6.4-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b667c51682fe9b9788c69465956baa8b6999531876ccedcafc895c74ad716cd8"}, + {file = "lxml-4.6.4-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:72e730d33fe2e302fd07285f14624fca5e5e2fb2bb4fb2c3941e318c41c443d1"}, + {file = "lxml-4.6.4-cp27-cp27m-win32.whl", hash = "sha256:433df8c7dde0f9e41cbf4f36b0829d50a378116ef5e962ba3881f2f5f025c7be"}, + {file = "lxml-4.6.4-cp27-cp27m-win_amd64.whl", hash = "sha256:35752ee40f7bbf6adc9ff4e1f4b84794a3593736dcce80db32e3c2aa85e294ac"}, + {file = "lxml-4.6.4-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ff5bb2a198ea67403bb6818705e9a4f90e0313f2215428ec51001ce56d939fb"}, + {file = "lxml-4.6.4-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9b87727561c1150c0cc91c5d9d389448b37a7d15f0ba939ed3d1acb2f11bf6c5"}, + {file = "lxml-4.6.4-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:45fdb2899c755138722797161547a40b3e2a06feda620cc41195ee7e97806d81"}, + {file = "lxml-4.6.4-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:38b9de0de3aa689fe9fb9877ae1be1e83b8cf9621f7e62049d0436b9ecf4ad64"}, + {file = "lxml-4.6.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:662523cd2a0246740225c7e32531f2e766544122e58bee70e700a024cfc0cf81"}, + {file = "lxml-4.6.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:4aa349c5567651f34d4eaae7de6ed5b523f6d70a288f9c6fbac22d13a0784e04"}, + {file = "lxml-4.6.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:08eb9200d88b376a8ed5e50f1dc1d1a45b49305169674002a3b5929943390591"}, + {file = "lxml-4.6.4-cp310-cp310-win32.whl", hash = "sha256:bdc224f216ead849e902151112efef6e96c41ee1322e15d4e5f7c8a826929aee"}, + {file = "lxml-4.6.4-cp310-cp310-win_amd64.whl", hash = "sha256:ab6db93a2b6b66cbf62b4e4a7135f476e708e8c5c990d186584142c77d7f975a"}, + {file = "lxml-4.6.4-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:50790313df028aa05cf22be9a8da033b86c42fa32523e4fd944827b482b17bf0"}, + {file = "lxml-4.6.4-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6764998345552b1dfc9326a932d2bad6367c6b37a176bb73ada6b9486bf602f7"}, + {file = "lxml-4.6.4-cp35-cp35m-win32.whl", hash = "sha256:543b239b191bb3b6d9bef5f09f1fb2be5b7eb09ab4d386aa655e4d53fbe9ff47"}, + {file = "lxml-4.6.4-cp35-cp35m-win_amd64.whl", hash = "sha256:a75c1ad05eedb1a3ff2a34a52a4f0836cfaa892e12796ba39a7732c82701eff4"}, + {file = "lxml-4.6.4-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:47e955112ce64241fdb357acf0216081f9f3255b3ac9c502ca4b3323ec1ca558"}, + {file = "lxml-4.6.4-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:20d7c8d90d449c6a353b15ee0459abae8395dbe59ad01e406ccbf30cd81c6f98"}, + {file = "lxml-4.6.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:240db6f3228d26e3c6f4fad914b9ddaaf8707254e8b3efd564dc680c8ec3c264"}, + {file = "lxml-4.6.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:351482da8dd028834028537f08724b1de22d40dcf3bb723b469446564f409074"}, + {file = "lxml-4.6.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e678a643177c0e5ec947b645fa7bc84260dfb9b6bf8fb1fdd83008dfc2ca5928"}, + {file = "lxml-4.6.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:15d0381feb56f08f78c5cc4fc385ddfe0bde1456e37f54a9322833371aec4060"}, + {file = "lxml-4.6.4-cp36-cp36m-win32.whl", hash = "sha256:4ba74afe5ee5cb5e28d83b513a6e8f0875fda1dc1a9aea42cc0065f029160d2a"}, + {file = "lxml-4.6.4-cp36-cp36m-win_amd64.whl", hash = "sha256:9c91a73971a922c13070fd8fa5a114c858251791ba2122a941e6aa781c713e44"}, + {file = "lxml-4.6.4-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:6020c70ff695106bf80651953a23e37718ef1fee9abd060dcad8e32ab2dc13f3"}, + {file = "lxml-4.6.4-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:f5dd358536b8a964bf6bd48de038754c1609e72e5f17f5d21efe2dda17594dbf"}, + {file = "lxml-4.6.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:7ae7089d81fc502df4b217ad77f03c54039fe90dac0acbe70448d7e53bfbc57e"}, + {file = "lxml-4.6.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:80d10d53d3184837445ff8562021bdd37f57c4cadacbf9d8726cc16220a00d54"}, + {file = "lxml-4.6.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e95da348d57eb448d226a44b868ff2ca5786fbcbe417ac99ff62d0a7d724b9c7"}, + {file = "lxml-4.6.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:ffd65cfa33fed01735c82aca640fde4cc63f0414775cba11e06f84fae2085a6e"}, + {file = "lxml-4.6.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:877666418598f6cb289546c77ff87590cfd212f903b522b0afa0b9fb73b3ccfb"}, + {file = "lxml-4.6.4-cp37-cp37m-win32.whl", hash = "sha256:e91d24623e747eeb2d8121f4a94c6a7ad27dc48e747e2dc95bfe88632bd028a2"}, + {file = "lxml-4.6.4-cp37-cp37m-win_amd64.whl", hash = "sha256:4ec9a80dd5704ecfde54319b6964368daf02848c8954d3bacb9b64d1c7659159"}, + {file = "lxml-4.6.4-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:2901625f4a878a055d275beedc20ba9cb359cefc4386a967222fee29eb236038"}, + {file = "lxml-4.6.4-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b567178a74a2261345890eac66fbf394692a6e002709d329f28a673ca6042473"}, + {file = "lxml-4.6.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:4717123f7c11c81e0da69989e5a64079c3f402b0efeb4c6241db6c369d657bd8"}, + {file = "lxml-4.6.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:cf201bf5594d1aab139fe53e3fca457e4f8204a5bbd65d48ab3b82a16f517868"}, + {file = "lxml-4.6.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a77a3470ba37e11872c75ca95baf9b3312133a3d5a5dc720803b23098c653976"}, + {file = "lxml-4.6.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:619c6d2b552bba00491e96c0518aad94002651c108a0f7364ff2d7798812c00e"}, + {file = "lxml-4.6.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:601f0ab75538b280aaf1e720eb9d68d4fa104ac274e1e9e6971df488f4dcdb0f"}, + {file = "lxml-4.6.4-cp38-cp38-win32.whl", hash = "sha256:75d3c5bbc0ddbad03bb68b9be638599f67e4b98ed3dcd0fec9f6f39e41ee96cb"}, + {file = "lxml-4.6.4-cp38-cp38-win_amd64.whl", hash = "sha256:4341d135f5660db10184963d9c3418c3e28d7f868aaf8b11a323ebf85813f7f4"}, + {file = "lxml-4.6.4-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:9db24803fa71e3305fe4a7812782b708da21a0b774b130dd1860cf40a6d7a3ee"}, + {file = "lxml-4.6.4-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:afd60230ad9d8bcba005945ec3a343722f09e0b7f8ae804246e5d2cfc6bd71a6"}, + {file = "lxml-4.6.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:0c15e1cd55055956e77b0732270f1c6005850696bc3ef3e03d01e78af84eaa42"}, + {file = "lxml-4.6.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6d422b3c729737d8a39279a25fa156c983a56458f8b2f97661ee6fb22b80b1d6"}, + {file = "lxml-4.6.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2eb90f6ec3c236ef2f1bb38aee7c0d23e77d423d395af6326e7cca637519a4cb"}, + {file = "lxml-4.6.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:51a0e5d243687596f46e24e464121d4b232ad772e2d1785b2a2c0eb413c285d4"}, + {file = "lxml-4.6.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d43bd68714049c84e297c005456a15ecdec818f7b5aa5868c8b0a865cfb78a44"}, + {file = "lxml-4.6.4-cp39-cp39-win32.whl", hash = "sha256:ee9e4b07b0eba4b6a521509e9e1877476729c1243246b6959de697ebea739643"}, + {file = "lxml-4.6.4-cp39-cp39-win_amd64.whl", hash = "sha256:48eaac2991b3036175b42ee8d3c23f4cca13f2be8426bf29401a690ab58c88f4"}, + {file = "lxml-4.6.4-pp37-pypy37_pp73-macosx_10_14_x86_64.whl", hash = "sha256:2b06a91cf7b8acea7793006e4ae50646cef0fe35ce5acd4f5cb1c77eb228e4a1"}, + {file = "lxml-4.6.4-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:523f195948a1ba4f9f5b7294d83c6cd876547dc741820750a7e5e893a24bbe38"}, + {file = "lxml-4.6.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:b0ca0ada9d3bc18bd6f611bd001a28abdd49ab9698bd6d717f7f5394c8e94628"}, + {file = "lxml-4.6.4-pp38-pypy38_pp73-macosx_10_14_x86_64.whl", hash = "sha256:197b7cb7a753cf553a45115739afd8458464a28913da00f5c525063f94cd3f48"}, + {file = "lxml-4.6.4-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:6298f5b42a26581206ef63fffa97c754245d329414108707c525512a5197f2ba"}, + {file = "lxml-4.6.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0b12c95542f04d10cba46b3ff28ea52ea56995b78cf918f0b11b05e75812bb79"}, + {file = "lxml-4.6.4.tar.gz", hash = "sha256:daf9bd1fee31f1c7a5928b3e1059e09a8d683ea58fb3ffc773b6c88cb8d1399c"}, ] mako = [ {file = "Mako-1.1.5-py2.py3-none-any.whl", hash = "sha256:6804ee66a7f6a6416910463b00d76a7b25194cd27f1918500c5bd7be2a088a23"}, @@ -1445,8 +1480,8 @@ orjson = [ {file = "orjson-3.6.4.tar.gz", hash = "sha256:f8dbc428fc6d7420f231a7133d8dff4c882e64acb585dcf2fda74bdcfe1a6d9d"}, ] packaging = [ - {file = "packaging-21.0-py3-none-any.whl", hash = "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"}, - {file = "packaging-21.0.tar.gz", hash = "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7"}, + {file = "packaging-21.2-py3-none-any.whl", hash = "sha256:14317396d1e8cdb122989b916fa2c7e9ca8e2be9e8060a6eff75b6b7b4d8a7e0"}, + {file = "packaging-21.2.tar.gz", hash = "sha256:096d689d78ca690e4cd8a89568ba06d07ca097e3306a4381635073ca91479966"}, ] paginate = [ {file = "paginate-0.5.6.tar.gz", hash = "sha256:5e6007b6a9398177a7e1648d04fdd9f8c9766a1a945bceac82f1929e8c78af2d"}, @@ -1472,42 +1507,42 @@ prometheus-fastapi-instrumentator = [ {file = "prometheus_fastapi_instrumentator-5.7.1-py3-none-any.whl", hash = "sha256:da40ea0df14b0e95d584769747fba777522a8df6a8c47cec2edf798f1fff49b5"}, ] protobuf = [ - {file = "protobuf-3.19.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:01a0645ef3acddfbc90237e1cdfae1086130fc7cb480b5874656193afd657083"}, - {file = "protobuf-3.19.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d3861c9721a90ba83ee0936a9cfcc4fa1c4b4144ac9658fb6f6343b38558e9b4"}, - {file = "protobuf-3.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b64be5d7270cf5e76375bac049846e8a9543a2d4368b69afe78ab725380a7487"}, - {file = "protobuf-3.19.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:2f6046b9e2feee0dce994493186e8715b4392ed5f50f356280ad9c2f9f93080a"}, - {file = "protobuf-3.19.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac2f8ec942d414609aba0331952ae12bb823e8f424bbb6b8c422f1cef32dc842"}, - {file = "protobuf-3.19.0-cp36-cp36m-win32.whl", hash = "sha256:3fea09aa04ef2f8b01fcc9bb87f19509934f8a35d177c865b8f9ee5c32b60c1b"}, - {file = "protobuf-3.19.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d1f4277d321f60456845ca9b882c4845736f1f5c1c69eb778eba22a97977d8af"}, - {file = "protobuf-3.19.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8488c2276f14f294e890cc1260ab342a13e90cd20dcc03319d2eea258f1fd321"}, - {file = "protobuf-3.19.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:36bf292f44966c67080e535321501717f4f1eba30faef8f2cd4b0c745a027211"}, - {file = "protobuf-3.19.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99af73ae34c93e0e2ace57ea2e70243f34fc015c8c23fd39ee93652e726f7e7"}, - {file = "protobuf-3.19.0-cp37-cp37m-win32.whl", hash = "sha256:f7a031cf8e2fc14acc0ba694f6dff0a01e06b70d817eba6edc72ee6cc20517ac"}, - {file = "protobuf-3.19.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d4ca5f0c7bc8d2e6966ca3bbd85e9ebe7191b6e21f067896d4af6b28ecff29fe"}, - {file = "protobuf-3.19.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9a8a880593015ef2c83f7af797fa4fbf583b2c98b4bd94e46c5b61fee319d84b"}, - {file = "protobuf-3.19.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:6f16925f5c977dd7787973a50c242e60c22b1d1182aba6bec7bd02862579c10f"}, - {file = "protobuf-3.19.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9097327d277b0aa4a3224e61cd6850aef3269172397715299bcffc9f90293c9"}, - {file = "protobuf-3.19.0-cp38-cp38-win32.whl", hash = "sha256:708d04394a63ee9bdc797938b6e15ed5bf24a1cb37743eb3886fd74a5a67a234"}, - {file = "protobuf-3.19.0-cp38-cp38-win_amd64.whl", hash = "sha256:ee4d07d596357f51316b6ecf1cc1927660e9d5e418385bb1c51fd2496cd9bee7"}, - {file = "protobuf-3.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:34a77b8fafdeb8f89fee2b7108ae60d8958d72e33478680cc1e05517892ecc46"}, - {file = "protobuf-3.19.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:4f93e0f6af796ddd1502225ff8ea25340ced186ca05b601c44d5c88b45ba80a0"}, - {file = "protobuf-3.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:942dd6bc8bd2a3c6a156d8ab0f80bd45313f22b78e1176283270054dcc8ca4c2"}, - {file = "protobuf-3.19.0-cp39-cp39-win32.whl", hash = "sha256:7b3867795708ac88fde8d6f34f0d9a50af56087e41f624bdb2e9ff808ea5dda7"}, - {file = "protobuf-3.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:a74432e9d28a6072a2359a0f49f81eb14dd718e7dbbfb6c0789b456c49e1f130"}, - {file = "protobuf-3.19.0-py2.py3-none-any.whl", hash = "sha256:c96e94d3e523a82caa3e5f74b35dd1c4884199358d01c950d95c341255ff48bc"}, - {file = "protobuf-3.19.0.tar.gz", hash = "sha256:6a1dc6584d24ef86f5b104bcad64fa0fe06ed36e5687f426e0445d363a041d18"}, + {file = "protobuf-3.19.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d80f80eb175bf5f1169139c2e0c5ada98b1c098e2b3c3736667f28cbbea39fc8"}, + {file = "protobuf-3.19.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:a529e7df52204565bcd33738a7a5f288f3d2d37d86caa5d78c458fa5fabbd54d"}, + {file = "protobuf-3.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28ccea56d4dc38d35cd70c43c2da2f40ac0be0a355ef882242e8586c6d66666f"}, + {file = "protobuf-3.19.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b30a7de128c46b5ecb343917d9fa737612a6e8280f440874e5cc2ba0d79b8f6"}, + {file = "protobuf-3.19.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5935c8ce02e3d89c7900140a8a42b35bc037ec07a6aeb61cc108be8d3c9438a6"}, + {file = "protobuf-3.19.1-cp36-cp36m-win32.whl", hash = "sha256:74f33edeb4f3b7ed13d567881da8e5a92a72b36495d57d696c2ea1ae0cfee80c"}, + {file = "protobuf-3.19.1-cp36-cp36m-win_amd64.whl", hash = "sha256:038daf4fa38a7e818dd61f51f22588d61755160a98db087a046f80d66b855942"}, + {file = "protobuf-3.19.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e51561d72efd5bd5c91490af1f13e32bcba8dab4643761eb7de3ce18e64a853"}, + {file = "protobuf-3.19.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:6e8ea9173403219239cdfd8d946ed101f2ab6ecc025b0fda0c6c713c35c9981d"}, + {file = "protobuf-3.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db3532d9f7a6ebbe2392041350437953b6d7a792de10e629c1e4f5a6b1fe1ac6"}, + {file = "protobuf-3.19.1-cp37-cp37m-win32.whl", hash = "sha256:615b426a177780ce381ecd212edc1e0f70db8557ed72560b82096bd36b01bc04"}, + {file = "protobuf-3.19.1-cp37-cp37m-win_amd64.whl", hash = "sha256:d8919368410110633717c406ab5c97e8df5ce93020cfcf3012834f28b1fab1ea"}, + {file = "protobuf-3.19.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:71b0250b0cfb738442d60cab68abc166de43411f2a4f791d31378590bfb71bd7"}, + {file = "protobuf-3.19.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3cd0458870ea7d1c58e948ac8078f6ba8a7ecc44a57e03032ed066c5bb318089"}, + {file = "protobuf-3.19.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:655264ed0d0efe47a523e2255fc1106a22f6faab7cc46cfe99b5bae085c2a13e"}, + {file = "protobuf-3.19.1-cp38-cp38-win32.whl", hash = "sha256:b691d996c6d0984947c4cf8b7ae2fe372d99b32821d0584f0b90277aa36982d3"}, + {file = "protobuf-3.19.1-cp38-cp38-win_amd64.whl", hash = "sha256:e7e8d2c20921f8da0dea277dfefc6abac05903ceac8e72839b2da519db69206b"}, + {file = "protobuf-3.19.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fd390367fc211cc0ffcf3a9e149dfeca78fecc62adb911371db0cec5c8b7472d"}, + {file = "protobuf-3.19.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d83e1ef8cb74009bebee3e61cc84b1c9cd04935b72bca0cbc83217d140424995"}, + {file = "protobuf-3.19.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36d90676d6f426718463fe382ec6274909337ca6319d375eebd2044e6c6ac560"}, + {file = "protobuf-3.19.1-cp39-cp39-win32.whl", hash = "sha256:e7b24c11df36ee8e0c085e5b0dc560289e4b58804746fb487287dda51410f1e2"}, + {file = "protobuf-3.19.1-cp39-cp39-win_amd64.whl", hash = "sha256:77d2fadcf369b3f22859ab25bd12bb8e98fb11e05d9ff9b7cd45b711c719c002"}, + {file = "protobuf-3.19.1-py2.py3-none-any.whl", hash = "sha256:e813b1c9006b6399308e917ac5d298f345d95bb31f46f02b60cd92970a9afa17"}, + {file = "protobuf-3.19.1.tar.gz", hash = "sha256:62a8e4baa9cb9e064eb62d1002eca820857ab2138440cb4b3ea4243830f94ca7"}, ] py = [ - {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, - {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] pycodestyle = [ {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, {file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"}, ] pycparser = [ - {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, - {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] pydantic = [ {file = "pydantic-1.8.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:05ddfd37c1720c392f4e0d43c484217b7521558302e7069ce8d318438d297739"}, @@ -1575,8 +1610,8 @@ pytest-cov = [ {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, ] pytest-tap = [ - {file = "pytest-tap-3.2.tar.gz", hash = "sha256:1b585c4a636458dbd958d136381bbabb1752c5877d05fac7d6a6001a8a9ddc29"}, - {file = "pytest_tap-3.2-py3-none-any.whl", hash = "sha256:18f59047f8bc68247d37f807fae7f2f8897d2c7397aea2fd2870f0421dc566cb"}, + {file = "pytest-tap-3.3.tar.gz", hash = "sha256:5f0919a147cf0396b2f10d64d365a0bf8062e06543e93c675c9d37f5605e983c"}, + {file = "pytest_tap-3.3-py3-none-any.whl", hash = "sha256:4fbbc0e090c2e94f6199bee4e4f68ab3c5e176b37a72a589ad84e0f72a2fce55"}, ] python-dateutil = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, @@ -1648,8 +1683,8 @@ sqlalchemy = [ {file = "SQLAlchemy-1.4.26.tar.gz", hash = "sha256:6bc7f9d7d90ef55e8c6db1308a8619cd8f40e24a34f759119b95e7284dca351a"}, ] starlette = [ - {file = "starlette-0.16.0-py3-none-any.whl", hash = "sha256:38eb24bf705a2c317e15868e384c1b8a12ca396e5a3c3a003db7e667c43f939f"}, - {file = "starlette-0.16.0.tar.gz", hash = "sha256:e1904b5d0007aee24bdd3c43994be9b3b729f4f58e740200de1d623f8c3a8870"}, + {file = "starlette-0.17.0-py3-none-any.whl", hash = "sha256:64ffd950183d474df2cf7a4018c8bbb31a481367691c70f5ace4b2d376235f72"}, + {file = "starlette-0.17.0.tar.gz", hash = "sha256:31a889e7d7bf487f70d9d197ed7efadb47fa938c58626ed93e381480833c5b84"}, ] "tap.py" = [ {file = "tap.py-3.0-py2.py3-none-any.whl", hash = "sha256:a598bfaa2e224d71f2e86147c2ef822c18ff2e1b8ef006397e5056b08f92f699"}, @@ -1660,8 +1695,8 @@ toml = [ {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] tomli = [ - {file = "tomli-1.2.1-py3-none-any.whl", hash = "sha256:8dd0e9524d6f386271a36b41dbf6c57d8e32fd96fd22b6584679dc569d20899f"}, - {file = "tomli-1.2.1.tar.gz", hash = "sha256:a5b75cb6f3968abb47af1b40c1819dc519ea82bcc065776a866e8d74c5ca9442"}, + {file = "tomli-1.2.2-py3-none-any.whl", hash = "sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade"}, + {file = "tomli-1.2.2.tar.gz", hash = "sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee"}, ] tomlkit = [ {file = "tomlkit-0.7.2-py2.py3-none-any.whl", hash = "sha256:173ad840fa5d2aac140528ca1933c29791b79a374a0861a80347f42ec9328117"}, diff --git a/pyproject.toml b/pyproject.toml index 20855fa6..1d4c858c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ bcrypt = "^3.2.0" bleach = "^4.1.0" email-validator = "^1.1.3" fakeredis = "^1.6.1" -fastapi = "^0.70.0" +fastapi = { git = "https://github.com/kevr/fastapi.git", branch = "upgrade-starlette-0.17.0" } feedgen = "^0.9.0" httpx = "^0.20.0" itsdangerous = "^2.0.1" From 85ebc72e8af542d73909b6f58f9bfb3b4f40ccd3 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 8 Nov 2021 18:18:41 -0800 Subject: [PATCH 531/844] fix(fastapi): only elevated users are allowed to suspend accounts Signed-off-by: Kevin Morris --- aurweb/auth.py | 3 ++ aurweb/routers/accounts.py | 7 +++- po/aurweb.pot | 4 +++ templates/partials/account_form.html | 18 +++++----- test/test_accounts_routes.py | 49 ++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index 5e45ee83..38754db0 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -51,6 +51,9 @@ class AnonymousUser: LangPreference = aurweb.config.get("options", "default_lang") Timezone = aurweb.config.get("options", "default_timezone") + Suspended = 0 + InactivityTS = 0 + # A stub ssh_pub_key relationship. ssh_pub_key = None diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 152b0a15..498568ad 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -143,6 +143,10 @@ def process_account_form(request: Request, user: models.User, args: dict): if not email or not username: return (False, ["Missing a required field."]) + inactive = args.get("J", False) + if not request.user.is_elevated() and inactive != bool(user.InactivityTS): + return (False, ["You do not have permission to suspend accounts."]) + username_min_len = aurweb.config.getint("options", "username_min_len") username_max_len = aurweb.config.getint("options", "username_max_len") if not util.valid_username(args.get("U")): @@ -528,7 +532,8 @@ async def account_edit_post(request: Request, user.Homepage = HP or user.Homepage user.IRCNick = I or user.IRCNick user.PGPKey = K or user.PGPKey - user.InactivityTS = datetime.utcnow().timestamp() if J else 0 + user.Suspended = J + user.InactivityTS = int(datetime.utcnow().timestamp()) * int(J) # If we update the language, update the cookie as well. if L and L != user.LangPreference: diff --git a/po/aurweb.pot b/po/aurweb.pot index 721f874e..f4deee70 100644 --- a/po/aurweb.pot +++ b/po/aurweb.pot @@ -879,6 +879,10 @@ msgstr "" msgid "Account suspended" msgstr "" +#: aurweb/routers/accounts.py +msgid "You do not have permission to suspend accounts." +msgstr "" + #: lib/acctfuncs.inc.php #, php-format msgid "" diff --git a/templates/partials/account_form.html b/templates/partials/account_form.html index f166c230..2e47a932 100644 --- a/templates/partials/account_form.html +++ b/templates/partials/account_form.html @@ -42,14 +42,16 @@ "account is inactive." | tr }}

    -

    - - -

    + {% if request.user.is_elevated() %} +

    + + +

    + {% endif %} {% if request.user.has_credential("CRED_ACCOUNT_CHANGE_TYPE") %}

    diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index 188f7048..5e855daf 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -780,6 +780,55 @@ def test_post_account_edit_error_invalid_password(): assert "Invalid password." in content +def test_post_account_edit_inactivity_unauthorized(): + cookies = {"AURSID": user.login(Request(), "testPassword")} + post_data = { + "U": "test", + "E": "test@example.org", + "J": True, + "passwd": "testPassword" + } + with client as request: + resp = request.post(f"/account/{user.Username}/edit", data=post_data, + cookies=cookies) + assert resp.status_code == int(HTTPStatus.BAD_REQUEST) + + errors = get_errors(resp.text) + expected = "You do not have permission to suspend accounts." + assert errors[0].text.strip() == expected + + +def test_post_account_edit_inactivity(): + with db.begin(): + user.AccountTypeID = TRUSTED_USER_ID + assert not user.Suspended + + cookies = {"AURSID": user.login(Request(), "testPassword")} + post_data = { + "U": "test", + "E": "test@example.org", + "J": True, + "passwd": "testPassword" + } + with client as request: + resp = request.post(f"/account/{user.Username}/edit", data=post_data, + cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + # Make sure the user record got updated correctly. + assert user.Suspended + assert user.InactivityTS > 0 + + post_data.update({"J": False}) + with client as request: + resp = request.post(f"/account/{user.Username}/edit", data=post_data, + cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + assert not user.Suspended + assert user.InactivityTS == 0 + + def test_post_account_edit_error_unauthorized(): request = Request() sid = user.login(request, "testPassword") From 464540c9a9f41aad52e633698647a9030092dc51 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 9 Nov 2021 00:14:24 -0800 Subject: [PATCH 532/844] fix: use https for aurblup's default mirror instead of ftp It seems the ftp mirror from kernel.org cannot be used anymore, but the https mirror can. So, the default config has been updated to reflect this; otherwise, aurblup bugs out. Signed-off-by: Kevin Morris --- conf/config.defaults | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/config.defaults b/conf/config.defaults index b078e57c..babfd482 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -98,7 +98,7 @@ max-blob-size = 256000 [aurblup] db-path = /srv/http/aurweb/aurblup/ sync-dbs = core extra community multilib testing community-testing -server = ftp://mirrors.kernel.org/archlinux/%s/os/x86_64 +server = https://mirrors.kernel.org/archlinux/%s/os/x86_64 [mkpkglists] packagesfile = /srv/http/aurweb/web/html/packages.gz From b8d7619dbc34c41d2dac59fd717c7553054eb9d9 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 9 Nov 2021 00:17:52 -0800 Subject: [PATCH 533/844] change: add mkpkglists options to config.dev Here, we default to using root as the storage directory. Primarily because it makes sense in Docker; config.dev can always be fixed up by developers to reflect local system changes. Signed-off-by: Kevin Morris --- conf/config.dev | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/config.dev b/conf/config.dev index fb43612e..6cbe97cc 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -65,3 +65,8 @@ session_secret = secret [devel] ;commit_hash = 1234567 + +[mkpkglists] +packagesfile = /packages.gz +pkgbasefile = /pkgbase.gz +userfile = /users.gz From 338a44839f02850890e69f2985827cab9a1aefb4 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 9 Nov 2021 00:18:54 -0800 Subject: [PATCH 534/844] fix: override aurblup's db-path option in config.dev Signed-off-by: Kevin Morris --- conf/config.dev | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/config.dev b/conf/config.dev index 6cbe97cc..dac85477 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -70,3 +70,6 @@ session_secret = secret packagesfile = /packages.gz pkgbasefile = /pkgbase.gz userfile = /users.gz + +[aurblup] +db-path = YOUR_AUR_ROOT/aurblup/ From 4b8963b7bac999e2effb9840b01c9c01b8218fc0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 9 Nov 2021 00:29:19 -0800 Subject: [PATCH 535/844] feat(docker): add cron service (aurblup + mkpkglists) Normally, these scripts are used to update official providers in the aurweb database along with archives that can be retrieved. Run both of these scripts in a 5 minute cron job, to both reflect the live instance database and production load. Signed-off-by: Kevin Morris --- docker-compose.yml | 17 +++++++++++++++++ docker/config/aurweb-cron | 2 ++ docker/cron-entrypoint.sh | 16 ++++++++++++++++ docker/scripts/install-deps.sh | 2 +- docker/scripts/run-cron.sh | 7 +++++++ 5 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 docker/config/aurweb-cron create mode 100755 docker/cron-entrypoint.sh create mode 100755 docker/scripts/run-cron.sh diff --git a/docker-compose.yml b/docker-compose.yml index 038eb65b..c2b14f91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -144,6 +144,19 @@ services: volumes: - git_data:/aurweb/aur.git + cron: + image: aurweb:latest + init: true + environment: + - AUR_CONFIG=/aurweb/conf/config + entrypoint: /docker/cron-entrypoint.sh + command: /docker/scripts/run-cron.sh + depends_on: + mariadb_init: + condition: service_started + volumes: + - mariadb_run:/var/run/mysqld + php-fpm: image: aurweb:latest init: true @@ -163,6 +176,8 @@ services: condition: service_healthy memcached: condition: service_healthy + cron: + condition: service_started volumes: - mariadb_run:/var/run/mysqld ports: @@ -190,6 +205,8 @@ services: condition: service_healthy redis: condition: service_healthy + cron: + condition: service_started volumes: - mariadb_run:/var/run/mysqld ports: diff --git a/docker/config/aurweb-cron b/docker/config/aurweb-cron new file mode 100644 index 00000000..1be7c13c --- /dev/null +++ b/docker/config/aurweb-cron @@ -0,0 +1,2 @@ +*/5 * * * * aurweb-aurblup +*/5 * * * * aurweb-mkpkglists diff --git a/docker/cron-entrypoint.sh b/docker/cron-entrypoint.sh new file mode 100755 index 00000000..d4173eaf --- /dev/null +++ b/docker/cron-entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -eou pipefail + +# Prepare AUR_CONFIG. +cp -vf conf/config.dev conf/config +sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config + +# Create directories we need. +mkdir -p /aurweb/aurblup + +# Install the cron configuration. +cp /docker/config/aurweb-cron /etc/cron.d/aurweb-cron +chmod 0644 /etc/cron.d/aurweb-cron +crontab /etc/cron.d/aurweb-cron + +exec "$@" diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh index 52ad6747..d64340e3 100755 --- a/docker/scripts/install-deps.sh +++ b/docker/scripts/install-deps.sh @@ -8,7 +8,7 @@ pacman -Syu --noconfirm --noprogressbar \ --cachedir .pkg-cache git gpgme nginx redis openssh \ mariadb mariadb-libs cgit-aurweb uwsgi uwsgi-plugin-cgi \ php php-fpm memcached php-memcached python-pip pyalpm \ - python-srcinfo curl libeatmydata + python-srcinfo curl libeatmydata cronie # https://python-poetry.org/docs/ Installation section. curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - diff --git a/docker/scripts/run-cron.sh b/docker/scripts/run-cron.sh new file mode 100755 index 00000000..d927a790 --- /dev/null +++ b/docker/scripts/run-cron.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cd /aurweb +aurweb-aurblup +aurweb-mkpkglists + +exec /usr/bin/crond -n From 10fcf939911cce4481bc1ec825329c8bc57ebd1d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 9 Nov 2021 01:51:23 -0800 Subject: [PATCH 536/844] fix(fastapi): use correct official pkg base url Signed-off-by: Kevin Morris --- aurweb/models/official_provider.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aurweb/models/official_provider.py b/aurweb/models/official_provider.py index a8282ff1..ff70fe43 100644 --- a/aurweb/models/official_provider.py +++ b/aurweb/models/official_provider.py @@ -3,8 +3,7 @@ from sqlalchemy.exc import IntegrityError from aurweb import schema from aurweb.models.declarative import Base -# TODO: Fix this! Official packages aren't from aur.archlinux.org... -OFFICIAL_BASE = "https://aur.archlinux.org" +OFFICIAL_BASE = "https://archlinux.org" class OfficialProvider(Base): From f6061400509fdd808d066fceb4572c2728a39fde Mon Sep 17 00:00:00 2001 From: Kristian Klausen Date: Fri, 15 Oct 2021 20:14:31 +0200 Subject: [PATCH 537/844] feat(PHP): Add packages dump file with more metadata --- aurweb/scripts/mkpkglists.py | 10 ++++++++++ conf/config.defaults | 1 + test/setup.sh | 1 + web/html/index.php | 1 + 4 files changed, 13 insertions(+) diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index 6724141a..c73cc3be 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -2,11 +2,13 @@ import datetime import gzip +import json import aurweb.config import aurweb.db packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') +packagesmetafile = aurweb.config.get('mkpkglists', 'packagesmetafile') pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') userfile = aurweb.config.get('mkpkglists', 'userfile') @@ -27,6 +29,14 @@ def main(): "WHERE PackageBases.PackagerUID IS NOT NULL") f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + with gzip.open(packagesmetafile, "wt") as f: + cur = conn.execute("SELECT * FROM Packages") + json.dump({ + "warning": "This is a experimental! It can be removed or modified without warning!", + "columns": [d[0] for d in cur.description], + "data": cur.fetchall() + }, f) + with gzip.open(pkgbasefile, "w") as f: f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) cur = conn.execute("SELECT Name FROM PackageBases " + diff --git a/conf/config.defaults b/conf/config.defaults index babfd482..6ccf42d0 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -102,6 +102,7 @@ server = https://mirrors.kernel.org/archlinux/%s/os/x86_64 [mkpkglists] packagesfile = /srv/http/aurweb/web/html/packages.gz +packagesmetafile = /srv/http/aurweb/web/html/packages-meta-v1.json.gz pkgbasefile = /srv/http/aurweb/web/html/pkgbase.gz userfile = /srv/http/aurweb/web/html/users.gz diff --git a/test/setup.sh b/test/setup.sh index 191a73d8..8ee9eef2 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -67,6 +67,7 @@ server = file://$(pwd)/remote/ [mkpkglists] packagesfile = packages.gz +packagesmetafile = packages-meta-v1.json.gz pkgbasefile = pkgbase.gz userfile = users.gz EOF diff --git a/web/html/index.php b/web/html/index.php index e57e7708..3163c3e8 100644 --- a/web/html/index.php +++ b/web/html/index.php @@ -189,6 +189,7 @@ if (!empty($tokens[1]) && '/' . $tokens[1] == get_pkg_route()) { readfile("./$path"); break; case "/packages.gz": + case "/packages-teapot.json.gz": case "/pkgbase.gz": case "/users.gz": header("Content-Type: text/plain"); From f3f662c696aaa35b57737c63bc506ded464a7d81 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 31 Oct 2021 16:52:30 -0700 Subject: [PATCH 538/844] fix(mkpkglists): improve package meta archive The SQL logic in this file for package metadata now exactly reflects RPC's search logic, without searching for specific packages. Two command line arguments are available: --extended | Include License, Keywords, Groups, relations and dependencies. When --extended is passed, the script will create a packages-meta-ext-v1.json.gz, configured via packagesmetaextfile. Archive JSON is in the following format: line-separated package objects enclosed in a list: [ {...}, {...}, {...} ] Signed-off-by: Kevin Morris --- aurweb/scripts/mkpkglists.py | 273 ++++++++++++++++++++++++++++++++--- conf/config.defaults | 1 + test/setup.sh | 2 + web/html/index.php | 3 +- 4 files changed, 255 insertions(+), 24 deletions(-) diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index c73cc3be..f2095a20 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -1,16 +1,192 @@ #!/usr/bin/env python3 +""" +Produces package, package base and user archives for the AUR +database. + +Archives: + + packages.gz | A line-separated list of package names + packages-meta-v1.json | A type=search RPC-formatted JSON dataset + packages-meta-ext-v1.json | An --extended archive + pkgbase.gz | A line-separated list of package base names + users.gz | A line-separated list of user names + +This script takes an optional argument: --extended. Based +on the following, right-hand side fields are added to each item. + + --extended | License, Keywords, Groups, relations and dependencies + +""" import datetime import gzip -import json +import os +import sys + +from collections import defaultdict +from decimal import Decimal +from typing import Tuple + +import orjson import aurweb.config import aurweb.db + +def state_path(archive: str) -> str: + # A hard-coded /tmp state directory. + # TODO: Use Redis cache to store this state after we merge + # FastAPI into master and removed PHP from the tree. + return os.path.join("/tmp", os.path.basename(archive) + ".state") + + packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') packagesmetafile = aurweb.config.get('mkpkglists', 'packagesmetafile') +packagesmetaextfile = aurweb.config.get('mkpkglists', 'packagesmetaextfile') +packages_state = state_path(packagesfile) + pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') +pkgbases_state = state_path(pkgbasefile) + userfile = aurweb.config.get('mkpkglists', 'userfile') +users_state = state_path(userfile) + + +def should_update(state: str, tablename: str) -> Tuple[bool, int]: + if aurweb.config.get("database", "backend") != "mysql": + return (False, 0) + + db_name = aurweb.config.get("database", "name") + conn = aurweb.db.Connection() + cur = conn.execute("SELECT auto_increment FROM information_schema.tables " + "WHERE table_schema = ? AND table_name = ?", + (db_name, tablename,)) + update_time = cur.fetchone()[0] + + saved_update_time = 0 + if os.path.exists(state): + with open(state) as f: + saved_update_time = int(f.read().strip()) + + return (saved_update_time == update_time, update_time) + + +def update_state(state: str, update_time: int) -> None: + with open(state, "w") as f: + f.write(str(update_time)) + + +TYPE_MAP = { + "depends": "Depends", + "makedepends": "MakeDepends", + "checkdepends": "CheckDepends", + "optdepends": "OptDepends", + "conflicts": "Conflicts", + "provides": "Provides", + "replaces": "Replaces", +} + + +def get_extended_dict(query: str): + """ + Produce data in the form in a single bulk SQL query: + + { + : { + "Depends": [...], + "Conflicts": [...], + "License": [...] + } + } + + The caller can then use this data to populate a dataset of packages. + + output = produce_base_output_data() + data = get_extended_dict(query) + for i in range(len(output)): + package_id = output[i].get("ID") + output[i].update(data.get(package_id)) + """ + + conn = aurweb.db.Connection() + + cursor = conn.execute(query) + + data = defaultdict(lambda: defaultdict(list)) + + for result in cursor.fetchall(): + + pkgid = result[0] + key = TYPE_MAP.get(result[1]) + output = result[2] + if result[3]: + output += result[3] + + # In all cases, we have at least an empty License list. + if "License" not in data[pkgid]: + data[pkgid]["License"] = [] + + # In all cases, we have at least an empty Keywords list. + if "Keywords" not in data[pkgid]: + data[pkgid]["Keywords"] = [] + + data[pkgid][key].append(output) + + conn.close() + return data + + +def get_extended_fields(): + # Returns: [ID, Type, Name, Cond] + query = """ + SELECT PackageDepends.PackageID AS ID, DependencyTypes.Name AS Type, + PackageDepends.DepName AS Name, PackageDepends.DepCondition AS Cond + FROM PackageDepends + LEFT JOIN DependencyTypes + ON DependencyTypes.ID = PackageDepends.DepTypeID + UNION SELECT PackageRelations.PackageID AS ID, RelationTypes.Name AS Type, + PackageRelations.RelName AS Name, + PackageRelations.RelCondition AS Cond + FROM PackageRelations + LEFT JOIN RelationTypes + ON RelationTypes.ID = PackageRelations.RelTypeID + UNION SELECT PackageGroups.PackageID AS ID, 'Groups' AS Type, + Groups.Name, '' AS Cond + FROM Groups + INNER JOIN PackageGroups ON PackageGroups.GroupID = Groups.ID + UNION SELECT PackageLicenses.PackageID AS ID, 'License' AS Type, + Licenses.Name, '' as Cond + FROM Licenses + INNER JOIN PackageLicenses ON PackageLicenses.LicenseID = Licenses.ID + UNION SELECT Packages.ID AS ID, 'Keywords' AS Type, + PackageKeywords.Keyword AS Name, '' as Cond + FROM PackageKeywords + INNER JOIN Packages ON Packages.PackageBaseID = PackageKeywords.PackageBaseID + """ + return get_extended_dict(query) + + +EXTENDED_FIELD_HANDLERS = { + "--extended": get_extended_fields +} + + +def is_decimal(column): + """ Check if an SQL column is of decimal.Decimal type. """ + if isinstance(column, Decimal): + return float(column) + return column + + +def write_archive(archive: str, output: list): + with gzip.open(archive, "wb") as f: + f.write(b"[\n") + for i, item in enumerate(output): + f.write(orjson.dumps(item)) + if i < len(output) - 1: + f.write(b",") + f.write(b"\n") + f.write(b"]") def main(): @@ -21,32 +197,83 @@ def main(): pkgbaselist_header = "# AUR package base list, generated on " + datestr userlist_header = "# AUR user name list, generated on " + datestr - with gzip.open(packagesfile, "w") as f: - f.write(bytes(pkglist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT Packages.Name FROM Packages " + - "INNER JOIN PackageBases " + - "ON PackageBases.ID = Packages.PackageBaseID " + + updated, update_time = should_update(packages_state, "Packages") + if not updated: + print("Updating Packages...") + + # Query columns; copied from RPC. + columns = ("Packages.ID, Packages.Name, " + "PackageBases.ID AS PackageBaseID, " + "PackageBases.Name AS PackageBase, " + "Version, Description, URL, NumVotes, " + "Popularity, OutOfDateTS AS OutOfDate, " + "Users.UserName AS Maintainer, " + "SubmittedTS AS FirstSubmitted, " + "ModifiedTS AS LastModified") + + # Perform query. + cur = conn.execute(f"SELECT {columns} FROM Packages " + "LEFT JOIN PackageBases " + "ON PackageBases.ID = Packages.PackageBaseID " + "LEFT JOIN Users " + "ON PackageBases.MaintainerUID = Users.ID " "WHERE PackageBases.PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - with gzip.open(packagesmetafile, "wt") as f: - cur = conn.execute("SELECT * FROM Packages") - json.dump({ - "warning": "This is a experimental! It can be removed or modified without warning!", - "columns": [d[0] for d in cur.description], - "data": cur.fetchall() - }, f) + # Produce packages-meta-v1.json.gz + output = list() + snapshot_uri = aurweb.config.get("options", "snapshot_uri") + for result in cur.fetchall(): + item = { + column[0]: is_decimal(result[i]) + for i, column in enumerate(cur.description) + } + item["URLPath"] = snapshot_uri % item.get("Name") + output.append(item) - with gzip.open(pkgbasefile, "w") as f: - f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT Name FROM PackageBases " + - "WHERE PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + write_archive(packagesmetafile, output) - with gzip.open(userfile, "w") as f: - f.write(bytes(userlist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT UserName FROM Users") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + # Produce packages-meta-ext-v1.json.gz + if len(sys.argv) > 1 and sys.argv[1] in EXTENDED_FIELD_HANDLERS: + f = EXTENDED_FIELD_HANDLERS.get(sys.argv[1]) + data = f() + + default_ = {"Groups": [], "License": [], "Keywords": []} + for i in range(len(output)): + data_ = data.get(output[i].get("ID"), default_) + output[i].update(data_) + + write_archive(packagesmetaextfile, output) + + # Produce packages.gz + with gzip.open(packagesfile, "wb") as f: + f.write(bytes(pkglist_header + "\n", "UTF-8")) + f.writelines([ + bytes(x.get("Name") + "\n", "UTF-8") + for x in output + ]) + + update_state(packages_state, update_time) + + updated, update_time = should_update(pkgbases_state, "PackageBases") + if not updated: + print("Updating PackageBases...") + # Produce pkgbase.gz + with gzip.open(pkgbasefile, "w") as f: + f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT Name FROM PackageBases " + + "WHERE PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + update_state(pkgbases_state, update_time) + + updated, update_time = should_update(users_state, "Users") + if not updated: + print("Updating Users...") + # Produce users.gz + with gzip.open(userfile, "w") as f: + f.write(bytes(userlist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT UserName FROM Users") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + update_state(users_state, update_time) conn.close() diff --git a/conf/config.defaults b/conf/config.defaults index 6ccf42d0..25d6dff9 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -103,6 +103,7 @@ server = https://mirrors.kernel.org/archlinux/%s/os/x86_64 [mkpkglists] packagesfile = /srv/http/aurweb/web/html/packages.gz packagesmetafile = /srv/http/aurweb/web/html/packages-meta-v1.json.gz +packagesmetaextfile = /srv/http/aurweb/web/html/packages-meta-ext-v1.json.gz pkgbasefile = /srv/http/aurweb/web/html/pkgbase.gz userfile = /srv/http/aurweb/web/html/users.gz diff --git a/test/setup.sh b/test/setup.sh index 8ee9eef2..d0c15b82 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -37,6 +37,7 @@ enable-maintenance = 0 maintenance-exceptions = 127.0.0.1 commit_uri = https://aur.archlinux.org/cgit/aur.git/log/?h=%s&id=%s localedir = $TOPLEVEL/web/locale/ +snapshot_uri = /cgit/aur.git/snapshot/%s.tar.gz [notifications] notify-cmd = $NOTIFY @@ -68,6 +69,7 @@ server = file://$(pwd)/remote/ [mkpkglists] packagesfile = packages.gz packagesmetafile = packages-meta-v1.json.gz +packagesmetaextfile = packages-meta-ext-v1.json.gz pkgbasefile = pkgbase.gz userfile = users.gz EOF diff --git a/web/html/index.php b/web/html/index.php index 3163c3e8..dc435162 100644 --- a/web/html/index.php +++ b/web/html/index.php @@ -189,7 +189,8 @@ if (!empty($tokens[1]) && '/' . $tokens[1] == get_pkg_route()) { readfile("./$path"); break; case "/packages.gz": - case "/packages-teapot.json.gz": + case "/packages-meta-v1.json.gz": + case "/packages-meta-ext-v1.json.gz": case "/pkgbase.gz": case "/users.gz": header("Content-Type: text/plain"); From d62af4ceb582c360a6d7bbb876418170efbd92fc Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 6 Nov 2021 16:23:08 -0700 Subject: [PATCH 539/844] feat(mkpkglists): added metadata archives Two new archives are available: - packages-meta-v1.json.gz - RPC search formatted data for all packages - ~2.1MB at the time of writing. - packages-meta-ext-v1.json.gz (via --extended) - RPC multiinfo formatted data for all packages. - ~9.8MB at the time of writing. New dependencies are required for this update: - `python-orjson` All archives served out by aur.archlinux.org distribute the Last-Modified header and support the If-Modified-Since header, which should be populated with Last-Modified's value. These should be used by clients to avoid redownloading the archive when unnecessary. Additionally, the new meta archives contain a format suitable for streaming the data as the file is retrieved. It is still in JSON format, however, users can parse package objects line by line after the first '[' found in the file, until the last ']'; both contained on their own lines. Note: This commit is a documentation change and commit body. Signed-off-by: Kevin Morris --- doc/maintenance.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/maintenance.txt b/doc/maintenance.txt index d6094545..2c5c9faf 100644 --- a/doc/maintenance.txt +++ b/doc/maintenance.txt @@ -70,7 +70,8 @@ computations and clean up the database: * aurweb-pkgmaint automatically removes empty repositories that were created within the last 24 hours but never populated. -* aurweb-mkpkglists generates the package list files. +* aurweb-mkpkglists generates the package list files; it takes an optional + --extended flag, which additionally produces multiinfo metadata. * aurweb-usermaint removes the last login IP address of all users that did not login within the past seven days. @@ -79,7 +80,7 @@ These scripts can be installed by running `python3 setup.py install` and are usually scheduled using Cron. The current setup is: ---- -*/5 * * * * aurweb-mkpkglists +*/5 * * * * aurweb-mkpkglists [--extended] 1 */2 * * * aurweb-popupdate 2 */2 * * * aurweb-aurblup 3 */2 * * * aurweb-pkgmaint From 0155f4ea84917ca12c905e9035a1d202ea91a3ff Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 6 Nov 2021 17:13:16 -0700 Subject: [PATCH 540/844] fix(mkpkglists): remove caching We really need caching for this; however, our current caching method will cause the script to bypass changes to columns if they have nothing to do with IDs. Signed-off-by: Kevin Morris --- aurweb/scripts/mkpkglists.py | 159 ++++++++++++----------------------- 1 file changed, 54 insertions(+), 105 deletions(-) diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index f2095a20..2566a146 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -20,60 +20,23 @@ on the following, right-hand side fields are added to each item. import datetime import gzip -import os import sys from collections import defaultdict from decimal import Decimal -from typing import Tuple import orjson import aurweb.config import aurweb.db - -def state_path(archive: str) -> str: - # A hard-coded /tmp state directory. - # TODO: Use Redis cache to store this state after we merge - # FastAPI into master and removed PHP from the tree. - return os.path.join("/tmp", os.path.basename(archive) + ".state") - - packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') packagesmetafile = aurweb.config.get('mkpkglists', 'packagesmetafile') packagesmetaextfile = aurweb.config.get('mkpkglists', 'packagesmetaextfile') -packages_state = state_path(packagesfile) pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') -pkgbases_state = state_path(pkgbasefile) userfile = aurweb.config.get('mkpkglists', 'userfile') -users_state = state_path(userfile) - - -def should_update(state: str, tablename: str) -> Tuple[bool, int]: - if aurweb.config.get("database", "backend") != "mysql": - return (False, 0) - - db_name = aurweb.config.get("database", "name") - conn = aurweb.db.Connection() - cur = conn.execute("SELECT auto_increment FROM information_schema.tables " - "WHERE table_schema = ? AND table_name = ?", - (db_name, tablename,)) - update_time = cur.fetchone()[0] - - saved_update_time = 0 - if os.path.exists(state): - with open(state) as f: - saved_update_time = int(f.read().strip()) - - return (saved_update_time == update_time, update_time) - - -def update_state(state: str, update_time: int) -> None: - with open(state, "w") as f: - f.write(str(update_time)) TYPE_MAP = { @@ -197,83 +160,69 @@ def main(): pkgbaselist_header = "# AUR package base list, generated on " + datestr userlist_header = "# AUR user name list, generated on " + datestr - updated, update_time = should_update(packages_state, "Packages") - if not updated: - print("Updating Packages...") + # Query columns; copied from RPC. + columns = ("Packages.ID, Packages.Name, " + "PackageBases.ID AS PackageBaseID, " + "PackageBases.Name AS PackageBase, " + "Version, Description, URL, NumVotes, " + "Popularity, OutOfDateTS AS OutOfDate, " + "Users.UserName AS Maintainer, " + "SubmittedTS AS FirstSubmitted, " + "ModifiedTS AS LastModified") - # Query columns; copied from RPC. - columns = ("Packages.ID, Packages.Name, " - "PackageBases.ID AS PackageBaseID, " - "PackageBases.Name AS PackageBase, " - "Version, Description, URL, NumVotes, " - "Popularity, OutOfDateTS AS OutOfDate, " - "Users.UserName AS Maintainer, " - "SubmittedTS AS FirstSubmitted, " - "ModifiedTS AS LastModified") + # Perform query. + cur = conn.execute(f"SELECT {columns} FROM Packages " + "LEFT JOIN PackageBases " + "ON PackageBases.ID = Packages.PackageBaseID " + "LEFT JOIN Users " + "ON PackageBases.MaintainerUID = Users.ID " + "WHERE PackageBases.PackagerUID IS NOT NULL") - # Perform query. - cur = conn.execute(f"SELECT {columns} FROM Packages " - "LEFT JOIN PackageBases " - "ON PackageBases.ID = Packages.PackageBaseID " - "LEFT JOIN Users " - "ON PackageBases.MaintainerUID = Users.ID " - "WHERE PackageBases.PackagerUID IS NOT NULL") + # Produce packages-meta-v1.json.gz + output = list() + snapshot_uri = aurweb.config.get("options", "snapshot_uri") + for result in cur.fetchall(): + item = { + column[0]: is_decimal(result[i]) + for i, column in enumerate(cur.description) + } + item["URLPath"] = snapshot_uri % item.get("Name") + output.append(item) - # Produce packages-meta-v1.json.gz - output = list() - snapshot_uri = aurweb.config.get("options", "snapshot_uri") - for result in cur.fetchall(): - item = { - column[0]: is_decimal(result[i]) - for i, column in enumerate(cur.description) - } - item["URLPath"] = snapshot_uri % item.get("Name") - output.append(item) + write_archive(packagesmetafile, output) - write_archive(packagesmetafile, output) + # Produce packages-meta-ext-v1.json.gz + if len(sys.argv) > 1 and sys.argv[1] in EXTENDED_FIELD_HANDLERS: + f = EXTENDED_FIELD_HANDLERS.get(sys.argv[1]) + data = f() - # Produce packages-meta-ext-v1.json.gz - if len(sys.argv) > 1 and sys.argv[1] in EXTENDED_FIELD_HANDLERS: - f = EXTENDED_FIELD_HANDLERS.get(sys.argv[1]) - data = f() + default_ = {"Groups": [], "License": [], "Keywords": []} + for i in range(len(output)): + data_ = data.get(output[i].get("ID"), default_) + output[i].update(data_) - default_ = {"Groups": [], "License": [], "Keywords": []} - for i in range(len(output)): - data_ = data.get(output[i].get("ID"), default_) - output[i].update(data_) + write_archive(packagesmetaextfile, output) - write_archive(packagesmetaextfile, output) + # Produce packages.gz + with gzip.open(packagesfile, "wb") as f: + f.write(bytes(pkglist_header + "\n", "UTF-8")) + f.writelines([ + bytes(x.get("Name") + "\n", "UTF-8") + for x in output + ]) - # Produce packages.gz - with gzip.open(packagesfile, "wb") as f: - f.write(bytes(pkglist_header + "\n", "UTF-8")) - f.writelines([ - bytes(x.get("Name") + "\n", "UTF-8") - for x in output - ]) + # Produce pkgbase.gz + with gzip.open(pkgbasefile, "w") as f: + f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT Name FROM PackageBases " + + "WHERE PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - update_state(packages_state, update_time) - - updated, update_time = should_update(pkgbases_state, "PackageBases") - if not updated: - print("Updating PackageBases...") - # Produce pkgbase.gz - with gzip.open(pkgbasefile, "w") as f: - f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT Name FROM PackageBases " + - "WHERE PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - update_state(pkgbases_state, update_time) - - updated, update_time = should_update(users_state, "Users") - if not updated: - print("Updating Users...") - # Produce users.gz - with gzip.open(userfile, "w") as f: - f.write(bytes(userlist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT UserName FROM Users") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - update_state(users_state, update_time) + # Produce users.gz + with gzip.open(userfile, "w") as f: + f.write(bytes(userlist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT UserName FROM Users") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) conn.close() From 0403b89f53302f7d58b170081ef7bbd2726d8fc1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 9 Nov 2021 02:08:03 -0800 Subject: [PATCH 541/844] feat: add packagesmeta[ext]file option to conf/config.dev Better defaults for Docker here. Signed-off-by: Kevin Morris --- conf/config.dev | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/config.dev b/conf/config.dev index dac85477..05566e8b 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -68,6 +68,8 @@ session_secret = secret [mkpkglists] packagesfile = /packages.gz +packagesmetafile = /packages-meta-v1.json.gz +packagesmetaextfile = /packages-meta-ext-v1.json.gz pkgbasefile = /pkgbase.gz userfile = /users.gz From 068b067e148cb4f39fdfb7aa2086568084ac6a0a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 9 Nov 2021 02:28:52 -0800 Subject: [PATCH 542/844] feat(docker): log cron executions Signed-off-by: Kevin Morris --- docker/config/aurweb-cron | 4 ++-- docker/scripts/run-cron.sh | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docker/config/aurweb-cron b/docker/config/aurweb-cron index 1be7c13c..87d7ed05 100644 --- a/docker/config/aurweb-cron +++ b/docker/config/aurweb-cron @@ -1,2 +1,2 @@ -*/5 * * * * aurweb-aurblup -*/5 * * * * aurweb-mkpkglists +*/5 * * * * bash -c 'aurweb-aurblup && echo "[$(date -u)] executed aurblup" >> /var/log/mkpkglists.log' +*/5 * * * * bash -c 'aurweb-mkpkglists && echo "[$(date -u)] executed mkpkglists" >> /var/log/mkpkglists.log' diff --git a/docker/scripts/run-cron.sh b/docker/scripts/run-cron.sh index d927a790..d227be94 100755 --- a/docker/scripts/run-cron.sh +++ b/docker/scripts/run-cron.sh @@ -2,6 +2,13 @@ cd /aurweb aurweb-aurblup +if [ $? -eq 0 ]; then + echo "[$(date -u)] executed aurblup" >> /var/log/aurblup.log +fi + aurweb-mkpkglists +if [ $? -eq 0 ]; then + echo "[$(date -u)] executed mkpkglists" >> /var/log/mkpkglists.log +fi exec /usr/bin/crond -n From 107367f958ef320d63952def915b65cef1ed31bd Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 9 Nov 2021 02:29:39 -0800 Subject: [PATCH 543/844] feat(docker): use mkpkglists --extended flag Signed-off-by: Kevin Morris --- docker/config/aurweb-cron | 2 +- docker/scripts/run-cron.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/config/aurweb-cron b/docker/config/aurweb-cron index 87d7ed05..149b9d19 100644 --- a/docker/config/aurweb-cron +++ b/docker/config/aurweb-cron @@ -1,2 +1,2 @@ */5 * * * * bash -c 'aurweb-aurblup && echo "[$(date -u)] executed aurblup" >> /var/log/mkpkglists.log' -*/5 * * * * bash -c 'aurweb-mkpkglists && echo "[$(date -u)] executed mkpkglists" >> /var/log/mkpkglists.log' +*/5 * * * * bash -c 'aurweb-mkpkglists --extended && echo "[$(date -u)] executed mkpkglists" >> /var/log/mkpkglists.log' diff --git a/docker/scripts/run-cron.sh b/docker/scripts/run-cron.sh index d227be94..83ad6566 100755 --- a/docker/scripts/run-cron.sh +++ b/docker/scripts/run-cron.sh @@ -6,7 +6,7 @@ if [ $? -eq 0 ]; then echo "[$(date -u)] executed aurblup" >> /var/log/aurblup.log fi -aurweb-mkpkglists +aurweb-mkpkglists --extended if [ $? -eq 0 ]; then echo "[$(date -u)] executed mkpkglists" >> /var/log/mkpkglists.log fi From abbecf5194b69d419e81b547eaa8b3e72d550524 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 10 Nov 2021 06:18:34 -0800 Subject: [PATCH 544/844] change(mkpkglists): remove header comments These comments change every time mkpkglists is run; which would invalidate the ETag headers disbursed by the gzip host. This commit removes those changing headers. Signed-off-by: Kevin Morris --- aurweb/scripts/mkpkglists.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index 2566a146..b2d37d85 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -18,7 +18,6 @@ on the following, right-hand side fields are added to each item. """ -import datetime import gzip import sys @@ -155,11 +154,6 @@ def write_archive(archive: str, output: list): def main(): conn = aurweb.db.Connection() - datestr = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") - pkglist_header = "# AUR package list, generated on " + datestr - pkgbaselist_header = "# AUR package base list, generated on " + datestr - userlist_header = "# AUR user name list, generated on " + datestr - # Query columns; copied from RPC. columns = ("Packages.ID, Packages.Name, " "PackageBases.ID AS PackageBaseID, " @@ -205,7 +199,6 @@ def main(): # Produce packages.gz with gzip.open(packagesfile, "wb") as f: - f.write(bytes(pkglist_header + "\n", "UTF-8")) f.writelines([ bytes(x.get("Name") + "\n", "UTF-8") for x in output @@ -213,14 +206,12 @@ def main(): # Produce pkgbase.gz with gzip.open(pkgbasefile, "w") as f: - f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) cur = conn.execute("SELECT Name FROM PackageBases " + "WHERE PackagerUID IS NOT NULL") f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) # Produce users.gz with gzip.open(userfile, "w") as f: - f.write(bytes(userlist_header + "\n", "UTF-8")) cur = conn.execute("SELECT UserName FROM Users") f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) From 4f7aeafa8d9fc835c604b82bd8f94308220ed0dd Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 10 Nov 2021 06:20:22 -0800 Subject: [PATCH 545/844] feat(docker): host gzip archive downloads - added config option [mkpkglists] archivedir - created by mkpkglists Signed-off-by: Kevin Morris --- aurweb/scripts/mkpkglists.py | 4 ++++ conf/config.defaults | 1 + conf/config.dev | 11 ++++++----- docker-compose.yml | 5 +++++ docker/config/nginx.conf | 19 ++++++++++++++++++- docker/php-entrypoint.sh | 4 ++++ 6 files changed, 38 insertions(+), 6 deletions(-) diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index b2d37d85..f4b0fbe5 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -19,6 +19,7 @@ on the following, right-hand side fields are added to each item. """ import gzip +import os import sys from collections import defaultdict @@ -29,6 +30,9 @@ import orjson import aurweb.config import aurweb.db +archivedir = aurweb.config.get("mkpkglists", "archivedir") +os.makedirs(archivedir, exist_ok=True) + packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') packagesmetafile = aurweb.config.get('mkpkglists', 'packagesmetafile') packagesmetaextfile = aurweb.config.get('mkpkglists', 'packagesmetaextfile') diff --git a/conf/config.defaults b/conf/config.defaults index 25d6dff9..c29d7045 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -101,6 +101,7 @@ sync-dbs = core extra community multilib testing community-testing server = https://mirrors.kernel.org/archlinux/%s/os/x86_64 [mkpkglists] +archivedir = /srv/http/aurweb/web/html packagesfile = /srv/http/aurweb/web/html/packages.gz packagesmetafile = /srv/http/aurweb/web/html/packages-meta-v1.json.gz packagesmetaextfile = /srv/http/aurweb/web/html/packages-meta-ext-v1.json.gz diff --git a/conf/config.dev b/conf/config.dev index 05566e8b..9467615e 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -67,11 +67,12 @@ session_secret = secret ;commit_hash = 1234567 [mkpkglists] -packagesfile = /packages.gz -packagesmetafile = /packages-meta-v1.json.gz -packagesmetaextfile = /packages-meta-ext-v1.json.gz -pkgbasefile = /pkgbase.gz -userfile = /users.gz +archivedir = /var/lib/aurweb/archives +packagesfile = /var/lib/aurweb/archives/packages.gz +packagesmetafile = /var/lib/aurweb/archives/packages-meta-v1.json.gz +packagesmetaextfile = /var/lib/aurweb/archives/packages-meta-ext-v1.json.gz +pkgbasefile = /var/lib/aurweb/archives/pkgbase.gz +userfile = /var/lib/aurweb/archives/users.gz [aurblup] db-path = YOUR_AUR_ROOT/aurblup/ diff --git a/docker-compose.yml b/docker-compose.yml index c2b14f91..2fba1305 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -156,6 +156,7 @@ services: condition: service_started volumes: - mariadb_run:/var/run/mysqld + - archives:/var/lib/aurweb/archives php-fpm: image: aurweb:latest @@ -180,6 +181,7 @@ services: condition: service_started volumes: - mariadb_run:/var/run/mysqld + - archives:/var/lib/aurweb/archives ports: - "19000:9000" @@ -236,6 +238,8 @@ services: condition: service_healthy php-fpm: condition: service_healthy + volumes: + - archives:/var/lib/aurweb/archives sharness: image: aurweb:latest @@ -343,3 +347,4 @@ volumes: mariadb_data: {} # Share /var/lib/mysql git_data: {} # Share aurweb/aur.git smartgit_run: {} + archives: {} diff --git a/docker/config/nginx.conf b/docker/config/nginx.conf index 4288a57d..c3ffd7fa 100644 --- a/docker/config/nginx.conf +++ b/docker/config/nginx.conf @@ -94,7 +94,24 @@ http { ssl_certificate /etc/ssl/certs/web.cert.pem; ssl_certificate_key /etc/ssl/private/web.key.pem; - root /aurweb/web/html; + location ~ ^/.*\.gz$ { + # Override mime type to text/plain. + types { text/plain gz; } + default_type text/plain; + + # Filesystem location of .gz archives. + root /var/lib/aurweb/archives; + + # When we match this block, fix-up trying without a trailing slash. + try_files $uri $uri/ =404; + + # Caching headers. + expires max; + add_header Content-Encoding gzip; + add_header Cache-Control public; + add_header Last-Modified ""; + add_header ETag ""; + } location / { try_files $uri @proxy_to_app; diff --git a/docker/php-entrypoint.sh b/docker/php-entrypoint.sh index 81f70673..274f8e17 100755 --- a/docker/php-entrypoint.sh +++ b/docker/php-entrypoint.sh @@ -1,6 +1,10 @@ #!/bin/bash set -eou pipefail +for archive in packages pkgbase users packages-meta-v1.json packages-meta-ext-v1.json; do + ln -vsf /var/lib/aurweb/archives/${archive}.gz /aurweb/web/html/${archive}.gz +done + # Setup a config for our mysql db. cp -vf conf/config.dev conf/config sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config From 0c57c53da18b36c470471a754e5ea892a46db505 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 10 Nov 2021 07:04:03 -0800 Subject: [PATCH 546/844] fix(sharness): fix AUR_CONFIG generation for mkpkglists test Signed-off-by: Kevin Morris --- test/setup.sh | 1 + test/t2100-mkpkglists.t | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/setup.sh b/test/setup.sh index d0c15b82..a09e0c6e 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -67,6 +67,7 @@ sync-dbs = test server = file://$(pwd)/remote/ [mkpkglists] +archivedir = $(pwd)/archive packagesfile = packages.gz packagesmetafile = packages-meta-v1.json.gz packagesmetaextfile = packages-meta-ext-v1.json.gz diff --git a/test/t2100-mkpkglists.t b/test/t2100-mkpkglists.t index 6bb6838d..d217c4f6 100755 --- a/test/t2100-mkpkglists.t +++ b/test/t2100-mkpkglists.t @@ -8,8 +8,8 @@ test_expect_success 'Test package list generation with no packages.' ' echo "DELETE FROM Packages;" | sqlite3 aur.db && echo "DELETE FROM PackageBases;" | sqlite3 aur.db && cover "$MKPKGLISTS" && - test $(zcat packages.gz | wc -l) -eq 1 && - test $(zcat pkgbase.gz | wc -l) -eq 1 + test $(zcat packages.gz | wc -l) -eq 0 && + test $(zcat pkgbase.gz | wc -l) -eq 0 ' test_expect_success 'Test package list generation.' ' From 4b2be7fff8f8e5203cd84a9f25f590d4bdf8ef5d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 10 Nov 2021 13:00:01 -0800 Subject: [PATCH 547/844] feat(docker): add poetry caching Signed-off-by: Kevin Morris --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index b490d2fa..3c12cbf8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,8 @@ FROM archlinux:base-devel +VOLUME /root/.cache/pypoetry/cache +VOLUME /root/.cache/pypoetry/artifacts + ENV PATH="/root/.poetry/bin:${PATH}" ENV PYTHONPATH=/aurweb ENV AUR_CONFIG=conf/config From daef98080e7f1f595dde09c20da1b7b57f5478ed Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 10 Nov 2021 13:05:19 -0800 Subject: [PATCH 548/844] fix(fastapi): fix broken official package query Signed-off-by: Kevin Morris --- aurweb/packages/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index be351ebe..cdec26f3 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -56,8 +56,8 @@ def dep_extra_desc(dep: models.PackageDependency) -> str: def pkgname_link(pkgname: str) -> str: base = "/".join([OFFICIAL_BASE, "packages"]) official = db.query(models.OfficialProvider).filter( - models.OfficialProvider.Name == pkgname) - if official.scalar(): + models.OfficialProvider.Name == pkgname).exists() + if db.query(official).scalar(): return f"{base}/?q={pkgname}" return f"/packages/{pkgname}" From 52110b7db5e9b6f3a1dc4fefdf60a31dc946d487 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 10 Nov 2021 13:40:19 -0800 Subject: [PATCH 549/844] fix(mkpkglists): default keys to result[1] Signed-off-by: Kevin Morris --- aurweb/scripts/mkpkglists.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index 2566a146..72efcd0a 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -21,14 +21,12 @@ on the following, right-hand side fields are added to each item. import datetime import gzip import sys - from collections import defaultdict from decimal import Decimal -import orjson - import aurweb.config import aurweb.db +import orjson packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') packagesmetafile = aurweb.config.get('mkpkglists', 'packagesmetafile') @@ -80,7 +78,7 @@ def get_extended_dict(query: str): for result in cursor.fetchall(): pkgid = result[0] - key = TYPE_MAP.get(result[1]) + key = TYPE_MAP.get(result[1], result[1]) output = result[2] if result[3]: output += result[3] From 6e344ce9da894cdb70fb02cc531145907ac28a48 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 10 Nov 2021 13:40:19 -0800 Subject: [PATCH 550/844] fix(mkpkglists): default keys to result[1] Signed-off-by: Kevin Morris --- aurweb/scripts/mkpkglists.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index f4b0fbe5..74d41c7c 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -21,7 +21,6 @@ on the following, right-hand side fields are added to each item. import gzip import os import sys - from collections import defaultdict from decimal import Decimal @@ -83,7 +82,7 @@ def get_extended_dict(query: str): for result in cursor.fetchall(): pkgid = result[0] - key = TYPE_MAP.get(result[1]) + key = TYPE_MAP.get(result[1], result[1]) output = result[2] if result[3]: output += result[3] From 8788f9900576bc7fa79b54566f23cdaeaeafa9eb Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 10 Nov 2021 13:54:18 -0800 Subject: [PATCH 551/844] fix(mkpkglists): restore isort order Signed-off-by: Kevin Morris --- aurweb/scripts/mkpkglists.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index 74d41c7c..307b2b12 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -21,6 +21,7 @@ on the following, right-hand side fields are added to each item. import gzip import os import sys + from collections import defaultdict from decimal import Decimal From 66978e40a46cd096a49c0c7cba1ad60010313ba1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 10 Nov 2021 13:57:33 -0800 Subject: [PATCH 552/844] fix(mkpkglists): fix isort order (master) Signed-off-by: Kevin Morris --- aurweb/scripts/mkpkglists.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index 72efcd0a..2d34a17b 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -21,12 +21,14 @@ on the following, right-hand side fields are added to each item. import datetime import gzip import sys + from collections import defaultdict from decimal import Decimal +import orjson + import aurweb.config import aurweb.db -import orjson packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') packagesmetafile = aurweb.config.get('mkpkglists', 'packagesmetafile') From 567090547d5f96b5e4266bb2d88c7344348bccf2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 13:09:05 -0800 Subject: [PATCH 553/844] add labels to gitlab issue templates Signed-off-by: Kevin Morris --- .gitlab/issue_templates/Account Request.md | 2 ++ .gitlab/issue_templates/Bug.md | 2 ++ .gitlab/issue_templates/Feature.md | 2 ++ .gitlab/issue_templates/Feedback.md | 2 ++ 4 files changed, 8 insertions(+) diff --git a/.gitlab/issue_templates/Account Request.md b/.gitlab/issue_templates/Account Request.md index 244cfbe8..6831d3ad 100644 --- a/.gitlab/issue_templates/Account Request.md +++ b/.gitlab/issue_templates/Account Request.md @@ -10,3 +10,5 @@ - Username: the_username_you_want - Email: valid@email.org - Account Type: (User|Trusted User) + +/label account-request diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md index d84a5181..d125e22b 100644 --- a/.gitlab/issue_templates/Bug.md +++ b/.gitlab/issue_templates/Bug.md @@ -32,3 +32,5 @@ aurweb's HTML render output. If you're testing locally, use the commit on which you are experiencing the bug. If you have found a bug which exists on live aur.archlinux.org, include the version located at the bottom of the webpage. + +/label bug unconfirmed diff --git a/.gitlab/issue_templates/Feature.md b/.gitlab/issue_templates/Feature.md index 5b1524b1..c907adcd 100644 --- a/.gitlab/issue_templates/Feature.md +++ b/.gitlab/issue_templates/Feature.md @@ -28,3 +28,5 @@ Example: - [Feature] Do not allow users to be Tyrants - \<(issue|merge_request)_link\> + +/label feature unconsidered diff --git a/.gitlab/issue_templates/Feedback.md b/.gitlab/issue_templates/Feedback.md index e32120aa..950ec0c6 100644 --- a/.gitlab/issue_templates/Feedback.md +++ b/.gitlab/issue_templates/Feedback.md @@ -54,3 +54,5 @@ That being said: please include an overall summary of your experience and how you felt about the current implementation which you're testing in comparison with PHP (current aur.archlinux.org, or https://localhost:8443 through docker). + +/label feedback From 0da11f068bd34bc534baf34027b2a925475f6666 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 16:22:30 -0800 Subject: [PATCH 554/844] fix(fastapi): check for prometheus info.response When this is unchecked, exceptions cause the resulting stack trace to be oblivious to the original exception thrown. This commit changes that behavior so that metrics are created only when info.response exists. Signed-off-by: Kevin Morris --- aurweb/prometheus.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/aurweb/prometheus.py b/aurweb/prometheus.py index a64f6b27..dae56320 100644 --- a/aurweb/prometheus.py +++ b/aurweb/prometheus.py @@ -79,9 +79,10 @@ def http_requests_total() -> Callable[[Info], None]: method = scope.get("method") path = get_matching_route_path(base_scope, scope.get("router").routes) - status = str(int(info.response.status_code))[:1] + "xx" - metric.labels(method=method, path=path, status=status).inc() + if info.response: + status = str(int(info.response.status_code))[:1] + "xx" + metric.labels(method=method, path=path, status=status).inc() return instrumentation @@ -95,7 +96,8 @@ def http_api_requests_total() -> Callable[[Info], None]: def instrumentation(info: Info) -> None: if info.request.url.path.rstrip("/") == "/rpc": type = info.request.query_params.get("type", "None") - status = str(info.response.status_code)[:1] + "xx" - metric.labels(type=type, status=status).inc() + if info.response: + status = str(info.response.status_code)[:1] + "xx" + metric.labels(type=type, status=status).inc() return instrumentation From cef217388adaa97e66e638ca4baf99f3b8d0f865 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 13:09:05 -0800 Subject: [PATCH 555/844] add labels to gitlab issue templates Signed-off-by: Kevin Morris --- .gitlab/issue_templates/Account Request.md | 2 ++ .gitlab/issue_templates/Bug.md | 2 ++ .gitlab/issue_templates/Feature.md | 2 ++ .gitlab/issue_templates/Feedback.md | 2 ++ 4 files changed, 8 insertions(+) diff --git a/.gitlab/issue_templates/Account Request.md b/.gitlab/issue_templates/Account Request.md index 244cfbe8..6831d3ad 100644 --- a/.gitlab/issue_templates/Account Request.md +++ b/.gitlab/issue_templates/Account Request.md @@ -10,3 +10,5 @@ - Username: the_username_you_want - Email: valid@email.org - Account Type: (User|Trusted User) + +/label account-request diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md index d84a5181..d125e22b 100644 --- a/.gitlab/issue_templates/Bug.md +++ b/.gitlab/issue_templates/Bug.md @@ -32,3 +32,5 @@ aurweb's HTML render output. If you're testing locally, use the commit on which you are experiencing the bug. If you have found a bug which exists on live aur.archlinux.org, include the version located at the bottom of the webpage. + +/label bug unconfirmed diff --git a/.gitlab/issue_templates/Feature.md b/.gitlab/issue_templates/Feature.md index 5b1524b1..c907adcd 100644 --- a/.gitlab/issue_templates/Feature.md +++ b/.gitlab/issue_templates/Feature.md @@ -28,3 +28,5 @@ Example: - [Feature] Do not allow users to be Tyrants - \<(issue|merge_request)_link\> + +/label feature unconsidered diff --git a/.gitlab/issue_templates/Feedback.md b/.gitlab/issue_templates/Feedback.md index e32120aa..950ec0c6 100644 --- a/.gitlab/issue_templates/Feedback.md +++ b/.gitlab/issue_templates/Feedback.md @@ -54,3 +54,5 @@ That being said: please include an overall summary of your experience and how you felt about the current implementation which you're testing in comparison with PHP (current aur.archlinux.org, or https://localhost:8443 through docker). + +/label feedback From 5f5fa44d0d3b1930ce22e64968392c8b595205c7 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 17:12:13 -0800 Subject: [PATCH 556/844] fix(fastapi): fix licenses check Signed-off-by: Kevin Morris --- templates/partials/packages/details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index e8414bf4..583149f8 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -61,7 +61,7 @@ {% endif %} - {% if licenses and licenses.scalar() and show_package_details %} + {% if licenses and licenses.count() and show_package_details %} {{ "Licenses" | tr }}: {{ licenses | join(', ', attribute='Name') | default('None' | tr) }} From 363afff33260b2e3ce6236a22c8d6c32f0ffdd70 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 17:36:08 -0800 Subject: [PATCH 557/844] feat(fastapi): add /pkgbase/{name}/keywords (post) Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 23 ++++++++++++++++++ templates/partials/packages/details.html | 4 ++-- test/test_packages_routes.py | 30 ++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index bcc0be56..e227fe23 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -816,6 +816,29 @@ async def requests_close_post(request: Request, id: int, return RedirectResponse("/requests", status_code=HTTPStatus.SEE_OTHER) +@router.post("/pkgbase/{name}/keywords") +async def pkgbase_keywords(request: Request, name: str, + keywords: str = Form(default=str())): + pkgbase = get_pkg_or_base(name, models.PackageBase) + keywords = set(keywords.split(" ")) + + # Delete all keywords which are not supplied by the user. + with db.begin(): + db.delete(models.PackageKeyword, + and_(models.PackageKeyword.PackageBaseID == pkgbase.ID, + ~models.PackageKeyword.Keyword.in_(keywords))) + + existing_keywords = set(kwd.Keyword for kwd in pkgbase.keywords.all()) + with db.begin(): + for keyword in keywords.difference(existing_keywords): + db.create(models.PackageKeyword, + PackageBase=pkgbase, + Keyword=keyword) + + return RedirectResponse(f"/pkgbase/{name}", + status_code=HTTPStatus.SEE_OTHER) + + @router.get("/pkgbase/{name}/flag") @auth_required(True, redirect="/pkgbase/{name}/flag") async def pkgbase_flag_get(request: Request, name: str): diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index 583149f8..d525f63b 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -37,7 +37,7 @@ {{ "Keywords" | tr }}: {% if request.user.has_credential("CRED_PKGBASE_SET_KEYWORDS", approved=[pkgbase.Maintainer]) %} -

    @@ -51,7 +51,7 @@ {% else %} - {% for keyword in pkgbase.keywords %} + {% for keyword in pkgbase.keywords.all() %} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index c4d9ab1c..887945d9 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -2604,3 +2604,33 @@ def test_account_comments(client: TestClient, user: User, package: Package): expected = rendered_comment.RenderedComment.replace( "

    ", "").replace("

    ", "") assert rendered[0].text.strip() == expected + + +def test_pkgbase_keywords(client: TestClient, user: User, package: Package): + endpoint = f"/pkgbase/{package.PackageBase.Name}" + with client as request: + resp = request.get(endpoint) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + keywords = root.xpath('//a[@class="keyword"]') + assert len(keywords) == 0 + + cookies = {"AURSID": user.login(Request(), "testPassword")} + post_endpoint = f"{endpoint}/keywords" + with client as request: + resp = request.post(post_endpoint, data={ + "keywords": "abc test" + }, cookies=cookies) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + with client as request: + resp = request.get(resp.headers.get("location")) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + keywords = root.xpath('//a[@class="keyword"]') + assert len(keywords) == 2 + expected = ["abc", "test"] + for i, keyword in enumerate(keywords): + assert keyword.text.strip() == expected[i] From 20f5519b99a29dc376a6d8a2325f58ca9305380c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 18:13:21 -0800 Subject: [PATCH 558/844] fix(fastapi): hide keywords when there are none or they can't be edited Signed-off-by: Kevin Morris --- templates/partials/packages/details.html | 56 ++++++++++++------------ 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index d525f63b..00859068 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -33,34 +33,36 @@ {% endif %} - - {{ "Keywords" | tr }}: - {% if request.user.has_credential("CRED_PKGBASE_SET_KEYWORDS", approved=[pkgbase.Maintainer]) %} - - -
    - - -
    - - - {% else %} - - {% for keyword in pkgbase.keywords.all() %} -
    + {{ "Keywords" | tr }}: + {% if request.user.has_credential("CRED_PKGBASE_SET_KEYWORDS", approved=[pkgbase.Maintainer]) %} + +
    - {{ keyword.Keyword }} - - {% endfor %} - - {% endif %} - +
    + + +
    +
    + + {% else %} + + {% for keyword in pkgbase.keywords.all() %} + + {{ keyword.Keyword }} + + {% endfor %} + + {% endif %} + + {% endif %} {% if licenses and licenses.count() and show_package_details %} {{ "Licenses" | tr }}: From 2dc6cfec23dcc8432db52f0676e7aa9f643d521c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 18:14:15 -0800 Subject: [PATCH 559/844] fix(fastapi): reorganize licenses display Signed-off-by: Kevin Morris --- templates/partials/packages/details.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index 00859068..1fbd47d9 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -63,10 +63,10 @@ {% endif %} {% endif %} - {% if licenses and licenses.count() and show_package_details %} + {% if show_package_details and licenses and licenses.count() %} {{ "Licenses" | tr }}: - {{ licenses | join(', ', attribute='Name') | default('None' | tr) }} + {{ licenses.all() | join(', ', attribute='Name') | default('None' | tr) }} {% endif %} {% if show_package_details %} From 2016b80ea97dc89d3170a91265abfd05763b81ab Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 18:14:50 -0800 Subject: [PATCH 560/844] fix(fastapi): hide conflicts when there are none Signed-off-by: Kevin Morris --- templates/partials/packages/details.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index 1fbd47d9..da99cf1b 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -69,11 +69,11 @@ {{ licenses.all() | join(', ', attribute='Name') | default('None' | tr) }} {% endif %} - {% if show_package_details %} + {% if show_package_details and conflicts and conflicts.count() %} {{ "Conflicts" | tr }}: - {{ conflicts | join(', ', attribute='RelName') }} + {{ conflicts.all() | join(', ', attribute='RelName') }} {% endif %} From 50a9690c2ddefeb3fe758ed998f2edbe3773d5f1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 19:09:24 -0800 Subject: [PATCH 561/844] feat(fastapi): add Provides field in package details Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 6 +++++- templates/partials/packages/details.html | 8 ++++++++ test/test_packages_routes.py | 14 +++++++++++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index e227fe23..fb91a8e3 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -12,7 +12,7 @@ import aurweb.packages.util from aurweb import db, defaults, l10n, logging, models, util from aurweb.auth import auth_required from aurweb.models.package_request import ACCEPTED_ID, PENDING_ID, REJECTED_ID -from aurweb.models.relation_type import CONFLICTS_ID +from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID from aurweb.models.request_type import DELETION_ID, MERGE, MERGE_ID from aurweb.packages.search import PackageSearch from aurweb.packages.util import get_pkg_or_base, get_pkgbase_comment, query_notified, query_voted @@ -258,6 +258,10 @@ async def package(request: Request, name: str) -> Response: ) context["conflicts"] = conflicts + provides = pkg.package_relations.filter( + models.PackageRelation.RelTypeID == PROVIDES_ID) + context["provides"] = provides + return render_template(request, "packages/show.html", context) diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index da99cf1b..70636926 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -77,6 +77,14 @@ {% endif %} + {% if show_package_details and provides and provides.count() %} + + {{ "Provides" | tr }}: + + {{ provides.all() | join(', ', attribute='RelName') }} + + + {% endif %} {{ "Submitter" | tr }}: diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 887945d9..464a7f60 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -217,8 +217,16 @@ def test_package_official_not_found(client: TestClient, package: Package): def test_package(client: TestClient, package: Package): """ Test a single / packages / {name} route. """ - with client as request: + with db.begin(): + db.create(PackageRelation, PackageID=package.ID, + RelTypeID=PROVIDES_ID, + RelName="test_provider1") + db.create(PackageRelation, PackageID=package.ID, + RelTypeID=PROVIDES_ID, + RelName="test_provider2") + + with client as request: resp = request.get(package_endpoint(package)) assert resp.status_code == int(HTTPStatus.OK) @@ -238,6 +246,10 @@ def test_package(client: TestClient, package: Package): pkgbase = row.find("./td/a") assert pkgbase.text.strip() == package.PackageBase.Name + provides = root.xpath('//tr[@id="provides"]/td') + expected = ["test_provider1", "test_provider2"] + assert provides[0].text.strip() == ", ".join(expected) + def test_package_comments(client: TestClient, user: User, package: Package): now = (datetime.utcnow().timestamp()) From a33e9bd571dfaf1bd2a719df0a80e66f0c33cc2e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 19:14:08 -0800 Subject: [PATCH 562/844] feat(fastapi): add Replaces field to package details Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 6 +++++- templates/partials/packages/details.html | 8 ++++++++ test/test_packages_routes.py | 13 ++++++++++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index fb91a8e3..f03be217 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -12,7 +12,7 @@ import aurweb.packages.util from aurweb import db, defaults, l10n, logging, models, util from aurweb.auth import auth_required from aurweb.models.package_request import ACCEPTED_ID, PENDING_ID, REJECTED_ID -from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID +from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID from aurweb.models.request_type import DELETION_ID, MERGE, MERGE_ID from aurweb.packages.search import PackageSearch from aurweb.packages.util import get_pkg_or_base, get_pkgbase_comment, query_notified, query_voted @@ -262,6 +262,10 @@ async def package(request: Request, name: str) -> Response: models.PackageRelation.RelTypeID == PROVIDES_ID) context["provides"] = provides + replaces = pkg.package_relations.filter( + models.PackageRelation.RelTypeID == REPLACES_ID) + context["replaces"] = replaces + return render_template(request, "packages/show.html", context) diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index 70636926..7516b324 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -85,6 +85,14 @@ {% endif %} + {% if show_package_details and replaces and replaces.count() %} + + {{ "Replaces" | tr }}: + + {{ replaces.all() | join(', ', attribute='RelName') }} + + + {% endif %} {{ "Submitter" | tr }}: diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 464a7f60..b00844c2 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -24,7 +24,7 @@ from aurweb.models.package_notification import PackageNotification from aurweb.models.package_relation import PackageRelation from aurweb.models.package_request import ACCEPTED_ID, REJECTED_ID, PackageRequest from aurweb.models.package_vote import PackageVote -from aurweb.models.relation_type import PROVIDES_ID, RelationType +from aurweb.models.relation_type import PROVIDES_ID, REPLACES_ID, RelationType from aurweb.models.request_type import DELETION_ID, MERGE_ID, RequestType from aurweb.models.user import User from aurweb.testing import setup_test_db @@ -226,6 +226,13 @@ def test_package(client: TestClient, package: Package): RelTypeID=PROVIDES_ID, RelName="test_provider2") + db.create(PackageRelation, PackageID=package.ID, + RelTypeID=REPLACES_ID, + RelName="test_replacer1") + db.create(PackageRelation, PackageID=package.ID, + RelTypeID=REPLACES_ID, + RelName="test_replacer2") + with client as request: resp = request.get(package_endpoint(package)) assert resp.status_code == int(HTTPStatus.OK) @@ -250,6 +257,10 @@ def test_package(client: TestClient, package: Package): expected = ["test_provider1", "test_provider2"] assert provides[0].text.strip() == ", ".join(expected) + replaces = root.xpath('//tr[@id="replaces"]/td') + expected = ["test_replacer1", "test_replacer2"] + assert replaces[0].text.strip() == ", ".join(expected) + def test_package_comments(client: TestClient, user: User, package: Package): now = (datetime.utcnow().timestamp()) From e8e9edbb21cfe5dd33f491acbdcc1d9b98845a69 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 19:30:21 -0800 Subject: [PATCH 563/844] change(fastapi): simplify package details database queries Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 16 ++++------------ templates/partials/packages/details.html | 2 +- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index f03be217..0949909e 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -229,9 +229,7 @@ async def package(request: Request, name: str) -> Response: context["package"] = pkg # Package sources. - context["sources"] = db.query(models.PackageSource).join( - models.Package).join(models.PackageBase).filter( - models.PackageBase.ID == pkgbase.ID) + context["sources"] = pkg.package_sources # Package dependencies. dependencies = db.query(models.PackageDependency).join( @@ -246,16 +244,10 @@ async def package(request: Request, name: str) -> Response: models.Package.Name.asc()) context["required_by"] = required_by - licenses = db.query(models.License).join(models.PackageLicense).join( - models.Package).join(models.PackageBase).filter( - models.PackageBase.ID == pkgbase.ID) - context["licenses"] = licenses + context["licenses"] = pkg.package_licenses - conflicts = db.query(models.PackageRelation).join(models.Package).join( - models.PackageBase).filter( - and_(models.PackageRelation.RelTypeID == CONFLICTS_ID, - models.PackageBase.ID == pkgbase.ID) - ) + conflicts = pkg.package_relations.filter( + models.PackageRelation.RelTypeID == CONFLICTS_ID) context["conflicts"] = conflicts provides = pkg.package_relations.filter( diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index 7516b324..7e20b082 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -66,7 +66,7 @@ {% if show_package_details and licenses and licenses.count() %} {{ "Licenses" | tr }}: - {{ licenses.all() | join(', ', attribute='Name') | default('None' | tr) }} + {{ licenses.all() | join(', ', attribute='License.Name') }} {% endif %} {% if show_package_details and conflicts and conflicts.count() %} From 7aa959150ec6614fbf0684917a3efe1c2e3918d4 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 19:53:50 -0800 Subject: [PATCH 564/844] feat(fastapi): add id="conflicts" to package details conflicts Signed-off-by: Kevin Morris --- templates/partials/packages/details.html | 2 +- test/test_packages_routes.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index 7e20b082..c9b95a26 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -70,7 +70,7 @@ {% endif %} {% if show_package_details and conflicts and conflicts.count() %} - + {{ "Conflicts" | tr }}: {{ conflicts.all() | join(', ', attribute='RelName') }} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index b00844c2..1dabada8 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -24,7 +24,7 @@ from aurweb.models.package_notification import PackageNotification from aurweb.models.package_relation import PackageRelation from aurweb.models.package_request import ACCEPTED_ID, REJECTED_ID, PackageRequest from aurweb.models.package_vote import PackageVote -from aurweb.models.relation_type import PROVIDES_ID, REPLACES_ID, RelationType +from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID, RelationType from aurweb.models.request_type import DELETION_ID, MERGE_ID, RequestType from aurweb.models.user import User from aurweb.testing import setup_test_db @@ -233,6 +233,13 @@ def test_package(client: TestClient, package: Package): RelTypeID=REPLACES_ID, RelName="test_replacer2") + db.create(PackageRelation, PackageID=package.ID, + RelTypeID=CONFLICTS_ID, + RelName="test_conflict1") + db.create(PackageRelation, PackageID=package.ID, + RelTypeID=CONFLICTS_ID, + RelName="test_conflict2") + with client as request: resp = request.get(package_endpoint(package)) assert resp.status_code == int(HTTPStatus.OK) @@ -261,6 +268,10 @@ def test_package(client: TestClient, package: Package): expected = ["test_replacer1", "test_replacer2"] assert replaces[0].text.strip() == ", ".join(expected) + conflicts = root.xpath('//tr[@id="conflicts"]/td') + expected = ["test_conflict1", "test_conflict2"] + assert conflicts[0].text.strip() == ", ".join(expected) + def test_package_comments(client: TestClient, user: User, package: Package): now = (datetime.utcnow().timestamp()) From 686c0322907ed3bb8015c42f050f845323549776 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 11 Nov 2021 19:55:04 -0800 Subject: [PATCH 565/844] feat(fastapi): add id="licenses" to package details licenses Signed-off-by: Kevin Morris --- templates/partials/packages/details.html | 2 +- test/test_packages_routes.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index c9b95a26..67c32170 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -64,7 +64,7 @@ {% endif %} {% if show_package_details and licenses and licenses.count() %} - + {{ "Licenses" | tr }}: {{ licenses.all() | join(', ', attribute='License.Name') }} diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 1dabada8..1bdb3ea3 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -11,6 +11,7 @@ from fastapi.testclient import TestClient from sqlalchemy import and_ from aurweb import asgi, db, defaults +from aurweb.models import License, PackageLicense from aurweb.models.account_type import USER_ID, AccountType from aurweb.models.dependency_type import DependencyType from aurweb.models.official_provider import OfficialProvider @@ -240,6 +241,17 @@ def test_package(client: TestClient, package: Package): RelTypeID=CONFLICTS_ID, RelName="test_conflict2") + # Create some licenses. + licenses = [ + db.create(License, Name="test_license1"), + db.create(License, Name="test_license2") + ] + + db.create(PackageLicense, PackageID=package.ID, + License=licenses[0]) + db.create(PackageLicense, PackageID=package.ID, + License=licenses[1]) + with client as request: resp = request.get(package_endpoint(package)) assert resp.status_code == int(HTTPStatus.OK) @@ -260,6 +272,10 @@ def test_package(client: TestClient, package: Package): pkgbase = row.find("./td/a") assert pkgbase.text.strip() == package.PackageBase.Name + licenses = root.xpath('//tr[@id="licenses"]/td') + expected = ["test_license1", "test_license2"] + assert licenses[0].text.strip() == ", ".join(expected) + provides = root.xpath('//tr[@id="provides"]/td') expected = ["test_provider1", "test_provider2"] assert provides[0].text.strip() == ", ".join(expected) From bd59adc886a6ce53fc5fe4c874a81cc9f8d4fcb5 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 12 Nov 2021 17:39:26 -0800 Subject: [PATCH 566/844] fix(fastapi): use NumVotes for votes field in package details Signed-off-by: Kevin Morris --- templates/partials/packages/details.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index 67c32170..dbb81c19 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -132,11 +132,11 @@ {{ "Votes" | tr }}: {% if not is_maintainer %} - {{ pkgbase.package_votes.count() }} + {{ pkgbase.NumVotes }} {% else %} - {{ pkgbase.package_votes.count() }} + {{ pkgbase.NumVotes }} {% endif %} From cee7512e4d843c771a1aee42277781fd949648cc Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 12 Nov 2021 20:49:25 -0800 Subject: [PATCH 567/844] cleanup(fastapi): simplify PackageDependency.is_package() Signed-off-by: Kevin Morris --- aurweb/models/package_dependency.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/aurweb/models/package_dependency.py b/aurweb/models/package_dependency.py index edaa6538..c4c5f6c1 100644 --- a/aurweb/models/package_dependency.py +++ b/aurweb/models/package_dependency.py @@ -1,9 +1,10 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship -from aurweb import schema +from aurweb import db, schema from aurweb.models.declarative import Base from aurweb.models.dependency_type import DependencyType as _DependencyType +from aurweb.models.official_provider import OfficialProvider as _OfficialProvider from aurweb.models.package import Package as _Package @@ -46,11 +47,7 @@ class PackageDependency(Base): params=("NULL")) def is_package(self) -> bool: - # TODO: Improve the speed of this query if possible. - from aurweb import db - from aurweb.models.official_provider import OfficialProvider - from aurweb.models.package import Package - pkg = db.query(Package, Package.Name == self.DepName) - official = db.query(OfficialProvider, - OfficialProvider.Name == self.DepName) - return pkg.scalar() or official.scalar() + pkg = db.query(_Package).filter(_Package.Name == self.DepName).exists() + official = db.query(_OfficialProvider).filter( + _OfficialProvider.Name == self.DepName).exists() + return db.query(pkg).scalar() or db.query(official).scalar() From f8ba2c53421050ab154040a7cbb2c0de554ae8d1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 14 Nov 2021 15:32:11 -0800 Subject: [PATCH 568/844] cleanup(fastapi): simplify aurweb.routers.accounts.accounts_post Signed-off-by: Kevin Morris --- aurweb/routers/accounts.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 498568ad..83c16ed0 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -662,7 +662,7 @@ async def accounts(request: Request): account_type.TRUSTED_USER_AND_DEV}) async def accounts_post(request: Request, O: int = Form(default=0), # Offset - SB: str = Form(default=str()), # Search By + SB: str = Form(default=str()), # Sort By U: str = Form(default=str()), # Username T: str = Form(default=str()), # Account Type S: bool = Form(default=False), # Suspended @@ -705,23 +705,19 @@ async def accounts_post(request: Request, # Populate this list with any additional statements to # be ANDed together. - statements = [] - if account_type_id is not None: - statements.append(models.AccountType.ID == account_type_id) - if U: - statements.append(models.User.Username.like(f"%{U}%")) - if S: - statements.append(models.User.Suspended == S) - if E: - statements.append(models.User.Email.like(f"%{E}%")) - if R: - statements.append(models.User.RealName.like(f"%{R}%")) - if I: - statements.append(models.User.IRCNick.like(f"%{I}%")) - if K: - statements.append(models.User.PGPKey.like(f"%{K}%")) + statements = [ + v for k, v in [ + (account_type_id is not None, models.AccountType.ID == account_type_id), + (bool(U), models.User.Username.like(f"%{U}%")), + (bool(S), models.User.Suspended == S), + (bool(E), models.User.Email.like(f"%{E}%")), + (bool(R), models.User.RealName.like(f"%{R}%")), + (bool(I), models.User.IRCNick.like(f"%{I}%")), + (bool(K), models.User.PGPKey.like(f"%{K}%")), + ] if k + ] - # Filter the query by combining all statements added above into + # Filter the query by coe-mbining all statements added above into # an AND statement, unless there's just one statement, which # we pass on to filter() as args. if statements: From 4103ab49c9c6922e89f89a25fc3b2b5b461c1bcb Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 14 Nov 2021 15:36:06 -0800 Subject: [PATCH 569/844] housekeep(fastapi): rework aurweb.db session API Changes: ------- - Add aurweb.db.get_session() - Returns aurweb.db's global `session` instance - Provides us a way to change the implementation of the session instance without interrupting user code. - Use aurweb.db.get_session() in session API methods - Add docstrings to session API methods - Refactor aurweb.db.delete - Normalize aurweb.db.delete to an alias of session.delete - Refresh instances in places we depend on their non-PK columns being up to date. Signed-off-by: Kevin Morris --- aurweb/auth.py | 8 ++- aurweb/db.py | 89 ++++++++++++++++++++------------- aurweb/models/ban.py | 9 ++-- aurweb/models/user.py | 2 +- aurweb/packages/util.py | 18 +++++-- aurweb/ratelimit.py | 4 +- aurweb/routers/accounts.py | 49 ++++++++---------- aurweb/routers/packages.py | 68 +++++++++++++------------ aurweb/rpc.py | 31 ++++++++++-- aurweb/scripts/popupdate.py | 2 +- aurweb/scripts/rendercomment.py | 12 +++-- aurweb/testing/__init__.py | 10 ++-- aurweb/users/__init__.py | 0 aurweb/users/util.py | 19 +++++++ aurweb/util.py | 1 + test/test_account_type.py | 4 +- test/test_db.py | 6 +-- test/test_dependency_type.py | 4 +- test/test_packages_util.py | 6 +++ test/test_ratelimit.py | 2 +- test/test_relation_type.py | 2 +- test/test_request_type.py | 4 +- 22 files changed, 212 insertions(+), 138 deletions(-) create mode 100644 aurweb/users/__init__.py create mode 100644 aurweb/users/util.py diff --git a/aurweb/auth.py b/aurweb/auth.py index 38754db0..98a43fd5 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -13,7 +13,7 @@ from starlette.requests import HTTPConnection import aurweb.config -from aurweb import l10n, util +from aurweb import db, l10n, util from aurweb.models import Session, User from aurweb.models.account_type import ACCOUNT_TYPE_ID from aurweb.templates import make_variable_context, render_template @@ -98,14 +98,12 @@ class AnonymousUser: class BasicAuthBackend(AuthenticationBackend): async def authenticate(self, conn: HTTPConnection): - from aurweb.db import session - sid = conn.cookies.get("AURSID") if not sid: return (None, AnonymousUser()) now_ts = datetime.utcnow().timestamp() - record = session.query(Session).filter( + record = db.query(Session).filter( and_(Session.SessionID == sid, Session.LastUpdateTS >= now_ts)).first() @@ -116,7 +114,7 @@ class BasicAuthBackend(AuthenticationBackend): # At this point, we cannot have an invalid user if the record # exists, due to ForeignKey constraints in the schema upheld # by mysqlclient. - user = session.query(User).filter(User.ID == record.UsersID).first() + user = db.query(User).filter(User.ID == record.UsersID).first() user.nonce = util.make_nonce() user.authenticated = True diff --git a/aurweb/db.py b/aurweb/db.py index c1e80751..39232d5a 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -2,10 +2,10 @@ import functools import math import re -from typing import Iterable +from typing import Iterable, NewType from sqlalchemy import event -from sqlalchemy.orm import scoped_session +from sqlalchemy.orm import Query, scoped_session import aurweb.config import aurweb.util @@ -22,6 +22,9 @@ session = None # Global introspected object memo. introspected = dict() +# A mocked up type. +Base = NewType("aurweb.models.declarative_base.Base", "Base") + def make_random_value(table: str, column: str): """ Generate a unique, random value for a string column in a table. @@ -58,55 +61,69 @@ def make_random_value(table: str, column: str): return string -def query(model, *args, **kwargs): - return session.query(model).filter(*args, **kwargs) +def get_session(): + """ Return aurweb.db's global session. """ + return session -def create(model, *args, **kwargs): - instance = model(*args, **kwargs) +def refresh(model: Base) -> Base: + """ Refresh the session's knowledge of `model`. """ + get_session().refresh(model) + return model + + +def query(Model: Base, *args, **kwargs) -> Query: + """ + Perform an ORM query against the database session. + + This method also runs Query.filter on the resulting model + query with *args and **kwargs. + + :param Model: Declarative ORM class + """ + return get_session().query(Model).filter(*args, **kwargs) + + +def create(Model: Base, *args, **kwargs) -> Base: + """ + Create a record and add() it to the database session. + + :param Model: Declarative ORM class + :return: Model instance + """ + instance = Model(*args, **kwargs) return add(instance) -def delete(model, *args, **kwargs): - instance = session.query(model).filter(*args, **kwargs) - for record in instance: - session.delete(record) +def delete(model: Base) -> None: + """ + Delete a set of records found by Query.filter(*args, **kwargs). + + :param Model: Declarative ORM class + """ + get_session().delete(model) -def delete_all(iterable: Iterable): - with begin(): - for obj in iterable: - session.delete(obj) +def delete_all(iterable: Iterable) -> None: + """ Delete each instance found in `iterable`. """ + session_ = get_session() + aurweb.util.apply_all(iterable, session_.delete) -def rollback(): - session.rollback() +def rollback() -> None: + """ Rollback the database session. """ + get_session().rollback() -def add(model): - session.add(model) +def add(model: Base) -> Base: + """ Add `model` to the database session. """ + get_session().add(model) return model def begin(): - """ Begin an SQLAlchemy SessionTransaction. - - This context is **required** to perform an modifications to the - database. - - Example: - - with db.begin(): - object = db.create(...) - # On __exit__, db.commit() is run. - - with db.begin(): - object = db.delete(...) - # On __exit__, db.commit() is run. - - :return: A new SessionTransaction based on session - """ - return session.begin() + """ Begin an SQLAlchemy SessionTransaction. """ + return get_session().begin() def get_sqlalchemy_url(): diff --git a/aurweb/models/ban.py b/aurweb/models/ban.py index a70be7b9..0fcb6d2e 100644 --- a/aurweb/models/ban.py +++ b/aurweb/models/ban.py @@ -1,6 +1,6 @@ from fastapi import Request -from aurweb import schema +from aurweb import db, schema from aurweb.models.declarative import Base @@ -10,11 +10,10 @@ class Ban(Base): __mapper_args__ = {"primary_key": [__table__.c.IPAddress]} def __init__(self, **kwargs): - self.IPAddress = kwargs.get("IPAddress") - self.BanTS = kwargs.get("BanTS") + super().__init__(**kwargs) def is_banned(request: Request): - from aurweb.db import session ip = request.client.host - return session.query(Ban).filter(Ban.IPAddress == ip).first() is not None + exists = db.query(Ban).filter(Ban.IPAddress == ip).exists() + return db.query(exists).scalar() diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 8db34c38..43910db9 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -146,7 +146,7 @@ class User(Base): self.authenticated = False if self.session: with db.begin(): - db.session.delete(self.session) + db.delete(self.session) def is_trusted_user(self): return self.AccountType.ID in { diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index cdec26f3..78f5bf18 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -110,18 +110,26 @@ def get_pkg_or_base( raise HTTPException(status_code=HTTPStatus.NOT_FOUND) instance = db.query(cls).filter(cls.Name == name).first() - if cls == models.PackageBase and not instance: + if not instance: raise HTTPException(status_code=HTTPStatus.NOT_FOUND) - return instance + return db.refresh(instance) -def get_pkgbase_comment( - pkgbase: models.PackageBase, id: int) -> models.PackageComment: +def get_pkgbase_comment(pkgbase: models.PackageBase, id: int) \ + -> models.PackageComment: comment = pkgbase.comments.filter(models.PackageComment.ID == id).first() if not comment: raise HTTPException(status_code=HTTPStatus.NOT_FOUND) - return comment + return db.refresh(comment) + + +def get_pkgreq_by_id(id: int): + pkgreq = db.query(models.PackageRequest).filter( + models.PackageRequest.ID == id).first() + if not pkgreq: + raise HTTPException(status_code=HTTPStatus.NOT_FOUND) + return db.refresh(pkgreq) @register_filter("out_of_date") diff --git a/aurweb/ratelimit.py b/aurweb/ratelimit.py index e306f7a7..a71cb1cc 100644 --- a/aurweb/ratelimit.py +++ b/aurweb/ratelimit.py @@ -40,8 +40,10 @@ def _update_ratelimit_db(request: Request): now = int(datetime.utcnow().timestamp()) time_to_delete = now - window_length + records = db.query(ApiRateLimit).filter( + ApiRateLimit.WindowStart < time_to_delete) with db.begin(): - db.delete(ApiRateLimit, ApiRateLimit.WindowStart < time_to_delete) + db.delete_all(records) host = request.client.host record = db.query(ApiRateLimit, ApiRateLimit.IP == host).first() diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 83c16ed0..aca322b5 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -4,7 +4,7 @@ import typing from datetime import datetime from http import HTTPStatus -from fastapi import APIRouter, Form, HTTPException, Request +from fastapi import APIRouter, Form, Request from fastapi.responses import HTMLResponse, RedirectResponse from sqlalchemy import and_, func, or_ @@ -20,6 +20,7 @@ from aurweb.models.account_type import (DEVELOPER, DEVELOPER_ID, TRUSTED_USER, T from aurweb.models.ssh_pub_key import get_fingerprint from aurweb.scripts.notify import ResetKeyNotification, WelcomeNotification from aurweb.templates import make_context, make_variable_context, render_template +from aurweb.users.util import get_user_by_name router = APIRouter() logger = logging.get_logger(__name__) @@ -49,6 +50,7 @@ async def passreset_post(request: Request, return render_template(request, "passreset.html", context, status_code=HTTPStatus.NOT_FOUND) + db.refresh(user) if resetkey: context["resetkey"] = resetkey @@ -83,7 +85,7 @@ async def passreset_post(request: Request, with db.begin(): user.ResetKey = str() if user.session: - db.session.delete(user.session) + db.delete(user.session) user.update_password(password) # Render ?step=complete. @@ -458,15 +460,15 @@ def cannot_edit(request, user): @router.get("/account/{username}/edit", response_class=HTMLResponse) @auth_required(True, redirect="/account/{username}") -async def account_edit(request: Request, - username: str): +async def account_edit(request: Request, username: str): user = db.query(models.User, models.User.Username == username).first() + response = cannot_edit(request, user) if response: return response context = await make_variable_context(request, "Accounts") - context["user"] = user + context["user"] = db.refresh(user) context = make_account_form_context(context, request, user, dict()) return render_template(request, "account/edit.html", context) @@ -497,16 +499,14 @@ async def account_edit_post(request: Request, ON: bool = Form(default=False), # Owner Notify T: int = Form(default=None), passwd: str = Form(default=str())): - from aurweb.db import session - - user = session.query(models.User).filter( + user = db.query(models.User).filter( models.User.Username == username).first() response = cannot_edit(request, user) if response: return response context = await make_variable_context(request, "Accounts") - context["user"] = user + context["user"] = db.refresh(user) args = dict(await request.form()) context = make_account_form_context(context, request, user, args) @@ -575,7 +575,7 @@ async def account_edit_post(request: Request, user.ssh_pub_key.Fingerprint = fingerprint elif user.ssh_pub_key: # Else, if the user has a public key already, delete it. - session.delete(user.ssh_pub_key) + db.delete(user.ssh_pub_key) if T and T != user.AccountTypeID: with db.begin(): @@ -617,27 +617,16 @@ account_template = ( status_code=HTTPStatus.UNAUTHORIZED) async def account(request: Request, username: str): _ = l10n.get_translator_for_request(request) - context = await make_variable_context(request, - _("Account") + " " + username) - - user = db.query(models.User, models.User.Username == username).first() - if not user: - raise HTTPException(status_code=HTTPStatus.NOT_FOUND) - - context["user"] = user - + context = await make_variable_context( + request, _("Account") + " " + username) + context["user"] = get_user_by_name(username) return render_template(request, "account/show.html", context) @router.get("/account/{username}/comments") @auth_required(redirect="/account/{username}/comments") async def account_comments(request: Request, username: str): - user = db.query(models.User).filter( - models.User.Username == username - ).first() - if not user: - raise HTTPException(status_code=HTTPStatus.NOT_FOUND) - + user = get_user_by_name(username) context = make_context(request, "Accounts") context["username"] = username context["comments"] = user.package_comments.order_by( @@ -725,7 +714,7 @@ async def accounts_post(request: Request, # Finally, order and truncate our users for the current page. users = query.order_by(*order_by).limit(pp).offset(offset) - context["users"] = users + context["users"] = util.apply_all(users, db.refresh) return render_template(request, "account/index.html", context) @@ -751,6 +740,9 @@ async def terms_of_service(request: Request): unaccepted = db.query(models.Term).filter( ~models.Term.ID.in_(db.query(models.AcceptedTerm.TermsID))).all() + for record in (diffs + unaccepted): + db.refresh(record) + # Translate the 'Terms of Service' part of our page title. _ = l10n.get_translator_for_request(request) title = f"AUR {_('Terms of Service')}" @@ -782,18 +774,21 @@ async def terms_of_service_post(request: Request, # We already did the database filters here, so let's just use # them instead of reiterating the process in terms_of_service. accept_needed = sorted(unaccepted + diffs) - return render_terms_of_service(request, context, accept_needed) + return render_terms_of_service( + request, context, util.apply_all(accept_needed, db.refresh)) with db.begin(): # For each term we found, query for the matching accepted term # and update its Revision to the term's current Revision. for term in diffs: + db.refresh(term) accepted_term = request.user.accepted_terms.filter( models.AcceptedTerm.TermsID == term.ID).first() accepted_term.Revision = term.Revision # For each term that was never accepted, accept it! for term in unaccepted: + db.refresh(term) db.create(models.AcceptedTerm, User=request.user, Term=term, Revision=term.Revision) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 0949909e..07e8af72 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -4,7 +4,7 @@ from typing import Any, Dict, List from fastapi import APIRouter, Form, HTTPException, Query, Request, Response from fastapi.responses import JSONResponse, RedirectResponse -from sqlalchemy import and_, case +from sqlalchemy import case import aurweb.filters import aurweb.packages.util @@ -15,9 +15,9 @@ from aurweb.models.package_request import ACCEPTED_ID, PENDING_ID, REJECTED_ID from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID from aurweb.models.request_type import DELETION_ID, MERGE, MERGE_ID from aurweb.packages.search import PackageSearch -from aurweb.packages.util import get_pkg_or_base, get_pkgbase_comment, query_notified, query_voted +from aurweb.packages.util import get_pkg_or_base, get_pkgbase_comment, get_pkgreq_by_id, query_notified, query_voted from aurweb.scripts import notify, popupdate -from aurweb.scripts.rendercomment import update_comment_render +from aurweb.scripts.rendercomment import update_comment_render_fastapi from aurweb.templates import make_context, make_variable_context, render_raw_template, render_template logger = logging.get_logger(__name__) @@ -92,7 +92,10 @@ async def packages_get(request: Request, context: Dict[str, Any], # Insert search results into the context. results = search.results() - context["packages"] = results.limit(per_page).offset(offset) + + packages = results.limit(per_page).offset(offset) + util.apply_all(packages, db.refresh) + context["packages"] = packages context["packages_voted"] = query_voted( context.get("packages"), request.user) context["packages_notified"] = query_notified( @@ -132,6 +135,7 @@ def create_request_if_missing(requests: List[models.PackageRequest], ClosedTS=now, Closer=user) requests.append(pkgreq) + return pkgreq def delete_package(deleter: models.User, package: models.Package): @@ -147,8 +151,9 @@ def delete_package(deleter: models.User, package: models.Package): ).first() with db.begin(): - create_request_if_missing( + pkgreq = create_request_if_missing( requests, reqtype, deleter, package) + db.refresh(pkgreq) bases_to_delete.append(package.PackageBase) @@ -171,8 +176,9 @@ def delete_package(deleter: models.User, package: models.Package): ) # Perform all the deletions. - db.delete_all([package]) - db.delete_all(bases_to_delete) + with db.begin(): + db.delete(package) + db.delete_all(bases_to_delete) # Send out all the notifications. util.apply_all(notifications, lambda n: n.send()) @@ -221,8 +227,7 @@ async def make_single_context(request: Request, async def package(request: Request, name: str) -> Response: # Get the Package. pkg = get_pkg_or_base(name, models.Package) - pkgbase = (get_pkg_or_base(name, models.PackageBase) - if not pkg else pkg.PackageBase) + pkgbase = pkg.PackageBase # Add our base information. context = await make_single_context(request, pkgbase) @@ -312,7 +317,7 @@ async def pkgbase_comments_post( db.create(models.PackageNotification, User=request.user, PackageBase=pkgbase) - update_comment_render(comment.ID) + update_comment_render_fastapi(comment) # Redirect to the pkgbase page. return RedirectResponse(f"/pkgbase/{pkgbase.Name}#comment-{comment.ID}", @@ -374,7 +379,7 @@ async def pkgbase_comment_post( db.create(models.PackageNotification, User=request.user, PackageBase=pkgbase) - update_comment_render(db_comment.ID) + update_comment_render_fastapi(db_comment) if not next: next = f"/pkgbase/{pkgbase.Name}" @@ -539,7 +544,7 @@ def remove_users(pkgbase, usernames): conn, comaintainer.User.ID, pkgbase.ID ) ) - db.session.delete(comaintainer) + db.delete(comaintainer) # Send out notifications if need be. for notify_ in notifications: @@ -679,14 +684,8 @@ async def requests(request: Request, @router.get("/pkgbase/{name}/request") @auth_required(True, redirect="/pkgbase/{name}/request") async def package_request(request: Request, name: str): + pkgbase = get_pkg_or_base(name, models.PackageBase) context = await make_variable_context(request, "Submit Request") - - pkgbase = db.query(models.PackageBase).filter( - models.PackageBase.Name == name).first() - - if not pkgbase: - raise HTTPException(status_code=HTTPStatus.NOT_FOUND) - context["pkgbase"] = pkgbase return render_template(request, "pkgbase/request.html", context) @@ -729,6 +728,7 @@ async def pkgbase_request_post(request: Request, name: str, ] return render_template(request, "pkgbase/request.html", context) + db.refresh(target) if target.ID == pkgbase.ID: # TODO: This error needs to be translated. context["errors"] = [ @@ -767,8 +767,7 @@ async def pkgbase_request_post(request: Request, name: str, @router.get("/requests/{id}/close") @auth_required(True, redirect="/requests/{id}/close") async def requests_close(request: Request, id: int): - pkgreq = db.query(models.PackageRequest).filter( - models.PackageRequest.ID == id).first() + pkgreq = get_pkgreq_by_id(id) if not request.user.is_elevated() and request.user != pkgreq.User: # Request user doesn't have permission here: redirect to '/'. return RedirectResponse("/", status_code=HTTPStatus.SEE_OTHER) @@ -783,8 +782,7 @@ async def requests_close(request: Request, id: int): async def requests_close_post(request: Request, id: int, reason: int = Form(default=0), comments: str = Form(default=str())): - pkgreq = db.query(models.PackageRequest).filter( - models.PackageRequest.ID == id).first() + pkgreq = get_pkgreq_by_id(id) if not request.user.is_elevated() and request.user != pkgreq.User: # Request user doesn't have permission here: redirect to '/'. return RedirectResponse("/", status_code=HTTPStatus.SEE_OTHER) @@ -823,13 +821,17 @@ async def pkgbase_keywords(request: Request, name: str, keywords = set(keywords.split(" ")) # Delete all keywords which are not supplied by the user. - with db.begin(): - db.delete(models.PackageKeyword, - and_(models.PackageKeyword.PackageBaseID == pkgbase.ID, - ~models.PackageKeyword.Keyword.in_(keywords))) + other_keywords = pkgbase.keywords.filter( + ~models.PackageKeyword.Keyword.in_(keywords)) + other_keyword_strings = [kwd.Keyword for kwd in other_keywords] - existing_keywords = set(kwd.Keyword for kwd in pkgbase.keywords.all()) + existing_keywords = set( + kwd.Keyword for kwd in + pkgbase.keywords.filter( + ~models.PackageKeyword.Keyword.in_(other_keyword_strings)) + ) with db.begin(): + db.delete_all(other_keywords) for keyword in keywords.difference(existing_keywords): db.create(models.PackageKeyword, PackageBase=pkgbase, @@ -940,7 +942,7 @@ def pkgbase_unnotify_instance(request: Request, pkgbase: models.PackageBase): has_cred = request.user.has_credential("CRED_PKGBASE_NOTIFY") if has_cred and notif: with db.begin(): - db.session.delete(notif) + db.delete(notif) @router.post("/pkgbase/{name}/unnotify") @@ -988,7 +990,7 @@ async def pkgbase_unvote(request: Request, name: str): has_cred = request.user.has_credential("CRED_PKGBASE_VOTE") if has_cred and vote: with db.begin(): - db.session.delete(vote) + db.delete(vote) # Update NumVotes/Popularity. conn = db.ConnectionExecutor(db.get_engine().raw_connection()) @@ -1015,7 +1017,7 @@ def pkgbase_disown_instance(request: Request, pkgbase: models.PackageBase): if co: with db.begin(): pkgbase.Maintainer = co.User - db.session.delete(co) + db.delete(co) else: pkgbase.Maintainer = None @@ -1463,8 +1465,8 @@ def pkgbase_merge_instance(request: Request, pkgbase: models.PackageBase, with db.begin(): # Delete pkgbase and its packages now that everything's merged. for pkg in pkgbase.packages: - db.session.delete(pkg) - db.session.delete(pkgbase) + db.delete(pkg) + db.delete(pkgbase) # Accept merge requests related to this pkgbase and target. for pkgreq in requests: diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 4ab005af..03662790 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Any, Dict, List +from typing import Any, Callable, Dict, List, NewType from sqlalchemy import and_ @@ -25,6 +25,10 @@ REL_TYPES = { } +DataGenerator = NewType("DataGenerator", + Callable[[models.Package], Dict[str, Any]]) + + class RPCError(Exception): pass @@ -188,15 +192,32 @@ class RPC: self._update_json_relations(package, data) return data - def _handle_multiinfo_type(self, args: List[str] = [], **kwargs): + def _assemble_json_data(self, packages: List[models.Package], + data_generator: DataGenerator) \ + -> List[Dict[str, Any]]: + """ + Assemble JSON data out of a list of packages. + + :param packages: A list of Package instances or a Package ORM query + :param data_generator: Generator callable of single-Package JSON data + """ + output = [] + for pkg in packages: + db.refresh(pkg) + output.append(data_generator(pkg)) + return output + + def _handle_multiinfo_type(self, args: List[str] = [], **kwargs) \ + -> List[Dict[str, Any]]: self._enforce_args(args) args = set(args) packages = db.query(models.Package).filter( models.Package.Name.in_(args)) - return [self._get_info_json_data(pkg) for pkg in packages] + return self._assemble_json_data(packages, self._get_info_json_data) def _handle_search_type(self, by: str = defaults.RPC_SEARCH_BY, - args: List[str] = []): + args: List[str] = []) \ + -> List[Dict[str, Any]]: # If `by` isn't maintainer and we don't have any args, raise an error. # In maintainer's case, return all orphans if there are no args, # so we need args to pass through to the handler without errors. @@ -212,7 +233,7 @@ class RPC: max_results = config.getint("options", "max_rpc_results") results = search.results().limit(max_results) - return [self._get_json_data(pkg) for pkg in results] + return self._assemble_json_data(results, self._get_json_data) def _handle_msearch_type(self, args: List[str] = [], **kwargs): return self._handle_search_type(by="m", args=args) diff --git a/aurweb/scripts/popupdate.py b/aurweb/scripts/popupdate.py index fa82208d..db4ba170 100755 --- a/aurweb/scripts/popupdate.py +++ b/aurweb/scripts/popupdate.py @@ -29,7 +29,7 @@ def run_single(conn, pkgbase): conn.commit() conn.close() - aurweb.db.session.refresh(pkgbase) + aurweb.db.refresh(pkgbase) def main(): diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index a00448d8..efa5357f 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -129,9 +129,14 @@ def save_rendered_comment(conn, commentid, html): [html, commentid]) -def update_comment_render(commentid): - conn = aurweb.db.Connection() +def update_comment_render_fastapi(comment): + conn = aurweb.db.ConnectionExecutor( + aurweb.db.get_engine().raw_connection()) + update_comment_render(conn, comment.ID) + aurweb.db.refresh(comment) + +def update_comment_render(conn, commentid): text, pkgbase = get_comment(conn, commentid) html = markdown.markdown(text, extensions=[ 'fenced_code', @@ -152,7 +157,8 @@ def update_comment_render(commentid): def main(): commentid = int(sys.argv[1]) - update_comment_render(commentid) + conn = aurweb.db.Connection() + update_comment_render(conn, commentid) if __name__ == '__main__': diff --git a/aurweb/testing/__init__.py b/aurweb/testing/__init__.py index 65d34253..2dd377e1 100644 --- a/aurweb/testing/__init__.py +++ b/aurweb/testing/__init__.py @@ -19,7 +19,7 @@ def references_graph(table): "regexp_1": r'(?i)\s+references\s+("|\')?', "regexp_2": r'("|\')?\s*\(', } - cursor = aurweb.db.session.execute(query, params=params) + cursor = aurweb.db.get_session().execute(query, params=params) return [row[0] for row in cursor.fetchall()] @@ -51,7 +51,7 @@ def setup_test_db(*args): db_backend = aurweb.config.get("database", "backend") if db_backend != "sqlite": # pragma: no cover - aurweb.db.session.execute("SET FOREIGN_KEY_CHECKS = 0") + aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 0") else: # We're using sqlite, setup tables to be deleted without violating # foreign key constraints by graphing references. @@ -59,10 +59,10 @@ def setup_test_db(*args): references_graph(table) for table in tables)) for table in tables: - aurweb.db.session.execute(f"DELETE FROM {table}") + aurweb.db.get_session().execute(f"DELETE FROM {table}") if db_backend != "sqlite": # pragma: no cover - aurweb.db.session.execute("SET FOREIGN_KEY_CHECKS = 1") + aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 1") # Expunge all objects from SQLAlchemy's IdentityMap. - aurweb.db.session.expunge_all() + aurweb.db.get_session().expunge_all() diff --git a/aurweb/users/__init__.py b/aurweb/users/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/aurweb/users/util.py b/aurweb/users/util.py new file mode 100644 index 00000000..e9635f08 --- /dev/null +++ b/aurweb/users/util.py @@ -0,0 +1,19 @@ +from http import HTTPStatus + +from fastapi import HTTPException + +from aurweb import db +from aurweb.models import User + + +def get_user_by_name(username: str) -> User: + """ + Query a user by its username. + + :param username: User.Username + :return: User instance + """ + user = db.query(User).filter(User.Username == username).first() + if not user: + raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) + return db.refresh(user) diff --git a/aurweb/util.py b/aurweb/util.py index 88142cbc..1c2042fa 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -155,6 +155,7 @@ def get_ssh_fingerprints(): def apply_all(iterable: Iterable, fn: Callable): for item in iterable: fn(item) + return iterable def sanitize_params(offset: str, per_page: str) -> Tuple[int, int]: diff --git a/test/test_account_type.py b/test/test_account_type.py index 86e68253..12472348 100644 --- a/test/test_account_type.py +++ b/test/test_account_type.py @@ -20,7 +20,7 @@ def setup(): yield account_type with begin(): - delete(AccountType, AccountType.ID == account_type.ID) + delete(account_type) def test_account_type(): @@ -50,4 +50,4 @@ def test_user_account_type_relationship(): # This must be deleted here to avoid foreign key issues when # deleting the temporary AccountType in the fixture. with begin(): - delete(User, User.ID == user.ID) + delete(user) diff --git a/test/test_db.py b/test/test_db.py index 7798d2f6..8283a957 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -279,13 +279,13 @@ def test_connection_execute_paramstyle_unsupported(): def test_create_delete(): with db.begin(): - db.create(AccountType, AccountType="test") + account_type = db.create(AccountType, AccountType="test") record = db.query(AccountType, AccountType.AccountType == "test").first() assert record is not None with db.begin(): - db.delete(AccountType, AccountType.AccountType == "test") + db.delete(account_type) record = db.query(AccountType, AccountType.AccountType == "test").first() assert record is None @@ -306,7 +306,7 @@ def test_add_commit(): # Remove the record. with db.begin(): - db.delete(AccountType, AccountType.ID == account_type.ID) + db.delete(account_type) def test_connection_executor_mysql_paramstyle(): diff --git a/test/test_dependency_type.py b/test/test_dependency_type.py index 4d555123..cb8dece4 100644 --- a/test/test_dependency_type.py +++ b/test/test_dependency_type.py @@ -24,7 +24,7 @@ def test_dependency_type_creation(): assert bool(dependency_type.ID) assert dependency_type.Name == "Test Type" with begin(): - delete(DependencyType, DependencyType.ID == dependency_type.ID) + delete(dependency_type) def test_dependency_type_null_name_uses_default(): @@ -32,4 +32,4 @@ def test_dependency_type_null_name_uses_default(): dependency_type = create(DependencyType) assert dependency_type.Name == str() with begin(): - delete(DependencyType, DependencyType.ID == dependency_type.ID) + delete(dependency_type) diff --git a/test/test_packages_util.py b/test/test_packages_util.py index 1396734b..622c08c2 100644 --- a/test/test_packages_util.py +++ b/test/test_packages_util.py @@ -2,6 +2,7 @@ from datetime import datetime import pytest +from fastapi import HTTPException from fastapi.testclient import TestClient from aurweb import asgi, db @@ -98,3 +99,8 @@ def test_query_notified(maintainer: User, package: Package): query = db.query(Package).filter(Package.ID == package.ID).all() query_notified = util.query_notified(query, maintainer) assert query_notified[package.PackageBase.ID] + + +def test_pkgreq_by_id_not_found(): + with pytest.raises(HTTPException): + util.get_pkgreq_by_id(0) diff --git a/test/test_ratelimit.py b/test/test_ratelimit.py index 2634b714..0a72a7e4 100644 --- a/test/test_ratelimit.py +++ b/test/test_ratelimit.py @@ -103,7 +103,7 @@ def test_ratelimit_db(get: mock.MagicMock, getboolean: mock.MagicMock, # Delete the ApiRateLimit record. with db.begin(): - db.delete(ApiRateLimit) + db.delete(db.query(ApiRateLimit).first()) # Should be good to go again! assert not check_ratelimit(request) diff --git a/test/test_relation_type.py b/test/test_relation_type.py index fbc22c71..d2dabceb 100644 --- a/test/test_relation_type.py +++ b/test/test_relation_type.py @@ -18,7 +18,7 @@ def test_relation_type_creation(): assert relation_type.Name == "test-relation" with db.begin(): - db.delete(RelationType, RelationType.ID == relation_type.ID) + db.delete(relation_type) def test_relation_types(): diff --git a/test/test_request_type.py b/test/test_request_type.py index 8d21c2d9..0db24921 100644 --- a/test/test_request_type.py +++ b/test/test_request_type.py @@ -18,7 +18,7 @@ def test_request_type_creation(): assert request_type.Name == "Test Request" with db.begin(): - db.delete(RequestType, RequestType.ID == request_type.ID) + db.delete(request_type) def test_request_type_null_name_returns_empty_string(): @@ -29,7 +29,7 @@ def test_request_type_null_name_returns_empty_string(): assert request_type.Name == str() with db.begin(): - db.delete(RequestType, RequestType.ID == request_type.ID) + db.delete(request_type) def test_request_type_name_display(): From 12400147fc0e1f3bed2bc54a92c4de76cf8312f2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 14 Nov 2021 16:02:00 -0800 Subject: [PATCH 570/844] fix: initialize engine and session in util/adduser.py Signed-off-by: Kevin Morris --- util/adduser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/util/adduser.py b/util/adduser.py index 7e35d13d..1853869a 100644 --- a/util/adduser.py +++ b/util/adduser.py @@ -33,6 +33,7 @@ def parse_args(): def main(): args = parse_args() + db.get_engine() type = db.query(AccountType, AccountType.AccountType == args.type).first() with db.begin(): From 9424341b55bf55b4b064cfc2b8e4d89536901e69 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 14 Nov 2021 23:33:58 -0800 Subject: [PATCH 571/844] fix(docker): fix cgit css config Signed-off-by: Kevin Morris --- docker-compose.yml | 2 ++ docker/cgit-entrypoint.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 2fba1305..bda4ddfb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,6 +112,7 @@ services: environment: - AUR_CONFIG=/aurweb/conf/config - CGIT_CLONE_PREFIX=${AURWEB_PHP_PREFIX} + - CGIT_CSS=/css/cgit.css entrypoint: /docker/cgit-entrypoint.sh command: /docker/scripts/run-cgit.sh 3000 healthcheck: @@ -131,6 +132,7 @@ services: environment: - AUR_CONFIG=/aurweb/conf/config - CGIT_CLONE_PREFIX=${AURWEB_FASTAPI_PREFIX} + - CGIT_CSS=/static/css/cgit.css entrypoint: /docker/cgit-entrypoint.sh command: /docker/scripts/run-cgit.sh 3000 healthcheck: diff --git a/docker/cgit-entrypoint.sh b/docker/cgit-entrypoint.sh index 3615ade5..f9ca86c0 100755 --- a/docker/cgit-entrypoint.sh +++ b/docker/cgit-entrypoint.sh @@ -8,5 +8,6 @@ sed -ri "s|clone-prefix=.*|clone-prefix=${CGIT_CLONE_PREFIX}|" /etc/cgitrc sed -ri 's|header=.*|header=/aurweb/web/template/cgit/header.html|' /etc/cgitrc sed -ri 's|footer=.*|footer=/aurweb/web/template/cgit/footer.html|' /etc/cgitrc sed -ri 's|repo\.path=.*|repo.path=/aurweb/aur.git|' /etc/cgitrc +sed -ri "s|^(css)=.*$|\1=${CGIT_CSS}|" /etc/cgitrc exec "$@" From 7f6d9966e585626da181a6e642e1a73710f2f817 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 14 Nov 2021 16:02:00 -0800 Subject: [PATCH 572/844] fix: initialize engine and session in util/adduser.py Signed-off-by: Kevin Morris --- util/adduser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/util/adduser.py b/util/adduser.py index 7e35d13d..1853869a 100644 --- a/util/adduser.py +++ b/util/adduser.py @@ -33,6 +33,7 @@ def parse_args(): def main(): args = parse_args() + db.get_engine() type = db.query(AccountType, AccountType.AccountType == args.type).first() with db.begin(): From b0b05df19341f5c4a75a78b69b3abc08d61fc238 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 16 Nov 2021 21:10:53 -0800 Subject: [PATCH 573/844] fix(fastapi): pin markdown to 3.3.4 Signed-off-by: Kevin Morris --- poetry.lock | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 37e2f8f9..550a91fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1003,7 +1003,7 @@ h11 = ">=0.9.0,<1" [metadata] lock-version = "1.1" python-versions = ">=3.9,<3.10" -content-hash = "356b37d545d78b8aa1e1939f42522207bcf79526abe8193308c5a2955897d6fd" +content-hash = "844618f499e19d6d20f8479d165be3f60495bfa66fcb1f462256b101f9d395f9" [metadata.files] aiofiles = [ diff --git a/pyproject.toml b/pyproject.toml index 1d4c858c..7b2e9ef3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ alembic = "^1.7.4" mysqlclient = "^2.0.3" Authlib = "^0.15.5" Jinja2 = "^3.0.2" -Markdown = "^3.3.4" +Markdown = "3.3.4" Werkzeug = "^2.0.2" SQLAlchemy = "^1.4.26" From cea9104efb3c496230bee60f6b76be42a8719c61 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 16 Nov 2021 01:45:32 -0800 Subject: [PATCH 574/844] feat(poetry): add pytest-xdist Signed-off-by: Kevin Morris --- poetry.lock | 67 ++++++++++++++++++++++++++++++++++++++++++++------ pyproject.toml | 1 + 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 550a91fa..8d42cc50 100644 --- a/poetry.lock +++ b/poetry.lock @@ -53,7 +53,7 @@ tests = ["pytest", "pytest-asyncio", "mypy (>=0.800)"] name = "atomicwrites" version = "1.4.0" description = "Atomic file writes." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -61,7 +61,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "attrs" version = "21.2.0" description = "Classes Without Boilerplate" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -234,6 +234,17 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" dnspython = ">=1.15.0" idna = ">=2.0.0" +[[package]] +name = "execnet" +version = "1.9.0" +description = "execnet: rapid multi-Python deployment" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +testing = ["pre-commit"] + [[package]] name = "fakeredis" version = "1.6.1" @@ -432,7 +443,7 @@ python-versions = ">=3.5" name = "iniconfig" version = "1.1.1" description = "iniconfig: brain-dead simple config-ini parsing" -category = "dev" +category = "main" optional = false python-versions = "*" @@ -575,7 +586,7 @@ python-versions = "*" name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -639,7 +650,7 @@ python-versions = ">=3.5" name = "py" version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -705,7 +716,7 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" name = "pytest" version = "6.2.5" description = "pytest: simple powerful testing with Python" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -751,6 +762,18 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"] +[[package]] +name = "pytest-forked" +version = "1.3.0" +description = "run tests in isolated forked subprocesses" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +py = "*" +pytest = ">=3.10" + [[package]] name = "pytest-tap" version = "3.3" @@ -763,6 +786,24 @@ python-versions = "*" pytest = ">=3.0" "tap.py" = ">=3.0,<4.0" +[[package]] +name = "pytest-xdist" +version = "2.4.0" +description = "pytest xdist plugin for distributed testing and loop-on-failing modes" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +execnet = ">=1.1" +pytest = ">=6.0.0" +pytest-forked = "*" + +[package.extras] +psutil = ["psutil (>=3.0)"] +setproctitle = ["setproctitle"] +testing = ["filelock"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -1003,7 +1044,7 @@ h11 = ">=0.9.0,<1" [metadata] lock-version = "1.1" python-versions = ">=3.9,<3.10" -content-hash = "844618f499e19d6d20f8479d165be3f60495bfa66fcb1f462256b101f9d395f9" +content-hash = "6ab137fb829b2a6d49552c4864d00be04a2d58d80a872f3cd3b9e5cc67f95b9d" [metadata.files] aiofiles = [ @@ -1200,6 +1241,10 @@ email-validator = [ {file = "email_validator-1.1.3-py2.py3-none-any.whl", hash = "sha256:5675c8ceb7106a37e40e2698a57c056756bf3f272cfa8682a4f87ebd95d8440b"}, {file = "email_validator-1.1.3.tar.gz", hash = "sha256:aa237a65f6f4da067119b7df3f13e89c25c051327b2b5b66dc075f33d62480d7"}, ] +execnet = [ + {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, + {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, +] fakeredis = [ {file = "fakeredis-1.6.1-py3-none-any.whl", hash = "sha256:5eb1516f1fe1813e9da8f6c482178fc067af09f53de587ae03887ef5d9d13024"}, {file = "fakeredis-1.6.1.tar.gz", hash = "sha256:0d06a9384fb79da9f2164ce96e34eb9d4e2ea46215070805ea6fd3c174590b47"}, @@ -1609,10 +1654,18 @@ pytest-cov = [ {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, ] +pytest-forked = [ + {file = "pytest-forked-1.3.0.tar.gz", hash = "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca"}, + {file = "pytest_forked-1.3.0-py2.py3-none-any.whl", hash = "sha256:dc4147784048e70ef5d437951728825a131b81714b398d5d52f17c7c144d8815"}, +] pytest-tap = [ {file = "pytest-tap-3.3.tar.gz", hash = "sha256:5f0919a147cf0396b2f10d64d365a0bf8062e06543e93c675c9d37f5605e983c"}, {file = "pytest_tap-3.3-py3-none-any.whl", hash = "sha256:4fbbc0e090c2e94f6199bee4e4f68ab3c5e176b37a72a589ad84e0f72a2fce55"}, ] +pytest-xdist = [ + {file = "pytest-xdist-2.4.0.tar.gz", hash = "sha256:89b330316f7fc475f999c81b577c2b926c9569f3d397ae432c0c2e2496d61ff9"}, + {file = "pytest_xdist-2.4.0-py3-none-any.whl", hash = "sha256:7b61ebb46997a0820a263553179d6d1e25a8c50d8a8620cd1aa1e20e3be99168"}, +] python-dateutil = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, diff --git a/pyproject.toml b/pyproject.toml index 7b2e9ef3..d296fb4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ gunicorn = "^20.1.0" Hypercorn = "^0.11.2" mysql-connector = "^2.2.9" prometheus-fastapi-instrumentator = "^5.7.1" +pytest-xdist = "^2.4.0" [tool.poetry.dev-dependencies] flake8 = "^4.0.1" From 40b21203ed8b1cd833fa0333ebf3d1985567fcbe Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 16 Nov 2021 01:46:07 -0800 Subject: [PATCH 575/844] feat(poetry): add filelock Signed-off-by: Kevin Morris --- poetry.lock | 18 +++++++++++++++++- pyproject.toml | 1 + 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 8d42cc50..ab559b77 100644 --- a/poetry.lock +++ b/poetry.lock @@ -300,6 +300,18 @@ python-versions = "*" lxml = "*" python-dateutil = "*" +[[package]] +name = "filelock" +version = "3.3.2" +description = "A platform independent file lock." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] +testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] + [[package]] name = "flake8" version = "4.0.1" @@ -1044,7 +1056,7 @@ h11 = ">=0.9.0,<1" [metadata] lock-version = "1.1" python-versions = ">=3.9,<3.10" -content-hash = "6ab137fb829b2a6d49552c4864d00be04a2d58d80a872f3cd3b9e5cc67f95b9d" +content-hash = "ca42bd35717062d6784025ed3956423502ac66adba059ccc080bcaaa666651cd" [metadata.files] aiofiles = [ @@ -1253,6 +1265,10 @@ fastapi = [] feedgen = [ {file = "feedgen-0.9.0.tar.gz", hash = "sha256:8e811bdbbed6570034950db23a4388453628a70e689a6e8303ccec430f5a804a"}, ] +filelock = [ + {file = "filelock-3.3.2-py3-none-any.whl", hash = "sha256:bb2a1c717df74c48a2d00ed625e5a66f8572a3a30baacb7657add1d7bac4097b"}, + {file = "filelock-3.3.2.tar.gz", hash = "sha256:7afc856f74fa7006a289fd10fa840e1eebd8bbff6bffb69c26c54a0512ea8cf8"}, +] flake8 = [ {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"}, diff --git a/pyproject.toml b/pyproject.toml index d296fb4e..8d14735a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ Hypercorn = "^0.11.2" mysql-connector = "^2.2.9" prometheus-fastapi-instrumentator = "^5.7.1" pytest-xdist = "^2.4.0" +filelock = "^3.3.2" [tool.poetry.dev-dependencies] flake8 = "^4.0.1" From 0abdf8d468579c5e98ddac29c5f97ec1e546bd5e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 00:22:52 -0800 Subject: [PATCH 576/844] fix(fastapi): close connection used for initdb Signed-off-by: Kevin Morris --- aurweb/initdb.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aurweb/initdb.py b/aurweb/initdb.py index 9a063ba4..a4a9f621 100644 --- a/aurweb/initdb.py +++ b/aurweb/initdb.py @@ -46,7 +46,9 @@ def run(args): engine = aurweb.db.get_engine(echo=(args.verbose >= 1)) aurweb.schema.metadata.create_all(engine) - feed_initial_data(engine.connect()) + conn = engine.connect() + feed_initial_data(conn) + conn.close() if args.use_alembic: alembic.command.stamp(alembic_config, 'head') From 07aac768d633288d61abd85d055629cfe65800b2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 00:27:44 -0800 Subject: [PATCH 577/844] change(fastapi): remove sqlite support Signed-off-by: Kevin Morris --- aurweb/asgi.py | 6 ++++++ test/test_asgi.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 16de771e..aafb00b2 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -48,6 +48,12 @@ async def app_startup(): "TEST_RECURSION_LIMIT", sys.getrecursionlimit())) sys.setrecursionlimit(recursion_limit) + backend = aurweb.config.get("database", "backend") + if backend not in aurweb.db.DRIVERS: + raise ValueError( + f"The configured database backend ({backend}) is unsupported. " + f"Supported backends: {str(aurweb.db.DRIVERS.keys())}") + session_secret = aurweb.config.get("fastapi", "session_secret") if not session_secret: raise Exception("[fastapi] session_secret must not be empty") diff --git a/test/test_asgi.py b/test/test_asgi.py index b8856741..fa2df5a1 100644 --- a/test/test_asgi.py +++ b/test/test_asgi.py @@ -45,3 +45,20 @@ async def test_asgi_http_exception_handler(): response = await aurweb.asgi.http_exception_handler(None, exc) assert response.body.decode() == \ f"

    {exc.status_code} {phrase}

    {exc.detail}

    " + + +@pytest.mark.asyncio +async def test_asgi_app_unsupported_backends(): + config_get = aurweb.config.get + + # Test that the previously supported "sqlite" backend is now + # unsupported by FastAPI. + def mock_sqlite_backend(section: str, key: str): + if section == "database" and key == "backend": + return "sqlite" + return config_get(section, key) + + with mock.patch("aurweb.config.get", side_effect=mock_sqlite_backend): + expr = r"^.*\(sqlite\) is unsupported.*$" + with pytest.raises(ValueError, match=expr): + await aurweb.asgi.app_startup() From fa43f6bc3ebebff7d97f3361ec07a61e92bd1ce5 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 00:33:41 -0800 Subject: [PATCH 578/844] change(aurweb): add parallel tests and improve aurweb.db This change utilizes pytest-xdist to perform a multiproc test run and reworks aurweb.db's code. We no longer use a global engine, session or Session, but we now use a memo of engines and sessions as they are requested, based on the PYTEST_CURRENT_TEST environment variable, which is available during testing. Additionally, this change strips several SQLite components out of the Python code-base. SQLite is still compatible with PHP and sharness tests, but not with our FastAPI implementation. More changes: ------------ - Remove use of aurweb.db.session global in other code. - Use new aurweb.db.name() dynamic db name function in env.py. - Added 'addopts' to pytest.ini which utilizes multiprocessing. - Highly recommended to leave this be or modify `-n auto` to `-n {cpu_threads}` where cpu_threads is at least 2. Signed-off-by: Kevin Morris --- aurweb/db.py | 251 +++++++++++++++++++++--------- aurweb/routers/auth.py | 6 +- aurweb/routers/packages.py | 8 +- aurweb/schema.py | 4 +- aurweb/testing/__init__.py | 68 ++++---- migrations/env.py | 8 +- pytest.ini | 6 + test/conftest.py | 178 +++++++++++++++++++++ test/test_accepted_term.py | 37 ++--- test/test_account_type.py | 54 ++++--- test/test_accounts_routes.py | 9 +- test/test_api_rate_limit.py | 19 +-- test/test_auth.py | 26 ++-- test/test_auth_routes.py | 5 +- test/test_ban.py | 9 +- test/test_cache.py | 7 +- test/test_captcha.py | 7 + test/test_db.py | 110 +------------ test/test_dependency_type.py | 5 +- test/test_group.py | 9 +- test/test_homepage.py | 11 +- test/test_html.py | 5 +- test/test_initdb.py | 9 ++ test/test_license.py | 9 +- test/test_official_provider.py | 23 +-- test/test_package.py | 5 +- test/test_package_base.py | 5 +- test/test_package_blacklist.py | 16 +- test/test_package_comaintainer.py | 28 ++-- test/test_package_comment.py | 59 +++---- test/test_package_dependency.py | 95 +++-------- test/test_package_group.py | 37 ++--- test/test_package_keyword.py | 37 ++--- test/test_package_license.py | 38 ++--- test/test_package_notification.py | 25 ++- test/test_package_relation.py | 90 +++-------- test/test_package_request.py | 107 +++++-------- test/test_package_source.py | 2 +- test/test_package_vote.py | 29 ++-- test/test_packages_routes.py | 23 +-- test/test_packages_util.py | 17 +- test/test_popupdate.py | 7 + test/test_ratelimit.py | 20 ++- test/test_relation_type.py | 5 +- test/test_request_type.py | 5 +- test/test_routes.py | 12 +- test/test_rpc.py | 8 +- test/test_rss.py | 8 +- test/test_session.py | 5 +- test/test_ssh_pub_key.py | 14 +- test/test_term.py | 18 +-- test/test_trusted_user_routes.py | 5 +- test/test_tu_vote.py | 43 +++-- test/test_tu_voteinfo.py | 5 +- test/test_user.py | 14 +- 55 files changed, 781 insertions(+), 884 deletions(-) create mode 100644 test/conftest.py diff --git a/aurweb/db.py b/aurweb/db.py index 39232d5a..b8b49e40 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -1,29 +1,34 @@ import functools +import hashlib import math +import os import re from typing import Iterable, NewType -from sqlalchemy import event -from sqlalchemy.orm import Query, scoped_session +import sqlalchemy + +from sqlalchemy import create_engine, event +from sqlalchemy.engine.base import Engine +from sqlalchemy.engine.url import URL +from sqlalchemy.orm import Query, Session, SessionTransaction, scoped_session, sessionmaker import aurweb.config import aurweb.util -# See get_engine. -engine = None +from aurweb import logging -# ORM Session class. -Session = None +logger = logging.get_logger(__name__) -# Global ORM Session object. -session = None +DRIVERS = { + "mysql": "mysql+mysqldb" +} # Global introspected object memo. introspected = dict() -# A mocked up type. -Base = NewType("aurweb.models.declarative_base.Base", "Base") +# Some types we don't get access to in this module. +Base = NewType("Base", "aurweb.models.declarative_base.Base") def make_random_value(table: str, column: str): @@ -56,14 +61,85 @@ def make_random_value(table: str, column: str): length = col.type.length string = aurweb.util.make_random_string(length) - while session.query(table).filter(column == string).first(): + while query(table).filter(column == string).first(): string = aurweb.util.make_random_string(length) return string -def get_session(): +def test_name() -> str: + """ + Return the unhashed database name. + + The unhashed database name is determined (lower = higher priority) by: + ------------------------------------------- + 1. {test_suite} portion of PYTEST_CURRENT_TEST + 2. aurweb.config.get("database", "name") + + During `pytest` runs, the PYTEST_CURRENT_TEST environment variable + is set to the current test in the format `{test_suite}::{test_func}`. + + This allows tests to use a suite-specific database for its runs, + which decouples database state from test suites. + + :return: Unhashed database name + """ + db = os.environ.get("PYTEST_CURRENT_TEST", + aurweb.config.get("database", "name")) + return db.split(":")[0] + + +def name() -> str: + """ + Return sanitized database name that can be used for tests or production. + + If test_name() starts with "test/", the database name is SHA-1 hashed, + prefixed with 'db', and returned. Otherwise, test_name() is passed + through and not hashed at all. + + :return: SHA1-hashed database name prefixed with 'db' + """ + dbname = test_name() + if not dbname.startswith("test/"): + return dbname + sha1 = hashlib.sha1(dbname.encode()).hexdigest() + return "db" + sha1 + + +# Module-private global memo used to store SQLAlchemy sessions. +_sessions = dict() + + +def get_session(engine: Engine = None) -> Session: """ Return aurweb.db's global session. """ - return session + dbname = name() + + global _sessions + if dbname not in _sessions: + + if not engine: # pragma: no cover + engine = get_engine() + + Session = scoped_session( + sessionmaker(autocommit=True, autoflush=False, bind=engine)) + _sessions[dbname] = Session() + + # If this is the first grab of this session, log out the + # database name used. + raw_dbname = test_name() + logger.debug(f"DBName({raw_dbname}): {dbname}") + + return _sessions.get(dbname) + + +def pop_session(dbname: str) -> None: + """ + Pop a Session out of the private _sessions memo. + + :param dbname: Database name + :raises KeyError: When `dbname` does not exist in the memo + """ + global _sessions + _sessions.pop(dbname) def refresh(model: Base) -> Base: @@ -121,41 +197,40 @@ def add(model: Base) -> Base: return model -def begin(): +def begin() -> SessionTransaction: """ Begin an SQLAlchemy SessionTransaction. """ return get_session().begin() -def get_sqlalchemy_url(): +def get_sqlalchemy_url() -> URL: """ - Build an SQLAlchemy for use with create_engine based on the aurweb configuration. - """ - import sqlalchemy + Build an SQLAlchemy URL for use with create_engine. - constructor = sqlalchemy.engine.url.URL + :return: sqlalchemy.engine.url.URL + """ + constructor = URL parts = sqlalchemy.__version__.split('.') major = int(parts[0]) minor = int(parts[1]) if major == 1 and minor >= 4: # pragma: no cover - constructor = sqlalchemy.engine.url.URL.create + constructor = URL.create aur_db_backend = aurweb.config.get('database', 'backend') if aur_db_backend == 'mysql': - if aurweb.config.get_with_fallback('database', 'port', fallback=None): - port = aurweb.config.get('database', 'port') - param_query = None - else: - port = None - param_query = { - 'unix_socket': aurweb.config.get('database', 'socket') - } + param_query = {} + port = aurweb.config.get_with_fallback("database", "port", None) + if not port: + param_query["unix_socket"] = aurweb.config.get( + "database", "socket") + return constructor( - 'mysql+mysqldb', + DRIVERS.get(aur_db_backend), username=aurweb.config.get('database', 'user'), - password=aurweb.config.get('database', 'password'), + password=aurweb.config.get_with_fallback('database', 'password', + fallback=None), host=aurweb.config.get('database', 'host'), - database=aurweb.config.get('database', 'name'), + database=name(), port=port, query=param_query ) @@ -168,58 +243,83 @@ def get_sqlalchemy_url(): raise ValueError('unsupported database backend') -def get_engine(echo: bool = False): +def sqlite_regexp(regex, item) -> bool: # pragma: no cover + """ Method which mimics SQL's REGEXP for SQLite. """ + return bool(re.search(regex, str(item))) + + +def setup_sqlite(engine: Engine) -> None: # pragma: no cover + """ Perform setup for an SQLite engine. """ + @event.listens_for(engine, "connect") + def do_begin(conn, record): + create_deterministic_function = functools.partial( + conn.create_function, + deterministic=True + ) + create_deterministic_function("REGEXP", 2, sqlite_regexp) + + +# Module-private global memo used to store SQLAlchemy engines. +_engines = dict() + + +def get_engine(dbname: str = None, echo: bool = False) -> Engine: """ - Return the global SQLAlchemy engine. + Return the SQLAlchemy engine for `dbname`. The engine is created on the first call to get_engine and then stored in the `engine` global variable for the next calls. + + :param dbname: Database name (default: aurweb.db.name()) + :param echo: Flag passed through to sqlalchemy.create_engine + :return: SQLAlchemy Engine instance """ - from sqlalchemy import create_engine - from sqlalchemy.orm import sessionmaker + if not dbname: + dbname = name() - global engine, session, Session - - if engine is None: + global _engines + if dbname not in _engines: + db_backend = aurweb.config.get("database", "backend") connect_args = dict() - db_backend = aurweb.config.get("database", "backend") - if db_backend == "sqlite": - # check_same_thread is for a SQLite technicality - # https://fastapi.tiangolo.com/tutorial/sql-databases/#note + is_sqlite = bool(db_backend == "sqlite") + if is_sqlite: # pragma: no cover connect_args["check_same_thread"] = False - engine = create_engine(get_sqlalchemy_url(), - connect_args=connect_args, - echo=echo) + kwargs = { + "echo": echo, + "connect_args": connect_args + } + _engines[dbname] = create_engine(get_sqlalchemy_url(), **kwargs) - Session = scoped_session( - sessionmaker(autocommit=True, autoflush=False, bind=engine)) - session = Session() + if is_sqlite: # pragma: no cover + setup_sqlite(_engines.get(dbname)) - if db_backend == "sqlite": - # For SQLite, we need to add some custom functions as - # they are used in the reference graph method. - def regexp(regex, item): - return bool(re.search(regex, str(item))) - - @event.listens_for(engine, "connect") - def do_begin(conn, record): - create_deterministic_function = functools.partial( - conn.create_function, - deterministic=True - ) - create_deterministic_function("REGEXP", 2, regexp) - - return engine + return _engines.get(dbname) -def kill_engine(): - global engine, Session, session - if engine: - session.close() - engine.dispose() - engine = Session = session = None +def pop_engine(dbname: str) -> None: + """ + Pop an Engine out of the private _engines memo. + + :param dbname: Database name + :raises KeyError: When `dbname` does not exist in the memo + """ + global _engines + _engines.pop(dbname) + + +def kill_engine() -> None: + """ Close the current session and dispose of the engine. """ + dbname = name() + + session = get_session() + session.close() + pop_session(dbname) + + engine = get_engine() + engine.dispose() + pop_engine(dbname) def connect(): @@ -248,7 +348,9 @@ class ConnectionExecutor: def paramstyle(self): return self._paramstyle - def execute(self, query, params=()): + def execute(self, query, params=()): # pragma: no cover + # TODO: SQLite support has been removed in FastAPI. It remains + # here to fund its support for PHP until it is removed. if self._paramstyle in ('format', 'pyformat'): query = query.replace('%', '%%').replace('?', '%s') elif self._paramstyle == 'qmark': @@ -278,16 +380,19 @@ class Connection: if aur_db_backend == 'mysql': import MySQLdb aur_db_host = aurweb.config.get('database', 'host') - aur_db_name = aurweb.config.get('database', 'name') + aur_db_name = name() aur_db_user = aurweb.config.get('database', 'user') - aur_db_pass = aurweb.config.get('database', 'password') + aur_db_pass = aurweb.config.get_with_fallback( + 'database', 'password', str()) aur_db_socket = aurweb.config.get('database', 'socket') self._conn = MySQLdb.connect(host=aur_db_host, user=aur_db_user, passwd=aur_db_pass, db=aur_db_name, unix_socket=aur_db_socket) - elif aur_db_backend == 'sqlite': + elif aur_db_backend == 'sqlite': # pragma: no cover + # TODO: SQLite support has been removed in FastAPI. It remains + # here to fund its support for PHP until it is removed. import sqlite3 aur_db_name = aurweb.config.get('database', 'name') self._conn = sqlite3.connect(aur_db_name) diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index b8e83c7d..055f0dca 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -6,7 +6,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse import aurweb.config -from aurweb import cookies +from aurweb import cookies, db from aurweb.auth import auth_required from aurweb.l10n import get_translator_for_request from aurweb.models import User @@ -45,9 +45,7 @@ async def login_post(request: Request, raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=_("Bad Referer header.")) - from aurweb.db import session - - user = session.query(User).filter(User.Username == user).first() + user = db.query(User).filter(User.Username == user).first() if not user: return await login_template(request, next, errors=["Bad username or password."]) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 07e8af72..c8ceb275 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -1014,12 +1014,12 @@ def pkgbase_disown_instance(request: Request, pkgbase: models.PackageBase): models.PackageComaintainer.Priority.asc() ).limit(1).first() - if co: - with db.begin(): + with db.begin(): + if co: pkgbase.Maintainer = co.User db.delete(co) - else: - pkgbase.Maintainer = None + else: + pkgbase.Maintainer = None notif.send() diff --git a/aurweb/schema.py b/aurweb/schema.py index fb8f0dee..43db920d 100644 --- a/aurweb/schema.py +++ b/aurweb/schema.py @@ -16,13 +16,13 @@ db_backend = aurweb.config.get("database", "backend") @compiles(TINYINT, 'sqlite') -def compile_tinyint_sqlite(type_, compiler, **kw): +def compile_tinyint_sqlite(type_, compiler, **kw): # pragma: no cover """TINYINT is not supported on SQLite. Substitute it with INTEGER.""" return 'INTEGER' @compiles(BIGINT, 'sqlite') -def compile_bigint_sqlite(type_, compiler, **kw): +def compile_bigint_sqlite(type_, compiler, **kw): # pragma: no cover """ For SQLite's AUTOINCREMENT to work on BIGINT columns, we need to map BIGINT to INTEGER. Aside from that, BIGINT is the same as INTEGER for SQLite. diff --git a/aurweb/testing/__init__.py b/aurweb/testing/__init__.py index 2dd377e1..8261051d 100644 --- a/aurweb/testing/__init__.py +++ b/aurweb/testing/__init__.py @@ -1,26 +1,6 @@ -from itertools import chain - import aurweb.db - -def references_graph(table): - """ Taken from Django's sqlite3/operations.py. """ - query = """ - WITH tables AS ( - SELECT :table name - UNION - SELECT sqlite_master.name - FROM sqlite_master - JOIN tables ON (sql REGEXP :regexp_1 || tables.name || :regexp_2) - ) SELECT name FROM tables; - """ - params = { - "table": table, - "regexp_1": r'(?i)\s+references\s+("|\')?', - "regexp_2": r'("|\')?\s*\(', - } - cursor = aurweb.db.get_session().execute(query, params=params) - return [row[0] for row in cursor.fetchall()] +from aurweb import models def setup_test_db(*args): @@ -47,22 +27,38 @@ def setup_test_db(*args): aurweb.db.get_engine() tables = list(args) + if not tables: + tables = [ + models.AcceptedTerm.__tablename__, + models.ApiRateLimit.__tablename__, + models.Ban.__tablename__, + models.Group.__tablename__, + models.License.__tablename__, + models.OfficialProvider.__tablename__, + models.Package.__tablename__, + models.PackageBase.__tablename__, + models.PackageBlacklist.__tablename__, + models.PackageComaintainer.__tablename__, + models.PackageComment.__tablename__, + models.PackageDependency.__tablename__, + models.PackageGroup.__tablename__, + models.PackageKeyword.__tablename__, + models.PackageLicense.__tablename__, + models.PackageNotification.__tablename__, + models.PackageRelation.__tablename__, + models.PackageRequest.__tablename__, + models.PackageSource.__tablename__, + models.PackageVote.__tablename__, + models.Session.__tablename__, + models.SSHPubKey.__tablename__, + models.Term.__tablename__, + models.TUVote.__tablename__, + models.TUVoteInfo.__tablename__, + models.User.__tablename__, + ] - db_backend = aurweb.config.get("database", "backend") - - if db_backend != "sqlite": # pragma: no cover - aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 0") - else: - # We're using sqlite, setup tables to be deleted without violating - # foreign key constraints by graphing references. - tables = set(chain.from_iterable( - references_graph(table) for table in tables)) - + aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 0") for table in tables: aurweb.db.get_session().execute(f"DELETE FROM {table}") - - if db_backend != "sqlite": # pragma: no cover - aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 1") - - # Expunge all objects from SQLAlchemy's IdentityMap. + aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 1") aurweb.db.get_session().expunge_all() diff --git a/migrations/env.py b/migrations/env.py index 7130d141..774ecdeb 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -41,8 +41,8 @@ def run_migrations_offline(): script output. """ - db_name = aurweb.config.get("database", "name") - logging.info(f"Performing offline migration on database '{db_name}'.") + dbname = aurweb.db.name() + logging.info(f"Performing offline migration on database '{dbname}'.") context.configure( url=aurweb.db.get_sqlalchemy_url(), target_metadata=target_metadata, @@ -61,8 +61,8 @@ def run_migrations_online(): and associate a connection with the context. """ - db_name = aurweb.config.get("database", "name") - logging.info(f"Performing online migration on database '{db_name}'.") + dbname = aurweb.db.name() + logging.info(f"Performing online migration on database '{dbname}'.") connectable = sqlalchemy.create_engine( aurweb.db.get_sqlalchemy_url(), poolclass=sqlalchemy.pool.NullPool, diff --git a/pytest.ini b/pytest.ini index 510447c9..9f70a2bd 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,3 +8,9 @@ # https://bugs.python.org/issue45097 filterwarnings = ignore::DeprecationWarning:asyncio.base_events + +# Build in coverage and pytest-xdist multiproc testing. +addopts = --cov=aurweb --cov-append --dist load --dist loadfile -n auto + +# Our pytest units are located in the ./test/ directory. +testpaths = test diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..47d9ca4b --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,178 @@ +""" +pytest configuration. + +The conftest.py file is used to define pytest-global fixtures +or actions run before tests. + +Module scoped fixtures: +---------------------- +- setup_database +- db_session (depends: setup_database) + +Function scoped fixtures: +------------------------ +- db_test (depends: db_session) + +Tests in aurweb which access the database **must** use the `db_test` +function fixture. Most database tests simply require this fixture in +an autouse=True setup fixture, or for fixtures used in DB tests example: + + # In scenarios which there are no other database fixtures + # or other database fixtures dependency paths don't always + # hit `db_test`. + @pytest.fixture(autouse=True) + def setup(db_test): + return + + # In scenarios where we can embed the `db_test` fixture in + # specific fixtures that already exist. + @pytest.fixture + def user(db_test): + with db.begin(): + user = db.create(User, ...) + yield user + +The `db_test` fixture triggers our module-level database fixtures, +then clears the database for each test function run in that module. +It is done this way because migration has a large cost; migrating +ahead of each function takes too long when compared to this method. +""" +import pytest + +from filelock import FileLock +from sqlalchemy import create_engine +from sqlalchemy.engine import URL +from sqlalchemy.engine.base import Engine +from sqlalchemy.orm import scoped_session + +import aurweb.config +import aurweb.db + +from aurweb import initdb, logging, testing + +logger = logging.get_logger(__name__) + + +def test_engine() -> Engine: + """ + Return a privileged SQLAlchemy engine with no database. + + This method is particularly useful for providing an engine that + can be used to create and drop databases from an SQL server. + + :return: SQLAlchemy Engine instance (not connected to a database) + """ + unix_socket = aurweb.config.get_with_fallback("database", "socket", None) + kwargs = { + "username": aurweb.config.get("database", "user"), + "password": aurweb.config.get_with_fallback( + "database", "password", None), + "host": aurweb.config.get("database", "host"), + "port": aurweb.config.get_with_fallback("database", "port", None), + "query": { + "unix_socket": unix_socket + } + } + + backend = aurweb.config.get("database", "backend") + driver = aurweb.db.DRIVERS.get(backend) + return create_engine(URL.create(driver, **kwargs)) + + +class AlembicArgs: + """ + Masquerade an ArgumentParser like structure. + + This structure is needed to pass conftest-specific arguments + to initdb.run duration database creation. + """ + verbose = False + use_alembic = True + + +def _create_database(engine: Engine, dbname: str) -> None: + """ + Create a test database. + + :param engine: Engine returned by test_engine() + :param dbname: Database name to create + """ + conn = engine.connect() + conn.execute(f"CREATE DATABASE {dbname}") + conn.close() + initdb.run(AlembicArgs) + + +def _drop_database(engine: Engine, dbname: str) -> None: + """ + Drop a test database. + + :param engine: Engine returned by test_engine() + :param dbname: Database name to drop + """ + aurweb.schema.metadata.drop_all(bind=engine) + conn = engine.connect() + conn.execute(f"DROP DATABASE {dbname}") + conn.close() + + +@pytest.fixture(scope="module") +def setup_database(tmp_path_factory: pytest.fixture, + worker_id: pytest.fixture) -> None: + """ Create and drop a database for the suite this fixture is used in. """ + engine = test_engine() + dbname = aurweb.db.name() + + if worker_id == "master": # pragma: no cover + # If we're not running tests through multiproc pytest-xdist. + yield _create_database(engine, dbname) + _drop_database(engine, dbname) + return + + root_tmp_dir = tmp_path_factory.getbasetemp().parent + fn = root_tmp_dir / dbname + + with FileLock(str(fn) + ".lock"): + if fn.is_file(): + # If the data file exists, skip database creation. + yield + else: + # Otherwise, create the data file and create the database. + fn.write_text("1") + yield _create_database(engine, dbname) + _drop_database(engine, dbname) + + +@pytest.fixture(scope="module") +def db_session(setup_database: pytest.fixture) -> scoped_session: + """ + Yield a database session based on aurweb.db.name(). + + The returned session is popped out of persistence after the test is run. + """ + # After the test runs, aurweb.db.name() ends up returning the + # configured database, because PYTEST_CURRENT_TEST is removed. + dbname = aurweb.db.name() + session = aurweb.db.get_session() + yield session + + # Close the session and pop it. + session.close() + aurweb.db.pop_session(dbname) + + +@pytest.fixture +def db_test(db_session: scoped_session) -> None: + """ + Database test fixture. + + This fixture should be included in any tests which access the + database. It ensures that a test database is created and + alembic migrated, takes care of dropping the database when + the module is complete, and runs setup_test_db() to clear out + tables for each test. + + Tests using this fixture should access the database + session via aurweb.db.get_session(). + """ + testing.setup_test_db() diff --git a/test/test_accepted_term.py b/test/test_accepted_term.py index cd8bd7af..de18c61a 100644 --- a/test/test_accepted_term.py +++ b/test/test_accepted_term.py @@ -2,38 +2,33 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, query +from aurweb import db from aurweb.models.accepted_term import AcceptedTerm -from aurweb.models.account_type import AccountType +from aurweb.models.account_type import USER_ID from aurweb.models.term import Term from aurweb.models.user import User -from aurweb.testing import setup_test_db user = term = accepted_term = None @pytest.fixture(autouse=True) -def setup(): - global user, term, accepted_term +def setup(db_test): + global user, term - setup_test_db("Users", "AcceptedTerms", "Terms") + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountTypeID=USER_ID) - account_type = query(AccountType, - AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - - term = create(Term, Description="Test term", URL="https://test.term") + term = db.create(Term, Description="Test term", + URL="https://test.term") yield term - # Eradicate any terms we created. - setup_test_db("AcceptedTerms", "Terms") - def test_accepted_term(): - accepted_term = create(AcceptedTerm, User=user, Term=term) + with db.begin(): + accepted_term = db.create(AcceptedTerm, User=user, Term=term) # Make sure our AcceptedTerm relationships got initialized properly. assert accepted_term.User == user @@ -42,14 +37,10 @@ def test_accepted_term(): def test_accepted_term_null_user_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(AcceptedTerm, Term=term) - session.rollback() + AcceptedTerm(Term=term) def test_accepted_term_null_term_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(AcceptedTerm, User=user) - session.rollback() + AcceptedTerm(User=user) diff --git a/test/test_account_type.py b/test/test_account_type.py index 12472348..1d71f878 100644 --- a/test/test_account_type.py +++ b/test/test_account_type.py @@ -1,31 +1,29 @@ import pytest -from aurweb.db import begin, create, delete, query +from aurweb import db from aurweb.models.account_type import AccountType from aurweb.models.user import User -from aurweb.testing import setup_test_db - -account_type = None @pytest.fixture(autouse=True) -def setup(): - setup_test_db("Users") - - global account_type - - with begin(): - account_type = create(AccountType, AccountType="TestUser") - - yield account_type - - with begin(): - delete(account_type) +def setup(db_test): + return -def test_account_type(): +@pytest.fixture +def account_type() -> AccountType: + with db.begin(): + account_type_ = db.create(AccountType, AccountType="TestUser") + + yield account_type_ + + with db.begin(): + db.delete(account_type_) + + +def test_account_type(account_type): """ Test creating an AccountType, and reading its columns. """ - # Make sure it got created and was given an ID. + # Make sure it got db.created and was given an ID. assert bool(account_type.ID) # Next, test our string functions. @@ -34,20 +32,20 @@ def test_account_type(): "" % ( account_type.ID) - record = query(AccountType, - AccountType.AccountType == "TestUser").first() + record = db.query(AccountType, + AccountType.AccountType == "TestUser").first() assert account_type == record -def test_user_account_type_relationship(): - with begin(): - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) +def test_user_account_type_relationship(account_type): + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountType=account_type) assert user.AccountType == account_type - # This must be deleted here to avoid foreign key issues when + # This must be db.deleted here to avoid foreign key issues when # deleting the temporary AccountType in the fixture. - with begin(): - delete(user) + with db.begin(): + db.delete(user) diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index 5e855daf..e828f70f 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -20,7 +20,6 @@ from aurweb.models.session import Session from aurweb.models.ssh_pub_key import SSHPubKey, get_fingerprint from aurweb.models.term import Term from aurweb.models.user import User -from aurweb.testing import setup_test_db from aurweb.testing.html import get_errors from aurweb.testing.requests import Request @@ -50,11 +49,9 @@ def make_ssh_pubkey(): @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user - setup_test_db("Users", "Sessions", "Bans", "Terms", "AcceptedTerms") - account_type = query(AccountType, AccountType.AccountType == "User").first() @@ -65,10 +62,6 @@ def setup(): yield user - # Remove term records so other tests don't get them - # and falsely redirect. - setup_test_db("Terms", "AcceptedTerms") - @pytest.fixture def tu_user(): diff --git a/test/test_api_rate_limit.py b/test/test_api_rate_limit.py index 25cb3e0f..82805ecf 100644 --- a/test/test_api_rate_limit.py +++ b/test/test_api_rate_limit.py @@ -3,19 +3,18 @@ import pytest from sqlalchemy.exc import IntegrityError from aurweb import db -from aurweb.db import create from aurweb.models.api_rate_limit import ApiRateLimit -from aurweb.testing import setup_test_db @pytest.fixture(autouse=True) -def setup(): - setup_test_db("ApiRateLimit") +def setup(db_test): + return def test_api_rate_key_creation(): with db.begin(): - rate = create(ApiRateLimit, IP="127.0.0.1", Requests=10, WindowStart=1) + rate = db.create(ApiRateLimit, IP="127.0.0.1", Requests=10, + WindowStart=1) assert rate.IP == "127.0.0.1" assert rate.Requests == 10 assert rate.WindowStart == 1 @@ -23,19 +22,15 @@ def test_api_rate_key_creation(): def test_api_rate_key_ip_default(): with db.begin(): - api_rate_limit = create(ApiRateLimit, Requests=10, WindowStart=1) + api_rate_limit = db.create(ApiRateLimit, Requests=10, WindowStart=1) assert api_rate_limit.IP == str() def test_api_rate_key_null_requests_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - create(ApiRateLimit, IP="127.0.0.1", WindowStart=1) - db.rollback() + ApiRateLimit(IP="127.0.0.1", WindowStart=1) def test_api_rate_key_null_window_start_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - create(ApiRateLimit, IP="127.0.0.1", Requests=1) - db.rollback() + ApiRateLimit(IP="127.0.0.1", Requests=1) diff --git a/test/test_auth.py b/test/test_auth.py index 7aea17a0..08eaac0b 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -6,28 +6,22 @@ from sqlalchemy.exc import IntegrityError from aurweb import db from aurweb.auth import AnonymousUser, BasicAuthBackend, account_type_required, has_credential -from aurweb.db import create, query -from aurweb.models.account_type import USER, USER_ID, AccountType +from aurweb.models.account_type import USER, USER_ID from aurweb.models.session import Session from aurweb.models.user import User -from aurweb.testing import setup_test_db from aurweb.testing.requests import Request user = backend = request = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, backend, request - setup_test_db("Users", "Sessions") - - account_type = query(AccountType, - AccountType.AccountType == "User").first() with db.begin(): - user = create(User, Username="test", Email="test@example.com", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) + user = db.create(User, Username="test", Email="test@example.com", + RealName="Test User", Passwd="testPassword", + AccountTypeID=USER_ID) backend = BasicAuthBackend() request = Request() @@ -56,10 +50,8 @@ async def test_auth_backend_invalid_user_id(): # Create a new session with a fake user id. now_ts = datetime.utcnow().timestamp() with pytest.raises(IntegrityError): - with db.begin(): - create(Session, UsersID=666, SessionID="realSession", - LastUpdateTS=now_ts + 5) - db.rollback() + Session(UsersID=666, SessionID="realSession", + LastUpdateTS=now_ts + 5) @pytest.mark.asyncio @@ -68,8 +60,8 @@ async def test_basic_auth_backend(): # equal the real_user. now_ts = datetime.utcnow().timestamp() with db.begin(): - create(Session, UsersID=user.ID, SessionID="realSession", - LastUpdateTS=now_ts + 5) + db.create(Session, UsersID=user.ID, SessionID="realSession", + LastUpdateTS=now_ts + 5) request.cookies["AURSID"] = "realSession" _, result = await backend.authenticate(request) assert result == user diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index 39afc6f9..a0bb8a7c 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -13,7 +13,6 @@ from aurweb.db import begin, create, query from aurweb.models.account_type import AccountType from aurweb.models.session import Session from aurweb.models.user import User -from aurweb.testing import setup_test_db # Some test global constants. TEST_USERNAME = "test" @@ -27,11 +26,9 @@ user = client = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, client - setup_test_db("Users", "Sessions", "Bans") - account_type = query(AccountType, AccountType.AccountType == "User").first() diff --git a/test/test_ban.py b/test/test_ban.py index f96e9d14..2c705410 100644 --- a/test/test_ban.py +++ b/test/test_ban.py @@ -9,18 +9,15 @@ from sqlalchemy import exc as sa_exc from aurweb import db from aurweb.db import create from aurweb.models.ban import Ban, is_banned -from aurweb.testing import setup_test_db from aurweb.testing.requests import Request ban = request = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global ban, request - setup_test_db("Bans") - ts = datetime.utcnow() + timedelta(seconds=30) with db.begin(): ban = create(Ban, IPAddress="127.0.0.1", BanTS=ts) @@ -33,8 +30,6 @@ def test_ban(): def test_invalid_ban(): - from aurweb.db import session - with pytest.raises(sa_exc.IntegrityError): bad_ban = Ban(BanTS=datetime.utcnow()) @@ -44,7 +39,7 @@ def test_invalid_ban(): with warnings.catch_warnings(): warnings.simplefilter("ignore", sa_exc.SAWarning) with db.begin(): - session.add(bad_ban) + db.add(bad_ban) # Since we got a transaction failure, we need to rollback. db.rollback() diff --git a/test/test_cache.py b/test/test_cache.py index 35346e52..b49ee386 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -3,14 +3,11 @@ import pytest from aurweb import cache, db from aurweb.models.account_type import USER_ID from aurweb.models.user import User -from aurweb.testing import setup_test_db @pytest.fixture(autouse=True) -def setup(): - setup_test_db( - User.__tablename__ - ) +def setup(db_test): + return class StubRedis: diff --git a/test/test_captcha.py b/test/test_captcha.py index ec19dee9..e5f8c71a 100644 --- a/test/test_captcha.py +++ b/test/test_captcha.py @@ -1,8 +1,15 @@ import hashlib +import pytest + from aurweb import captcha +@pytest.fixture(autouse=True) +def setup(db_test): + return + + def test_captcha_salts(): """ Make sure we can get some captcha salts. """ salts = captcha.get_captcha_salts() diff --git a/test/test_db.py b/test/test_db.py index 8283a957..f36fff2c 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -12,7 +12,6 @@ import aurweb.initdb from aurweb import db from aurweb.models.account_type import AccountType -from aurweb.testing import setup_test_db class Args: @@ -96,16 +95,10 @@ def make_temp_mysql_config(): @pytest.fixture(autouse=True) -def setup_db(): +def setup(db_test): if os.path.exists("/tmp/aurweb.sqlite3"): os.remove("/tmp/aurweb.sqlite3") - # In various places in this test, we reinitialize the engine. - # Make sure we kill the previous engine before initializing - # it via setup_test_db(). - aurweb.db.kill_engine() - setup_test_db() - def test_sqlalchemy_sqlite_url(): tmpctx, tmp = make_temp_sqlite_config() @@ -159,24 +152,6 @@ def test_sqlalchemy_unknown_backend(): def test_db_connects_without_fail(): """ This only tests the actual config supplied to pytest. """ db.connect() - assert db.engine is not None - - -def test_connection_class_sqlite_without_fail(): - tmpctx, tmp = make_temp_sqlite_config() - with tmpctx: - with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): - aurweb.config.rehash() - - aurweb.db.kill_engine() - aurweb.initdb.run(Args()) - - conn = db.Connection() - cur = conn.execute( - "SELECT AccountType FROM AccountTypes WHERE ID = ?", (1,)) - account_type = cur.fetchone()[0] - assert account_type == "User" - aurweb.config.rehash() def test_connection_class_unsupported_backend(): @@ -200,83 +175,6 @@ def test_connection_mysql(): aurweb.config.rehash() -@mock.patch("sqlite3.connect", mock.MagicMock(return_value=DBConnection())) -@mock.patch.object(sqlite3, "paramstyle", "qmark") -def test_connection_sqlite(): - db.Connection() - - -@mock.patch("sqlite3.connect", mock.MagicMock(return_value=DBConnection())) -@mock.patch.object(sqlite3, "paramstyle", "format") -def test_connection_execute_paramstyle_format(): - tmpctx, tmp = make_temp_sqlite_config() - - with tmpctx: - with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): - aurweb.config.rehash() - - aurweb.db.kill_engine() - aurweb.initdb.run(Args()) - - # Test SQLite route of clearing tables. - setup_test_db("Users", "Bans") - - conn = db.Connection() - - # First, test ? to %s format replacement. - account_types = conn\ - .execute("SELECT * FROM AccountTypes WHERE AccountType = ?", - ["User"]).fetchall() - assert account_types == \ - ["SELECT * FROM AccountTypes WHERE AccountType = %s", ["User"]] - - # Test other format replacement. - account_types = conn\ - .execute("SELECT * FROM AccountTypes WHERE AccountType = %", - ["User"]).fetchall() - assert account_types == \ - ["SELECT * FROM AccountTypes WHERE AccountType = %%", ["User"]] - aurweb.config.rehash() - - -@mock.patch("sqlite3.connect", mock.MagicMock(return_value=DBConnection())) -@mock.patch.object(sqlite3, "paramstyle", "qmark") -def test_connection_execute_paramstyle_qmark(): - tmpctx, tmp = make_temp_sqlite_config() - - with tmpctx: - with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): - aurweb.config.rehash() - - aurweb.db.kill_engine() - aurweb.initdb.run(Args()) - - conn = db.Connection() - # We don't modify anything when using qmark, so test equality. - account_types = conn\ - .execute("SELECT * FROM AccountTypes WHERE AccountType = ?", - ["User"]).fetchall() - assert account_types == \ - ["SELECT * FROM AccountTypes WHERE AccountType = ?", ["User"]] - aurweb.config.rehash() - - -@mock.patch("sqlite3.connect", mock.MagicMock(return_value=DBConnection())) -@mock.patch.object(sqlite3, "paramstyle", "unsupported") -def test_connection_execute_paramstyle_unsupported(): - tmpctx, tmp = make_temp_sqlite_config() - with tmpctx: - with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): - aurweb.config.rehash() - conn = db.Connection() - with pytest.raises(ValueError, match="unsupported paramstyle"): - conn.execute( - "SELECT * FROM AccountTypes WHERE AccountType = ?", - ["User"] - ).fetchall() - aurweb.config.rehash() - - def test_create_delete(): with db.begin(): account_type = db.create(AccountType, AccountType="test") @@ -318,3 +216,9 @@ def test_connection_executor_mysql_paramstyle(): def test_connection_executor_sqlite_paramstyle(): executor = db.ConnectionExecutor(None, backend="sqlite") assert executor.paramstyle() == sqlite3.paramstyle + + +def test_name_without_pytest_current_test(): + with mock.patch.dict("os.environ", {}, clear=True): + dbname = aurweb.db.name() + assert dbname == aurweb.config.get("database", "name") diff --git a/test/test_dependency_type.py b/test/test_dependency_type.py index cb8dece4..c5afd38d 100644 --- a/test/test_dependency_type.py +++ b/test/test_dependency_type.py @@ -2,12 +2,11 @@ import pytest from aurweb.db import begin, create, delete, query from aurweb.models.dependency_type import DependencyType -from aurweb.testing import setup_test_db @pytest.fixture(autouse=True) -def setup(): - setup_test_db() +def setup(db_test): + return def test_dependency_types(): diff --git a/test/test_group.py b/test/test_group.py index cea69b68..82b82464 100644 --- a/test/test_group.py +++ b/test/test_group.py @@ -4,12 +4,11 @@ from sqlalchemy.exc import IntegrityError from aurweb import db from aurweb.models.group import Group -from aurweb.testing import setup_test_db @pytest.fixture(autouse=True) -def setup(): - setup_test_db("Groups") +def setup(db_test): + return def test_group_creation(): @@ -21,6 +20,4 @@ def test_group_creation(): def test_group_null_name_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - db.create(Group) - db.rollback() + Group() diff --git a/test/test_homepage.py b/test/test_homepage.py index 5c678b71..2e6d53c9 100644 --- a/test/test_homepage.py +++ b/test/test_homepage.py @@ -18,7 +18,6 @@ from aurweb.models.package_request import PackageRequest from aurweb.models.request_type import DELETION_ID, RequestType from aurweb.models.user import User from aurweb.redis import redis_connection -from aurweb.testing import setup_test_db from aurweb.testing.html import parse_root from aurweb.testing.requests import Request @@ -26,14 +25,8 @@ client = TestClient(app) @pytest.fixture(autouse=True) -def setup(): - yield setup_test_db( - User.__tablename__, - Package.__tablename__, - PackageBase.__tablename__, - PackageComaintainer.__tablename__, - PackageRequest.__tablename__ - ) +def setup(db_test): + return @pytest.fixture diff --git a/test/test_html.py b/test/test_html.py index 8e7cb2d1..db47c5e5 100644 --- a/test/test_html.py +++ b/test/test_html.py @@ -8,14 +8,13 @@ from fastapi.testclient import TestClient from aurweb import asgi, db from aurweb.models.account_type import TRUSTED_USER_ID, USER_ID, AccountType from aurweb.models.user import User -from aurweb.testing import setup_test_db from aurweb.testing.html import get_errors, get_successes, parse_root from aurweb.testing.requests import Request @pytest.fixture(autouse=True) -def setup(): - setup_test_db(User.__tablename__) +def setup(db_test): + return @pytest.fixture diff --git a/test/test_initdb.py b/test/test_initdb.py index c7d29ee2..44681d8e 100644 --- a/test/test_initdb.py +++ b/test/test_initdb.py @@ -1,3 +1,5 @@ +import pytest + import aurweb.config import aurweb.db import aurweb.initdb @@ -5,6 +7,11 @@ import aurweb.initdb from aurweb.models.account_type import AccountType +@pytest.fixture(autouse=True) +def setup(db_test): + return + + class Args: use_alembic = True verbose = True @@ -15,6 +22,8 @@ def test_run(): aurweb.db.kill_engine() metadata.drop_all(aurweb.db.get_engine()) aurweb.initdb.run(Args()) + + # Check that constant table rows got added via initdb. record = aurweb.db.query(AccountType, AccountType.AccountType == "User").first() assert record is not None diff --git a/test/test_license.py b/test/test_license.py index 2c52f058..b34bd260 100644 --- a/test/test_license.py +++ b/test/test_license.py @@ -4,12 +4,11 @@ from sqlalchemy.exc import IntegrityError from aurweb import db from aurweb.models.license import License -from aurweb.testing import setup_test_db @pytest.fixture(autouse=True) -def setup(): - setup_test_db("Licenses") +def setup(db_test): + return def test_license_creation(): @@ -21,6 +20,4 @@ def test_license_creation(): def test_license_null_name_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - db.create(License) - db.rollback() + License() diff --git a/test/test_official_provider.py b/test/test_official_provider.py index 0aa4f1d1..9287ea2d 100644 --- a/test/test_official_provider.py +++ b/test/test_official_provider.py @@ -4,12 +4,11 @@ from sqlalchemy.exc import IntegrityError from aurweb import db from aurweb.models.official_provider import OfficialProvider -from aurweb.testing import setup_test_db @pytest.fixture(autouse=True) -def setup(): - setup_test_db("OfficialProviders") +def setup(db_test): + return def test_official_provider_creation(): @@ -53,26 +52,14 @@ def test_official_provider_cs(): def test_official_provider_null_name_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - db.create(OfficialProvider, - Repo="some-repo", - Provides="some-provides") - db.rollback() + OfficialProvider(Repo="some-repo", Provides="some-provides") def test_official_provider_null_repo_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - db.create(OfficialProvider, - Name="some-name", - Provides="some-provides") - db.rollback() + OfficialProvider(Name="some-name", Provides="some-provides") def test_official_provider_null_provides_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - db.create(OfficialProvider, - Name="some-name", - Repo="some-repo") - db.rollback() + OfficialProvider(Name="some-name", Repo="some-repo") diff --git a/test/test_package.py b/test/test_package.py index 112ca9b4..c2afa660 100644 --- a/test/test_package.py +++ b/test/test_package.py @@ -8,17 +8,14 @@ from aurweb.models.account_type import AccountType from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.user import User -from aurweb.testing import setup_test_db user = pkgbase = package = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, pkgbase, package - setup_test_db("Packages", "PackageBases", "Users") - account_type = db.query(AccountType, AccountType.AccountType == "User").first() diff --git a/test/test_package_base.py b/test/test_package_base.py index 2bc6278f..8e4b2edf 100644 --- a/test/test_package_base.py +++ b/test/test_package_base.py @@ -8,17 +8,14 @@ from aurweb import db from aurweb.models.account_type import AccountType from aurweb.models.package_base import PackageBase from aurweb.models.user import User -from aurweb.testing import setup_test_db user = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user - setup_test_db("Users", "PackageBases") - account_type = db.query(AccountType, AccountType.AccountType == "User").first() with db.begin(): diff --git a/test/test_package_blacklist.py b/test/test_package_blacklist.py index 93f15de7..6f4c36d7 100644 --- a/test/test_package_blacklist.py +++ b/test/test_package_blacklist.py @@ -6,20 +6,18 @@ from aurweb import db from aurweb.models.package_base import PackageBase from aurweb.models.package_blacklist import PackageBlacklist from aurweb.models.user import User -from aurweb.testing import setup_test_db user = pkgbase = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, pkgbase - setup_test_db("PackageBlacklist", "PackageBases", "Users") - - user = db.create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword") - pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword") + pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) def test_package_blacklist_creation(): @@ -31,6 +29,4 @@ def test_package_blacklist_creation(): def test_package_blacklist_null_name_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - db.create(PackageBlacklist) - db.rollback() + PackageBlacklist() diff --git a/test/test_package_comaintainer.py b/test/test_package_comaintainer.py index cba99ba0..ff74cddf 100644 --- a/test/test_package_comaintainer.py +++ b/test/test_package_comaintainer.py @@ -2,29 +2,28 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, rollback +from aurweb import db from aurweb.models.package_base import PackageBase from aurweb.models.package_comaintainer import PackageComaintainer from aurweb.models.user import User -from aurweb.testing import setup_test_db user = pkgbase = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, pkgbase - setup_test_db("Users", "PackageBases", "PackageComaintainers") - - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword") - pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword") + pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) def test_package_comaintainer_creation(): - package_comaintainer = create(PackageComaintainer, User=user, - PackageBase=pkgbase, Priority=5) + with db.begin(): + package_comaintainer = db.create(PackageComaintainer, User=user, + PackageBase=pkgbase, Priority=5) assert bool(package_comaintainer) assert package_comaintainer.User == user assert package_comaintainer.PackageBase == pkgbase @@ -33,17 +32,14 @@ def test_package_comaintainer_creation(): def test_package_comaintainer_null_user_raises_exception(): with pytest.raises(IntegrityError): - create(PackageComaintainer, PackageBase=pkgbase, Priority=1) - rollback() + PackageComaintainer(PackageBase=pkgbase, Priority=1) def test_package_comaintainer_null_pkgbase_raises_exception(): with pytest.raises(IntegrityError): - create(PackageComaintainer, User=user, Priority=1) - rollback() + PackageComaintainer(User=user, Priority=1) def test_package_comaintainer_null_priority_raises_exception(): with pytest.raises(IntegrityError): - create(PackageComaintainer, User=user, PackageBase=pkgbase) - rollback() + PackageComaintainer(User=user, PackageBase=pkgbase) diff --git a/test/test_package_comment.py b/test/test_package_comment.py index 60f0333d..b00e08c3 100644 --- a/test/test_package_comment.py +++ b/test/test_package_comment.py @@ -2,70 +2,55 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import begin, create, query, rollback -from aurweb.models.account_type import AccountType +from aurweb import db +from aurweb.models.account_type import USER_ID from aurweb.models.package_base import PackageBase from aurweb.models.package_comment import PackageComment from aurweb.models.user import User -from aurweb.testing import setup_test_db user = pkgbase = None @pytest.fixture(autouse=True) -def setup(): - setup_test_db("PackageBases", "PackageComments", "Users") - +def setup(db_test): global user, pkgbase - account_type = query(AccountType, - AccountType.AccountType == "User").first() - with begin(): - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountTypeID=USER_ID) + pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) def test_package_comment_creation(): - with begin(): - package_comment = create(PackageComment, - PackageBase=pkgbase, - User=user, - Comments="Test comment.", - RenderedComment="Test rendered comment.") + with db.begin(): + package_comment = db.create(PackageComment, PackageBase=pkgbase, + User=user, Comments="Test comment.", + RenderedComment="Test rendered comment.") assert bool(package_comment.ID) def test_package_comment_null_package_base_raises_exception(): with pytest.raises(IntegrityError): - with begin(): - create(PackageComment, User=user, Comments="Test comment.", - RenderedComment="Test rendered comment.") - rollback() + PackageComment(User=user, Comments="Test comment.", + RenderedComment="Test rendered comment.") def test_package_comment_null_user_raises_exception(): with pytest.raises(IntegrityError): - with begin(): - create(PackageComment, PackageBase=pkgbase, - Comments="Test comment.", - RenderedComment="Test rendered comment.") - rollback() + PackageComment(PackageBase=pkgbase, + Comments="Test comment.", + RenderedComment="Test rendered comment.") def test_package_comment_null_comments_raises_exception(): with pytest.raises(IntegrityError): - with begin(): - create(PackageComment, PackageBase=pkgbase, User=user, - RenderedComment="Test rendered comment.") - rollback() + PackageComment(PackageBase=pkgbase, User=user, + RenderedComment="Test rendered comment.") def test_package_comment_null_renderedcomment_defaults(): - with begin(): - record = create(PackageComment, - PackageBase=pkgbase, - User=user, - Comments="Test comment.") + with db.begin(): + record = db.create(PackageComment, PackageBase=pkgbase, + User=user, Comments="Test comment.") assert record.RenderedComment == str() diff --git a/test/test_package_dependency.py b/test/test_package_dependency.py index 2ddef68e..e6125669 100644 --- a/test/test_package_dependency.py +++ b/test/test_package_dependency.py @@ -3,117 +3,70 @@ import pytest from sqlalchemy.exc import IntegrityError from aurweb import db -from aurweb.db import create, query -from aurweb.models.account_type import AccountType -from aurweb.models.dependency_type import DependencyType +from aurweb.models.account_type import USER_ID +from aurweb.models.dependency_type import CHECKDEPENDS_ID, DEPENDS_ID, MAKEDEPENDS_ID, OPTDEPENDS_ID from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.package_dependency import PackageDependency from aurweb.models.user import User -from aurweb.testing import setup_test_db user = pkgbase = package = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, pkgbase, package - setup_test_db("Users", "PackageBases", "Packages", "PackageDepends") - - account_type = query(AccountType, - AccountType.AccountType == "User").first() - with db.begin(): - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - pkgbase = create(PackageBase, - Name="test-package", - Maintainer=user) - package = create(Package, - PackageBase=pkgbase, - Name=pkgbase.Name, - Description="Test description.", - URL="https://test.package") + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountTypeID=USER_ID) + pkgbase = db.create(PackageBase, + Name="test-package", + Maintainer=user) + package = db.create(Package, + PackageBase=pkgbase, + Name=pkgbase.Name, + Description="Test description.", + URL="https://test.package") def test_package_dependencies(): - depends = query(DependencyType, DependencyType.Name == "depends").first() - with db.begin(): - pkgdep = create(PackageDependency, Package=package, - DependencyType=depends, - DepName="test-dep") + pkgdep = db.create(PackageDependency, Package=package, + DepTypeID=DEPENDS_ID, DepName="test-dep") assert pkgdep.DepName == "test-dep" assert pkgdep.Package == package - assert pkgdep.DependencyType == depends - assert pkgdep in depends.package_dependencies assert pkgdep in package.package_dependencies - makedepends = query(DependencyType, - DependencyType.Name == "makedepends").first() with db.begin(): - pkgdep.DependencyType = makedepends - assert pkgdep.DepName == "test-dep" - assert pkgdep.Package == package - assert pkgdep.DependencyType == makedepends - assert pkgdep in makedepends.package_dependencies - assert pkgdep in package.package_dependencies + pkgdep.DepTypeID = MAKEDEPENDS_ID - checkdepends = query(DependencyType, - DependencyType.Name == "checkdepends").first() with db.begin(): - pkgdep.DependencyType = checkdepends - assert pkgdep.DepName == "test-dep" - assert pkgdep.Package == package - assert pkgdep.DependencyType == checkdepends - assert pkgdep in checkdepends.package_dependencies - assert pkgdep in package.package_dependencies + pkgdep.DepTypeID = CHECKDEPENDS_ID - optdepends = query(DependencyType, - DependencyType.Name == "optdepends").first() with db.begin(): - pkgdep.DependencyType = optdepends - assert pkgdep.DepName == "test-dep" - assert pkgdep.Package == package - assert pkgdep.DependencyType == optdepends - assert pkgdep in optdepends.package_dependencies - assert pkgdep in package.package_dependencies + pkgdep.DepTypeID = OPTDEPENDS_ID assert not pkgdep.is_package() with db.begin(): - base = create(PackageBase, Name=pkgdep.DepName, Maintainer=user) - create(Package, PackageBase=base, Name=pkgdep.DepName) + base = db.create(PackageBase, Name=pkgdep.DepName, Maintainer=user) + db.create(Package, PackageBase=base, Name=pkgdep.DepName) assert pkgdep.is_package() def test_package_dependencies_null_package_raises_exception(): - depends = query(DependencyType, DependencyType.Name == "depends").first() with pytest.raises(IntegrityError): - with db.begin(): - create(PackageDependency, - DependencyType=depends, - DepName="test-dep") - db.rollback() + PackageDependency(DepTypeID=DEPENDS_ID, DepName="test-dep") def test_package_dependencies_null_dependency_type_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - create(PackageDependency, - Package=package, - DepName="test-dep") - db.rollback() + PackageDependency(Package=package, DepName="test-dep") def test_package_dependencies_null_depname_raises_exception(): - depends = query(DependencyType, DependencyType.Name == "depends").first() with pytest.raises(IntegrityError): - with db.begin(): - create(PackageDependency, - Package=package, - DependencyType=depends) - db.rollback() + PackageDependency(DepTypeID=DEPENDS_ID, Package=package) diff --git a/test/test_package_group.py b/test/test_package_group.py index 0e6e41e3..2c91e0b1 100644 --- a/test/test_package_group.py +++ b/test/test_package_group.py @@ -2,51 +2,44 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, query -from aurweb.models.account_type import AccountType +from aurweb import db +from aurweb.models.account_type import USER_ID from aurweb.models.group import Group from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.package_group import PackageGroup from aurweb.models.user import User -from aurweb.testing import setup_test_db user = group = pkgbase = package = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, group, pkgbase, package - setup_test_db("Users", "PackageBases", "Packages", - "Groups", "PackageGroups") + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountTypeID=USER_ID) + group = db.create(Group, Name="Test Group") - account_type = query(AccountType, - AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - - group = create(Group, Name="Test Group") - pkgbase = create(PackageBase, Name="test-package", Maintainer=user) - package = create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + with db.begin(): + pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) + package = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) def test_package_group(): - package_group = create(PackageGroup, Package=package, Group=group) + with db.begin(): + package_group = db.create(PackageGroup, Package=package, Group=group) assert package_group.Group == group assert package_group.Package == package def test_package_group_null_package_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(PackageGroup, Group=group) - session.rollback() + PackageGroup(Group=group) def test_package_group_null_group_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(PackageGroup, Package=package) - session.rollback() + PackageGroup(Package=package) diff --git a/test/test_package_keyword.py b/test/test_package_keyword.py index 316e7ca8..88ccb734 100644 --- a/test/test_package_keyword.py +++ b/test/test_package_keyword.py @@ -2,44 +2,37 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, query -from aurweb.models.account_type import AccountType +from aurweb import db +from aurweb.models.account_type import USER_ID from aurweb.models.package_base import PackageBase from aurweb.models.package_keyword import PackageKeyword from aurweb.models.user import User -from aurweb.testing import setup_test_db user = pkgbase = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, pkgbase - setup_test_db("Users", "PackageBases", "PackageKeywords") - - account_type = query(AccountType, - AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - pkgbase = create(PackageBase, - Name="beautiful-package", - Maintainer=user) + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountTypeID=USER_ID) + pkgbase = db.create(PackageBase, + Name="beautiful-package", + Maintainer=user) def test_package_keyword(): - pkg_keyword = create(PackageKeyword, - PackageBase=pkgbase, - Keyword="test") + with db.begin(): + pkg_keyword = db.create(PackageKeyword, + PackageBase=pkgbase, + Keyword="test") assert pkg_keyword in pkgbase.keywords assert pkgbase == pkg_keyword.PackageBase def test_package_keyword_null_pkgbase_raises_exception(): - from aurweb.db import session - with pytest.raises(IntegrityError): - create(PackageKeyword, - Keyword="test") - session.rollback() + PackageKeyword(Keyword="test") diff --git a/test/test_package_license.py b/test/test_package_license.py index f7654dee..965d0c6f 100644 --- a/test/test_package_license.py +++ b/test/test_package_license.py @@ -2,51 +2,45 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, query -from aurweb.models.account_type import AccountType +from aurweb import db +from aurweb.models.account_type import USER_ID from aurweb.models.license import License from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.package_license import PackageLicense from aurweb.models.user import User -from aurweb.testing import setup_test_db user = license = pkgbase = package = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, license, pkgbase, package - setup_test_db("Users", "PackageBases", "Packages", - "Licenses", "PackageLicenses") + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountTypeID=USER_ID) + license = db.create(License, Name="Test License") - account_type = query(AccountType, - AccountType.AccountType == "User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - - license = create(License, Name="Test License") - pkgbase = create(PackageBase, Name="test-package", Maintainer=user) - package = create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + with db.begin(): + pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) + package = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) def test_package_license(): - package_license = create(PackageLicense, Package=package, License=license) + with db.begin(): + package_license = db.create(PackageLicense, Package=package, + License=license) assert package_license.License == license assert package_license.Package == package def test_package_license_null_package_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(PackageLicense, License=license) - session.rollback() + PackageLicense(License=license) def test_package_license_null_license_raises_exception(): - from aurweb.db import session with pytest.raises(IntegrityError): - create(PackageLicense, Package=package) - session.rollback() + PackageLicense(Package=package) diff --git a/test/test_package_notification.py b/test/test_package_notification.py index 2898a904..2e505dd8 100644 --- a/test/test_package_notification.py +++ b/test/test_package_notification.py @@ -2,29 +2,28 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, rollback +from aurweb import db from aurweb.models.package_base import PackageBase from aurweb.models.package_notification import PackageNotification from aurweb.models.user import User -from aurweb.testing import setup_test_db user = pkgbase = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, pkgbase - setup_test_db("Users", "PackageBases", "PackageNotifications") - - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword") - pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword") + pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) def test_package_notification_creation(): - package_notification = create(PackageNotification, User=user, - PackageBase=pkgbase) + with db.begin(): + package_notification = db.create( + PackageNotification, User=user, PackageBase=pkgbase) assert bool(package_notification) assert package_notification.User == user assert package_notification.PackageBase == pkgbase @@ -32,11 +31,9 @@ def test_package_notification_creation(): def test_package_notification_null_user_raises_exception(): with pytest.raises(IntegrityError): - create(PackageNotification, PackageBase=pkgbase) - rollback() + PackageNotification(PackageBase=pkgbase) def test_package_notification_null_pkgbase_raises_exception(): with pytest.raises(IntegrityError): - create(PackageNotification, User=user) - rollback() + PackageNotification(User=user) diff --git a/test/test_package_relation.py b/test/test_package_relation.py index edb67078..e5f7f453 100644 --- a/test/test_package_relation.py +++ b/test/test_package_relation.py @@ -1,103 +1,63 @@ import pytest -from sqlalchemy.exc import IntegrityError, OperationalError +from sqlalchemy.exc import IntegrityError from aurweb import db -from aurweb.db import create, query -from aurweb.models.account_type import AccountType +from aurweb.models.account_type import USER_ID from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.package_relation import PackageRelation -from aurweb.models.relation_type import RelationType +from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID from aurweb.models.user import User -from aurweb.testing import setup_test_db user = pkgbase = package = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, pkgbase, package - setup_test_db("Users", "PackageBases", "Packages", "PackageRelations") - - account_type = query(AccountType, - AccountType.AccountType == "User").first() - with db.begin(): - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - pkgbase = create(PackageBase, - Name="test-package", - Maintainer=user) - package = create(Package, - PackageBase=pkgbase, - Name=pkgbase.Name, - Description="Test description.", - URL="https://test.package") + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountTypeID=USER_ID) + pkgbase = db.create(PackageBase, + Name="test-package", + Maintainer=user) + package = db.create(Package, + PackageBase=pkgbase, + Name=pkgbase.Name, + Description="Test description.", + URL="https://test.package") def test_package_relation(): - conflicts = query(RelationType, RelationType.Name == "conflicts").first() - with db.begin(): - pkgrel = create(PackageRelation, Package=package, - RelationType=conflicts, - RelName="test-relation") + pkgrel = db.create(PackageRelation, Package=package, + RelTypeID=CONFLICTS_ID, + RelName="test-relation") + assert pkgrel.RelName == "test-relation" assert pkgrel.Package == package - assert pkgrel.RelationType == conflicts - assert pkgrel in conflicts.package_relations assert pkgrel in package.package_relations - provides = query(RelationType, RelationType.Name == "provides").first() with db.begin(): - pkgrel.RelationType = provides - assert pkgrel.RelName == "test-relation" - assert pkgrel.Package == package - assert pkgrel.RelationType == provides - assert pkgrel in provides.package_relations - assert pkgrel in package.package_relations + pkgrel.RelTypeID = PROVIDES_ID - replaces = query(RelationType, RelationType.Name == "replaces").first() with db.begin(): - pkgrel.RelationType = replaces - assert pkgrel.RelName == "test-relation" - assert pkgrel.Package == package - assert pkgrel.RelationType == replaces - assert pkgrel in replaces.package_relations - assert pkgrel in package.package_relations + pkgrel.RelTypeID = REPLACES_ID def test_package_relation_null_package_raises_exception(): - conflicts = query(RelationType, RelationType.Name == "conflicts").first() - assert conflicts is not None - with pytest.raises(IntegrityError): - with db.begin(): - create(PackageRelation, - RelationType=conflicts, - RelName="test-relation") - db.rollback() + PackageRelation(RelTypeID=CONFLICTS_ID, RelName="test-relation") def test_package_relation_null_relation_type_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - create(PackageRelation, - Package=package, - RelName="test-relation") - db.rollback() + PackageRelation(Package=package, RelName="test-relation") def test_package_relation_null_relname_raises_exception(): - depends = query(RelationType, RelationType.Name == "conflicts").first() - assert depends is not None - - with pytest.raises((OperationalError, IntegrityError)): - with db.begin(): - create(PackageRelation, - Package=package, - RelationType=depends) - db.rollback() + with pytest.raises(IntegrityError): + PackageRelation(Package=package, RelTypeID=CONFLICTS_ID) diff --git a/test/test_package_request.py b/test/test_package_request.py index 1589ffc2..4b5dfb2b 100644 --- a/test/test_package_request.py +++ b/test/test_package_request.py @@ -5,41 +5,35 @@ import pytest from sqlalchemy.exc import IntegrityError from aurweb import db -from aurweb.db import create, query, rollback +from aurweb.models.account_type import USER_ID from aurweb.models.package_base import PackageBase from aurweb.models.package_request import (ACCEPTED, ACCEPTED_ID, CLOSED, CLOSED_ID, PENDING, PENDING_ID, REJECTED, REJECTED_ID, PackageRequest) -from aurweb.models.request_type import RequestType +from aurweb.models.request_type import MERGE_ID from aurweb.models.user import User -from aurweb.testing import setup_test_db user = pkgbase = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, pkgbase - setup_test_db("PackageRequests", "PackageBases", "Users") - with db.begin(): - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword") - pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountTypeID=USER_ID) + pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) def test_package_request_creation(): - request_type = query(RequestType, RequestType.Name == "merge").first() - assert request_type.Name == "merge" - with db.begin(): - package_request = create(PackageRequest, RequestType=request_type, - User=user, PackageBase=pkgbase, - PackageBaseName=pkgbase.Name, - Comments=str(), ClosureComment=str()) + package_request = db.create(PackageRequest, ReqTypeID=MERGE_ID, + User=user, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str()) assert bool(package_request.ID) - assert package_request.RequestType == request_type assert package_request.User == user assert package_request.PackageBase == pkgbase assert package_request.PackageBaseName == pkgbase.Name @@ -47,22 +41,18 @@ def test_package_request_creation(): assert package_request.ClosureComment == str() # Make sure that everything is cross-referenced with relationships. - assert package_request in request_type.package_requests assert package_request in user.package_requests assert package_request in pkgbase.requests def test_package_request_closed(): - request_type = query(RequestType, RequestType.Name == "merge").first() - assert request_type.Name == "merge" - ts = int(datetime.utcnow().timestamp()) with db.begin(): - package_request = create(PackageRequest, RequestType=request_type, - User=user, PackageBase=pkgbase, - PackageBaseName=pkgbase.Name, - Closer=user, ClosedTS=ts, - Comments=str(), ClosureComment=str()) + package_request = db.create(PackageRequest, ReqTypeID=MERGE_ID, + User=user, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Closer=user, ClosedTS=ts, + Comments=str(), ClosureComment=str()) assert package_request.Closer == user assert package_request.ClosedTS == ts @@ -73,73 +63,54 @@ def test_package_request_closed(): def test_package_request_null_request_type_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - create(PackageRequest, User=user, PackageBase=pkgbase, - PackageBaseName=pkgbase.Name, - Comments=str(), ClosureComment=str()) - rollback() + PackageRequest(User=user, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str()) def test_package_request_null_user_raises_exception(): - request_type = query(RequestType, RequestType.Name == "merge").first() with pytest.raises(IntegrityError): - with db.begin(): - create(PackageRequest, RequestType=request_type, - PackageBase=pkgbase, PackageBaseName=pkgbase.Name, - Comments=str(), ClosureComment=str()) - rollback() + PackageRequest(ReqTypeID=MERGE_ID, + PackageBase=pkgbase, PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str()) def test_package_request_null_package_base_raises_exception(): - request_type = query(RequestType, RequestType.Name == "merge").first() with pytest.raises(IntegrityError): - with db.begin(): - create(PackageRequest, RequestType=request_type, - User=user, PackageBaseName=pkgbase.Name, - Comments=str(), ClosureComment=str()) - rollback() + PackageRequest(ReqTypeID=MERGE_ID, + User=user, PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str()) def test_package_request_null_package_base_name_raises_exception(): - request_type = query(RequestType, RequestType.Name == "merge").first() with pytest.raises(IntegrityError): - with db.begin(): - create(PackageRequest, RequestType=request_type, - User=user, PackageBase=pkgbase, - Comments=str(), ClosureComment=str()) - rollback() + PackageRequest(ReqTypeID=MERGE_ID, + User=user, PackageBase=pkgbase, + Comments=str(), ClosureComment=str()) def test_package_request_null_comments_raises_exception(): - request_type = query(RequestType, RequestType.Name == "merge").first() with pytest.raises(IntegrityError): - with db.begin(): - create(PackageRequest, RequestType=request_type, User=user, - PackageBase=pkgbase, PackageBaseName=pkgbase.Name, - ClosureComment=str()) - rollback() + PackageRequest(ReqTypeID=MERGE_ID, User=user, + PackageBase=pkgbase, PackageBaseName=pkgbase.Name, + ClosureComment=str()) def test_package_request_null_closure_comment_raises_exception(): - request_type = query(RequestType, RequestType.Name == "merge").first() with pytest.raises(IntegrityError): - with db.begin(): - create(PackageRequest, RequestType=request_type, User=user, - PackageBase=pkgbase, PackageBaseName=pkgbase.Name, - Comments=str()) - rollback() + PackageRequest(ReqTypeID=MERGE_ID, User=user, + PackageBase=pkgbase, PackageBaseName=pkgbase.Name, + Comments=str()) def test_package_request_status_display(): """ Test status_display() based on the Status column value. """ - request_type = query(RequestType, RequestType.Name == "merge").first() - with db.begin(): - pkgreq = create(PackageRequest, RequestType=request_type, - User=user, PackageBase=pkgbase, - PackageBaseName=pkgbase.Name, - Comments=str(), ClosureComment=str(), - Status=PENDING_ID) + pkgreq = db.create(PackageRequest, ReqTypeID=MERGE_ID, + User=user, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + Comments=str(), ClosureComment=str(), + Status=PENDING_ID) assert pkgreq.status_display() == PENDING with db.begin(): diff --git a/test/test_package_source.py b/test/test_package_source.py index d1adcf9c..b83c9d48 100644 --- a/test/test_package_source.py +++ b/test/test_package_source.py @@ -14,7 +14,7 @@ user = pkgbase = package = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, pkgbase, package setup_test_db("PackageSources", "Packages", "PackageBases", "Users") diff --git a/test/test_package_vote.py b/test/test_package_vote.py index cb15e217..d1ec203b 100644 --- a/test/test_package_vote.py +++ b/test/test_package_vote.py @@ -4,30 +4,30 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, rollback +from aurweb import db from aurweb.models.package_base import PackageBase from aurweb.models.package_vote import PackageVote from aurweb.models.user import User -from aurweb.testing import setup_test_db user = pkgbase = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, pkgbase - setup_test_db("Users", "PackageBases", "PackageVotes") - - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword") - pkgbase = create(PackageBase, Name="test-package", Maintainer=user) + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword") + pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) def test_package_vote_creation(): ts = int(datetime.utcnow().timestamp()) - package_vote = create(PackageVote, User=user, PackageBase=pkgbase, - VoteTS=ts) + + with db.begin(): + package_vote = db.create(PackageVote, User=user, + PackageBase=pkgbase, VoteTS=ts) assert bool(package_vote) assert package_vote.User == user assert package_vote.PackageBase == pkgbase @@ -36,17 +36,14 @@ def test_package_vote_creation(): def test_package_vote_null_user_raises_exception(): with pytest.raises(IntegrityError): - create(PackageVote, PackageBase=pkgbase, VoteTS=1) - rollback() + PackageVote(PackageBase=pkgbase, VoteTS=1) def test_package_vote_null_pkgbase_raises_exception(): with pytest.raises(IntegrityError): - create(PackageVote, User=user, VoteTS=1) - rollback() + PackageVote(User=user, VoteTS=1) def test_package_vote_null_votets_raises_exception(): with pytest.raises(IntegrityError): - create(PackageVote, User=user, PackageBase=pkgbase) - rollback() + PackageVote(User=user, PackageBase=pkgbase) diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 1bdb3ea3..02c22d9d 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -28,7 +28,6 @@ from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID, RelationType from aurweb.models.request_type import DELETION_ID, MERGE_ID, RequestType from aurweb.models.user import User -from aurweb.testing import setup_test_db from aurweb.testing.html import get_errors, get_successes, parse_root from aurweb.testing.requests import Request @@ -65,21 +64,8 @@ def create_package_rel(package: Package, @pytest.fixture(autouse=True) -def setup(): - setup_test_db( - User.__tablename__, - Package.__tablename__, - PackageBase.__tablename__, - PackageDependency.__tablename__, - PackageRelation.__tablename__, - PackageKeyword.__tablename__, - PackageVote.__tablename__, - PackageNotification.__tablename__, - PackageComaintainer.__tablename__, - PackageComment.__tablename__, - PackageRequest.__tablename__, - OfficialProvider.__tablename__ - ) +def setup(db_test): + return @pytest.fixture @@ -91,12 +77,11 @@ def client() -> TestClient: @pytest.fixture def user() -> User: """ Yield a user. """ - account_type = db.query(AccountType, AccountType.ID == USER_ID).first() with db.begin(): user = db.create(User, Username="test", Email="test@example.org", Passwd="testPassword", - AccountType=account_type) + AccountTypeID=USER_ID) yield user @@ -1173,7 +1158,7 @@ def test_pkgbase_comments(client: TestClient, maintainer: User, user: User, PackageNotification.UserID == maintainer.ID ).first() with db.begin(): - db.session.delete(db_notif) + db.delete(db_notif) # Now, let's edit the comment we just created. comment_id = int(headers[0].attrib["id"].split("-")[-1]) diff --git a/test/test_packages_util.py b/test/test_packages_util.py index 622c08c2..cd0982b2 100644 --- a/test/test_packages_util.py +++ b/test/test_packages_util.py @@ -6,7 +6,7 @@ from fastapi import HTTPException from fastapi.testclient import TestClient from aurweb import asgi, db -from aurweb.models.account_type import USER_ID, AccountType +from aurweb.models.account_type import USER_ID from aurweb.models.official_provider import OFFICIAL_BASE, OfficialProvider from aurweb.models.package import Package from aurweb.models.package_base import PackageBase @@ -15,29 +15,20 @@ from aurweb.models.package_vote import PackageVote from aurweb.models.user import User from aurweb.packages import util from aurweb.redis import kill_redis -from aurweb.testing import setup_test_db @pytest.fixture(autouse=True) -def setup(): - setup_test_db( - User.__tablename__, - Package.__tablename__, - PackageBase.__tablename__, - PackageVote.__tablename__, - PackageNotification.__tablename__, - OfficialProvider.__tablename__ - ) +def setup(db_test): + return @pytest.fixture def maintainer() -> User: - account_type = db.query(AccountType, AccountType.ID == USER_ID).first() with db.begin(): maintainer = db.create(User, Username="test_maintainer", Email="test_maintainer@examepl.org", Passwd="testPassword", - AccountType=account_type) + AccountTypeID=USER_ID) yield maintainer diff --git a/test/test_popupdate.py b/test/test_popupdate.py index 93f86f10..ce3f9f11 100644 --- a/test/test_popupdate.py +++ b/test/test_popupdate.py @@ -1,5 +1,12 @@ +import pytest + from aurweb.scripts import popupdate +@pytest.fixture(autouse=True) +def setup(db_test): + return + + def test_popupdate(): popupdate.main() diff --git a/test/test_ratelimit.py b/test/test_ratelimit.py index 0a72a7e4..859adea9 100644 --- a/test/test_ratelimit.py +++ b/test/test_ratelimit.py @@ -8,15 +8,14 @@ from aurweb import config, db, logging from aurweb.models import ApiRateLimit from aurweb.ratelimit import check_ratelimit from aurweb.redis import redis_connection -from aurweb.testing import setup_test_db from aurweb.testing.requests import Request logger = logging.get_logger(__name__) @pytest.fixture(autouse=True) -def setup(): - setup_test_db(ApiRateLimit.__tablename__) +def setup(db_test): + return @pytest.fixture @@ -31,27 +30,36 @@ def pipeline(): yield pipeline +config_getint = config.getint + + def mock_config_getint(section: str, key: str): if key == "request_limit": return 4 elif key == "window_length": return 100 - return config.getint(section, key) + return config_getint(section, key) + + +config_getboolean = config.getboolean def mock_config_getboolean(return_value: int = 0): def fn(section: str, key: str): if section == "ratelimit" and key == "cache": return return_value - return config.getboolean(section, key) + return config_getboolean(section, key) return fn +config_get = config.get + + def mock_config_get(return_value: str = "none"): def fn(section: str, key: str): if section == "options" and key == "cache": return return_value - return config.get(section, key) + return config_get(section, key) return fn diff --git a/test/test_relation_type.py b/test/test_relation_type.py index d2dabceb..263ae1ec 100644 --- a/test/test_relation_type.py +++ b/test/test_relation_type.py @@ -2,12 +2,11 @@ import pytest from aurweb import db from aurweb.models.relation_type import RelationType -from aurweb.testing import setup_test_db @pytest.fixture(autouse=True) -def setup(): - setup_test_db() +def setup(db_test): + return def test_relation_type_creation(): diff --git a/test/test_request_type.py b/test/test_request_type.py index 0db24921..0bc86319 100644 --- a/test/test_request_type.py +++ b/test/test_request_type.py @@ -2,12 +2,11 @@ import pytest from aurweb import db from aurweb.models.request_type import DELETION_ID, MERGE_ID, ORPHAN_ID, RequestType -from aurweb.testing import setup_test_db @pytest.fixture(autouse=True) -def setup(): - setup_test_db() +def setup(db_test): + return def test_request_type_creation(): diff --git a/test/test_routes.py b/test/test_routes.py index e3f69d7a..32f507f3 100644 --- a/test/test_routes.py +++ b/test/test_routes.py @@ -10,27 +10,21 @@ from fastapi.testclient import TestClient from aurweb import db from aurweb.asgi import app -from aurweb.models.account_type import AccountType +from aurweb.models.account_type import USER_ID from aurweb.models.user import User -from aurweb.testing import setup_test_db from aurweb.testing.requests import Request user = client = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, client - setup_test_db("Users", "Sessions") - - account_type = db.query(AccountType, - AccountType.AccountType == "User").first() - with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", - AccountType=account_type) + AccountTypeID=USER_ID) client = TestClient(app) diff --git a/test/test_rpc.py b/test/test_rpc.py index 055baa33..f20c9b02 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -26,7 +26,6 @@ from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import RelationType from aurweb.models.user import User from aurweb.redis import redis_connection -from aurweb.testing import setup_test_db def make_request(path, headers: Dict[str, str] = {}): @@ -35,11 +34,8 @@ def make_request(path, headers: Dict[str, str] = {}): @pytest.fixture(autouse=True) -def setup(): - # Set up tables. - setup_test_db("Users", "PackageBases", "Packages", "Licenses", - "PackageDepends", "PackageRelations", "PackageLicenses", - "PackageKeywords", "PackageVotes", "ApiRateLimit") +def setup(db_test): + # TODO: Rework this into organized fixtures. # Create test package details. with begin(): diff --git a/test/test_rss.py b/test/test_rss.py index 40607ade..7123fbf1 100644 --- a/test/test_rss.py +++ b/test/test_rss.py @@ -12,17 +12,13 @@ from aurweb.models.account_type import AccountType from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.user import User -from aurweb.testing import setup_test_db logger = logging.get_logger(__name__) @pytest.fixture(autouse=True) -def setup(): - setup_test_db( - Package.__tablename__, - PackageBase.__tablename__, - User.__tablename__) +def setup(db_test): + return @pytest.fixture diff --git a/test/test_session.py b/test/test_session.py index 4e6f4db4..7d3037a1 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -8,17 +8,14 @@ from aurweb import db from aurweb.models.account_type import AccountType from aurweb.models.session import Session, generate_unique_sid from aurweb.models.user import User -from aurweb.testing import setup_test_db account_type = user = session = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global account_type, user, session - setup_test_db("Users", "Sessions") - account_type = db.query(AccountType, AccountType.AccountType == "User").first() with db.begin(): diff --git a/test/test_ssh_pub_key.py b/test/test_ssh_pub_key.py index 12a3e1ce..bb787759 100644 --- a/test/test_ssh_pub_key.py +++ b/test/test_ssh_pub_key.py @@ -1,10 +1,9 @@ import pytest from aurweb import db -from aurweb.models.account_type import AccountType +from aurweb.models.account_type import USER_ID from aurweb.models.ssh_pub_key import SSHPubKey, get_fingerprint from aurweb.models.user import User -from aurweb.testing import setup_test_db TEST_SSH_PUBKEY = """ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCycoCi5yGCvSclH2wmNBUuwsYEzRZZBJaQquRc4ysl+Tg+/jiDkR3Zn9fIznC4KnFoyrIHzkKuePZ3bNDYwkZxkJKoWBCh4hXKDXSm87FMN0+VDC+1QxF/z0XaAGr/P6f4XukabyddypBdnHcZiplbw+YOSqcAE2TCqOlSXwNMOcF9U89UsR/Q9i9I52hlvU0q8+fZVGhou1KCowFSnHYtrr5KYJ04CXkJ13DkVf3+pjQWyrByvBcf1hGEaczlgfobrrv/y96jDhgfXucxliNKLdufDPPkii3LhhsNcDmmI1VZ3v0irKvd9WZuauqloobY84zEFcDTyjn0hxGjVeYFejm4fBnvjga0yZXORuWksdNfXWLDxFk6MDDd1jF0ExRbP+OxDuU4IVyIuDL7S3cnbf2YjGhkms/8voYT2OBE7FwNlfv98Kr0NUp51zpf55Arxn9j0Rz9xTA7FiODQgCn6iQ0SDtzUNL0IKTCw26xJY5gzMxbfpvzPQGeulx/ioM= kevr@volcano @@ -14,21 +13,16 @@ user = ssh_pub_key = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, ssh_pub_key - setup_test_db("Users", "SSHPubKeys") - - account_type = db.query(AccountType, - AccountType.AccountType == "User").first() with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", - AccountType=account_type) + AccountTypeID=USER_ID) with db.begin(): - ssh_pub_key = db.create(SSHPubKey, - UserID=user.ID, + ssh_pub_key = db.create(SSHPubKey, UserID=user.ID, Fingerprint="testFingerprint", PubKey="testPubKey") diff --git a/test/test_term.py b/test/test_term.py index 3f28311f..bfa73a76 100644 --- a/test/test_term.py +++ b/test/test_term.py @@ -4,17 +4,11 @@ from sqlalchemy.exc import IntegrityError from aurweb import db from aurweb.models.term import Term -from aurweb.testing import setup_test_db @pytest.fixture(autouse=True) -def setup(): - setup_test_db("Terms") - - yield None - - # Wipe em out just in case records are leftover. - setup_test_db("Terms") +def setup(db_test): + return def test_term_creation(): @@ -29,13 +23,9 @@ def test_term_creation(): def test_term_null_description_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - db.create(Term, URL="https://fake_url.io") - db.rollback() + Term(URL="https://fake_url.io") def test_term_null_url_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - db.create(Term, Description="Term description") - db.rollback() + Term(Description="Term description") diff --git a/test/test_trusted_user_routes.py b/test/test_trusted_user_routes.py index 0579247e..43a3443b 100644 --- a/test/test_trusted_user_routes.py +++ b/test/test_trusted_user_routes.py @@ -14,7 +14,6 @@ from aurweb.models.account_type import AccountType from aurweb.models.tu_vote import TUVote from aurweb.models.tu_voteinfo import TUVoteInfo from aurweb.models.user import User -from aurweb.testing import setup_test_db from aurweb.testing.requests import Request DATETIME_REGEX = r'^[0-9]{4}-[0-9]{2}-[0-9]{2}$' @@ -76,8 +75,8 @@ def assert_past_vote_html(row, expected): @pytest.fixture(autouse=True) -def setup(): - setup_test_db("TU_Votes", "TU_VoteInfo", "Users") +def setup(db_test): + return @pytest.fixture diff --git a/test/test_tu_vote.py b/test/test_tu_vote.py index 9ff4a8d9..1dd33387 100644 --- a/test/test_tu_vote.py +++ b/test/test_tu_vote.py @@ -4,53 +4,48 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import create, query, rollback -from aurweb.models.account_type import AccountType +from aurweb import db +from aurweb.models.account_type import TRUSTED_USER_ID from aurweb.models.tu_vote import TUVote from aurweb.models.tu_voteinfo import TUVoteInfo from aurweb.models.user import User -from aurweb.testing import setup_test_db user = tu_voteinfo = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user, tu_voteinfo - setup_test_db("Users", "TU_VoteInfo", "TU_Votes") - - tu_type = query(AccountType, - AccountType.AccountType == "Trusted User").first() - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=tu_type) - ts = int(datetime.utcnow().timestamp()) - tu_voteinfo = create(TUVoteInfo, - Agenda="Blah blah.", - User=user.Username, - Submitted=ts, End=ts + 5, - Quorum=0.5, - Submitter=user) + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountTypeID=TRUSTED_USER_ID) + + tu_voteinfo = db.create(TUVoteInfo, + Agenda="Blah blah.", + User=user.Username, + Submitted=ts, End=ts + 5, + Quorum=0.5, + Submitter=user) def test_tu_vote_creation(): - tu_vote = create(TUVote, User=user, VoteInfo=tu_voteinfo) + with db.begin(): + tu_vote = db.create(TUVote, User=user, VoteInfo=tu_voteinfo) + assert tu_vote.VoteInfo == tu_voteinfo assert tu_vote.User == user - assert tu_vote in user.tu_votes assert tu_vote in tu_voteinfo.tu_votes def test_tu_vote_null_user_raises_exception(): with pytest.raises(IntegrityError): - create(TUVote, VoteInfo=tu_voteinfo) - rollback() + TUVote(VoteInfo=tu_voteinfo) def test_tu_vote_null_voteinfo_raises_exception(): with pytest.raises(IntegrityError): - create(TUVote, User=user) - rollback() + TUVote(User=user) diff --git a/test/test_tu_voteinfo.py b/test/test_tu_voteinfo.py index b60e2e6a..5926fbf9 100644 --- a/test/test_tu_voteinfo.py +++ b/test/test_tu_voteinfo.py @@ -9,17 +9,14 @@ from aurweb.db import create, query, rollback from aurweb.models.account_type import AccountType from aurweb.models.tu_voteinfo import TUVoteInfo from aurweb.models.user import User -from aurweb.testing import setup_test_db user = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global user - setup_test_db("Users", "PackageBases", "TU_VoteInfo") - tu_type = query(AccountType, AccountType.AccountType == "Trusted User").first() with db.begin(): diff --git a/test/test_user.py b/test/test_user.py index 771611d8..07f10487 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -19,27 +19,15 @@ from aurweb.models.package_vote import PackageVote from aurweb.models.session import Session from aurweb.models.ssh_pub_key import SSHPubKey from aurweb.models.user import User -from aurweb.testing import setup_test_db from aurweb.testing.requests import Request account_type = user = None @pytest.fixture(autouse=True) -def setup(): +def setup(db_test): global account_type, user - setup_test_db( - User.__tablename__, - Session.__tablename__, - Ban.__tablename__, - SSHPubKey.__tablename__, - Package.__tablename__, - PackageBase.__tablename__, - PackageVote.__tablename__, - PackageNotification.__tablename__ - ) - account_type = db.query(AccountType, AccountType.AccountType == "User").first() From fa26c8078b5dd305fcc62e349a2856ddd29d0b68 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 00:44:35 -0800 Subject: [PATCH 579/844] fix(docker): modify db configuration for new tests A user that can create databases is now required for tests, we use the 'root' user in Docker. Added docker services: --------------------- - mariadb_test - host localhost:13307 Signed-off-by: Kevin Morris --- conf/config.defaults | 2 +- conf/config.dev | 7 ++-- docker-compose.yml | 65 ++++++++++++++----------------- docker/fastapi-entrypoint.sh | 4 ++ docker/mariadb-entrypoint.sh | 15 +++---- docker/mariadb-init-entrypoint.sh | 2 + docker/php-entrypoint.sh | 7 +++- docker/scripts/run-php.sh | 3 -- docker/scripts/run-pytests.sh | 11 +----- docker/scripts/run-tests.sh | 6 --- docker/test-mysql-entrypoint.sh | 9 ----- docker/tests-entrypoint.sh | 1 - 12 files changed, 53 insertions(+), 79 deletions(-) diff --git a/conf/config.defaults b/conf/config.defaults index c29d7045..68e235be 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -5,7 +5,7 @@ socket = /var/run/mysqld/mysqld.sock ;port = 3306 name = AUR user = aur -password = aur +;password = aur [options] username_min_len = 3 diff --git a/conf/config.dev b/conf/config.dev index 9467615e..e97f6f12 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -6,7 +6,8 @@ ; development-specific options too. [database] -; Options: mysql, sqlite. +; PHP options: mysql, sqlite. +; FastAPI options: mysql. backend = mysql ; If using sqlite, set name to the database file path. @@ -14,8 +15,8 @@ name = aurweb ; MySQL database information. User defaults to root for containerized ; testing with mysqldb. This should be set to a non-root user. -user = aur -password = aur +user = root +;password = aur host = localhost ;port = 3306 socket = /var/run/mysqld/mysqld.sock diff --git a/docker-compose.yml b/docker-compose.yml index bda4ddfb..c39d38bf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,6 +77,24 @@ services: mariadb: condition: service_healthy + mariadb_test: + # Test database. + image: aurweb:latest + init: true + environment: + - MARIADB_PRIVILEGED=1 + entrypoint: /docker/mariadb-entrypoint.sh + command: /usr/bin/mysqld_safe --datadir=/var/lib/mysql + ports: + # This will expose mariadbd on 127.0.0.1:13307 in the host. + # Ex: `mysql -uaur -paur -h 127.0.0.1 -P 13306 aurweb` + - "13307:3306" + volumes: + - mariadb_test_run:/var/run/mysqld # Bind socket in this volume. + healthcheck: + test: "bash /docker/health/mariadb.sh" + interval: 3s + git: image: aurweb:latest init: true @@ -254,10 +272,9 @@ services: stdin_open: true tty: true depends_on: - git: + mariadb_test: condition: service_healthy volumes: - - git_data:/aurweb/aur.git - ./cache:/cache - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations @@ -280,34 +297,12 @@ services: stdin_open: true tty: true depends_on: - mariadb_init: - condition: service_started + mariadb_test: + condition: service_healthy + tmpfs: + - /tmp volumes: - - mariadb_run:/var/run/mysqld - - git_data:/aurweb/aur.git - - ./cache:/cache - - ./aurweb:/aurweb/aurweb - - ./migrations:/aurweb/migrations - - ./test:/aurweb/test - - ./web/html:/aurweb/web/html - - ./web/template:/aurweb/web/template - - ./web/lib:/aurweb/web/lib - - ./templates:/aurweb/templates - - pytest-sqlite: - image: aurweb:latest - profiles: ["dev"] - init: true - environment: - - AUR_CONFIG=conf/config.sqlite - - TEST_RECURSION_LIMIT=${TEST_RECURSION_LIMIT} - - PROMETHEUS_MULTIPROC_DIR=/tmp_prometheus - entrypoint: /docker/test-sqlite-entrypoint.sh - command: setup-sqlite.sh run-pytests.sh clean - stdin_open: true - tty: true - volumes: - - git_data:/aurweb/aur.git + - mariadb_test_run:/var/run/mysqld - ./cache:/cache - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations @@ -325,16 +320,15 @@ services: - AUR_CONFIG=conf/config - TEST_RECURSION_LIMIT=${TEST_RECURSION_LIMIT} - PROMETHEUS_MULTIPROC_DIR=/tmp_prometheus - entrypoint: /docker/tests-entrypoint.sh - command: setup-sqlite.sh run-tests.sh + entrypoint: /docker/test-mysql-entrypoint.sh + command: /docker/scripts/run-tests.sh stdin_open: true tty: true depends_on: - mariadb_init: - condition: service_started + mariadb_test: + condition: service_healthy volumes: - - mariadb_run:/var/run/mysqld - - git_data:/aurweb/aur.git + - mariadb_test_run:/var/run/mysqld - ./cache:/cache - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations @@ -345,6 +339,7 @@ services: - ./templates:/aurweb/templates volumes: + mariadb_test_run: {} mariadb_run: {} # Share /var/run/mysqld/mysqld.sock mariadb_data: {} # Share /var/lib/mysql git_data: {} # Share aurweb/aur.git diff --git a/docker/fastapi-entrypoint.sh b/docker/fastapi-entrypoint.sh index f4ceaafa..9df6382d 100755 --- a/docker/fastapi-entrypoint.sh +++ b/docker/fastapi-entrypoint.sh @@ -5,6 +5,10 @@ set -eou pipefail cp -vf conf/config.dev conf/config sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config +# Change database user/password. +sed -ri "s/^;?(user) = .*$/\1 = aur/" conf/config +sed -ri "s/^;?(password) = .*$/\1 = aur/" conf/config + sed -ri "s;^(aur_location) = .+;\1 = ${AURWEB_FASTAPI_PREFIX};" conf/config # Setup Redis for FastAPI. diff --git a/docker/mariadb-entrypoint.sh b/docker/mariadb-entrypoint.sh index e1ebfa6a..a00f6106 100755 --- a/docker/mariadb-entrypoint.sh +++ b/docker/mariadb-entrypoint.sh @@ -13,23 +13,18 @@ done # Configure databases. DATABASE="aurweb" # Persistent database for fastapi/php-fpm. -TEST_DB="aurweb_test" # Test database (ephemereal). echo "Taking care of primary database '${DATABASE}'..." mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'localhost' IDENTIFIED BY 'aur';" mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'%' IDENTIFIED BY 'aur';" mysql -u root -e "CREATE DATABASE IF NOT EXISTS $DATABASE;" -mysql -u root -e "GRANT ALL ON ${DATABASE}.* TO 'aur'@'localhost';" -mysql -u root -e "GRANT ALL ON ${DATABASE}.* TO 'aur'@'%';" -# Drop and create our test database. -echo "Dropping test database '$TEST_DB'..." -mysql -u root -e "DROP DATABASE IF EXISTS $TEST_DB;" -mysql -u root -e "CREATE DATABASE $TEST_DB;" -mysql -u root -e "GRANT ALL ON ${TEST_DB}.* TO 'aur'@'localhost';" -mysql -u root -e "GRANT ALL ON ${TEST_DB}.* TO 'aur'@'%';" +mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'%' IDENTIFIED BY 'aur';" +mysql -u root -e "GRANT ALL ON aurweb.* TO 'aur'@'localhost';" +mysql -u root -e "GRANT ALL ON aurweb.* TO 'aur'@'%';" -echo "Created new '$TEST_DB'!" +mysql -u root -e "CREATE USER IF NOT EXISTS 'root'@'%' IDENTIFIED BY 'aur';" +mysql -u root -e "GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION;" mysqladmin -uroot shutdown diff --git a/docker/mariadb-init-entrypoint.sh b/docker/mariadb-init-entrypoint.sh index 413227b9..6df98e4f 100755 --- a/docker/mariadb-init-entrypoint.sh +++ b/docker/mariadb-init-entrypoint.sh @@ -4,6 +4,8 @@ set -eou pipefail # Setup a config for our mysql db. cp -vf conf/config.dev conf/config sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config +sed -ri "s/^;?(user) = .*$/\1 = aur/g" conf/config +sed -ri "s/^;?(password) = .*$/\1 = aur/g" conf/config python -m aurweb.initdb 2>/dev/null || /bin/true diff --git a/docker/php-entrypoint.sh b/docker/php-entrypoint.sh index 274f8e17..05b76408 100755 --- a/docker/php-entrypoint.sh +++ b/docker/php-entrypoint.sh @@ -9,14 +9,19 @@ done cp -vf conf/config.dev conf/config sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config -sed -ri "s;^(aur_location) = .+;\1 = ${AURWEB_PHP_PREFIX};" conf/config +# Change database user/password. +sed -ri "s/^;?(user) = .*$/\1 = aur/" conf/config +sed -ri "s/^;?(password) = .*$/\1 = aur/" conf/config # Enable memcached. sed -ri 's/^(cache) = .+$/\1 = memcache/' conf/config +# Setup various location configurations. +sed -ri "s;^(aur_location) = .+;\1 = ${AURWEB_PHP_PREFIX};" conf/config sed -ri "s|^(git_clone_uri_anon) = .+|\1 = ${AURWEB_PHP_PREFIX}/%s.git|" conf/config.defaults sed -ri "s|^(git_clone_uri_priv) = .+|\1 = ${AURWEB_SSHD_PREFIX}/%s.git|" conf/config.defaults +# Listen on :9000. sed -ri 's/^(listen).*/\1 = 0.0.0.0:9000/' /etc/php/php-fpm.d/www.conf sed -ri 's/^;?(clear_env).*/\1 = no/' /etc/php/php-fpm.d/www.conf diff --git a/docker/scripts/run-php.sh b/docker/scripts/run-php.sh index 22346b47..b86f8ce5 100755 --- a/docker/scripts/run-php.sh +++ b/docker/scripts/run-php.sh @@ -1,7 +1,4 @@ #!/bin/bash set -eou pipefail -# Initialize the new database; ignore errors. -python -m aurweb.initdb 2>/dev/null || /bin/true - exec php-fpm --fpm-config /etc/php/php-fpm.conf --nodaemonize diff --git a/docker/scripts/run-pytests.sh b/docker/scripts/run-pytests.sh index ee546fb7..b8f695df 100755 --- a/docker/scripts/run-pytests.sh +++ b/docker/scripts/run-pytests.sh @@ -25,17 +25,8 @@ done rm -rf $PROMETHEUS_MULTIPROC_DIR mkdir -p $PROMETHEUS_MULTIPROC_DIR -# Initialize the new database; ignore errors. -python -m aurweb.initdb 2>/dev/null || \ - (echo "Error: aurweb.initdb failed; already initialized?" && /bin/true) - -# Run test_initdb ahead of time, which clears out the database, -# in case of previous failures which stopped the test suite before -# finishing the ends of some test fixtures. -eatmydata -- pytest test/test_initdb.py - # Run pytest with optional targets in front of it. -eatmydata -- make -C test "${PARAMS[@]}" pytest +pytest # By default, report coverage and move it into cache. if [ $COVERAGE -eq 1 ]; then diff --git a/docker/scripts/run-tests.sh b/docker/scripts/run-tests.sh index 3181a623..45c7835f 100755 --- a/docker/scripts/run-tests.sh +++ b/docker/scripts/run-tests.sh @@ -12,12 +12,6 @@ bash $dir/run-sharness.sh # Pass --silence to avoid reporting coverage. We will do that below. bash $dir/run-pytests.sh --no-coverage -# Export SQLite aurweb configuration. -export AUR_CONFIG=conf/config.sqlite - -# Run Python tests. -bash $dir/run-pytests.sh --no-coverage - make -C test coverage # /cache is mounted as a volume. Copy coverage into it. diff --git a/docker/test-mysql-entrypoint.sh b/docker/test-mysql-entrypoint.sh index 7be3626b..a46b2572 100755 --- a/docker/test-mysql-entrypoint.sh +++ b/docker/test-mysql-entrypoint.sh @@ -1,17 +1,8 @@ #!/bin/bash set -eou pipefail -DB_NAME="aurweb_test" - # Setup a config for our mysql db. cp -vf conf/config.dev conf/config sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config -sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" conf/config -# The port can be excluded from use if properly using -# volumes to share the mysql socket from the mariadb service. -# Example port sed: -# sed -i "s/^;?(port = .+)$/\1/" conf/config - -# Continue onto the main command. exec "$@" diff --git a/docker/tests-entrypoint.sh b/docker/tests-entrypoint.sh index ca3d3d9a..145bee6e 100755 --- a/docker/tests-entrypoint.sh +++ b/docker/tests-entrypoint.sh @@ -3,6 +3,5 @@ set -eou pipefail dir="$(dirname $0)" bash $dir/test-mysql-entrypoint.sh -bash $dir/test-sqlite-entrypoint.sh exec "$@" From a025118344ff857f5eab7a869b56b952cb46e6d8 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 00:47:33 -0800 Subject: [PATCH 580/844] change(docker): get python-poetry from arch instead of poetry Signed-off-by: Kevin Morris --- docker/scripts/install-deps.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh index d64340e3..ad0157f8 100755 --- a/docker/scripts/install-deps.sh +++ b/docker/scripts/install-deps.sh @@ -8,9 +8,7 @@ pacman -Syu --noconfirm --noprogressbar \ --cachedir .pkg-cache git gpgme nginx redis openssh \ mariadb mariadb-libs cgit-aurweb uwsgi uwsgi-plugin-cgi \ php php-fpm memcached php-memcached python-pip pyalpm \ - python-srcinfo curl libeatmydata cronie - -# https://python-poetry.org/docs/ Installation section. -curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - + python-srcinfo curl libeatmydata cronie python-poetry \ + python-poetry-core exec "$@" From 60f63876c42b5900cfa7af3442d4feaced3c88c4 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 00:48:31 -0800 Subject: [PATCH 581/844] change(.gitignore): ignore archives Signed-off-by: Kevin Morris --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e3201e94..8388694c 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ schema/aur-schema-sqlite.sql test/test-results/ test/trash directory* web/locale/*/ +web/html/*.gz # Do not stage compiled asciidoc: make -C doc doc/rpc.html From fb92fb509b4a5769154096969cbfb90f8f69e798 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 00:49:00 -0800 Subject: [PATCH 582/844] change(fastapi): use sys.getrecursionlimit() + 1000 as default Without the increment, we've seen tests failed due to recursion errors caused by starlette's base middleware. Just make it safe in case nobody supplies TEST_RECURSION_LIMIT. Signed-off-by: Kevin Morris --- aurweb/asgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aurweb/asgi.py b/aurweb/asgi.py index aafb00b2..b399cfb1 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -45,7 +45,7 @@ async def app_startup(): # when running test suites. # TODO: Find a proper fix to this issue. recursion_limit = int(os.environ.get( - "TEST_RECURSION_LIMIT", sys.getrecursionlimit())) + "TEST_RECURSION_LIMIT", sys.getrecursionlimit() + 1000)) sys.setrecursionlimit(recursion_limit) backend = aurweb.config.get("database", "backend") From a5c0c47e5b82334b6416b13ffcc0f2948377b582 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 00:59:10 -0800 Subject: [PATCH 583/844] change(.gitlab-ci): adapt for new conftest No longer do we need to create any database in .gitlab-ci. Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1590bf34..739c9408 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,14 +22,11 @@ test: - (cd '/usr' && /usr/bin/mysqld_safe --datadir='/var/lib/mysql') & - 'until : > /dev/tcp/127.0.0.1/3306; do sleep 1s; done' - ./docker/test-mysql-entrypoint.sh # Create mysql AUR_CONFIG. - - ./docker/test-sqlite-entrypoint.sh # Create sqlite AUR_CONFIG. - make -C po all install - - python -m aurweb.initdb # Initialize MySQL tables. - - AUR_CONFIG=conf/config.sqlite python -m aurweb.initdb - make -C test clean script: - - make -C test sh pytest # sharness tests use sqlite & pytest w/ mysql. - - AUR_CONFIG=conf/config.sqlite make -C test pytest + - make -C test sh # sharness tests use sqlite. + - pytest # Run pytest suites. - make -C test coverage # Produce coverage reports. - flake8 --count aurweb # Assert no flake8 violations in aurweb. - flake8 --count test # Assert no flake8 violations in test. @@ -65,8 +62,11 @@ deploy: # Set secure login config for aurweb. - sed -ri "s/^(disable_http_login).*$/\1 = 1/" conf/config.dev - docker-compose build - - docker system prune -f - docker-compose -f docker-compose.yml -f docker-compose.aur-dev.yml up -d + - docker image prune -f + - docker container prune -f + - docker volume prune -f + environment: name: development url: https://aur-dev.archlinux.org From 912b7e0c118c5717d96fc2770770be8b91b2f81e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 02:19:43 -0800 Subject: [PATCH 584/844] fix(docker): fix database user/password for git-entrypoint Signed-off-by: Kevin Morris --- docker/git-entrypoint.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/git-entrypoint.sh b/docker/git-entrypoint.sh index 3fee426a..296c1e47 100755 --- a/docker/git-entrypoint.sh +++ b/docker/git-entrypoint.sh @@ -42,6 +42,9 @@ EOF cp -vf conf/config.dev $AUR_CONFIG sed -i "s;YOUR_AUR_ROOT;$(pwd);g" $AUR_CONFIG +sed -ri "s/^;?(user) = .*$/\1 = aur/" $AUR_CONFIG +sed -ri "s/^;?(password) = .*$/\1 = aur/" $AUR_CONFIG + AUR_CONFIG_DEFAULTS="${AUR_CONFIG}.defaults" if [[ "$AUR_CONFIG_DEFAULTS" != "/aurweb/conf/config.defaults" ]]; then From abe8c0630c52bc5694f9284347045ae353c67507 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 3 Nov 2021 19:34:20 -0700 Subject: [PATCH 585/844] fix(rpc): improve type=info performance Now, we use an equivalent query to PHP's query, yet we grab every piece of data we need for all packages asked for in one database query. At this time, local benchmarks have shown a slight performance improvement when compared to PHP. fastapi 262 requests/sec php 250 requests/sec Extras: - Moved RPCError to the aurweb.exceptions module Signed-off-by: Kevin Morris --- aurweb/exceptions.py | 4 + aurweb/rpc.py | 220 +++++++++++++++++++++++++++---------------- 2 files changed, 141 insertions(+), 83 deletions(-) diff --git a/aurweb/exceptions.py b/aurweb/exceptions.py index 62015284..82628b0a 100644 --- a/aurweb/exceptions.py +++ b/aurweb/exceptions.py @@ -73,3 +73,7 @@ class NotVotedException(AurwebException): class InvalidArgumentsException(AurwebException): def __init__(self, msg): super(InvalidArgumentsException, self).__init__(msg) + + +class RPCError(AurwebException): + pass diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 03662790..c70ddf1a 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -1,38 +1,28 @@ from collections import defaultdict -from typing import Any, Callable, Dict, List, NewType +from typing import Any, Callable, Dict, List, NewType, Union -from sqlalchemy import and_ +from sqlalchemy import and_, literal import aurweb.config as config from aurweb import db, defaults, models, util -from aurweb.models import dependency_type, relation_type +from aurweb.exceptions import RPCError from aurweb.packages.search import RPCSearch -# Define dependency type mappings from ID to RPC-compatible keys. -DEP_TYPES = { - dependency_type.DEPENDS_ID: "Depends", - dependency_type.MAKEDEPENDS_ID: "MakeDepends", - dependency_type.CHECKDEPENDS_ID: "CheckDepends", - dependency_type.OPTDEPENDS_ID: "OptDepends" +TYPE_MAPPING = { + "depends": "Depends", + "makedepends": "MakeDepends", + "checkdepends": "CheckDepends", + "optdepends": "OptDepends", + "conflicts": "Conflicts", + "provides": "Provides", + "replaces": "Replaces", } -# Define relationship type mappings from ID to RPC-compatible keys. -REL_TYPES = { - relation_type.CONFLICTS_ID: "Conflicts", - relation_type.PROVIDES_ID: "Provides", - relation_type.REPLACES_ID: "Replaces" -} - - DataGenerator = NewType("DataGenerator", Callable[[models.Package], Dict[str, Any]]) -class RPCError(Exception): - pass - - class RPC: """ RPC API handler class. @@ -76,11 +66,11 @@ class RPC: # A mapping of by aliases. BY_ALIASES = {"name-desc": "nd", "name": "n", "maintainer": "m"} - def __init__(self, version: int = 0, type: str = None): + def __init__(self, version: int = 0, type: str = None) -> "RPC": self.version = version - self.type = type + self.type = RPC.TYPE_ALIASES.get(type, type) - def error(self, message: str) -> dict: + def error(self, message: str) -> Dict[str, Any]: return { "version": self.version, "results": [], @@ -89,7 +79,7 @@ class RPC: "error": message } - def _verify_inputs(self, by: str = [], args: List[str] = []): + def _verify_inputs(self, by: str = [], args: List[str] = []) -> None: if self.version is None: raise RPCError("Please specify an API version.") @@ -105,39 +95,11 @@ class RPC: if self.type not in RPC.EXPOSED_TYPES: raise RPCError("Incorrect request type specified.") - def _enforce_args(self, args: List[str]): + def _enforce_args(self, args: List[str]) -> None: if not args: raise RPCError("No request type/data specified.") - def _update_json_depends(self, package: models.Package, - data: Dict[str, Any]): - # Walk through all related PackageDependencies and produce - # the appropriate dict entries. - for dep in package.package_dependencies: - if dep.DepTypeID in DEP_TYPES: - key = DEP_TYPES.get(dep.DepTypeID) - - display = dep.DepName - if dep.DepCondition: - display += dep.DepCondition - - data[key].append(display) - - def _update_json_relations(self, package: models.Package, - data: Dict[str, Any]): - # Walk through all related PackageRelations and produce - # the appropriate dict entries. - for rel in package.package_relations: - if rel.RelTypeID in REL_TYPES: - key = REL_TYPES.get(rel.RelTypeID) - - display = rel.RelName - if rel.RelCondition: - display += rel.RelCondition - - data[key].append(display) - - def _get_json_data(self, package: models.Package): + def _get_json_data(self, package: models.Package) -> Dict[str, Any]: """ Produce dictionary data of one Package that can be JSON-serialized. :param package: Package instance @@ -175,21 +137,21 @@ class RPC: return data - def _get_info_json_data(self, package: models.Package): + def _get_info_json_data(self, package: models.Package) -> Dict[str, Any]: data = self._get_json_data(package) - # Add licenses and keywords to info output. + # All info results have _at least_ an empty list of + # License and Keywords. data.update({ - "License": [ - lic.License.Name for lic in package.package_licenses - ], - "Keywords": [ - keyword.Keyword for keyword in package.PackageBase.keywords - ] + "License": [], + "Keywords": [] }) - self._update_json_depends(package, data) - self._update_json_relations(package, data) + # If we actually got extra_info records, update data with + # them for this particular package. + if self.extra_info: + data.update(self.extra_info.get(package.ID, {})) + return data def _assemble_json_data(self, packages: List[models.Package], @@ -211,13 +173,97 @@ class RPC: -> List[Dict[str, Any]]: self._enforce_args(args) args = set(args) - packages = db.query(models.Package).filter( + + packages = db.query(models.Package).join(models.PackageBase).filter( models.Package.Name.in_(args)) + ids = {pkg.ID for pkg in packages} + + # Aliases for 80-width. + Package = models.Package + PackageKeyword = models.PackageKeyword + + subqueries = [ + # PackageDependency + db.query( + models.PackageDependency + ).join(models.DependencyType).filter( + models.PackageDependency.PackageID.in_(ids) + ).with_entities( + models.PackageDependency.PackageID.label("ID"), + models.DependencyType.Name.label("Type"), + models.PackageDependency.DepName.label("Name"), + models.PackageDependency.DepCondition.label("Cond") + ).distinct().order_by("ID"), + + # PackageRelation + db.query( + models.PackageRelation + ).join(models.RelationType).filter( + models.PackageRelation.PackageID.in_(ids) + ).with_entities( + models.PackageRelation.PackageID.label("ID"), + models.RelationType.Name.label("Type"), + models.PackageRelation.RelName.label("Name"), + models.PackageRelation.RelCondition.label("Cond") + ).distinct().order_by("ID"), + + # Groups + db.query(models.PackageGroup).join( + models.Group, + and_(models.PackageGroup.GroupID == models.Group.ID, + models.PackageGroup.PackageID.in_(ids)) + ).with_entities( + models.PackageGroup.PackageID.label("ID"), + literal("Groups").label("Type"), + models.Group.Name.label("Name"), + literal(str()).label("Cond") + ).distinct().order_by("ID"), + + # Licenses + db.query(models.PackageLicense).join( + models.License, + models.PackageLicense.LicenseID == models.License.ID + ).filter( + models.PackageLicense.PackageID.in_(ids) + ).with_entities( + models.PackageLicense.PackageID.label("ID"), + literal("License").label("Type"), + models.License.Name.label("Name"), + literal(str()).label("Cond") + ).distinct().order_by("ID"), + + # Keywords + db.query(models.PackageKeyword).join( + models.Package, + and_(Package.PackageBaseID == PackageKeyword.PackageBaseID, + Package.ID.in_(ids)) + ).with_entities( + models.Package.ID.label("ID"), + literal("Keywords").label("Type"), + models.PackageKeyword.Keyword.label("Name"), + literal(str()).label("Cond") + ).distinct().order_by("ID") + ] + + # Union all subqueries together. + query = subqueries[0].union_all(*subqueries[1:]) + + # Store our extra information in a class-wise dictionary, + # which contains package id -> extra info dict mappings. + self.extra_info = defaultdict(lambda: defaultdict(list)) + for record in query: + type_ = TYPE_MAPPING.get(record.Type, record.Type) + + name = record.Name + if record.Cond: + name += record.Cond + + self.extra_info[record.ID][type_].append(name) + return self._assemble_json_data(packages, self._get_info_json_data) def _handle_search_type(self, by: str = defaults.RPC_SEARCH_BY, - args: List[str] = []) \ - -> List[Dict[str, Any]]: + args: List[str] = []) -> List[Dict[str, Any]]: # If `by` isn't maintainer and we don't have any args, raise an error. # In maintainer's case, return all orphans if there are no args, # so we need args to pass through to the handler without errors. @@ -235,10 +281,12 @@ class RPC: results = search.results().limit(max_results) return self._assemble_json_data(results, self._get_json_data) - def _handle_msearch_type(self, args: List[str] = [], **kwargs): + def _handle_msearch_type(self, args: List[str] = [], **kwargs)\ + -> List[Dict[str, Any]]: return self._handle_search_type(by="m", args=args) - def _handle_suggest_type(self, args: List[str] = [], **kwargs): + def _handle_suggest_type(self, args: List[str] = [], **kwargs)\ + -> List[str]: if not args: return [] @@ -251,7 +299,8 @@ class RPC: ).order_by(models.Package.Name.asc()).limit(20) return [pkg.Name for pkg in packages] - def _handle_suggest_pkgbase_type(self, args: List[str] = [], **kwargs): + def _handle_suggest_pkgbase_type(self, args: List[str] = [], **kwargs)\ + -> List[str]: if not args: return [] @@ -261,7 +310,19 @@ class RPC: ).order_by(models.PackageBase.Name.asc()).limit(20) return [pkg.Name for pkg in packages] - def handle(self, by: str = defaults.RPC_SEARCH_BY, args: List[str] = []): + def _is_suggestion(self) -> bool: + return self.type.startswith("suggest") + + def _handle_callback(self, by: str, args: List[str])\ + -> Union[List[Dict[str, Any]], List[str]]: + # Get a handle to our callback and trap an RPCError with + # an empty list of results based on callback's execution. + callback = getattr(self, f"_handle_{self.type.replace('-', '_')}_type") + results = callback(by=by, args=args) + return results + + def handle(self, by: str = defaults.RPC_SEARCH_BY, args: List[str] = [])\ + -> Union[List[Dict[str, Any]], Dict[str, Any]]: """ Request entrypoint. A router should pass v, type and args to this function and expect an output dictionary to be returned. @@ -269,10 +330,6 @@ class RPC: :param type: RPC type argument :param args: Deciphered list of arguments based on arg/arg[] inputs """ - # Convert type aliased types. - if self.type in RPC.TYPE_ALIASES: - self.type = RPC.TYPE_ALIASES.get(self.type) - # Prepare our output data dictionary with some basic keys. data = {"version": self.version, "type": self.type} @@ -283,20 +340,17 @@ class RPC: return self.error(str(exc)) # Convert by to its aliased value if it has one. - if by in RPC.BY_ALIASES: - by = RPC.BY_ALIASES.get(by) + by = RPC.BY_ALIASES.get(by, by) - # Get a handle to our callback and trap an RPCError with - # an empty list of results based on callback's execution. - callback = getattr(self, f"_handle_{self.type.replace('-', '_')}_type") + # Process the requested handler. try: - results = callback(by=by, args=args) + results = self._handle_callback(by, args) except RPCError as exc: return self.error(str(exc)) # These types are special: we produce a different kind of # successful JSON output: a list of results. - if self.type in ("suggest", "suggest-pkgbase"): + if self._is_suggestion(): return results # Return JSON output. From ccf50cbdf5135978dbb86e2c37e04b72911d4732 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 3 Nov 2021 19:36:08 -0700 Subject: [PATCH 586/844] change: rework test_rpc's TestClient usage into a fixture This is the first step on our path to reworking the test suite in general. Signed-off-by: Kevin Morris --- test/test_rpc.py | 186 ++++++++++++++++++++++++++++------------------- 1 file changed, 113 insertions(+), 73 deletions(-) diff --git a/test/test_rpc.py b/test/test_rpc.py index f20c9b02..a4cdb5da 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -1,7 +1,6 @@ import re from http import HTTPStatus -from typing import Dict from unittest import mock import orjson @@ -10,8 +9,7 @@ import pytest from fastapi.testclient import TestClient from redis.client import Pipeline -from aurweb import config, db, scripts -from aurweb.asgi import app +from aurweb import asgi, config, db, scripts from aurweb.db import begin, create, query from aurweb.models.account_type import AccountType from aurweb.models.dependency_type import DependencyType @@ -28,9 +26,9 @@ from aurweb.models.user import User from aurweb.redis import redis_connection -def make_request(path, headers: Dict[str, str] = {}): - with TestClient(app) as request: - return request.get(path, headers=headers) +@pytest.fixture +def client() -> TestClient: + yield TestClient(app=asgi.app) @pytest.fixture(autouse=True) @@ -205,7 +203,7 @@ def pipeline(): yield pipeline -def test_rpc_singular_info(): +def test_rpc_singular_info(client: TestClient): # Define expected response. expected_data = { "version": 5, @@ -239,7 +237,9 @@ def test_rpc_singular_info(): } # Make dummy request. - response_arg = make_request("/rpc/?v=5&type=info&arg=chungy-chungus&arg=big-chungus") + with client as request: + response_arg = request.get( + "/rpc/?v=5&type=info&arg=chungy-chungus&arg=big-chungus") # Load request response into Python dictionary. response_info_arg = orjson.loads(response_arg.content.decode()) @@ -254,9 +254,10 @@ def test_rpc_singular_info(): assert response_info_arg == expected_data -def test_rpc_nonexistent_package(): +def test_rpc_nonexistent_package(client: TestClient): # Make dummy request. - response = make_request("/rpc/?v=5&type=info&arg=nonexistent-package") + with client as request: + response = request.get("/rpc/?v=5&type=info&arg=nonexistent-package") # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -265,10 +266,12 @@ def test_rpc_nonexistent_package(): assert response_data["resultcount"] == 0 -def test_rpc_multiinfo(): +def test_rpc_multiinfo(client: TestClient): # Make dummy request. request_packages = ["big-chungus", "chungy-chungus"] - response = make_request("/rpc/?v=5&type=info&arg[]=big-chungus&arg[]=chungy-chungus") + with client as request: + response = request.get( + "/rpc/?v=5&type=info&arg[]=big-chungus&arg[]=chungy-chungus") # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -280,13 +283,20 @@ def test_rpc_multiinfo(): assert request_packages == [] -def test_rpc_mixedargs(): +def test_rpc_mixedargs(client: TestClient): # Make dummy request. response1_packages = ["gluggly-chungus"] response2_packages = ["gluggly-chungus", "chungy-chungus"] - response1 = make_request("/rpc/?v=5&arg[]=big-chungus&arg=gluggly-chungus&type=info") - response2 = make_request("/rpc/?v=5&arg=big-chungus&arg[]=gluggly-chungus&type=info&arg[]=chungy-chungus") + with client as request: + response1 = request.get( + "/rpc?v=5&arg[]=big-chungus&arg=gluggly-chungus&type=info") + assert response1.status_code == int(HTTPStatus.OK) + + with client as request: + response2 = request.get( + "/rpc?v=5&arg=big-chungus&arg[]=gluggly-chungus&type=info&arg[]=chungy-chungus") + assert response1.status_code == int(HTTPStatus.OK) # Load request response into Python dictionary. response1_data = orjson.loads(response1.content.decode()) @@ -303,7 +313,7 @@ def test_rpc_mixedargs(): assert i == [] -def test_rpc_no_dependencies(): +def test_rpc_no_dependencies(client: TestClient): """This makes sure things like 'MakeDepends' get removed from JSON strings when they don't have set values.""" @@ -330,7 +340,8 @@ def test_rpc_no_dependencies(): } # Make dummy request. - response = make_request("/rpc/?v=5&type=info&arg=chungy-chungus") + with client as request: + response = request.get("/rpc/?v=5&type=info&arg=chungy-chungus") response_data = orjson.loads(response.content.decode()) # Remove inconsistent keys. @@ -340,7 +351,7 @@ def test_rpc_no_dependencies(): assert response_data == expected_response -def test_rpc_bad_type(): +def test_rpc_bad_type(client: TestClient): # Define expected response. expected_data = { 'version': 5, @@ -351,7 +362,8 @@ def test_rpc_bad_type(): } # Make dummy request. - response = make_request("/rpc/?v=5&type=invalid-type&arg=big-chungus") + with client as request: + response = request.get("/rpc/?v=5&type=invalid-type&arg=big-chungus") # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -360,7 +372,7 @@ def test_rpc_bad_type(): assert expected_data == response_data -def test_rpc_bad_version(): +def test_rpc_bad_version(client: TestClient): # Define expected response. expected_data = { 'version': 0, @@ -371,7 +383,8 @@ def test_rpc_bad_version(): } # Make dummy request. - response = make_request("/rpc/?v=0&type=info&arg=big-chungus") + with client as request: + response = request.get("/rpc/?v=0&type=info&arg=big-chungus") # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -380,7 +393,7 @@ def test_rpc_bad_version(): assert expected_data == response_data -def test_rpc_no_version(): +def test_rpc_no_version(client: TestClient): # Define expected response. expected_data = { 'version': None, @@ -391,7 +404,8 @@ def test_rpc_no_version(): } # Make dummy request. - response = make_request("/rpc/?type=info&arg=big-chungus") + with client as request: + response = request.get("/rpc/?type=info&arg=big-chungus") # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -400,7 +414,7 @@ def test_rpc_no_version(): assert expected_data == response_data -def test_rpc_no_type(): +def test_rpc_no_type(client: TestClient): # Define expected response. expected_data = { 'version': 5, @@ -411,7 +425,8 @@ def test_rpc_no_type(): } # Make dummy request. - response = make_request("/rpc/?v=5&arg=big-chungus") + with client as request: + response = request.get("/rpc/?v=5&arg=big-chungus") # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -420,7 +435,7 @@ def test_rpc_no_type(): assert expected_data == response_data -def test_rpc_no_args(): +def test_rpc_no_args(client: TestClient): # Define expected response. expected_data = { 'version': 5, @@ -431,7 +446,8 @@ def test_rpc_no_args(): } # Make dummy request. - response = make_request("/rpc/?v=5&type=info") + with client as request: + response = request.get("/rpc/?v=5&type=info") # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -440,9 +456,10 @@ def test_rpc_no_args(): assert expected_data == response_data -def test_rpc_no_maintainer(): +def test_rpc_no_maintainer(client: TestClient): # Make dummy request. - response = make_request("/rpc/?v=5&type=info&arg=woogly-chungus") + with client as request: + response = request.get("/rpc/?v=5&type=info&arg=woogly-chungus") # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -451,33 +468,39 @@ def test_rpc_no_maintainer(): assert response_data["results"][0]["Maintainer"] is None -def test_rpc_suggest_pkgbase(): - response = make_request("/rpc?v=5&type=suggest-pkgbase&arg=big") +def test_rpc_suggest_pkgbase(client: TestClient): + with client as request: + response = request.get("/rpc?v=5&type=suggest-pkgbase&arg=big") data = response.json() assert data == ["big-chungus"] - response = make_request("/rpc?v=5&type=suggest-pkgbase&arg=chungy") + with client as request: + response = request.get("/rpc?v=5&type=suggest-pkgbase&arg=chungy") data = response.json() assert data == ["chungy-chungus"] # Test no arg supplied. - response = make_request("/rpc?v=5&type=suggest-pkgbase") + with client as request: + response = request.get("/rpc?v=5&type=suggest-pkgbase") data = response.json() assert data == [] -def test_rpc_suggest(): - response = make_request("/rpc?v=5&type=suggest&arg=other") +def test_rpc_suggest(client: TestClient): + with client as request: + response = request.get("/rpc?v=5&type=suggest&arg=other") data = response.json() assert data == ["other-pkg"] # Test non-existent Package. - response = make_request("/rpc?v=5&type=suggest&arg=nonexistent") + with client as request: + response = request.get("/rpc?v=5&type=suggest&arg=nonexistent") data = response.json() assert data == [] # Test no arg supplied. - response = make_request("/rpc?v=5&type=suggest") + with client as request: + response = request.get("/rpc?v=5&type=suggest") data = response.json() assert data == [] @@ -491,14 +514,17 @@ def mock_config_getint(section: str, key: str): @mock.patch("aurweb.config.getint", side_effect=mock_config_getint) -def test_rpc_ratelimit(getint: mock.MagicMock, pipeline: Pipeline): +def test_rpc_ratelimit(getint: mock.MagicMock, client: TestClient, + pipeline: Pipeline): for i in range(4): # The first 4 requests should be good. - response = make_request("/rpc?v=5&type=suggest-pkgbase&arg=big") + with client as request: + response = request.get("/rpc?v=5&type=suggest-pkgbase&arg=big") assert response.status_code == int(HTTPStatus.OK) # The fifth request should be banned. - response = make_request("/rpc?v=5&type=suggest-pkgbase&arg=big") + with client as request: + response = request.get("/rpc?v=5&type=suggest-pkgbase&arg=big") assert response.status_code == int(HTTPStatus.TOO_MANY_REQUESTS) # Delete the cached records. @@ -508,26 +534,32 @@ def test_rpc_ratelimit(getint: mock.MagicMock, pipeline: Pipeline): assert one and two # The new first request should be good. - response = make_request("/rpc?v=5&type=suggest-pkgbase&arg=big") + with client as request: + response = request.get("/rpc?v=5&type=suggest-pkgbase&arg=big") assert response.status_code == int(HTTPStatus.OK) -def test_rpc_etag(): - response1 = make_request("/rpc?v=5&type=suggest-pkgbase&arg=big") - response2 = make_request("/rpc?v=5&type=suggest-pkgbase&arg=big") +def test_rpc_etag(client: TestClient): + with client as request: + response1 = request.get("/rpc?v=5&type=suggest-pkgbase&arg=big") + + with client as request: + response2 = request.get("/rpc?v=5&type=suggest-pkgbase&arg=big") assert response1.headers.get("ETag") is not None assert response1.headers.get("ETag") != str() assert response1.headers.get("ETag") == response2.headers.get("ETag") -def test_rpc_search_arg_too_small(): - response = make_request("/rpc?v=5&type=search&arg=b") +def test_rpc_search_arg_too_small(client: TestClient): + with client as request: + response = request.get("/rpc?v=5&type=search&arg=b") assert response.status_code == int(HTTPStatus.OK) assert response.json().get("error") == "Query arg too small." -def test_rpc_search(): - response = make_request("/rpc?v=5&type=search&arg=big") +def test_rpc_search(client: TestClient): + with client as request: + response = request.get("/rpc?v=5&type=search&arg=big") assert response.status_code == int(HTTPStatus.OK) data = response.json() @@ -539,17 +571,18 @@ def test_rpc_search(): # Test the If-None-Match headers. etag = response.headers.get("ETag").strip('"') headers = {"If-None-Match": etag} - response = make_request("/rpc?v=5&type=search&arg=big", headers=headers) + response = request.get("/rpc?v=5&type=search&arg=big", headers=headers) assert response.status_code == int(HTTPStatus.NOT_MODIFIED) assert response.content == b'' # No args on non-m by types return an error. - response = make_request("/rpc?v=5&type=search") + response = request.get("/rpc?v=5&type=search") assert response.json().get("error") == "No request type/data specified." -def test_rpc_msearch(): - response = make_request("/rpc?v=5&type=msearch&arg=user1") +def test_rpc_msearch(client: TestClient): + with client as request: + response = request.get("/rpc?v=5&type=msearch&arg=user1") data = response.json() # user1 maintains 4 packages; assert that we got them all. @@ -564,73 +597,80 @@ def test_rpc_msearch(): assert names == expected_results # Search for a non-existent maintainer, giving us zero packages. - response = make_request("/rpc?v=5&type=msearch&arg=blah-blah") + response = request.get("/rpc?v=5&type=msearch&arg=blah-blah") data = response.json() assert data.get("resultcount") == 0 # A missing arg still succeeds, but it returns all orphans. # Just verify that we receive no error and the orphaned result. - response = make_request("/rpc?v=5&type=msearch") + response = request.get("/rpc?v=5&type=msearch") data = response.json() assert data.get("resultcount") == 1 result = data.get("results")[0] assert result.get("Name") == "woogly-chungus" -def test_rpc_search_depends(): - response = make_request( - "/rpc?v=5&type=search&by=depends&arg=chungus-depends") +def test_rpc_search_depends(client: TestClient): + with client as request: + response = request.get( + "/rpc?v=5&type=search&by=depends&arg=chungus-depends") data = response.json() assert data.get("resultcount") == 1 result = data.get("results")[0] assert result.get("Name") == "big-chungus" -def test_rpc_search_makedepends(): - response = make_request( - "/rpc?v=5&type=search&by=makedepends&arg=chungus-makedepends") +def test_rpc_search_makedepends(client: TestClient): + with client as request: + response = request.get( + "/rpc?v=5&type=search&by=makedepends&arg=chungus-makedepends") data = response.json() assert data.get("resultcount") == 1 result = data.get("results")[0] assert result.get("Name") == "big-chungus" -def test_rpc_search_optdepends(): - response = make_request( - "/rpc?v=5&type=search&by=optdepends&arg=chungus-optdepends") +def test_rpc_search_optdepends(client: TestClient): + with client as request: + response = request.get( + "/rpc?v=5&type=search&by=optdepends&arg=chungus-optdepends") data = response.json() assert data.get("resultcount") == 1 result = data.get("results")[0] assert result.get("Name") == "big-chungus" -def test_rpc_search_checkdepends(): - response = make_request( - "/rpc?v=5&type=search&by=checkdepends&arg=chungus-checkdepends") +def test_rpc_search_checkdepends(client: TestClient): + with client as request: + response = request.get( + "/rpc?v=5&type=search&by=checkdepends&arg=chungus-checkdepends") data = response.json() assert data.get("resultcount") == 1 result = data.get("results")[0] assert result.get("Name") == "big-chungus" -def test_rpc_incorrect_by(): - response = make_request("/rpc?v=5&type=search&by=fake&arg=big") +def test_rpc_incorrect_by(client: TestClient): + with client as request: + response = request.get("/rpc?v=5&type=search&by=fake&arg=big") assert response.json().get("error") == "Incorrect by field specified." -def test_rpc_jsonp_callback(): +def test_rpc_jsonp_callback(client: TestClient): """ Test the callback parameter. For end-to-end verification, the `examples/jsonp.html` file can be used to submit jsonp callback requests to the RPC. """ - response = make_request( - "/rpc?v=5&type=search&arg=big&callback=jsonCallback") + with client as request: + response = request.get( + "/rpc?v=5&type=search&arg=big&callback=jsonCallback") assert response.headers.get("content-type") == "text/javascript" assert re.search(r'^/\*\*/jsonCallback\(.*\)$', response.text) is not None # Test an invalid callback name; we get an application/json error. - response = make_request( - "/rpc?v=5&type=search&arg=big&callback=jsonCallback!") + with client as request: + response = request.get( + "/rpc?v=5&type=search&arg=big&callback=jsonCallback!") assert response.headers.get("content-type") == "application/json" assert response.json().get("error") == "Invalid callback name." From 94972841d6c6330503cdf33d53336c1bd47f9469 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 04:36:39 -0800 Subject: [PATCH 587/844] change(fastapi): decouple error logic from process_account_form Signed-off-by: Kevin Morris --- aurweb/exceptions.py | 9 ++ aurweb/routers/accounts.py | 209 ++++++++--------------------------- aurweb/users/validate.py | 204 ++++++++++++++++++++++++++++++++++ test/test_accounts_routes.py | 25 ++++- 4 files changed, 278 insertions(+), 169 deletions(-) create mode 100644 aurweb/users/validate.py diff --git a/aurweb/exceptions.py b/aurweb/exceptions.py index 82628b0a..31212676 100644 --- a/aurweb/exceptions.py +++ b/aurweb/exceptions.py @@ -1,3 +1,6 @@ +from typing import Any + + class AurwebException(Exception): pass @@ -77,3 +80,9 @@ class InvalidArgumentsException(AurwebException): class RPCError(AurwebException): pass + + +class ValidationError(AurwebException): + def __init__(self, data: Any, *args, **kwargs): + super().__init__(*args, **kwargs) + self.data = data diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index aca322b5..47483acc 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -6,20 +6,20 @@ from http import HTTPStatus from fastapi import APIRouter, Form, Request from fastapi.responses import HTMLResponse, RedirectResponse -from sqlalchemy import and_, func, or_ +from sqlalchemy import and_, or_ import aurweb.config -from aurweb import cookies, db, l10n, logging, models, time, util +from aurweb import cookies, db, l10n, logging, models, util from aurweb.auth import account_type_required, auth_required -from aurweb.captcha import get_captcha_answer, get_captcha_salts, get_captcha_token +from aurweb.captcha import get_captcha_salts +from aurweb.exceptions import ValidationError from aurweb.l10n import get_translator_for_request -from aurweb.models import account_type -from aurweb.models.account_type import (DEVELOPER, DEVELOPER_ID, TRUSTED_USER, TRUSTED_USER_AND_DEV, TRUSTED_USER_AND_DEV_ID, - TRUSTED_USER_ID, USER_ID) +from aurweb.models import account_type as at from aurweb.models.ssh_pub_key import get_fingerprint from aurweb.scripts.notify import ResetKeyNotification, WelcomeNotification from aurweb.templates import make_context, make_variable_context, render_template +from aurweb.users import validate from aurweb.users.util import get_user_by_name router = APIRouter() @@ -126,146 +126,31 @@ def process_account_form(request: Request, user: models.User, args: dict): # Get a local translator. _ = get_translator_for_request(request) - host = request.client.host - ban = db.query(models.Ban, models.Ban.IPAddress == host).first() - if ban: - return (False, [ - "Account registration has been disabled for your " - "IP address, probably due to sustained spam attacks. " - "Sorry for the inconvenience." - ]) + checks = [ + validate.is_banned, + validate.invalid_user_password, + validate.invalid_fields, + validate.invalid_suspend_permission, + validate.invalid_username, + validate.invalid_password, + validate.invalid_email, + validate.invalid_backup_email, + validate.invalid_homepage, + validate.invalid_pgp_key, + validate.invalid_ssh_pubkey, + validate.invalid_language, + validate.invalid_timezone, + validate.username_in_use, + validate.email_in_use, + validate.invalid_account_type, + validate.invalid_captcha + ] - if request.user.is_authenticated(): - if not request.user.valid_password(args.get("passwd", None)): - return (False, ["Invalid password."]) - - email = args.get("E", None) - username = args.get("U", None) - - if not email or not username: - return (False, ["Missing a required field."]) - - inactive = args.get("J", False) - if not request.user.is_elevated() and inactive != bool(user.InactivityTS): - return (False, ["You do not have permission to suspend accounts."]) - - username_min_len = aurweb.config.getint("options", "username_min_len") - username_max_len = aurweb.config.getint("options", "username_max_len") - if not util.valid_username(args.get("U")): - return (False, [ - "The username is invalid.", - [ - _("It must be between %s and %s characters long") % ( - username_min_len, username_max_len), - "Start and end with a letter or number", - "Can contain only one period, underscore or hyphen.", - ] - ]) - - password = args.get("P", None) - if password: - confirmation = args.get("C", None) - if not util.valid_password(password): - return (False, [ - _("Your password must be at least %s characters.") % ( - username_min_len) - ]) - elif not confirmation: - return (False, ["Please confirm your new password."]) - elif password != confirmation: - return (False, ["Password fields do not match."]) - - backup_email = args.get("BE", None) - homepage = args.get("HP", None) - pgp_key = args.get("K", None) - ssh_pubkey = args.get("PK", None) - language = args.get("L", None) - timezone = args.get("TZ", None) - - def username_exists(username): - return and_(models.User.ID != user.ID, - func.lower(models.User.Username) == username.lower()) - - def email_exists(email): - return and_(models.User.ID != user.ID, - func.lower(models.User.Email) == email.lower()) - - if not util.valid_email(email): - return (False, ["The email address is invalid."]) - elif backup_email and not util.valid_email(backup_email): - return (False, ["The backup email address is invalid."]) - elif homepage and not util.valid_homepage(homepage): - return (False, [ - "The home page is invalid, please specify the full HTTP(s) URL."]) - elif pgp_key and not util.valid_pgp_fingerprint(pgp_key): - return (False, ["The PGP key fingerprint is invalid."]) - elif ssh_pubkey and not util.valid_ssh_pubkey(ssh_pubkey): - return (False, ["The SSH public key is invalid."]) - elif language and language not in l10n.SUPPORTED_LANGUAGES: - return (False, ["Language is not currently supported."]) - elif timezone and timezone not in time.SUPPORTED_TIMEZONES: - return (False, ["Timezone is not currently supported."]) - elif db.query(models.User, username_exists(username)).first(): - # If the username already exists... - return (False, [ - _("The username, %s%s%s, is already in use.") % ( - "", username, "") - ]) - elif db.query(models.User, email_exists(email)).first(): - # If the email already exists... - return (False, [ - _("The address, %s%s%s, is already in use.") % ( - "", email, "") - ]) - - def ssh_fingerprint_exists(fingerprint): - return and_(models.SSHPubKey.UserID != user.ID, - models.SSHPubKey.Fingerprint == fingerprint) - - if ssh_pubkey: - fingerprint = get_fingerprint(ssh_pubkey.strip().rstrip()) - if fingerprint is None: - return (False, ["The SSH public key is invalid."]) - - if db.query(models.SSHPubKey, - ssh_fingerprint_exists(fingerprint)).first(): - return (False, [ - _("The SSH public key, %s%s%s, is already in use.") % ( - "", fingerprint, "") - ]) - - T = int(args.get("T", user.AccountTypeID)) - if T != user.AccountTypeID: - if T not in account_type.ACCOUNT_TYPE_NAME: - return (False, - ["Invalid account type provided."]) - elif not request.user.is_elevated(): - return (False, - ["You do not have permission to change account types."]) - - credential_checks = { - DEVELOPER_ID: request.user.is_developer, - TRUSTED_USER_AND_DEV_ID: request.user.is_developer, - TRUSTED_USER_ID: request.user.is_elevated, - USER_ID: request.user.is_elevated - } - credential_check = credential_checks.get(T) - - if not credential_check(): - name = account_type.ACCOUNT_TYPE_NAME.get(T) - error = _("You do not have permission to change " - "this user's account type to %s.") % name - return (False, [error]) - - captcha_salt = args.get("captcha_salt", None) - if captcha_salt and captcha_salt not in get_captcha_salts(): - return (False, ["This CAPTCHA has expired. Please try again."]) - - captcha = args.get("captcha", None) - if captcha: - answer = get_captcha_answer(get_captcha_token(captcha_salt)) - if captcha != answer: - return (False, ["The entered CAPTCHA answer is invalid."]) + try: + for check in checks: + check(**args, request=request, user=user, _=_) + except ValidationError as exc: + return (False, exc.data) return (True, []) @@ -286,16 +171,16 @@ def make_account_form_context(context: dict, context = copy.copy(context) context["account_types"] = [ - (USER_ID, "Normal User"), - (TRUSTED_USER_ID, TRUSTED_USER) + (at.USER_ID, "Normal User"), + (at.TRUSTED_USER_ID, at.TRUSTED_USER) ] user_account_type_id = context.get("account_types")[0][0] if request.user.has_credential("CRED_ACCOUNT_EDIT_DEV"): - context["account_types"].append((DEVELOPER_ID, DEVELOPER)) - context["account_types"].append((TRUSTED_USER_AND_DEV_ID, - TRUSTED_USER_AND_DEV)) + context["account_types"].append((at.DEVELOPER_ID, at.DEVELOPER)) + context["account_types"].append((at.TRUSTED_USER_AND_DEV_ID, + at.TRUSTED_USER_AND_DEV)) if request.user.is_authenticated(): context["username"] = args.get("U", user.Username) @@ -389,12 +274,10 @@ async def account_register_post(request: Request, captcha: str = Form(default=None), captcha_salt: str = Form(...)): context = await make_variable_context(request, "Register") - args = dict(await request.form()) + context = make_account_form_context(context, request, None, args) - ok, errors = process_account_form(request, request.user, args) - if not ok: # If the field values given do not meet the requirements, # return HTTP 400 with an error. @@ -636,9 +519,9 @@ async def account_comments(request: Request, username: str): @router.get("/accounts") @auth_required(True, redirect="/accounts") -@account_type_required({account_type.TRUSTED_USER, - account_type.DEVELOPER, - account_type.TRUSTED_USER_AND_DEV}) +@account_type_required({at.TRUSTED_USER, + at.DEVELOPER, + at.TRUSTED_USER_AND_DEV}) async def accounts(request: Request): context = make_context(request, "Accounts") return render_template(request, "account/search.html", context) @@ -646,9 +529,9 @@ async def accounts(request: Request): @router.post("/accounts") @auth_required(True, redirect="/accounts") -@account_type_required({account_type.TRUSTED_USER, - account_type.DEVELOPER, - account_type.TRUSTED_USER_AND_DEV}) +@account_type_required({at.TRUSTED_USER, + at.DEVELOPER, + at.TRUSTED_USER_AND_DEV}) async def accounts_post(request: Request, O: int = Form(default=0), # Offset SB: str = Form(default=str()), # Sort By @@ -680,10 +563,10 @@ async def accounts_post(request: Request, # Convert parameter T to an AccountType ID. account_types = { - "u": account_type.USER_ID, - "t": account_type.TRUSTED_USER_ID, - "d": account_type.DEVELOPER_ID, - "td": account_type.TRUSTED_USER_AND_DEV_ID + "u": at.USER_ID, + "t": at.TRUSTED_USER_ID, + "d": at.DEVELOPER_ID, + "td": at.TRUSTED_USER_AND_DEV_ID } account_type_id = account_types.get(T, None) diff --git a/aurweb/users/validate.py b/aurweb/users/validate.py new file mode 100644 index 00000000..4959e316 --- /dev/null +++ b/aurweb/users/validate.py @@ -0,0 +1,204 @@ +""" +Validation functions for account registration and edit fields. +Each of these functions extracts a subset of keyword arguments +out of form data from /account/register or /account/{username}/edit. + +All functions in this module raise aurweb.exceptions.ValidationError +when encountering invalid criteria and return silently otherwise. +""" +from typing import List, Optional, Tuple + +from fastapi import Request +from sqlalchemy import and_ + +from aurweb import config, db, l10n, models, time, util +from aurweb.captcha import get_captcha_answer, get_captcha_salts, get_captcha_token +from aurweb.exceptions import ValidationError +from aurweb.models import account_type as at +from aurweb.models.account_type import ACCOUNT_TYPE_NAME +from aurweb.models.ssh_pub_key import get_fingerprint + + +def invalid_fields(E: str = str(), U: str = str(), **kwargs) \ + -> Optional[Tuple[bool, List[str]]]: + if not E or not U: + raise ValidationError(["Missing a required field."]) + + +def invalid_suspend_permission(request: Request = None, + user: models.User = None, + J: bool = False, + **kwargs) \ + -> Optional[Tuple[bool, List[str]]]: + if not request.user.is_elevated() and J != bool(user.InactivityTS): + raise ValidationError([ + "You do not have permission to suspend accounts."]) + + +def invalid_username(request: Request = None, U: str = str(), _=None, + **kwargs): + if not util.valid_username(U): + username_min_len = config.getint("options", "username_min_len") + username_max_len = config.getint("options", "username_max_len") + raise ValidationError([ + "The username is invalid.", + [ + _("It must be between %s and %s characters long") % ( + username_min_len, username_max_len), + "Start and end with a letter or number", + "Can contain only one period, underscore or hyphen.", + ] + ]) + + +def invalid_password(P: str = str(), C: str = str(), + _: l10n.Translator = None, **kwargs) -> None: + if P: + if not util.valid_password(P): + username_min_len = config.getint( + "options", "username_min_len") + raise ValidationError([ + _("Your password must be at least %s characters.") % ( + username_min_len) + ]) + elif not C: + raise ValidationError(["Please confirm your new password."]) + elif P != C: + raise ValidationError(["Password fields do not match."]) + + +def is_banned(request: Request = None, **kwargs) -> None: + host = request.client.host + exists = db.query(models.Ban, models.Ban.IPAddress == host).exists() + if db.query(exists).scalar(): + raise ValidationError([ + "Account registration has been disabled for your " + "IP address, probably due to sustained spam attacks. " + "Sorry for the inconvenience." + ]) + + +def invalid_user_password(request: Request = None, passwd: str = str(), + **kwargs) -> None: + if request.user.is_authenticated(): + if not request.user.valid_password(passwd): + raise ValidationError(["Invalid password."]) + + +def invalid_email(E: str = str(), **kwargs) -> None: + if not util.valid_email(E): + raise ValidationError(["The email address is invalid."]) + + +def invalid_backup_email(BE: str = str(), **kwargs) -> None: + if BE and not util.valid_email(BE): + raise ValidationError(["The backup email address is invalid."]) + + +def invalid_homepage(HP: str = str(), **kwargs) -> None: + if HP and not util.valid_homepage(HP): + raise ValidationError([ + "The home page is invalid, please specify the full HTTP(s) URL."]) + + +def invalid_pgp_key(K: str = str(), **kwargs) -> None: + if K and not util.valid_pgp_fingerprint(K): + raise ValidationError(["The PGP key fingerprint is invalid."]) + + +def invalid_ssh_pubkey(PK: str = str(), user: models.User = None, + _: l10n.Translator = None, **kwargs) -> None: + if PK: + invalid_exc = ValidationError(["The SSH public key is invalid."]) + if not util.valid_ssh_pubkey(PK): + raise invalid_exc + + fingerprint = get_fingerprint(PK.strip().rstrip()) + if not fingerprint: + raise invalid_exc + + exists = db.query(models.SSHPubKey).filter( + and_(models.SSHPubKey.UserID != user.ID, + models.SSHPubKey.Fingerprint == fingerprint) + ).exists() + if db.query(exists).scalar(): + raise ValidationError([ + _("The SSH public key, %s%s%s, is already in use.") % ( + "", fingerprint, "") + ]) + + +def invalid_language(L: str = str(), **kwargs) -> None: + if L and L not in l10n.SUPPORTED_LANGUAGES: + raise ValidationError(["Language is not currently supported."]) + + +def invalid_timezone(TZ: str = str(), **kwargs) -> None: + if TZ and TZ not in time.SUPPORTED_TIMEZONES: + raise ValidationError(["Timezone is not currently supported."]) + + +def username_in_use(U: str = str(), user: models.User = None, + _: l10n.Translator = None, **kwargs) -> None: + exists = db.query(models.User).filter( + and_(models.User.ID != user.ID, + models.User.Username == U) + ).exists() + if db.query(exists).scalar(): + # If the username already exists... + raise ValidationError([ + _("The username, %s%s%s, is already in use.") % ( + "", U, "") + ]) + + +def email_in_use(E: str = str(), user: models.User = None, + _: l10n.Translator = None, **kwargs) -> None: + exists = db.query(models.User).filter( + and_(models.User.ID != user.ID, + models.User.Email == E) + ).exists() + if db.query(exists).scalar(): + # If the email already exists... + raise ValidationError([ + _("The address, %s%s%s, is already in use.") % ( + "", E, "") + ]) + + +def invalid_account_type(T: int = None, request: Request = None, + user: models.User = None, + _: l10n.Translator = None, + **kwargs) -> None: + if T is not None and (T := int(T)) != user.AccountTypeID: + if T not in ACCOUNT_TYPE_NAME: + raise ValidationError(["Invalid account type provided."]) + elif not request.user.is_elevated(): + raise ValidationError([ + "You do not have permission to change account types."]) + + credential_checks = { + at.USER_ID: request.user.is_trusted_user, + at.TRUSTED_USER_ID: request.user.is_trusted_user, + at.DEVELOPER_ID: lambda: request.user.is_developer(), + at.TRUSTED_USER_AND_DEV_ID: (lambda: request.user.is_trusted_user() + and request.user.is_developer()) + } + credential_check = credential_checks.get(T) + + if not credential_check(): + name = ACCOUNT_TYPE_NAME.get(T) + error = _("You do not have permission to change " + "this user's account type to %s.") % name + raise ValidationError([error]) + + +def invalid_captcha(captcha_salt: str = None, captcha: str = None, **kwargs) \ + -> None: + if captcha_salt and captcha_salt not in get_captcha_salts(): + raise ValidationError(["This CAPTCHA has expired. Please try again."]) + + if captcha: + answer = get_captcha_answer(get_captcha_token(captcha_salt)) + if captcha != answer: + raise ValidationError(["The entered CAPTCHA answer is invalid."]) diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index e828f70f..be929e97 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -1035,12 +1035,25 @@ def test_post_account_edit_account_types(): # Make sure it got changed to USER_ID as we intended. assert user.AccountTypeID == USER_ID - # Change user to a Developer. + # Change user to a TU & Dev, which can change themselves to a Developer. with db.begin(): - user.AccountTypeID = DEVELOPER_ID + user.AccountTypeID = TRUSTED_USER_AND_DEV_ID - # As a developer, we can absolutely change all account types. - # For example, from DEVELOPER_ID to TRUSTED_USER_AND_DEV_ID: + # As a TU & Dev, we can absolutely change all account types. + # For example, from TRUSTED_USER_AND_DEV_ID to DEVELOPER_ID: + post_data = { + "U": user.Username, + "E": user.Email, + "T": DEVELOPER_ID, + "passwd": "testPassword" + } + with client as request: + resp = request.post(endpoint, data=post_data, cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + assert user.AccountTypeID == DEVELOPER_ID + + # But we can't change a user to a Trusted User & Developer when + # we're just a Developer. post_data = { "U": user.Username, "E": user.Email, @@ -1049,8 +1062,8 @@ def test_post_account_edit_account_types(): } with client as request: resp = request.post(endpoint, data=post_data, cookies=cookies) - assert resp.status_code == int(HTTPStatus.OK) - assert user.AccountTypeID == TRUSTED_USER_AND_DEV_ID + assert resp.status_code == int(HTTPStatus.BAD_REQUEST) + assert user.AccountTypeID == DEVELOPER_ID def test_get_account(): From 303585cdbf50ffa70c7bc6f579c17d5e6bc08a42 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 05:40:11 -0800 Subject: [PATCH 588/844] change(fastapi): decouple update logic from account edit Signed-off-by: Kevin Morris --- aurweb/routers/accounts.py | 85 ++++------------------------ aurweb/users/update.py | 110 +++++++++++++++++++++++++++++++++++++ aurweb/util.py | 7 +++ 3 files changed, 128 insertions(+), 74 deletions(-) create mode 100644 aurweb/users/update.py diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 47483acc..02a7f4c6 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -1,7 +1,6 @@ import copy import typing -from datetime import datetime from http import HTTPStatus from fastapi import APIRouter, Form, Request @@ -19,7 +18,7 @@ from aurweb.models import account_type as at from aurweb.models.ssh_pub_key import get_fingerprint from aurweb.scripts.notify import ResetKeyNotification, WelcomeNotification from aurweb.templates import make_context, make_variable_context, render_template -from aurweb.users import validate +from aurweb.users import update, validate from aurweb.users.util import get_user_by_name router = APIRouter() @@ -405,79 +404,17 @@ async def account_edit_post(request: Request, return render_template(request, "account/edit.html", context, status_code=HTTPStatus.BAD_REQUEST) - # Set all updated fields as needed. - with db.begin(): - user.Username = U or user.Username - user.Email = E or user.Email - user.HideEmail = bool(H) - user.BackupEmail = BE or user.BackupEmail - user.RealName = R or user.RealName - user.Homepage = HP or user.Homepage - user.IRCNick = I or user.IRCNick - user.PGPKey = K or user.PGPKey - user.Suspended = J - user.InactivityTS = int(datetime.utcnow().timestamp()) * int(J) + updates = [ + update.simple, + update.language, + update.timezone, + update.ssh_pubkey, + update.account_type, + update.password + ] - # If we update the language, update the cookie as well. - if L and L != user.LangPreference: - request.cookies["AURLANG"] = L - with db.begin(): - user.LangPreference = L - context["language"] = L - - # If we update the timezone, also update the cookie. - if TZ and TZ != user.Timezone: - with db.begin(): - user.Timezone = TZ - request.cookies["AURTZ"] = TZ - context["timezone"] = TZ - - with db.begin(): - user.CommentNotify = bool(CN) - user.UpdateNotify = bool(UN) - user.OwnershipNotify = bool(ON) - - # If a PK is given, compare it against the target user's PK. - with db.begin(): - if PK: - # Get the second token in the public key, which is the actual key. - pubkey = PK.strip().rstrip() - parts = pubkey.split(" ") - if len(parts) == 3: - # Remove the host part. - pubkey = parts[0] + " " + parts[1] - fingerprint = get_fingerprint(pubkey) - if not user.ssh_pub_key: - # No public key exists, create one. - user.ssh_pub_key = models.SSHPubKey(UserID=user.ID, - PubKey=pubkey, - Fingerprint=fingerprint) - elif user.ssh_pub_key.PubKey != pubkey: - # A public key already exists, update it. - user.ssh_pub_key.PubKey = pubkey - user.ssh_pub_key.Fingerprint = fingerprint - elif user.ssh_pub_key: - # Else, if the user has a public key already, delete it. - db.delete(user.ssh_pub_key) - - if T and T != user.AccountTypeID: - with db.begin(): - user.AccountTypeID = T - - if P and not user.valid_password(P): - # Remove the fields we consumed for passwords. - context["P"] = context["C"] = str() - - # If a password was given and it doesn't match the user's, update it. - with db.begin(): - user.update_password(P) - - if user == request.user: - remember_me = request.cookies.get("AURREMEMBER", False) - - # If the target user is the request user, login with - # the updated password to update the Session record. - user.login(request, P, cookies.timeout(remember_me)) + for f in updates: + f(**args, request=request, user=user, context=context) if not errors: context["complete"] = True diff --git a/aurweb/users/update.py b/aurweb/users/update.py new file mode 100644 index 00000000..60a6184e --- /dev/null +++ b/aurweb/users/update.py @@ -0,0 +1,110 @@ +from datetime import datetime +from typing import Any, Dict + +from fastapi import Request + +from aurweb import cookies, db, models +from aurweb.models.ssh_pub_key import get_fingerprint +from aurweb.util import strtobool + + +def simple(U: str = str(), E: str = str(), H: bool = False, + BE: str = str(), R: str = str(), HP: str = str(), + I: str = str(), K: str = str(), J: bool = False, + CN: bool = False, UN: bool = False, ON: bool = False, + user: models.User = None, + **kwargs) -> None: + now = int(datetime.utcnow().timestamp()) + with db.begin(): + user.Username = U or user.Username + user.Email = E or user.Email + user.HideEmail = strtobool(H) + user.BackupEmail = BE or user.BackupEmail + user.RealName = R or user.RealName + user.Homepage = HP or user.Homepage + user.IRCNick = I or user.IRCNick + user.PGPKey = K or user.PGPKey + user.Suspended = strtobool(J) + user.InactivityTS = now * int(strtobool(J)) + user.CommentNotify = strtobool(CN) + user.UpdateNotify = strtobool(UN) + user.OwnershipNotify = strtobool(ON) + + +def language(L: str = str(), + request: Request = None, + user: models.User = None, + context: Dict[str, Any] = {}, + **kwargs) -> None: + if L and L != user.LangPreference: + with db.begin(): + user.LangPreference = L + context["language"] = L + + +def timezone(TZ: str = str(), + request: Request = None, + user: models.User = None, + context: Dict[str, Any] = {}, + **kwargs) -> None: + if TZ and TZ != user.Timezone: + with db.begin(): + user.Timezone = TZ + context["language"] = TZ + + +def ssh_pubkey(PK: str = str(), + user: models.User = None, + **kwargs) -> None: + # If a PK is given, compare it against the target user's PK. + if PK: + # Get the second token in the public key, which is the actual key. + pubkey = PK.strip().rstrip() + parts = pubkey.split(" ") + if len(parts) == 3: + # Remove the host part. + pubkey = parts[0] + " " + parts[1] + fingerprint = get_fingerprint(pubkey) + if not user.ssh_pub_key: + # No public key exists, create one. + with db.begin(): + db.create(models.SSHPubKey, UserID=user.ID, + PubKey=pubkey, Fingerprint=fingerprint) + elif user.ssh_pub_key.PubKey != pubkey: + # A public key already exists, update it. + with db.begin(): + user.ssh_pub_key.PubKey = pubkey + user.ssh_pub_key.Fingerprint = fingerprint + elif user.ssh_pub_key: + # Else, if the user has a public key already, delete it. + with db.begin(): + db.delete(user.ssh_pub_key) + + +def account_type(T: int = None, + user: models.User = None, + **kwargs) -> None: + if T is not None and (T := int(T)) != user.AccountTypeID: + with db.begin(): + user.AccountTypeID = T + + +def password(P: str = str(), + request: Request = None, + user: models.User = None, + context: Dict[str, Any] = {}, + **kwargs) -> None: + if P and not user.valid_password(P): + # Remove the fields we consumed for passwords. + context["P"] = context["C"] = str() + + # If a password was given and it doesn't match the user's, update it. + with db.begin(): + user.update_password(P) + + if user == request.user: + remember_me = request.cookies.get("AURREMEMBER", False) + + # If the target user is the request user, login with + # the updated password to update the Session record. + user.login(request, P, cookies.timeout(remember_me)) diff --git a/aurweb/util.py b/aurweb/util.py index 1c2042fa..b95fc6a3 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -7,6 +7,7 @@ import secrets import string from datetime import datetime +from distutils.util import strtobool as _strtobool from typing import Any, Callable, Dict, Iterable, Tuple from urllib.parse import urlencode, urlparse from zoneinfo import ZoneInfo @@ -170,3 +171,9 @@ def sanitize_params(offset: str, per_page: str) -> Tuple[int, int]: per_page = defaults.PP return (offset, per_page) + + +def strtobool(value: str) -> bool: + if isinstance(value, str): + return _strtobool(value) + return value From 2892d21ff173af8f484274b1b175447be2ae4bab Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 05:47:22 -0800 Subject: [PATCH 589/844] remove global aurweb.models flake8 F401 ignore Signed-off-by: Kevin Morris --- aurweb/models/__init__.py | 60 +++++++++++++++++++-------------------- setup.cfg | 1 - 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/aurweb/models/__init__.py b/aurweb/models/__init__.py index b430acd2..a06077ad 100644 --- a/aurweb/models/__init__.py +++ b/aurweb/models/__init__.py @@ -1,31 +1,31 @@ """ Collection of all aurweb SQLAlchemy declarative models. """ -from .accepted_term import AcceptedTerm -from .account_type import AccountType -from .api_rate_limit import ApiRateLimit -from .ban import Ban -from .dependency_type import DependencyType -from .group import Group -from .license import License -from .official_provider import OfficialProvider -from .package import Package -from .package_base import PackageBase -from .package_blacklist import PackageBlacklist -from .package_comaintainer import PackageComaintainer -from .package_comment import PackageComment -from .package_dependency import PackageDependency -from .package_group import PackageGroup -from .package_keyword import PackageKeyword -from .package_license import PackageLicense -from .package_notification import PackageNotification -from .package_relation import PackageRelation -from .package_request import PackageRequest -from .package_source import PackageSource -from .package_vote import PackageVote -from .relation_type import RelationType -from .request_type import RequestType -from .session import Session -from .ssh_pub_key import SSHPubKey -from .term import Term -from .tu_vote import TUVote -from .tu_voteinfo import TUVoteInfo -from .user import User +from .accepted_term import AcceptedTerm # noqa: F401 +from .account_type import AccountType # noqa: F401 +from .api_rate_limit import ApiRateLimit # noqa: F401 +from .ban import Ban # noqa: F401 +from .dependency_type import DependencyType # noqa: F401 +from .group import Group # noqa: F401 +from .license import License # noqa: F401 +from .official_provider import OfficialProvider # noqa: F401 +from .package import Package # noqa: F401 +from .package_base import PackageBase # noqa: F401 +from .package_blacklist import PackageBlacklist # noqa: F401 +from .package_comaintainer import PackageComaintainer # noqa: F401 +from .package_comment import PackageComment # noqa: F401 +from .package_dependency import PackageDependency # noqa: F401 +from .package_group import PackageGroup # noqa: F401 +from .package_keyword import PackageKeyword # noqa: F401 +from .package_license import PackageLicense # noqa: F401 +from .package_notification import PackageNotification # noqa: F401 +from .package_relation import PackageRelation # noqa: F401 +from .package_request import PackageRequest # noqa: F401 +from .package_source import PackageSource # noqa: F401 +from .package_vote import PackageVote # noqa: F401 +from .relation_type import RelationType # noqa: F401 +from .request_type import RequestType # noqa: F401 +from .session import Session # noqa: F401 +from .ssh_pub_key import SSHPubKey # noqa: F401 +from .term import Term # noqa: F401 +from .tu_vote import TUVote # noqa: F401 +from .tu_voteinfo import TUVoteInfo # noqa: F401 +from .user import User # noqa: F401 diff --git a/setup.cfg b/setup.cfg index 7c64a01f..cec1bcf5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,6 @@ per-file-ignores = aurweb/routers/accounts.py:C901 test/test_ssh_pub_key.py:E501 aurweb/routers/packages.py:E741 - aurweb/models/__init__.py:F401 [isort] line_length = 127 From 2df7187514a64d0e0ff88130562a9fc95c0f6611 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 05:48:43 -0800 Subject: [PATCH 590/844] fix global test_ssh_pub_key E501 flake8 violation Signed-off-by: Kevin Morris --- setup.cfg | 1 - test/test_ssh_pub_key.py | 9 ++++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index cec1bcf5..997bf4b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,6 @@ ignore = E741, W503, W504 # per-file-ignores = aurweb/routers/accounts.py:C901 - test/test_ssh_pub_key.py:E501 aurweb/routers/packages.py:E741 [isort] diff --git a/test/test_ssh_pub_key.py b/test/test_ssh_pub_key.py index bb787759..e17af5a7 100644 --- a/test/test_ssh_pub_key.py +++ b/test/test_ssh_pub_key.py @@ -6,7 +6,14 @@ from aurweb.models.ssh_pub_key import SSHPubKey, get_fingerprint from aurweb.models.user import User TEST_SSH_PUBKEY = """ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCycoCi5yGCvSclH2wmNBUuwsYEzRZZBJaQquRc4ysl+Tg+/jiDkR3Zn9fIznC4KnFoyrIHzkKuePZ3bNDYwkZxkJKoWBCh4hXKDXSm87FMN0+VDC+1QxF/z0XaAGr/P6f4XukabyddypBdnHcZiplbw+YOSqcAE2TCqOlSXwNMOcF9U89UsR/Q9i9I52hlvU0q8+fZVGhou1KCowFSnHYtrr5KYJ04CXkJ13DkVf3+pjQWyrByvBcf1hGEaczlgfobrrv/y96jDhgfXucxliNKLdufDPPkii3LhhsNcDmmI1VZ3v0irKvd9WZuauqloobY84zEFcDTyjn0hxGjVeYFejm4fBnvjga0yZXORuWksdNfXWLDxFk6MDDd1jF0ExRbP+OxDuU4IVyIuDL7S3cnbf2YjGhkms/8voYT2OBE7FwNlfv98Kr0NUp51zpf55Arxn9j0Rz9xTA7FiODQgCn6iQ0SDtzUNL0IKTCw26xJY5gzMxbfpvzPQGeulx/ioM= kevr@volcano +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCycoCi5yGCvSclH2wmNBUuwsYEzRZZBJaQquRc4y\ +sl+Tg+/jiDkR3Zn9fIznC4KnFoyrIHzkKuePZ3bNDYwkZxkJKoWBCh4hXKDXSm87FMN0+VDC+1QxF/\ +z0XaAGr/P6f4XukabyddypBdnHcZiplbw+YOSqcAE2TCqOlSXwNMOcF9U89UsR/Q9i9I52hlvU0q8+\ +fZVGhou1KCowFSnHYtrr5KYJ04CXkJ13DkVf3+pjQWyrByvBcf1hGEaczlgfobrrv/y96jDhgfXucx\ +liNKLdufDPPkii3LhhsNcDmmI1VZ3v0irKvd9WZuauqloobY84zEFcDTyjn0hxGjVeYFejm4fBnvjg\ +a0yZXORuWksdNfXWLDxFk6MDDd1jF0ExRbP+OxDuU4IVyIuDL7S3cnbf2YjGhkms/8voYT2OBE7FwN\ +lfv98Kr0NUp51zpf55Arxn9j0Rz9xTA7FiODQgCn6iQ0SDtzUNL0IKTCw26xJY5gzMxbfpvzPQGeul\ +x/ioM= kevr@volcano """ user = ssh_pub_key = None From 672af707ad34a014b1119a9104db4050b706678b Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 05:49:08 -0800 Subject: [PATCH 591/844] remove C901 and E741 per-file-ignores exclusion We no longer have C901 violations and we're already ignoring E741 (short variable names) in the overall `ignore` option. Signed-off-by: Kevin Morris --- setup.cfg | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/setup.cfg b/setup.cfg index 997bf4b7..08be9186 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,19 +19,6 @@ max-complexity = 10 # do this, so we're ignoring it here. ignore = E741, W503, W504 -# aurweb/routers/accounts.py -# Ignore over-reaching complexity. -# TODO: This should actually be addressed so we do not ignore C901. -# -# test/test_ssh_pub_key.py -# E501 is detected due to our >127 width test constant. Ignore it. -# Due to this, line width should _always_ be looked at in code reviews. -# Anything like this should be questioned. -# -per-file-ignores = - aurweb/routers/accounts.py:C901 - aurweb/routers/packages.py:E741 - [isort] line_length = 127 lines_between_types = 1 From dbe5cb4a33066c90333c32aab077081fec3ba426 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 18 Nov 2021 16:42:26 -0800 Subject: [PATCH 592/844] fix(fastapi): only include comment-edit.js where needed Closes: #178 Signed-off-by: Kevin Morris --- templates/account/comments.html | 3 +++ templates/packages/show.html | 3 +++ templates/partials/head.html | 3 --- templates/pkgbase.html | 3 +++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/templates/account/comments.html b/templates/account/comments.html index 95585180..8dff53e4 100644 --- a/templates/account/comments.html +++ b/templates/account/comments.html @@ -17,6 +17,9 @@
    + + + {% for comment in comments %} {% include "partials/account/comment.html" %} {% endfor %} diff --git a/templates/packages/show.html b/templates/packages/show.html index fbc9c0ea..25083020 100644 --- a/templates/packages/show.html +++ b/templates/packages/show.html @@ -15,6 +15,9 @@
    + + + {% set pkgname = package.Name %} {% set pkgbase_id = pkgbase.ID %} {% include "partials/packages/comments.html" %} diff --git a/templates/partials/head.html b/templates/partials/head.html index 21c79887..8bfde020 100644 --- a/templates/partials/head.html +++ b/templates/partials/head.html @@ -15,8 +15,5 @@ - - - AUR ({{ language }}) - {{ title | tr }} diff --git a/templates/pkgbase.html b/templates/pkgbase.html index 315cdf67..cdf23c35 100644 --- a/templates/pkgbase.html +++ b/templates/pkgbase.html @@ -14,6 +14,9 @@
    + + + {% set pkgname = result.Name %} {% set pkgbase_id = result.ID %} {% set comments = comments %} From 7739b2178ec01828666daf271e8451852af22d2e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 18 Nov 2021 16:43:10 -0800 Subject: [PATCH 593/844] fix(fastapi): fix comment edit image sources These were using the old comment image sources. Slipped in due to cache and not checking without cache. Fixed them to use src="/static/images/...". Signed-off-by: Kevin Morris --- templates/partials/comment_actions.html | 10 +++++----- web/html/js/comment-edit.js | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/partials/comment_actions.html b/templates/partials/comment_actions.html index b21b90bd..b8ccf945 100644 --- a/templates/partials/comment_actions.html +++ b/templates/partials/comment_actions.html @@ -12,7 +12,7 @@ value="{{ request.url.path }}" /> - {{ 'Edit comment' | tr }} @@ -57,7 +57,7 @@ diff --git a/web/html/js/comment-edit.js b/web/html/js/comment-edit.js index 4898c8d4..23ffdd34 100644 --- a/web/html/js/comment-edit.js +++ b/web/html/js/comment-edit.js @@ -1,6 +1,6 @@ function add_busy_indicator(sibling) { const img = document.createElement('img'); - img.src = "/images/ajax-loader.gif"; + img.src = "/static/images/ajax-loader.gif"; img.classList.add('ajax-loader'); img.style.height = 11; img.style.width = 16; From a348cdaac3a36c53ed4e9355f977cf63c4052801 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 18 Nov 2021 16:44:13 -0800 Subject: [PATCH 594/844] housekeep(fastapi): cleanup unneeded jinja set statement Signed-off-by: Kevin Morris --- templates/pkgbase.html | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/pkgbase.html b/templates/pkgbase.html index cdf23c35..05583494 100644 --- a/templates/pkgbase.html +++ b/templates/pkgbase.html @@ -19,6 +19,5 @@ {% set pkgname = result.Name %} {% set pkgbase_id = result.ID %} - {% set comments = comments %} {% include "partials/packages/comments.html" %} {% endblock %} From 7f981b9ed7d0ce90d23d4051d743accd0228b515 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 18 Nov 2021 21:15:57 -0800 Subject: [PATCH 595/844] fix(fastapi): utilize auto_{orphan,deletion}_age Didn't get this in when the initial request port went down; here it is. Auto-accept orphan requests when the package has been out of date for longer than auto_orphan_age. Auto-accept deletion requests by the package's maintainer if the package has been uploaded within auto_deletion_age seconds ago. Signed-off-by: Kevin Morris --- aurweb/packages/validate.py | 34 +++++++++++++++++++ aurweb/routers/packages.py | 65 ++++++++++++++++++------------------ test/test_packages_routes.py | 43 +++++++++++++++++++++++- 3 files changed, 109 insertions(+), 33 deletions(-) create mode 100644 aurweb/packages/validate.py diff --git a/aurweb/packages/validate.py b/aurweb/packages/validate.py new file mode 100644 index 00000000..e730e98b --- /dev/null +++ b/aurweb/packages/validate.py @@ -0,0 +1,34 @@ +from typing import Any, Dict + +from aurweb import db, models +from aurweb.exceptions import ValidationError + + +def request(pkgbase: models.PackageBase, + type: str, comments: str, merge_into: str, + context: Dict[str, Any]) -> None: + if not comments: + raise ValidationError(["The comment field must not be empty."]) + + if type == "merge": + # Perform merge-related checks. + if not merge_into: + # TODO: This error needs to be translated. + raise ValidationError( + ['The "Merge into" field must not be empty.']) + + target = db.query(models.PackageBase).filter( + models.PackageBase.Name == merge_into + ).first() + if not target: + # TODO: This error needs to be translated. + raise ValidationError([ + "The package base you want to merge into does not exist." + ]) + + db.refresh(target) + if target.ID == pkgbase.ID: + # TODO: This error needs to be translated. + raise ValidationError([ + "You cannot merge a package base into itself." + ]) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index c8ceb275..dfb8e108 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -11,9 +11,11 @@ import aurweb.packages.util from aurweb import db, defaults, l10n, logging, models, util from aurweb.auth import auth_required +from aurweb.exceptions import ValidationError from aurweb.models.package_request import ACCEPTED_ID, PENDING_ID, REJECTED_ID from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID from aurweb.models.request_type import DELETION_ID, MERGE, MERGE_ID +from aurweb.packages import validate from aurweb.packages.search import PackageSearch from aurweb.packages.util import get_pkg_or_base, get_pkgbase_comment, get_pkgreq_by_id, query_notified, query_voted from aurweb.scripts import notify, popupdate @@ -153,7 +155,7 @@ def delete_package(deleter: models.User, package: models.Package): with db.begin(): pkgreq = create_request_if_missing( requests, reqtype, deleter, package) - db.refresh(pkgreq) + pkgreq.Status = ACCEPTED_ID bases_to_delete.append(package.PackageBase) @@ -707,35 +709,13 @@ async def pkgbase_request_post(request: Request, name: str, return render_template(request, "pkgbase/request.html", context, status_code=HTTPStatus.BAD_REQUEST) - if not comments: - context["errors"] = ["The comment field must not be empty."] + try: + validate.request(pkgbase, type, comments, merge_into, context) + except ValidationError as exc: + logger.error(f"Request Validation Error: {str(exc.data)}") + context["errors"] = exc.data return render_template(request, "pkgbase/request.html", context) - if type == "merge": - # Perform merge-related checks. - if not merge_into: - # TODO: This error needs to be translated. - context["errors"] = ['The "Merge into" field must not be empty.'] - return render_template(request, "pkgbase/request.html", context) - - target = db.query(models.PackageBase).filter( - models.PackageBase.Name == merge_into - ).first() - if not target: - # TODO: This error needs to be translated. - context["errors"] = [ - "The package base you want to merge into does not exist." - ] - return render_template(request, "pkgbase/request.html", context) - - db.refresh(target) - if target.ID == pkgbase.ID: - # TODO: This error needs to be translated. - context["errors"] = [ - "You cannot merge a package base into itself." - ] - return render_template(request, "pkgbase/request.html", context) - # All good. Create a new PackageRequest based on the given type. now = int(datetime.utcnow().timestamp()) reqtype = db.query(models.RequestType).filter( @@ -748,16 +728,37 @@ async def pkgbase_request_post(request: Request, name: str, PackageBase=pkgbase, PackageBaseName=pkgbase.Name, MergeBaseName=merge_into, - Comments=comments, ClosureComment=str()) + Comments=comments, + ClosureComment=str()) - # Prepare notification object. conn = db.ConnectionExecutor(db.get_engine().raw_connection()) - notify_ = notify.RequestOpenNotification( + # Prepare notification object. + notif = notify.RequestOpenNotification( conn, request.user.ID, pkgreq.ID, reqtype.Name, pkgreq.PackageBase.ID, merge_into=merge_into or None) # Send the notification now that we're out of the DB scope. - notify_.send() + notif.send() + + auto_orphan_age = aurweb.config.getint("options", "auto_orphan_age") + auto_delete_age = aurweb.config.getint("options", "auto_delete_age") + + flagged = pkgbase.OutOfDateTS and pkgbase.OutOfDateTS >= auto_orphan_age + is_maintainer = pkgbase.Maintainer == request.user + outdated = now - pkgbase.SubmittedTS <= auto_delete_age + + if type == "orphan" and flagged: + with db.begin(): + pkgbase.Maintainer = None + pkgreq.Status = ACCEPTED_ID + db.refresh(pkgreq) + notif = notify.RequestCloseNotification( + conn, request.user.ID, pkgreq.ID, pkgreq.status_display()) + notif.send() + elif type == "deletion" and is_maintainer and outdated: + packages = pkgbase.packages.all() + for package in packages: + delete_package(request.user, package) # Redirect the submitting user to /packages. return RedirectResponse("/packages", diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 02c22d9d..64ee38d0 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -10,7 +10,7 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy import and_ -from aurweb import asgi, db, defaults +from aurweb import asgi, config, db, defaults from aurweb.models import License, PackageLicense from aurweb.models.account_type import USER_ID, AccountType from aurweb.models.dependency_type import DependencyType @@ -1536,6 +1536,24 @@ def test_pkgbase_request_post_deletion(client: TestClient, user: User, assert pkgreq.Comments == "We want to delete this." +def test_pkgbase_request_post_maintainer_deletion( + client: TestClient, maintainer: User, package: Package): + pkgbasename = package.PackageBase.Name + endpoint = f"/pkgbase/{package.PackageBase.Name}/request" + cookies = {"AURSID": maintainer.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, data={ + "type": "deletion", + "comments": "We want to delete this." + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + pkgreq = db.query(PackageRequest).filter( + PackageRequest.PackageBaseName == pkgbasename + ).first() + assert pkgreq.Status == ACCEPTED_ID + + def test_pkgbase_request_post_orphan(client: TestClient, user: User, package: Package): endpoint = f"/pkgbase/{package.PackageBase.Name}/request" @@ -1556,6 +1574,29 @@ def test_pkgbase_request_post_orphan(client: TestClient, user: User, assert pkgreq.Comments == "We want to disown this." +def test_pkgbase_request_post_auto_orphan(client: TestClient, user: User, + package: Package): + now = int(datetime.utcnow().timestamp()) + auto_orphan_age = config.getint("options", "auto_orphan_age") + with db.begin(): + package.PackageBase.OutOfDateTS = now - auto_orphan_age - 1 + + endpoint = f"/pkgbase/{package.PackageBase.Name}/request" + cookies = {"AURSID": user.login(Request(), "testPassword")} + with client as request: + resp = request.post(endpoint, data={ + "type": "orphan", + "comments": "We want to disown this." + }, cookies=cookies, allow_redirects=False) + assert resp.status_code == int(HTTPStatus.SEE_OTHER) + + pkgreq = db.query(PackageRequest).filter( + PackageRequest.PackageBaseID == package.PackageBase.ID + ).first() + assert pkgreq is not None + assert pkgreq.Status == ACCEPTED_ID + + def test_pkgbase_request_post_merge(client: TestClient, user: User, package: Package): with db.begin(): From f897411ddf7849123a3df72ae841b27a81bd12bf Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 18 Nov 2021 21:17:40 -0800 Subject: [PATCH 596/844] change(fastapi): let conftest bypass create database errors Signed-off-by: Kevin Morris --- test/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index 47d9ca4b..aa44831a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -43,6 +43,7 @@ from filelock import FileLock from sqlalchemy import create_engine from sqlalchemy.engine import URL from sqlalchemy.engine.base import Engine +from sqlalchemy.exc import OperationalError from sqlalchemy.orm import scoped_session import aurweb.config @@ -98,7 +99,10 @@ def _create_database(engine: Engine, dbname: str) -> None: :param dbname: Database name to create """ conn = engine.connect() - conn.execute(f"CREATE DATABASE {dbname}") + try: + conn.execute(f"CREATE DATABASE {dbname}") + except OperationalError: # pragma: no cover + pass conn.close() initdb.run(AlembicArgs) From 008a8824ceb7f243f58c7d228f7cd9c1152d7386 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 20 Nov 2021 13:19:34 -0800 Subject: [PATCH 597/844] housekeep(fastapi): simplify package_base_comaintainers_post Signed-off-by: Kevin Morris --- aurweb/packages/util.py | 88 ++++++++++++++++++++++++++++++++++++-- aurweb/routers/packages.py | 73 ++----------------------------- 2 files changed, 88 insertions(+), 73 deletions(-) diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index 78f5bf18..7c48f4e4 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -4,13 +4,14 @@ from typing import Dict, List, Union import orjson -from fastapi import HTTPException +from fastapi import HTTPException, Request from sqlalchemy import and_, orm -from aurweb import db, models +from aurweb import db, l10n, models, util from aurweb.models.official_provider import OFFICIAL_BASE from aurweb.models.relation_type import PROVIDES_ID from aurweb.redis import redis_connection +from aurweb.scripts import notify from aurweb.templates import register_filter @@ -223,6 +224,85 @@ def query_notified(query: List[models.Package], ).filter( models.PackageNotification.UserID == user.ID ) - for notify in notified: - output[notify.PackageBase.ID] = True + for notif in notified: + output[notif.PackageBase.ID] = True return output + + +def remove_comaintainers(pkgbase: models.PackageBase, + usernames: List[str]) -> None: + """ + Remove comaintainers from `pkgbase`. + + :param pkgbase: PackageBase instance + :param usernames: Iterable of username strings + :return: None + """ + conn = db.ConnectionExecutor(db.get_engine().raw_connection()) + notifications = [] + with db.begin(): + for username in usernames: + # We know that the users we passed here are in the DB. + # No need to check for their existence. + comaintainer = pkgbase.comaintainers.join(models.User).filter( + models.User.Username == username + ).first() + notifications.append( + notify.ComaintainerRemoveNotification( + conn, comaintainer.User.ID, pkgbase.ID + ) + ) + db.delete(comaintainer) + + # Send out notifications if need be. + util.apply_all(notifications, lambda n: n.send()) + + +def add_comaintainers(request: Request, pkgbase: models.PackageBase, + priority: int, usernames: List[str]) -> None: + """ + Add comaintainers to `pkgbase`. + + :param request: FastAPI request + :param pkgbase: PackageBase instance + :param priority: Initial priority value + :param usernames: Iterable of username strings + :return: None on success, an error string on failure + """ + + # First, perform a check against all usernames given; for each + # username, add its related User object to memo. + _ = l10n.get_translator_for_request(request) + memo = {} + for username in usernames: + user = db.query(models.User).filter( + models.User.Username == username).first() + if not user: + return _("Invalid user name: %s") % username + memo[username] = user + + # Alright, now that we got past the check, add them all to the DB. + conn = db.ConnectionExecutor(db.get_engine().raw_connection()) + notifications = [] + with db.begin(): + for username in usernames: + user = memo.get(username) + if pkgbase.Maintainer == user: + # Already a maintainer. Move along. + continue + + # If we get here, our user model object is in the memo. + comaintainer = db.create( + models.PackageComaintainer, + PackageBase=pkgbase, + User=user, + Priority=priority) + priority += 1 + + notifications.append( + notify.ComaintainerAddNotification( + conn, comaintainer.User.ID, pkgbase.ID) + ) + + # Send out notifications. + util.apply_all(notifications, lambda n: n.send()) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index dfb8e108..23f44ee3 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -15,6 +15,7 @@ from aurweb.exceptions import ValidationError from aurweb.models.package_request import ACCEPTED_ID, PENDING_ID, REJECTED_ID from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID from aurweb.models.request_type import DELETION_ID, MERGE, MERGE_ID +from aurweb.packages import util as pkgutil from aurweb.packages import validate from aurweb.packages.search import PackageSearch from aurweb.packages.util import get_pkg_or_base, get_pkgbase_comment, get_pkgreq_by_id, query_notified, query_voted @@ -531,28 +532,6 @@ async def package_base_comaintainers(request: Request, name: str) -> Response: return render_template(request, "pkgbase/comaintainers.html", context) -def remove_users(pkgbase, usernames): - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) - notifications = [] - with db.begin(): - for username in usernames: - # We know that the users we passed here are in the DB. - # No need to check for their existence. - comaintainer = pkgbase.comaintainers.join(models.User).filter( - models.User.Username == username - ).first() - notifications.append( - notify.ComaintainerRemoveNotification( - conn, comaintainer.User.ID, pkgbase.ID - ) - ) - db.delete(comaintainer) - - # Send out notifications if need be. - for notify_ in notifications: - notify_.send() - - @router.post("/pkgbase/{name}/comaintainers") @auth_required(True, redirect="/pkgbase/{name}/comaintainers") async def package_base_comaintainers_post( @@ -573,7 +552,7 @@ async def package_base_comaintainers_post( users.remove(str()) # Remove any empty strings from the set. records = {c.User.Username for c in pkgbase.comaintainers} - remove_users(pkgbase, records.difference(users)) + pkgutil.remove_comaintainers(pkgbase, records.difference(users)) # Default priority (lowest value; most preferred). priority = 1 @@ -590,52 +569,8 @@ async def package_base_comaintainers_post( if last_priority: priority = last_priority.Priority + 1 - def add_users(usernames): - """ Add users as comaintainers to pkgbase. - - :param usernames: An iterable of username strings - :return: None on success, an error string on failure. """ - nonlocal request, pkgbase, priority - - # First, perform a check against all usernames given; for each - # username, add its related User object to memo. - _ = l10n.get_translator_for_request(request) - memo = {} - for username in usernames: - user = db.query(models.User).filter( - models.User.Username == username).first() - if not user: - return _("Invalid user name: %s") % username - memo[username] = user - - # Alright, now that we got past the check, add them all to the DB. - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) - notifications = [] - with db.begin(): - for username in usernames: - user = memo.get(username) - if pkgbase.Maintainer == user: - # Already a maintainer. Move along. - continue - - # If we get here, our user model object is in the memo. - comaintainer = db.create( - models.PackageComaintainer, - PackageBase=pkgbase, - User=user, - Priority=priority) - priority += 1 - - notifications.append( - notify.ComaintainerAddNotification( - conn, comaintainer.User.ID, pkgbase.ID) - ) - - # Send out notifications. - for notify_ in notifications: - notify_.send() - - error = add_users(users.difference(records)) + error = pkgutil.add_comaintainers(request, pkgbase, priority, + users.difference(records)) if error: context = make_context(request, "Manage Co-maintainers") context["pkgbase"] = pkgbase From 0b5d08801615c3afa604d15d0f5db4f03d600b3d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 20 Nov 2021 13:18:48 -0800 Subject: [PATCH 598/844] fix(fastapi): catch ProgrammingError instead of OperationalError in conftest Signed-off-by: Kevin Morris --- test/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index aa44831a..db2e5997 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -43,7 +43,7 @@ from filelock import FileLock from sqlalchemy import create_engine from sqlalchemy.engine import URL from sqlalchemy.engine.base import Engine -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import ProgrammingError from sqlalchemy.orm import scoped_session import aurweb.config @@ -101,7 +101,7 @@ def _create_database(engine: Engine, dbname: str) -> None: conn = engine.connect() try: conn.execute(f"CREATE DATABASE {dbname}") - except OperationalError: # pragma: no cover + except ProgrammingError: # pragma: no cover pass conn.close() initdb.run(AlembicArgs) From 191198ca41a10358804a5d1cdc37a35ca9be7bd6 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 20 Nov 2021 13:31:09 -0800 Subject: [PATCH 599/844] housekeep(fastapi): simplify aurweb.spawn.stop() Signed-off-by: Kevin Morris --- aurweb/spawn.py | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/aurweb/spawn.py b/aurweb/spawn.py index 6d553dde..568a8a1d 100644 --- a/aurweb/spawn.py +++ b/aurweb/spawn.py @@ -18,6 +18,8 @@ import tempfile import time import urllib +from typing import Iterable, List + import aurweb.config import aurweb.schema @@ -127,17 +129,15 @@ def start(): spawn_child(["nginx", "-p", temporary_dir, "-c", generate_nginx_config()]) -def stop(): +def _kill_children(children: Iterable, exceptions: List[Exception] = []) \ + -> List[Exception]: """ - Stop all the child processes. + Kill each process found in `children`. - If an exception occurs during the process, the process continues anyway - because we don’t want to leave runaway processes around, and all the - exceptions are finally raised as a single ProcessExceptions. + :param children: Iterable of child processes + :param exceptions: Exception memo + :return: `exceptions` """ - global children - atexit.unregister(stop) - exceptions = [] for p in children: try: p.terminate() @@ -145,6 +145,18 @@ def stop(): print(f":: Sent SIGTERM to {p.args}", file=sys.stderr) except Exception as e: exceptions.append(e) + return exceptions + + +def _wait_for_children(children: Iterable, exceptions: List[Exception] = []) \ + -> List[Exception]: + """ + Wait for each process to end found in `children`. + + :param children: Iterable of child processes + :param exceptions: Exception memo + :return: `exceptions` + """ for p in children: try: rc = p.wait() @@ -154,6 +166,24 @@ def stop(): raise Exception(f"Process {p.args} exited with {rc}") except Exception as e: exceptions.append(e) + return exceptions + + +def stop() -> None: + """ + Stop all the child processes. + + If an exception occurs during the process, the process continues anyway + because we don’t want to leave runaway processes around, and all the + exceptions are finally raised as a single ProcessExceptions. + + :raises: ProcessException + :return: None + """ + global children + atexit.unregister(stop) + exceptions = _kill_children(children) + exceptions = _wait_for_children(children, exceptions) children = [] if exceptions: raise ProcessExceptions("Errors terminating the child processes:", From 82ca4ad9a0c399e25a2369509df005ba5a5c6860 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 20 Nov 2021 15:33:19 -0800 Subject: [PATCH 600/844] feat: check php configuration in aurweb.spawn Signed-off-by: Kevin Morris --- aurweb/spawn.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/aurweb/spawn.py b/aurweb/spawn.py index 568a8a1d..ecb759a5 100644 --- a/aurweb/spawn.py +++ b/aurweb/spawn.py @@ -23,11 +23,16 @@ from typing import Iterable, List import aurweb.config import aurweb.schema +from aurweb.exceptions import AurwebException + children = [] temporary_dir = None verbosity = 0 asgi_backend = '' +PHP_BINARY = os.environ.get("PHP_BINARY", "php") +PHP_MODULES = ["pdo_mysql", "pdo_sqlite"] + class ProcessExceptions(Exception): """ @@ -42,6 +47,35 @@ class ProcessExceptions(Exception): super().__init__("\n- ".join(messages)) +def validate_php_config() -> None: + """ + Perform a validation check against PHP_BINARY's configuration. + + AurwebException is raised here if checks fail to pass. We require + the 'pdo_mysql' and 'pdo_sqlite' modules to be enabled. + + :raises: AurwebException + :return: None + """ + try: + proc = subprocess.Popen([PHP_BINARY, "-m"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, _ = proc.communicate() + except FileNotFoundError: + raise AurwebException(f"Unable to locate the '{PHP_BINARY}' " + "executable.") + + assert proc.returncode == 0, ("Received non-zero error code " + f"{proc.returncode} from '{PHP_BINARY}'.") + + modules = out.decode().splitlines() + for module in PHP_MODULES: + if module not in modules: + raise AurwebException( + f"PHP does not have the '{module}' module enabled.") + + def generate_nginx_config(): """ Generate an nginx configuration based on aurweb's configuration. @@ -199,6 +233,13 @@ if __name__ == '__main__': parser.add_argument('-b', '--backend', choices=['hypercorn', 'uvicorn'], default='hypercorn', help='asgi backend used to launch the python server') args = parser.parse_args() + + try: + validate_php_config() + except AurwebException as exc: + print(f"error: {str(exc)}") + sys.exit(1) + verbosity = args.verbose asgi_backend = args.backend with tempfile.TemporaryDirectory(prefix="aurweb-") as tmpdirname: From 47d0df76e6f377a360903f48a96bb32e54884dfc Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 20 Nov 2021 15:37:47 -0800 Subject: [PATCH 601/844] feat: support gunicorn in aurweb.spawn This also comes with a -w|--workers argument that allows the caller to set the number of gunicorn workers. Signed-off-by: Kevin Morris --- aurweb/spawn.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/aurweb/spawn.py b/aurweb/spawn.py index ecb759a5..5b4dbe94 100644 --- a/aurweb/spawn.py +++ b/aurweb/spawn.py @@ -29,6 +29,7 @@ children = [] temporary_dir = None verbosity = 0 asgi_backend = '' +workers = 1 PHP_BINARY = os.environ.get("PHP_BINARY", "php") PHP_MODULES = ["pdo_mysql", "pdo_sqlite"] @@ -152,12 +153,25 @@ def start(): spawn_child(["php", "-S", php_address, "-t", htmldir]) # FastAPI - host, port = aurweb.config.get("fastapi", "bind_address").rsplit(":", 1) - if asgi_backend == "hypercorn": - portargs = ["-b", f"{host}:{port}"] - elif asgi_backend == "uvicorn": - portargs = ["--host", host, "--port", port] - spawn_child(["python", "-m", asgi_backend] + portargs + ["aurweb.asgi:app"]) + fastapi_host, fastapi_port = aurweb.config.get( + "fastapi", "bind_address").rsplit(":", 1) + + # Logging config. + aurwebdir = aurweb.config.get("options", "aurwebdir") + fastapi_log_config = os.path.join(aurwebdir, "logging.conf") + + backend_args = { + "hypercorn": ["-b", f"{fastapi_host}:{fastapi_port}"], + "uvicorn": ["--host", fastapi_host, "--port", fastapi_port], + "gunicorn": ["--bind", f"{fastapi_host}:{fastapi_port}", + "-k", "uvicorn.workers.UvicornWorker", + "-w", str(workers)] + } + backend_args = backend_args.get(asgi_backend) + spawn_child([ + "python", "-m", asgi_backend, + "--log-config", fastapi_log_config, + ] + backend_args + ["aurweb.asgi:app"]) # nginx spawn_child(["nginx", "-p", temporary_dir, "-c", generate_nginx_config()]) @@ -230,8 +244,11 @@ if __name__ == '__main__': description='Start aurweb\'s test server.') parser.add_argument('-v', '--verbose', action='count', default=0, help='increase verbosity') - parser.add_argument('-b', '--backend', choices=['hypercorn', 'uvicorn'], default='hypercorn', + choices = ['hypercorn', 'gunicorn', 'uvicorn'] + parser.add_argument('-b', '--backend', choices=choices, default='uvicorn', help='asgi backend used to launch the python server') + parser.add_argument("-w", "--workers", default=1, type=int, + help="number of workers to use in gunicorn") args = parser.parse_args() try: @@ -242,6 +259,7 @@ if __name__ == '__main__': verbosity = args.verbose asgi_backend = args.backend + workers = args.workers with tempfile.TemporaryDirectory(prefix="aurweb-") as tmpdirname: temporary_dir = tmpdirname start() From 19191fa8b56c47d256b2c8992853d96f82cabc5a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 20 Nov 2021 15:38:20 -0800 Subject: [PATCH 602/844] fix: update nginx config in aurweb.spawn Host a specific FastAPI nginx frontend as well as a PHP nginx frontend, configurable by the (PHP|FASTAPI)_NGINX_PORT environment variables. Signed-off-by: Kevin Morris --- aurweb/spawn.py | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/aurweb/spawn.py b/aurweb/spawn.py index 5b4dbe94..46f2f021 100644 --- a/aurweb/spawn.py +++ b/aurweb/spawn.py @@ -16,7 +16,6 @@ import subprocess import sys import tempfile import time -import urllib from typing import Iterable, List @@ -33,6 +32,8 @@ workers = 1 PHP_BINARY = os.environ.get("PHP_BINARY", "php") PHP_MODULES = ["pdo_mysql", "pdo_sqlite"] +PHP_NGINX_PORT = int(os.environ.get("PHP_NGINX_PORT", 8001)) +FASTAPI_NGINX_PORT = int(os.environ.get("FASTAPI_NGINX_PORT", 8002)) class ProcessExceptions(Exception): @@ -83,8 +84,10 @@ def generate_nginx_config(): The file is generated under `temporary_dir`. Returns the path to the created configuration file. """ - aur_location = aurweb.config.get("options", "aur_location") - aur_location_parts = urllib.parse.urlsplit(aur_location) + php_bind = aurweb.config.get("php", "bind_address") + php_host = php_bind.split(":")[0] + fastapi_bind = aurweb.config.get("fastapi", "bind_address") + fastapi_host = fastapi_bind.split(":")[0] config_path = os.path.join(temporary_dir, "nginx.conf") config = open(config_path, "w") # We double nginx's braces because they conflict with Python's f-strings. @@ -101,12 +104,23 @@ def generate_nginx_config(): uwsgi_temp_path {os.path.join(temporary_dir, "uwsgi")}; scgi_temp_path {os.path.join(temporary_dir, "scgi")}; server {{ - listen {aur_location_parts.netloc}; + listen {php_host}:{PHP_NGINX_PORT}; location / {{ - proxy_pass http://{aurweb.config.get("php", "bind_address")}; + proxy_pass http://{php_bind}; }} - location /sso {{ - proxy_pass http://{aurweb.config.get("fastapi", "bind_address")}; + }} + server {{ + listen {fastapi_host}:{FASTAPI_NGINX_PORT}; + location / {{ + try_files $uri @proxy_to_app; + }} + location @proxy_to_app {{ + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + proxy_buffering off; + proxy_pass http://{fastapi_bind}; }} }} }} @@ -149,6 +163,7 @@ def start(): # PHP php_address = aurweb.config.get("php", "bind_address") + php_host = php_address.split(":")[0] htmldir = aurweb.config.get("php", "htmldir") spawn_child(["php", "-S", php_address, "-t", htmldir]) @@ -176,6 +191,18 @@ def start(): # nginx spawn_child(["nginx", "-p", temporary_dir, "-c", generate_nginx_config()]) + print(f""" + > Started nginx. + > + > PHP backend: http://{php_address} + > FastAPI backend: http://{fastapi_host}:{fastapi_port} + > + > PHP frontend: http://{php_host}:{PHP_NGINX_PORT} + > FastAPI frontend: http://{fastapi_host}:{FASTAPI_NGINX_PORT} + > + > Frontends are hosted via nginx and should be preferred. +""") + def _kill_children(children: Iterable, exceptions: List[Exception] = []) \ -> List[Exception]: From 233d25b1c3434221871a7ccb04a7897c3213c33d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 20 Nov 2021 15:39:15 -0800 Subject: [PATCH 603/844] feat: add test_spawn, an aurweb.spawn test Signed-off-by: Kevin Morris --- test/test_spawn.py | 149 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 test/test_spawn.py diff --git a/test/test_spawn.py b/test/test_spawn.py new file mode 100644 index 00000000..195eb897 --- /dev/null +++ b/test/test_spawn.py @@ -0,0 +1,149 @@ +import os +import tempfile + +from typing import Tuple +from unittest import mock + +import pytest + +import aurweb.config +import aurweb.spawn + +from aurweb.exceptions import AurwebException + +# Some os.environ overrides we use in this suite. +TEST_ENVIRONMENT = { + "PHP_NGINX_PORT": "8001", + "FASTAPI_NGINX_PORT": "8002" +} + + +class FakeProcess: + """ Fake a subprocess.Popen return object. """ + + returncode = 0 + stdout = b'' + stderr = b'' + + def __init__(self, *args, **kwargs): + """ We need this constructor to remain compatible with Popen. """ + pass + + def communicate(self) -> Tuple[bytes, bytes]: + return (self.stdout, self.stderr) + + def terminate(self) -> None: + raise Exception("Fake termination.") + + def wait(self) -> int: + return self.returncode + + +class MockFakeProcess: + """ FakeProcess construction helper to be used in mocks. """ + + def __init__(self, return_code: int = 0, stdout: bytes = b'', + stderr: bytes = b''): + self.returncode = return_code + self.stdout = stdout + self.stderr = stderr + + def process(self, *args, **kwargs) -> FakeProcess: + proc = FakeProcess() + proc.returncode = self.returncode + proc.stdout = self.stdout + proc.stderr = self.stderr + return proc + + +@mock.patch("aurweb.spawn.PHP_BINARY", "does-not-exist") +def test_spawn(): + match = r"^Unable to locate the '.*' executable\.$" + with pytest.raises(AurwebException, match=match): + aurweb.spawn.validate_php_config() + + +@mock.patch("subprocess.Popen", side_effect=MockFakeProcess(1).process) +def test_spawn_non_zero_php_binary(fake_process: FakeProcess): + match = r"^Received non-zero error code.*$" + with pytest.raises(AssertionError, match=match): + aurweb.spawn.validate_php_config() + + +def test_spawn_missing_modules(): + side_effect = MockFakeProcess(stdout=b"pdo_sqlite").process + with mock.patch("subprocess.Popen", side_effect=side_effect): + match = r"PHP does not have the 'pdo_mysql' module enabled\.$" + with pytest.raises(AurwebException, match=match): + aurweb.spawn.validate_php_config() + + side_effect = MockFakeProcess(stdout=b"pdo_mysql").process + with mock.patch("subprocess.Popen", side_effect=side_effect): + match = r"PHP does not have the 'pdo_sqlite' module enabled\.$" + with pytest.raises(AurwebException, match=match): + aurweb.spawn.validate_php_config() + + +@mock.patch.dict("os.environ", TEST_ENVIRONMENT) +def test_spawn_generate_nginx_config(): + ctx = tempfile.TemporaryDirectory() + with ctx and mock.patch("aurweb.spawn.temporary_dir", ctx.name): + aurweb.spawn.generate_nginx_config() + nginx_config_path = os.path.join(ctx.name, "nginx.conf") + with open(nginx_config_path) as f: + nginx_config = f.read().rstrip() + + php_address = aurweb.config.get("php", "bind_address") + php_host = php_address.split(":")[0] + fastapi_address = aurweb.config.get("fastapi", "bind_address") + fastapi_host = fastapi_address.split(":")[0] + expected_content = [ + f'listen {php_host}:{TEST_ENVIRONMENT.get("PHP_NGINX_PORT")}', + f"proxy_pass http://{php_address}", + f'listen {fastapi_host}:{TEST_ENVIRONMENT.get("FASTAPI_NGINX_PORT")}', + f"proxy_pass http://{fastapi_address}" + ] + for expected in expected_content: + assert expected in nginx_config + + +@mock.patch("aurweb.spawn.asgi_backend", "uvicorn") +@mock.patch("aurweb.spawn.verbosity", 1) +@mock.patch("aurweb.spawn.workers", 1) +def test_spawn_start_stop(): + ctx = tempfile.TemporaryDirectory() + with ctx and mock.patch("aurweb.spawn.temporary_dir", ctx.name): + aurweb.spawn.start() + aurweb.spawn.stop() + + +@mock.patch("aurweb.spawn.asgi_backend", "uvicorn") +@mock.patch("aurweb.spawn.verbosity", 1) +@mock.patch("aurweb.spawn.workers", 1) +@mock.patch("aurweb.spawn.children", [MockFakeProcess().process()]) +def test_spawn_start_noop_with_children(): + aurweb.spawn.start() + + +@mock.patch("aurweb.spawn.asgi_backend", "uvicorn") +@mock.patch("aurweb.spawn.verbosity", 1) +@mock.patch("aurweb.spawn.workers", 1) +@mock.patch("aurweb.spawn.children", [MockFakeProcess().process()]) +def test_spawn_stop_terminate_failure(): + ctx = tempfile.TemporaryDirectory() + with ctx and mock.patch("aurweb.spawn.temporary_dir", ctx.name): + match = r"^Errors terminating the child processes" + with pytest.raises(aurweb.spawn.ProcessExceptions, match=match): + aurweb.spawn.stop() + + +@mock.patch("aurweb.spawn.asgi_backend", "uvicorn") +@mock.patch("aurweb.spawn.verbosity", 1) +@mock.patch("aurweb.spawn.workers", 1) +@mock.patch("aurweb.spawn.children", [MockFakeProcess(1).process()]) +def test_spawn_stop_wait_failure(): + ctx = tempfile.TemporaryDirectory() + with ctx and mock.patch("aurweb.spawn.temporary_dir", ctx.name): + match = r"^Errors terminating the child processes" + with pytest.raises(aurweb.spawn.ProcessExceptions, match=match): + aurweb.spawn.stop() From ba3ef742ceec27a2667d579ef78eb0fc36f1a364 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 20 Nov 2021 18:40:32 -0800 Subject: [PATCH 604/844] feat(docker): allow user-customizable ssh host keys There is a new ./data bind mount used here. If ssh_host_* keys are in ./data when the git service starts, they'll override the container-generated host keys. Signed-off-by: Kevin Morris --- docker-compose.aur-dev.yml | 1 + docker/git-entrypoint.sh | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/docker-compose.aur-dev.yml b/docker-compose.aur-dev.yml index 1db306cc..484f353a 100644 --- a/docker-compose.aur-dev.yml +++ b/docker-compose.aur-dev.yml @@ -18,6 +18,7 @@ services: restart: always volumes: - ${GIT_DATA_DIR}:/aurweb/aur.git + - ./data:/aurweb/data - cache:/cache smartgit: diff --git a/docker/git-entrypoint.sh b/docker/git-entrypoint.sh index 296c1e47..4d15bcb9 100755 --- a/docker/git-entrypoint.sh +++ b/docker/git-entrypoint.sh @@ -60,6 +60,13 @@ sed -ri "s|^(ssh-cmdline) = .+|\1 = $ssh_cmdline|" $AUR_CONFIG_DEFAULTS # Setup SSH Keys. ssh-keygen -A +# In docker-compose.aur-dev.yml, we bind ./data to /aurweb/data. +# Production users wishing to include their own SSH keys should +# supply them in ./data. +if [ -d /aurweb/data ]; then + find /aurweb/data -type f -name 'ssh_host_*' -exec cp -vf "{}" /etc/ssh/ \; +fi + # Taken from INSTALL. mkdir -pv $GIT_REPO From a1e547c057da8a2391c94a6d51c7c04fe37ad71b Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 20 Nov 2021 19:03:35 -0800 Subject: [PATCH 605/844] feat(docker): allow configurable SSH_CMDLINE in git service Signed-off-by: Kevin Morris --- docker-compose.aur-dev.yml | 4 ++++ docker-compose.yml | 1 + docker/git-entrypoint.sh | 4 ++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker-compose.aur-dev.yml b/docker-compose.aur-dev.yml index 484f353a..62109deb 100644 --- a/docker-compose.aur-dev.yml +++ b/docker-compose.aur-dev.yml @@ -16,6 +16,10 @@ services: git: restart: always + environment: + - AUR_CONFIG=/aurweb/conf/config + # SSH_CMDLINE should be updated to production's ssh cmdline. + - SSH_CMDLINE=${SSH_CMDLINE:-ssh ssh://aur@localhost:2222} volumes: - ${GIT_DATA_DIR}:/aurweb/aur.git - ./data:/aurweb/data diff --git a/docker-compose.yml b/docker-compose.yml index c39d38bf..26b7d62c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,6 +100,7 @@ services: init: true environment: - AUR_CONFIG=/aurweb/conf/config + - SSH_CMDLINE=${SSH_CMDLINE:-ssh ssh://aur@localhost:2222} entrypoint: /docker/git-entrypoint.sh command: /docker/scripts/run-sshd.sh ports: diff --git a/docker/git-entrypoint.sh b/docker/git-entrypoint.sh index 4d15bcb9..cfa1879b 100755 --- a/docker/git-entrypoint.sh +++ b/docker/git-entrypoint.sh @@ -54,8 +54,8 @@ fi # Set some defaults needed for pathing and ssh uris. sed -ri "s|^(repo-path) = .+|\1 = /aurweb/aur.git/|" $AUR_CONFIG_DEFAULTS -ssh_cmdline='ssh ssh://aur@localhost:2222' -sed -ri "s|^(ssh-cmdline) = .+|\1 = $ssh_cmdline|" $AUR_CONFIG_DEFAULTS +# SSH_CMDLINE can be provided via override in docker-compose.aur-dev.yml. +sed -ri "s|^(ssh-cmdline) = .+$|\1 = ${SSH_CMDLINE}|" $AUR_CONFIG_DEFAULTS # Setup SSH Keys. ssh-keygen -A From c7feecd4b83fde3aa0e1a1e392035be8fb32385e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 20 Nov 2021 19:34:33 -0800 Subject: [PATCH 606/844] housekeep(docker): remove configuration regexes in the nginx service Signed-off-by: Kevin Morris --- docker/nginx-entrypoint.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docker/nginx-entrypoint.sh b/docker/nginx-entrypoint.sh index 63307948..a58e67b7 100755 --- a/docker/nginx-entrypoint.sh +++ b/docker/nginx-entrypoint.sh @@ -11,17 +11,6 @@ KEY=/cache/production.key.pem DEST_CERT=/etc/ssl/certs/web.cert.pem DEST_KEY=/etc/ssl/private/web.key.pem -# Setup a config for our mysql db. -cp -vf conf/config.dev conf/config -sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config -sed -ri 's/^(host) = .+/\1 = mariadb/' conf/config -sed -ri 's/^(user) = .+/\1 = aur/' conf/config -sed -ri 's/^;?(password) = .+/\1 = aur/' conf/config - -# Setup http(s) stuff. -sed -ri "s|^(aur_location) = .+|\1 = https://localhost:8444|" conf/config -sed -ri 's/^(disable_http_login) = .+/\1 = 1/' conf/config - if [ -f "$CERT" ]; then cp -vf "$CERT" "$DEST_CERT" cp -vf "$KEY" "$DEST_KEY" From 604901fe7475912705e040ab40a509c80d109289 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 20 Nov 2021 20:00:53 -0800 Subject: [PATCH 607/844] fix(docker): fix nginx .gz match against cgit snapshots This only deals with .gz files in the root of the request_uri and now more. That is: /packages.gz goes through the nginx regex, but now /cgit/.../snapshot/package.tar.gz is served by the cgit block. Signed-off-by: Kevin Morris --- docker/config/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/config/nginx.conf b/docker/config/nginx.conf index c3ffd7fa..e51bd64f 100644 --- a/docker/config/nginx.conf +++ b/docker/config/nginx.conf @@ -94,7 +94,7 @@ http { ssl_certificate /etc/ssl/certs/web.cert.pem; ssl_certificate_key /etc/ssl/private/web.key.pem; - location ~ ^/.*\.gz$ { + location ~ ^/[^\/]+\.gz$ { # Override mime type to text/plain. types { text/plain gz; } default_type text/plain; From d4d9f50b8f540062a3191e254a16f2e4497b7b8f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 20 Nov 2021 20:05:04 -0800 Subject: [PATCH 608/844] change(docker): use ./data instead of ./cache For the `git` service, ./data is always used to provide an optional overriding of ssh host keys. In aur-dev production containers, most services which use the data mount use an internal Docker `data` volume instead. Signed-off-by: Kevin Morris --- docker-compose.aur-dev.yml | 13 ++++++------ docker-compose.override.yml | 12 +++++------ docker-compose.yml | 6 +++--- docker/ca-entrypoint.sh | 38 +++++++++++++++++------------------ docker/cgit-entrypoint.sh | 2 +- docker/nginx-entrypoint.sh | 8 ++++---- docker/scripts/run-fastapi.sh | 12 +++++------ docker/scripts/run-nginx.sh | 2 +- docker/scripts/run-pytests.sh | 10 ++++----- docker/scripts/run-tests.sh | 10 ++++----- 10 files changed, 56 insertions(+), 57 deletions(-) diff --git a/docker-compose.aur-dev.yml b/docker-compose.aur-dev.yml index 62109deb..ab4ff124 100644 --- a/docker-compose.aur-dev.yml +++ b/docker-compose.aur-dev.yml @@ -3,7 +3,7 @@ version: "3.8" services: ca: volumes: - - cache:/cache + - data:/data memcached: restart: always @@ -23,13 +23,12 @@ services: volumes: - ${GIT_DATA_DIR}:/aurweb/aur.git - ./data:/aurweb/data - - cache:/cache smartgit: restart: always volumes: - ${GIT_DATA_DIR}:/aurweb/aur.git - - cache:/cache + - data:/data - smartgit_run:/var/run/smartgit cgit-php: @@ -48,7 +47,7 @@ services: - AURWEB_PHP_PREFIX=${AURWEB_PHP_PREFIX} - AURWEB_SSHD_PREFIX=${AURWEB_SSHD_PREFIX} volumes: - - cache:/cache + - data:/data fastapi: restart: always @@ -60,13 +59,13 @@ services: - AURWEB_SSHD_PREFIX=${AURWEB_SSHD_PREFIX} - PROMETHEUS_MULTIPROC_DIR=/tmp_prometheus volumes: - - cache:/cache + - data:/data nginx: restart: always volumes: - ${GIT_DATA_DIR}:/aurweb/aur.git - - cache:/cache + - data:/data - logs:/var/log/nginx - smartgit_run:/var/run/smartgit @@ -75,5 +74,5 @@ volumes: mariadb_data: {} # Share /var/lib/mysql git_data: {} # Share aurweb/aur.git smartgit_run: {} - cache: {} + data: {} logs: {} diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 7349ac66..eae12a92 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -8,17 +8,17 @@ services: ca: volumes: - - ./cache:/cache + - ./data:/data git: volumes: - git_data:/aurweb/aur.git - - ./cache:/cache + - ./data:/aurweb/data smartgit: volumes: - git_data:/aurweb/aur.git - - ./cache:/cache + - ./data:/data - smartgit_run:/var/run/smartgit depends_on: mariadb: @@ -26,7 +26,7 @@ services: php-fpm: volumes: - - ./cache:/cache + - ./data:/data - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations - ./test:/aurweb/test @@ -37,7 +37,7 @@ services: fastapi: volumes: - - ./cache:/cache + - ./data:/data - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations - ./test:/aurweb/test @@ -49,7 +49,7 @@ services: nginx: volumes: - git_data:/aurweb/aur.git - - ./cache:/cache + - ./data:/data - ./logs:/var/log/nginx - ./web/html:/aurweb/web/html - ./web/template:/aurweb/web/template diff --git a/docker-compose.yml b/docker-compose.yml index 26b7d62c..e3bfacdc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -276,7 +276,7 @@ services: mariadb_test: condition: service_healthy volumes: - - ./cache:/cache + - ./data:/data - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations - ./test:/aurweb/test @@ -304,7 +304,7 @@ services: - /tmp volumes: - mariadb_test_run:/var/run/mysqld - - ./cache:/cache + - ./data:/data - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations - ./test:/aurweb/test @@ -330,7 +330,7 @@ services: condition: service_healthy volumes: - mariadb_test_run:/var/run/mysqld - - ./cache:/cache + - ./data:/data - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations - ./test:/aurweb/test diff --git a/docker/ca-entrypoint.sh b/docker/ca-entrypoint.sh index e95d267c..42d8bd14 100755 --- a/docker/ca-entrypoint.sh +++ b/docker/ca-entrypoint.sh @@ -1,58 +1,58 @@ #!/bin/bash set -eou pipefail -if [ -f /cache/ca.root.pem ]; then +if [ -f /data/ca.root.pem ]; then echo "Already have certs, skipping." exit 0 fi # Generate a new 2048-bit RSA key for the Root CA. -openssl genrsa -des3 -out /cache/ca.key -passout pass:devca 2048 +openssl genrsa -des3 -out /data/ca.key -passout pass:devca 2048 # Request and self-sign a new Root CA certificate, using # the RSA key. Output Root CA PEM-format certificate and key: -# /cache/ca.root.pem and /cache/ca.key.pem +# /data/ca.root.pem and /data/ca.key.pem openssl req -x509 -new -nodes -sha256 -days 1825 \ -passin pass:devca \ -subj "/C=US/ST=California/L=Authority/O=aurweb/CN=localhost" \ - -in /cache/ca.key -out /cache/ca.root.pem -keyout /cache/ca.key.pem + -in /data/ca.key -out /data/ca.root.pem -keyout /data/ca.key.pem # Generate a new 2048-bit RSA key for a localhost server. -openssl genrsa -out /cache/localhost.key 2048 +openssl genrsa -out /data/localhost.key 2048 # Generate a Certificate Signing Request (CSR) for the localhost server # using the RSA key we generated above. -openssl req -new -key /cache/localhost.key -passout pass:devca \ +openssl req -new -key /data/localhost.key -passout pass:devca \ -subj "/C=US/ST=California/L=Server/O=aurweb/CN=localhost" \ - -out /cache/localhost.csr + -out /data/localhost.csr # Get our CSR signed by our Root CA PEM-formatted certificate and key -# to produce a fresh /cache/localhost.cert.pem PEM-formatted certificate. -openssl x509 -req -in /cache/localhost.csr \ - -CA /cache/ca.root.pem -CAkey /cache/ca.key.pem \ +# to produce a fresh /data/localhost.cert.pem PEM-formatted certificate. +openssl x509 -req -in /data/localhost.csr \ + -CA /data/ca.root.pem -CAkey /data/ca.key.pem \ -CAcreateserial \ - -out /cache/localhost.cert.pem \ + -out /data/localhost.cert.pem \ -days 825 -sha256 \ -passin pass:devca \ -extfile /docker/localhost.ext -# Convert RSA key to a PEM-formatted key: /cache/localhost.key.pem -openssl rsa -in /cache/localhost.key -text > /cache/localhost.key.pem +# Convert RSA key to a PEM-formatted key: /data/localhost.key.pem +openssl rsa -in /data/localhost.key -text > /data/localhost.key.pem # At the end here, our notable certificates and keys are: -# - /cache/ca.root.pem -# - /cache/ca.key.pem -# - /cache/localhost.key.pem -# - /cache/localhost.cert.pem +# - /data/ca.root.pem +# - /data/ca.key.pem +# - /data/localhost.key.pem +# - /data/localhost.cert.pem # # When running a server which uses the localhost certificate, a chain # should be used, starting with localhost.cert.pem: -# - cat /cache/localhost.cert.pem /cache/ca.root.pem > localhost.chain.pem +# - cat /data/localhost.cert.pem /data/ca.root.pem > localhost.chain.pem # # The Root CA (ca.root.pem) should be imported into browsers or # ca-certificates on machines wishing to verify localhost. # -chmod 666 /cache/* +chmod 666 /data/* exec "$@" diff --git a/docker/cgit-entrypoint.sh b/docker/cgit-entrypoint.sh index f9ca86c0..a44675e2 100755 --- a/docker/cgit-entrypoint.sh +++ b/docker/cgit-entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/bash set -eou pipefail -mkdir -p /var/cache/cgit +mkdir -p /var/data/cgit cp -vf conf/cgitrc.proto /etc/cgitrc sed -ri "s|clone-prefix=.*|clone-prefix=${CGIT_CLONE_PREFIX}|" /etc/cgitrc diff --git a/docker/nginx-entrypoint.sh b/docker/nginx-entrypoint.sh index a58e67b7..6b9a6954 100755 --- a/docker/nginx-entrypoint.sh +++ b/docker/nginx-entrypoint.sh @@ -5,8 +5,8 @@ set -eou pipefail # user customization of the certificates that FastAPI uses. # Otherwise, fallback to localhost.{cert,key}.pem, generated by `ca`. -CERT=/cache/production.cert.pem -KEY=/cache/production.key.pem +CERT=/data/production.cert.pem +KEY=/data/production.key.pem DEST_CERT=/etc/ssl/certs/web.cert.pem DEST_KEY=/etc/ssl/private/web.key.pem @@ -15,8 +15,8 @@ if [ -f "$CERT" ]; then cp -vf "$CERT" "$DEST_CERT" cp -vf "$KEY" "$DEST_KEY" else - cat /cache/localhost.cert.pem /cache/ca.root.pem > "$DEST_CERT" - cp -vf /cache/localhost.key.pem "$DEST_KEY" + cat /data/localhost.cert.pem /data/ca.root.pem > "$DEST_CERT" + cp -vf /data/localhost.key.pem "$DEST_KEY" fi cp -vf /docker/config/nginx.conf /etc/nginx/nginx.conf diff --git a/docker/scripts/run-fastapi.sh b/docker/scripts/run-fastapi.sh index effc7fe4..ac54aedc 100755 --- a/docker/scripts/run-fastapi.sh +++ b/docker/scripts/run-fastapi.sh @@ -1,15 +1,15 @@ #!/bin/bash -CERT=/cache/localhost.cert.pem -KEY=/cache/localhost.key.pem +CERT=/data/localhost.cert.pem +KEY=/data/localhost.key.pem # If production.{cert,key}.pem exists, prefer them. This allows # user customization of the certificates that FastAPI uses. -if [ -f /cache/production.cert.pem ]; then - CERT=/cache/production.cert.pem +if [ -f /data/production.cert.pem ]; then + CERT=/data/production.cert.pem fi -if [ -f /cache/production.key.pem ]; then - KEY=/cache/production.key.pem +if [ -f /data/production.key.pem ]; then + KEY=/data/production.key.pem fi # By default, set FASTAPI_WORKERS to 2. In production, this should diff --git a/docker/scripts/run-nginx.sh b/docker/scripts/run-nginx.sh index 7780dae8..6ece3303 100755 --- a/docker/scripts/run-nginx.sh +++ b/docker/scripts/run-nginx.sh @@ -8,7 +8,7 @@ echo " (cgit) : https://localhost:8444/cgit/" echo " - PHP : https://localhost:8443/" echo " (cgit) : https://localhost:8443/cgit/" echo -echo " Note: Copy root CA (./cache/ca.root.pem) to ca-certificates or browser." +echo " Note: Copy root CA (./data/ca.root.pem) to ca-certificates or browser." echo echo " Thanks for using aurweb!" echo diff --git a/docker/scripts/run-pytests.sh b/docker/scripts/run-pytests.sh index b8f695df..2eadee42 100755 --- a/docker/scripts/run-pytests.sh +++ b/docker/scripts/run-pytests.sh @@ -32,10 +32,10 @@ pytest if [ $COVERAGE -eq 1 ]; then make -C test coverage - # /cache is mounted as a volume. Copy coverage into it. + # /data is mounted as a volume. Copy coverage into it. # Users can then sanitize the coverage locally in their - # aurweb root directory: ./util/fix-coverage ./cache/.coverage - rm -f /cache/.coverage - cp -v .coverage /cache/.coverage - chmod 666 /cache/.coverage + # aurweb root directory: ./util/fix-coverage ./data/.coverage + rm -f /data/.coverage + cp -v .coverage /data/.coverage + chmod 666 /data/.coverage fi diff --git a/docker/scripts/run-tests.sh b/docker/scripts/run-tests.sh index 45c7835f..a726c957 100755 --- a/docker/scripts/run-tests.sh +++ b/docker/scripts/run-tests.sh @@ -14,12 +14,12 @@ bash $dir/run-pytests.sh --no-coverage make -C test coverage -# /cache is mounted as a volume. Copy coverage into it. +# /data is mounted as a volume. Copy coverage into it. # Users can then sanitize the coverage locally in their -# aurweb root directory: ./util/fix-coverage ./cache/.coverage -rm -f /cache/.coverage -cp -v .coverage /cache/.coverage -chmod 666 /cache/.coverage +# aurweb root directory: ./util/fix-coverage ./data/.coverage +rm -f /data/.coverage +cp -v .coverage /data/.coverage +chmod 666 /data/.coverage # Run flake8 and isort checks. for dir in aurweb test migrations; do From e8f4c9cf69161076a2cc71fcab060f664b327045 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 21 Nov 2021 00:51:05 -0800 Subject: [PATCH 609/844] fix(fastapi): remove aurweb logger definition Both the root and aurweb loggers are included in output, causing repeated log messages. Now, just rely on the root logger for aurweb logging. Signed-off-by: Kevin Morris --- logging.conf | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/logging.conf b/logging.conf index ba41fb7b..310ac76e 100644 --- a/logging.conf +++ b/logging.conf @@ -1,5 +1,5 @@ [loggers] -keys=root,aurweb,test,uvicorn,hypercorn,alembic +keys=root,test,uvicorn,hypercorn,alembic [handlers] keys=simpleHandler,detailedHandler @@ -9,13 +9,7 @@ keys=simpleFormatter,detailedFormatter [logger_root] level=INFO -handlers=simpleHandler - -[logger_aurweb] -level=DEBUG handlers=detailedHandler -qualname=aurweb -propagate=1 [logger_test] level=DEBUG @@ -43,13 +37,13 @@ propagate=0 [handler_simpleHandler] class=StreamHandler -level=DEBUG +level=INFO formatter=simpleFormatter args=(sys.stdout,) [handler_detailedHandler] class=StreamHandler -level=DEBUG +level=INFO formatter=detailedFormatter args=(sys.stdout,) From bc7bf9866ad1f7b3e0ccc9c7b00ffac5d6f72524 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 21 Nov 2021 00:48:53 -0800 Subject: [PATCH 610/844] docker: bind ./aurweb in cron service by default Signed-off-by: Kevin Morris --- docker-compose.aur-dev.yml | 6 ++++++ docker-compose.yml | 1 + 2 files changed, 7 insertions(+) diff --git a/docker-compose.aur-dev.yml b/docker-compose.aur-dev.yml index ab4ff124..4b522e56 100644 --- a/docker-compose.aur-dev.yml +++ b/docker-compose.aur-dev.yml @@ -41,6 +41,12 @@ services: volumes: - ${GIT_DATA_DIR}:/aurweb/aur.git + cron: + volumes: + # Exclude ./aurweb:/aurweb in production. + - mariadb_run:/var/run/mysqld + - archives:/var/lib/aurweb/archives + php-fpm: restart: always environment: diff --git a/docker-compose.yml b/docker-compose.yml index e3bfacdc..ea0e8d1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -176,6 +176,7 @@ services: mariadb_init: condition: service_started volumes: + - ./aurweb:/aurweb/aurweb - mariadb_run:/var/run/mysqld - archives:/var/lib/aurweb/archives From 41e0eaaece5df78b4f9abbb17c4ff702e854ab8a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 21 Nov 2021 21:43:14 -0800 Subject: [PATCH 611/844] fix(docker): force bind ports to localhost only Signed-off-by: Kevin Morris --- docker-compose.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ea0e8d1b..5ff031e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,7 @@ services: test: "bash /docker/health/redis.sh" interval: 3s ports: - - "16379:6379" + - "127.0.0.1:16379:6379" mariadb: image: aurweb:latest @@ -58,7 +58,7 @@ services: ports: # This will expose mariadbd on 127.0.0.1:13306 in the host. # Ex: `mysql -uaur -paur -h 127.0.0.1 -P 13306 aurweb` - - "13306:3306" + - "127.0.0.1:13306:3306" volumes: - mariadb_run:/var/run/mysqld # Bind socket in this volume. - mariadb_data:/var/lib/mysql @@ -88,7 +88,7 @@ services: ports: # This will expose mariadbd on 127.0.0.1:13307 in the host. # Ex: `mysql -uaur -paur -h 127.0.0.1 -P 13306 aurweb` - - "13307:3306" + - "127.0.0.1:13307:3306" volumes: - mariadb_test_run:/var/run/mysqld # Bind socket in this volume. healthcheck: @@ -104,7 +104,7 @@ services: entrypoint: /docker/git-entrypoint.sh command: /docker/scripts/run-sshd.sh ports: - - "2222:2222" + - "127.0.0.1:2222:2222" healthcheck: test: "bash /docker/health/sshd.sh" interval: 3s @@ -141,7 +141,7 @@ services: git: condition: service_healthy ports: - - "13000:3000" + - "127.0.0.1:13000:3000" volumes: - git_data:/aurweb/aur.git @@ -161,7 +161,7 @@ services: git: condition: service_healthy ports: - - "13001:3000" + - "127.0.0.1:13001:3000" volumes: - git_data:/aurweb/aur.git @@ -205,7 +205,7 @@ services: - mariadb_run:/var/run/mysqld - archives:/var/lib/aurweb/archives ports: - - "19000:9000" + - "127.0.0.1:19000:9000" fastapi: image: aurweb:latest @@ -234,7 +234,7 @@ services: volumes: - mariadb_run:/var/run/mysqld ports: - - "18000:8000" + - "127.0.0.1:18000:8000" nginx: image: aurweb:latest @@ -244,8 +244,8 @@ services: entrypoint: /docker/nginx-entrypoint.sh command: /docker/scripts/run-nginx.sh ports: - - "8443:8443" # PHP - - "8444:8444" # FastAPI + - "127.0.0.1:8443:8443" # PHP + - "127.0.0.1:8444:8444" # FastAPI healthcheck: test: "bash /docker/health/nginx.sh" interval: 3s From 34747359ba599d2dda02a404d47a720b7363d367 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 21 Nov 2021 23:11:02 -0800 Subject: [PATCH 612/844] fix(docker): expose git service's 2222 through 0.0.0.0 Other ports we use are locked to 127.0.0.1. The `git` service, however, already promotes security in its sshd service and can't really be abused from an external source. This simplifies the need to forward to localhost if deploy targets want the sshd to be available. Signed-off-by: Kevin Morris --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5ff031e4..acb5dd65 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -104,7 +104,7 @@ services: entrypoint: /docker/git-entrypoint.sh command: /docker/scripts/run-sshd.sh ports: - - "127.0.0.1:2222:2222" + - "2222:2222" healthcheck: test: "bash /docker/health/sshd.sh" interval: 3s From e891d7c8e86b344af705580c9049d59570bed6f3 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 22 Nov 2021 10:18:02 -0800 Subject: [PATCH 613/844] change(docker): allow run-pytests to collect coverage Additionally fix up the argument parsing to be a bit less flexible. Signed-off-by: Kevin Morris --- docker/scripts/run-pytests.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docker/scripts/run-pytests.sh b/docker/scripts/run-pytests.sh index 2eadee42..d8c093d5 100755 --- a/docker/scripts/run-pytests.sh +++ b/docker/scripts/run-pytests.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -eou pipefail COVERAGE=1 PARAMS=() @@ -11,13 +10,13 @@ while [ $# -ne 0 ]; do COVERAGE=0 shift ;; - -*) - echo "usage: $0 [--no-coverage] targets ..." - exit 1 + clean) + rm -f .coverage + shift ;; *) - PARAMS+=("$key") - shift + echo "usage: $0 [--no-coverage] targets ..." + exit 1 ;; esac done @@ -30,7 +29,7 @@ pytest # By default, report coverage and move it into cache. if [ $COVERAGE -eq 1 ]; then - make -C test coverage + make -C test coverage || /bin/true # /data is mounted as a volume. Copy coverage into it. # Users can then sanitize the coverage locally in their From 39fd3b891e4c3f86dad74ea8b66b516abdcf45e7 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 21 Nov 2021 01:41:10 -0800 Subject: [PATCH 614/844] change: set -v for sh tests Signed-off-by: Kevin Morris --- test/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index 4a8207f8..a6abc9de 100644 --- a/test/Makefile +++ b/test/Makefile @@ -26,6 +26,6 @@ clean: rm -f ../.coverage $(T): - @echo "*** $@ ***"; $(SHELL) $@ + @echo "*** $@ ***"; $(SHELL) $@ -v .PHONY: check coverage $(FOREIGN_TARGETS) clean $(T) From 3b686c475d605309abc94d1655369ec4cf44deed Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 22 Nov 2021 07:34:35 -0800 Subject: [PATCH 615/844] fix: default detailed loglevel to DEBUG Signed-off-by: Kevin Morris --- logging.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logging.conf b/logging.conf index 310ac76e..3b96e827 100644 --- a/logging.conf +++ b/logging.conf @@ -43,7 +43,7 @@ args=(sys.stdout,) [handler_detailedHandler] class=StreamHandler -level=INFO +level=DEBUG formatter=detailedFormatter args=(sys.stdout,) From 47d83244bbd415b769b151dc64de18c9b62568b9 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 22 Nov 2021 22:21:45 -0800 Subject: [PATCH 616/844] change(gitlab-ci): add 'fast-single-thread' tag to the test stage Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 739c9408..8980fa78 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,4 @@ image: archlinux:base-devel - cache: key: system-v1 paths: @@ -13,6 +12,8 @@ variables: test: stage: test + tags: + - fast-single-thread before_script: - export PATH="$HOME/.poetry/bin:${PATH}" - ./docker/scripts/install-deps.sh From 6bb002e70889777024384529f37907f595894bf2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 24 Nov 2021 21:23:01 -0800 Subject: [PATCH 617/844] fix: use correct u2f ssh key prefixes Signed-off-by: Kevin Morris --- conf/config.defaults | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conf/config.defaults b/conf/config.defaults index a04f21bc..dd9bfd2f 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -62,7 +62,9 @@ ECDSA = SHA256:L71Q91yHwmHPYYkJMDgj0xmUuw16qFOhJbBr1mzsiOI RSA = SHA256:Ju+yWiMb/2O+gKQ9RJCDqvRg7l+Q95KFAeqM5sr6l2s [auth] -valid-keytypes = ssh-rsa ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 sk-ssh-ecdsa@openssh.com sk-ssh-ed25519@openssh.com +; For U2F key prefixes, see the following documentation from openssh: +; https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.u2f +valid-keytypes = ssh-rsa ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 sk-ecdsa-sha2-nistp256@openssh.com sk-ecdsa-sha2-nistp256-cert-v01@openssh.com sk-ssh-ed25519@openssh.com sk-ssh-ed25519-cert-v01@openssh.com username-regex = [a-zA-Z0-9]+[.\-_]?[a-zA-Z0-9]+$ git-serve-cmd = /usr/local/bin/aurweb-git-serve ssh-options = restrict From 1aab9604010e9b58c7bed8586931841751bbab68 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 24 Nov 2021 21:28:29 -0800 Subject: [PATCH 618/844] fix: use corrent u2f ssh key prefixes Signed-off-by: Kevin Morris --- conf/config.defaults | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conf/config.defaults b/conf/config.defaults index 68e235be..a589997b 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -72,7 +72,9 @@ ECDSA = SHA256:L71Q91yHwmHPYYkJMDgj0xmUuw16qFOhJbBr1mzsiOI RSA = SHA256:Ju+yWiMb/2O+gKQ9RJCDqvRg7l+Q95KFAeqM5sr6l2s [auth] -valid-keytypes = ssh-rsa ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 sk-ssh-ecdsa@openssh.com sk-ssh-ed25519@openssh.com +; For U2F key prefixes, see the following documentation from openssh: +; https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.u2f +valid-keytypes = ssh-rsa ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 sk-ecdsa-sha2-nistp256@openssh.com sk-ecdsa-sha2-nistp256-cert-v01@openssh.com sk-ssh-ed25519@openssh.com sk-ssh-ed25519-cert-v01@openssh.com username-regex = [a-zA-Z0-9]+[.\-_]?[a-zA-Z0-9]+$ git-serve-cmd = /usr/bin/aurweb-git-serve ssh-options = restrict From e558e979ff481148bb903ca21c7659b7ca43208d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 24 Nov 2021 21:28:49 -0800 Subject: [PATCH 619/844] fix(fastapi): check ssh key prefixes against configured valid-keytypes Signed-off-by: Kevin Morris --- aurweb/util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/aurweb/util.py b/aurweb/util.py index b95fc6a3..62575c71 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -84,9 +84,8 @@ def valid_pgp_fingerprint(fp): def valid_ssh_pubkey(pk): - valid_prefixes = ("ssh-rsa", "ecdsa-sha2-nistp256", - "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521", - "ssh-ed25519") + valid_prefixes = aurweb.config.get("auth", "valid-keytypes") + valid_prefixes = set(valid_prefixes.split(" ")) has_valid_prefix = False for prefix in valid_prefixes: From b98159d5b90fe0fd609a694257bb25a4fa579b0e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 16:43:29 -0800 Subject: [PATCH 620/844] change(docker): use step-ca for CA + cert generation Signed-off-by: Kevin Morris --- Dockerfile | 3 +- docker-compose.aur-dev.yml | 3 +- docker-compose.override.yml | 14 --- docker-compose.yml | 14 ++- docker/ca-entrypoint.sh | 163 +++++++++++++++++++++--------- docker/health/ca.sh | 2 + docker/nginx-entrypoint.sh | 2 +- docker/scripts/install-deps.sh | 2 +- docker/scripts/run-ca.sh | 7 ++ docker/scripts/update-step-config | 19 ++++ 10 files changed, 160 insertions(+), 69 deletions(-) create mode 100755 docker/health/ca.sh create mode 100755 docker/scripts/run-ca.sh create mode 100755 docker/scripts/update-step-config diff --git a/Dockerfile b/Dockerfile index 3c12cbf8..9af78c3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,8 @@ RUN /install-deps.sh # Copy Docker scripts COPY ./docker /docker -COPY ./docker/scripts/*.sh /usr/local/bin/ +COPY ./docker/scripts/* /usr/local/bin/ + # Copy over all aurweb files. COPY . /aurweb diff --git a/docker-compose.aur-dev.yml b/docker-compose.aur-dev.yml index 4b522e56..f27b2b19 100644 --- a/docker-compose.aur-dev.yml +++ b/docker-compose.aur-dev.yml @@ -70,9 +70,8 @@ services: nginx: restart: always volumes: - - ${GIT_DATA_DIR}:/aurweb/aur.git - data:/data - - logs:/var/log/nginx + - archives:/var/lib/aurweb/archives - smartgit_run:/var/run/smartgit volumes: diff --git a/docker-compose.override.yml b/docker-compose.override.yml index eae12a92..8c74f947 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -6,10 +6,6 @@ services: mariadb: condition: service_healthy - ca: - volumes: - - ./data:/data - git: volumes: - git_data:/aurweb/aur.git @@ -45,13 +41,3 @@ services: - ./web/template:/aurweb/web/template - ./web/lib:/aurweb/web/lib - ./templates:/aurweb/templates - - nginx: - volumes: - - git_data:/aurweb/aur.git - - ./data:/data - - ./logs:/var/log/nginx - - ./web/html:/aurweb/web/html - - ./web/template:/aurweb/web/template - - ./web/lib:/aurweb/web/lib - - smartgit_run:/var/run/smartgit diff --git a/docker-compose.yml b/docker-compose.yml index acb5dd65..c1f93319 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,16 @@ services: image: aurweb:latest init: true entrypoint: /docker/ca-entrypoint.sh - command: echo + command: /docker/scripts/run-ca.sh + healthcheck: + test: "bash /docker/health/run-ca.sh" + interval: 3s + tmpfs: + - /tmp + volumes: + - ./docker:/docker + - ./data:/data + - step:/root/.step memcached: image: aurweb:latest @@ -261,7 +270,9 @@ services: php-fpm: condition: service_healthy volumes: + - ./data:/data - archives:/var/lib/aurweb/archives + - smartgit_run:/var/run/smartgit sharness: image: aurweb:latest @@ -347,3 +358,4 @@ volumes: git_data: {} # Share aurweb/aur.git smartgit_run: {} archives: {} + step: {} diff --git a/docker/ca-entrypoint.sh b/docker/ca-entrypoint.sh index 42d8bd14..d03efbbc 100755 --- a/docker/ca-entrypoint.sh +++ b/docker/ca-entrypoint.sh @@ -1,58 +1,123 @@ #!/bin/bash +# Initialize step-ca and request certificates from it. +# +# Certificates created by this service are meant to be used in +# aurweb Docker's nginx service. +# +# If ./data/root_ca.crt is present, CA generation is skipped. +# If ./data/${host}.{cert,key}.pem is available, host certificate +# generation is skipped. +# set -eou pipefail -if [ -f /data/ca.root.pem ]; then - echo "Already have certs, skipping." - exit 0 +# /data-based variables. +DATA_DIR="/data" +DATA_ROOT_CA="$DATA_DIR/root_ca.crt" +DATA_CERT="$DATA_DIR/localhost.cert.pem" +DATA_CERT_KEY="$DATA_DIR/localhost.key.pem" + +# Host certificates requested from the CA (separated by spaces). +DATA_CERT_HOSTS='localhost' + +# Local step paths and CA configuration values. +STEP_DIR="$(step-cli path)" +STEP_CA_CONFIG="$STEP_DIR/config/ca.json" +STEP_CA_ADDR='127.0.0.1:8443' +STEP_CA_URL='https://localhost:8443' +STEP_CA_PROVISIONER='admin@localhost' + +# Password file used for both --password-file and --provisioner-password-file. +STEP_PASSWD_FILE="$STEP_DIR/password.txt" + +# Hostnames supported by the CA. +STEP_CA_NAME='aurweb' +STEP_CA_DNS='localhost' + +make_password() { + # Create a random 20-length password and write it to $1. + openssl rand -hex 20 > $1 +} + +setup_step_ca() { + # Cleanup and setup step ca configuration. + rm -rf $STEP_DIR/* + + # Initialize `step` + make_password "$STEP_PASSWD_FILE" + step-cli ca init \ + --name="$STEP_CA_NAME" \ + --dns="$STEP_CA_DNS" \ + --address="$STEP_CA_ADDR" \ + --password-file="$STEP_PASSWD_FILE" \ + --provisioner="$STEP_CA_PROVISIONER" \ + --provisioner-password-file="$STEP_PASSWD_FILE" \ + --with-ca-url="$STEP_CA_URL" + + # Update ca.json max TLS certificate duration to a year. + update-step-config "$STEP_CA_CONFIG" + + # Install root_ca.crt as read/writable to /data/root_ca.crt. + install -m666 "$STEP_DIR/certs/root_ca.crt" "$DATA_ROOT_CA" +} + +start_step_ca() { + # Start the step-ca web server. + step-ca "$STEP_CA_CONFIG" \ + --password-file="$STEP_PASSWD_FILE" & + until printf "" 2>>/dev/null >>/dev/tcp/127.0.0.1/8443; do + sleep 1 + done +} + +kill_step_ca() { + # Stop the step-ca web server. + killall step-ca >/dev/null 2>&1 || /bin/true +} + +install_step_ca() { + # Install step-ca certificate authority to the system. + step-cli certificate install "$STEP_DIR/certs/root_ca.crt" +} + +step_cert_request() { + # Request a certificate from the step ca. + step-cli ca certificate \ + --not-after=8800h \ + --provisioner="$STEP_CA_PROVISIONER" \ + --provisioner-password-file="$STEP_PASSWD_FILE" \ + $1 $2 $3 + chmod 666 /data/${1}.*.pem +} + +if [ ! -f $DATA_ROOT_CA ]; then + setup_step_ca + install_step_ca fi -# Generate a new 2048-bit RSA key for the Root CA. -openssl genrsa -des3 -out /data/ca.key -passout pass:devca 2048 +# For all hosts separated by spaces in $DATA_CERT_HOSTS, perform a check +# for their existence in /data and react accordingly. +for host in $DATA_CERT_HOSTS; do + if [ -f /data/${host}.cert.pem ] && [ -f /data/${host}.key.pem ]; then + # Found an override. Move on to running the service after + # printing a notification to the user. + echo "Found '${host}.{cert,key}.pem' override, skipping..." + echo -n "Note: If you need to regenerate certificates, run " + echo '`rm -f data/*.{cert,key}.pem` before starting this service.' + exec "$@" + else + # Otherwise, we had a missing cert or key, so remove both. + rm -f /data/${host}.cert.pem + rm -f /data/${host}.key.pem + fi +done -# Request and self-sign a new Root CA certificate, using -# the RSA key. Output Root CA PEM-format certificate and key: -# /data/ca.root.pem and /data/ca.key.pem -openssl req -x509 -new -nodes -sha256 -days 1825 \ - -passin pass:devca \ - -subj "/C=US/ST=California/L=Authority/O=aurweb/CN=localhost" \ - -in /data/ca.key -out /data/ca.root.pem -keyout /data/ca.key.pem +start_step_ca +for host in $DATA_CERT_HOSTS; do + step_cert_request $host /data/${host}.cert.pem /data/${host}.key.pem +done +kill_step_ca -# Generate a new 2048-bit RSA key for a localhost server. -openssl genrsa -out /data/localhost.key 2048 - -# Generate a Certificate Signing Request (CSR) for the localhost server -# using the RSA key we generated above. -openssl req -new -key /data/localhost.key -passout pass:devca \ - -subj "/C=US/ST=California/L=Server/O=aurweb/CN=localhost" \ - -out /data/localhost.csr - -# Get our CSR signed by our Root CA PEM-formatted certificate and key -# to produce a fresh /data/localhost.cert.pem PEM-formatted certificate. -openssl x509 -req -in /data/localhost.csr \ - -CA /data/ca.root.pem -CAkey /data/ca.key.pem \ - -CAcreateserial \ - -out /data/localhost.cert.pem \ - -days 825 -sha256 \ - -passin pass:devca \ - -extfile /docker/localhost.ext - -# Convert RSA key to a PEM-formatted key: /data/localhost.key.pem -openssl rsa -in /data/localhost.key -text > /data/localhost.key.pem - -# At the end here, our notable certificates and keys are: -# - /data/ca.root.pem -# - /data/ca.key.pem -# - /data/localhost.key.pem -# - /data/localhost.cert.pem -# -# When running a server which uses the localhost certificate, a chain -# should be used, starting with localhost.cert.pem: -# - cat /data/localhost.cert.pem /data/ca.root.pem > localhost.chain.pem -# -# The Root CA (ca.root.pem) should be imported into browsers or -# ca-certificates on machines wishing to verify localhost. -# - -chmod 666 /data/* +# Set permissions to /data to rwx for everybody. +chmod 777 /data exec "$@" diff --git a/docker/health/ca.sh b/docker/health/ca.sh new file mode 100755 index 00000000..3e4bbe8e --- /dev/null +++ b/docker/health/ca.sh @@ -0,0 +1,2 @@ + +exec printf "" 2>>/dev/null >>/dev/tcp/127.0.0.1/8443 diff --git a/docker/nginx-entrypoint.sh b/docker/nginx-entrypoint.sh index 6b9a6954..1527cda7 100755 --- a/docker/nginx-entrypoint.sh +++ b/docker/nginx-entrypoint.sh @@ -15,7 +15,7 @@ if [ -f "$CERT" ]; then cp -vf "$CERT" "$DEST_CERT" cp -vf "$KEY" "$DEST_KEY" else - cat /data/localhost.cert.pem /data/ca.root.pem > "$DEST_CERT" + cat /data/localhost.cert.pem /data/root_ca.crt > "$DEST_CERT" cp -vf /data/localhost.key.pem "$DEST_KEY" fi diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh index ad0157f8..372b6e0c 100755 --- a/docker/scripts/install-deps.sh +++ b/docker/scripts/install-deps.sh @@ -9,6 +9,6 @@ pacman -Syu --noconfirm --noprogressbar \ mariadb mariadb-libs cgit-aurweb uwsgi uwsgi-plugin-cgi \ php php-fpm memcached php-memcached python-pip pyalpm \ python-srcinfo curl libeatmydata cronie python-poetry \ - python-poetry-core + python-poetry-core step-cli step-ca exec "$@" diff --git a/docker/scripts/run-ca.sh b/docker/scripts/run-ca.sh new file mode 100755 index 00000000..1ef45ef7 --- /dev/null +++ b/docker/scripts/run-ca.sh @@ -0,0 +1,7 @@ +#!/bin/bash +STEP_DIR="$(step-cli path)" +STEP_PASSWD_FILE="$STEP_DIR/password.txt" +STEP_CA_CONFIG="$STEP_DIR/config/ca.json" + +# Start the step-ca https server. +exec step-ca "$STEP_CA_CONFIG" --password-file="$STEP_PASSWD_FILE" diff --git a/docker/scripts/update-step-config b/docker/scripts/update-step-config new file mode 100755 index 00000000..bbdb2680 --- /dev/null +++ b/docker/scripts/update-step-config @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +import json +import sys + +CA_CONFIG = sys.argv[1] + +with open(CA_CONFIG) as f: + data = json.load(f) + +if "authority" not in data: + data["authority"] = dict() +if "claims" not in data["authority"]: + data["authority"]["claims"] = dict() + +# One year of certificate duration. +data["authority"]["claims"] = {"maxTLSCertDuration": "8800h"} + +with open(CA_CONFIG, "w") as f: + json.dump(data, f) From 759f18ea75a5581cefa5ca6fe323bdc56944f47a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 16:44:56 -0800 Subject: [PATCH 621/844] feat: add aurweb-config console script This can be used to update config values for the entirety of a config. When config values are set through this tool, $AUR_CONFIG is overridden with a copy of the config file with all sections and options found in $AUR_CONFIG + $AUR_CONFIG_DEFAULTS. Signed-off-by: Kevin Morris --- aurweb/config.py | 12 ++++ aurweb/scripts/config.py | 61 +++++++++++++++++++ pyproject.toml | 1 + test/test_config.py | 125 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 199 insertions(+) create mode 100644 aurweb/scripts/config.py diff --git a/aurweb/config.py b/aurweb/config.py index aa111f15..0d0cf676 100644 --- a/aurweb/config.py +++ b/aurweb/config.py @@ -1,6 +1,8 @@ import configparser import os +from typing import Any + # Publicly visible version of aurweb. This is used to display # aurweb versioning in the footer and must be maintained. # Todo: Make this dynamic/automated. @@ -52,3 +54,13 @@ def getint(section, option, fallback=None): def get_section(section): if section in _get_parser().sections(): return _get_parser()[section] + + +def replace_key(section: str, option: str, value: Any) -> Any: + _get_parser().set(section, option, value) + + +def save() -> None: + aur_config = os.environ.get("AUR_CONFIG", "/etc/aurweb/config") + with open(aur_config, "w") as fp: + _get_parser().write(fp) diff --git a/aurweb/scripts/config.py b/aurweb/scripts/config.py new file mode 100644 index 00000000..dd7bcf5f --- /dev/null +++ b/aurweb/scripts/config.py @@ -0,0 +1,61 @@ +""" +Perform an action on the aurweb config. + +When AUR_CONFIG_IMMUTABLE is set, the `set` action is noop. +""" +import argparse +import configparser +import os +import sys + +import aurweb.config + + +def action_set(args): + # If AUR_CONFIG_IMMUTABLE is defined, skip out on config setting. + if os.environ.get("AUR_CONFIG_IMMUTABLE", 0): + return + + if not args.value: + print("error: no value provided", file=sys.stderr) + return + + try: + aurweb.config.replace_key(args.section, args.option, args.value) + aurweb.config.save() + except configparser.NoSectionError: + print("error: no section found", file=sys.stderr) + + +def action_get(args): + try: + value = aurweb.config.get(args.section, args.option) + print(value) + except (configparser.NoSectionError): + print("error: no section found", file=sys.stderr) + except (configparser.NoOptionError): + print("error: no option found", file=sys.stderr) + + +def parse_args(): + fmt_cls = argparse.RawDescriptionHelpFormatter + actions = ["get", "set"] + parser = argparse.ArgumentParser( + description="aurweb configuration tool", + formatter_class=lambda prog: fmt_cls(prog=prog, max_help_position=80)) + parser.add_argument("action", choices=actions, help="script action") + parser.add_argument("section", help="config section") + parser.add_argument("option", help="config option") + parser.add_argument("value", nargs="?", default=0, + help="config option value") + return parser.parse_args() + + +def main(): + args = parse_args() + action = getattr(sys.modules[__name__], f"action_{args.action}") + return action(args) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 8d14735a..82c439bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,3 +108,4 @@ aurweb-popupdate = "aurweb.scripts.popupdate:main" aurweb-rendercomment = "aurweb.scripts.rendercomment:main" aurweb-tuvotereminder = "aurweb.scripts.tuvotereminder:main" aurweb-usermaint = "aurweb.scripts.usermaint:main" +aurweb-config = "aurweb.scripts.config:main" diff --git a/test/test_config.py b/test/test_config.py index 4f10b60d..7e9d24b5 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -1,4 +1,16 @@ +import configparser +import io +import os +import re + +from unittest import mock + from aurweb import config +from aurweb.scripts.config import main + + +def noop(*args, **kwargs) -> None: + return def test_get(): @@ -11,3 +23,116 @@ def test_getboolean(): def test_getint(): assert config.getint("options", "disable_http_login") == 0 + + +def mock_config_get(): + config_get = config.get + + def _mock_config_get(section: str, option: str): + if section == "options": + if option == "salt_rounds": + return "666" + return config_get(section, option) + return _mock_config_get + + +@mock.patch("aurweb.config.get", side_effect=mock_config_get()) +def test_config_main_get(get: str): + stdout = io.StringIO() + args = ["aurweb-config", "get", "options", "salt_rounds"] + with mock.patch("sys.argv", args): + with mock.patch("sys.stdout", stdout): + main() + + expected = "666" + assert stdout.getvalue().strip() == expected + + +@mock.patch("aurweb.config.get", side_effect=mock_config_get()) +def test_config_main_get_unknown_section(get: str): + stderr = io.StringIO() + args = ["aurweb-config", "get", "fakeblahblah", "salt_rounds"] + with mock.patch("sys.argv", args): + with mock.patch("sys.stderr", stderr): + main() + + # With an invalid section, we should get a usage error. + expected = r'^error: no section found$' + assert re.match(expected, stderr.getvalue().strip()) + + +@mock.patch("aurweb.config.get", side_effect=mock_config_get()) +def test_config_main_get_unknown_option(get: str): + stderr = io.StringIO() + args = ["aurweb-config", "get", "options", "fakeblahblah"] + with mock.patch("sys.argv", args): + with mock.patch("sys.stderr", stderr): + main() + + expected = "error: no option found" + assert stderr.getvalue().strip() == expected + + +@mock.patch("aurweb.config.save", side_effect=noop) +def test_config_main_set(save: None): + data = None + + def mock_replace_key(section: str, option: str, value: str) -> None: + nonlocal data + data = value + + args = ["aurweb-config", "set", "options", "salt_rounds", "666"] + with mock.patch("sys.argv", args): + with mock.patch("aurweb.config.replace_key", + side_effect=mock_replace_key): + main() + + expected = "666" + assert data == expected + + +def test_config_main_set_immutable(): + data = None + + def mock_replace_key(section: str, option: str, value: str) -> None: + nonlocal data + data = value + + args = ["aurweb-config", "set", "options", "salt_rounds", "666"] + with mock.patch.dict(os.environ, {"AUR_CONFIG_IMMUTABLE": "1"}): + with mock.patch("sys.argv", args): + with mock.patch("aurweb.config.replace_key", + side_effect=mock_replace_key): + main() + + expected = None + assert data == expected + + +def test_config_main_set_invalid_value(): + stderr = io.StringIO() + + args = ["aurweb-config", "set", "options", "salt_rounds"] + with mock.patch("sys.argv", args): + with mock.patch("sys.stderr", stderr): + main() + + expected = "error: no value provided" + assert stderr.getvalue().strip() == expected + + +@mock.patch("aurweb.config.save", side_effect=noop) +def test_config_main_set_unknown_section(save: None): + stderr = io.StringIO() + + def mock_replace_key(section: str, option: str, value: str) -> None: + raise configparser.NoSectionError(section=section) + + args = ["aurweb-config", "set", "options", "salt_rounds", "666"] + with mock.patch("sys.argv", args): + with mock.patch("sys.stderr", stderr): + with mock.patch("aurweb.config.replace_key", + side_effect=mock_replace_key): + main() + + assert stderr.getvalue().strip() == "error: no section found" From d658627e992dd0fc16e5e2aa76d52dda76de4380 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 19:10:59 -0800 Subject: [PATCH 622/844] fix(fastapi): don't redirect to login on authed /login Closes #184 Signed-off-by: Kevin Morris --- aurweb/routers/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index 055f0dca..c5a99419 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -24,13 +24,13 @@ async def login_template(request: Request, next: str, errors: list = None): @router.get("/login", response_class=HTMLResponse) -@auth_required(False) +@auth_required(False, login=False) async def login_get(request: Request, next: str = "/"): return await login_template(request, next) @router.post("/login", response_class=HTMLResponse) -@auth_required(False) +@auth_required(False, login=False) async def login_post(request: Request, next: str = Form(...), user: str = Form(default=str()), From 47feb72f48cb0d1c36fffff39160e48b8e870488 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 20:04:26 -0800 Subject: [PATCH 623/844] fix(fastapi): fix SessionID (and ResetKey) generation Signed-off-by: Kevin Morris --- aurweb/db.py | 29 +---------------------------- aurweb/models/session.py | 9 ++++----- aurweb/models/user.py | 4 ++++ aurweb/routers/accounts.py | 5 +++-- 4 files changed, 12 insertions(+), 35 deletions(-) diff --git a/aurweb/db.py b/aurweb/db.py index b8b49e40..70ad58d1 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -24,42 +24,15 @@ DRIVERS = { "mysql": "mysql+mysqldb" } -# Global introspected object memo. -introspected = dict() - # Some types we don't get access to in this module. Base = NewType("Base", "aurweb.models.declarative_base.Base") -def make_random_value(table: str, column: str): +def make_random_value(table: str, column: str, length: int): """ Generate a unique, random value for a string column in a table. - This can be used to generate for example, session IDs that - align with the properties of the database column with regards - to size. - - Internally, we use SQLAlchemy introspection to look at column - to decide which length to use for random string generation. - :return: A unique string that is not in the database """ - global introspected - - # Make sure column is converted to a string for memo interaction. - scolumn = str(column) - - # If the target column is not yet introspected, store its introspection - # object into our global `introspected` memo. - if scolumn not in introspected: - from sqlalchemy import inspect - target_column = scolumn.split('.')[-1] - col = list(filter(lambda c: c.name == target_column, - inspect(table).columns))[0] - introspected[scolumn] = col - - col = introspected.get(scolumn) - length = col.type.length - string = aurweb.util.make_random_string(length) while query(table).filter(column == string).first(): string = aurweb.util.make_random_string(length) diff --git a/aurweb/models/session.py b/aurweb/models/session.py index 96f88d85..7a06eddc 100644 --- a/aurweb/models/session.py +++ b/aurweb/models/session.py @@ -1,8 +1,7 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship -from aurweb import schema -from aurweb.db import make_random_value, query +from aurweb import db, schema from aurweb.models.declarative import Base from aurweb.models.user import User as _User @@ -19,8 +18,8 @@ class Session(Base): def __init__(self, **kwargs): super().__init__(**kwargs) - user_exists = query( - query(_User).filter(_User.ID == self.UsersID).exists() + user_exists = db.query( + db.query(_User).filter(_User.ID == self.UsersID).exists() ).scalar() if not user_exists: raise IntegrityError( @@ -31,4 +30,4 @@ class Session(Base): def generate_unique_sid(): - return make_random_value(Session, Session.SessionID) + return db.make_random_value(Session, Session.SessionID, 32) diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 43910db9..03634a36 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -230,3 +230,7 @@ class User(Base): def __repr__(self): return "" % ( self.ID, str(self.AccountType), self.Username) + + +def generate_unique_resetkey(): + return db.make_random_value(User, User.ResetKey, 32) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 02a7f4c6..ddee1764 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -16,6 +16,7 @@ from aurweb.exceptions import ValidationError from aurweb.l10n import get_translator_for_request from aurweb.models import account_type as at from aurweb.models.ssh_pub_key import get_fingerprint +from aurweb.models.user import generate_unique_resetkey from aurweb.scripts.notify import ResetKeyNotification, WelcomeNotification from aurweb.templates import make_context, make_variable_context, render_template from aurweb.users import update, validate @@ -92,7 +93,7 @@ async def passreset_post(request: Request, status_code=HTTPStatus.SEE_OTHER) # If we got here, we continue with issuing a resetkey for the user. - resetkey = db.make_random_value(models.User, models.User.ResetKey) + resetkey = generate_unique_resetkey() with db.begin(): user.ResetKey = resetkey @@ -291,7 +292,7 @@ async def account_register_post(request: Request, # Create a user with no password with a resetkey, then send # an email off about it. - resetkey = db.make_random_value(models.User, models.User.ResetKey) + resetkey = generate_unique_resetkey() # By default, we grab the User account type to associate with. atype = db.query(models.AccountType, From 7b0d664bc0c4d9f75abc2ed659e14ca5f66dba1c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 21:03:24 -0800 Subject: [PATCH 624/844] fix(docker): reorg ./data mounts Signed-off-by: Kevin Morris --- docker-compose.aur-dev.yml | 3 ++- docker-compose.override.yml | 11 +++++++++++ docker-compose.yml | 10 ---------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/docker-compose.aur-dev.yml b/docker-compose.aur-dev.yml index f27b2b19..0b91dd93 100644 --- a/docker-compose.aur-dev.yml +++ b/docker-compose.aur-dev.yml @@ -4,6 +4,7 @@ services: ca: volumes: - data:/data + - step:/root/.step memcached: restart: always @@ -22,7 +23,7 @@ services: - SSH_CMDLINE=${SSH_CMDLINE:-ssh ssh://aur@localhost:2222} volumes: - ${GIT_DATA_DIR}:/aurweb/aur.git - - ./data:/aurweb/data + - data:/aurweb/data smartgit: restart: always diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 8c74f947..1e466730 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,6 +1,11 @@ version: "3.8" services: + ca: + volumes: + - ./data:/data + - step:/root/.step + mariadb_init: depends_on: mariadb: @@ -41,3 +46,9 @@ services: - ./web/template:/aurweb/web/template - ./web/lib:/aurweb/web/lib - ./templates:/aurweb/templates + + nginx: + volumes: + - ./data:/data + - archives:/var/lib/aurweb/archives + - smartgit_run:/var/run/smartgit diff --git a/docker-compose.yml b/docker-compose.yml index c1f93319..5d8f7d78 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,12 +33,6 @@ services: healthcheck: test: "bash /docker/health/run-ca.sh" interval: 3s - tmpfs: - - /tmp - volumes: - - ./docker:/docker - - ./data:/data - - step:/root/.step memcached: image: aurweb:latest @@ -269,10 +263,6 @@ services: condition: service_healthy php-fpm: condition: service_healthy - volumes: - - ./data:/data - - archives:/var/lib/aurweb/archives - - smartgit_run:/var/run/smartgit sharness: image: aurweb:latest From 199622c53f65260670868ca3712d2f9e30de4461 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 21:35:35 -0800 Subject: [PATCH 625/844] fix(fastapi): refresh records when fetching updated packages Signed-off-by: Kevin Morris --- aurweb/packages/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index 7c48f4e4..55af3a34 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -167,6 +167,7 @@ def updated_packages(limit: int = 0, for pkg in query: # For each Package returned by the query, append a dict # containing Package columns we're interested in. + db.refresh(pkg) packages.append({ "Name": pkg.Name, "Version": pkg.Version, From 0e938209afbf25748cf95bd27b32ad2814f8d77b Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 22:34:15 -0800 Subject: [PATCH 626/844] feat(aurweb-config): add unset action and simplify Signed-off-by: Kevin Morris --- aurweb/config.py | 7 ++++- aurweb/scripts/config.py | 38 ++++++++++++++++---------- test/test_config.py | 59 +++++++++++++++++++++++++++++++++------- 3 files changed, 78 insertions(+), 26 deletions(-) diff --git a/aurweb/config.py b/aurweb/config.py index 0d0cf676..86f8ddf7 100644 --- a/aurweb/config.py +++ b/aurweb/config.py @@ -56,8 +56,13 @@ def get_section(section): return _get_parser()[section] -def replace_key(section: str, option: str, value: Any) -> Any: +def unset_option(section: str, option: str) -> None: + _get_parser().remove_option(section, option) + + +def set_option(section: str, option: str, value: Any) -> None: _get_parser().set(section, option, value) + return value def save() -> None: diff --git a/aurweb/scripts/config.py b/aurweb/scripts/config.py index dd7bcf5f..e7c91dd1 100644 --- a/aurweb/scripts/config.py +++ b/aurweb/scripts/config.py @@ -11,35 +11,43 @@ import sys import aurweb.config -def action_set(args): +def do_action(func, *args, save: bool = True): # If AUR_CONFIG_IMMUTABLE is defined, skip out on config setting. - if os.environ.get("AUR_CONFIG_IMMUTABLE", 0): + if int(os.environ.get("AUR_CONFIG_IMMUTABLE", 0)): return + value = None + try: + value = func(*args) + if save: + aurweb.config.save() + except configparser.NoSectionError: + print("error: no section found", file=sys.stderr) + except configparser.NoOptionError: + print("error: no option found", file=sys.stderr) + + return value + + +def action_set(args): if not args.value: print("error: no value provided", file=sys.stderr) return + do_action(aurweb.config.set_option, args.section, args.option, args.value) - try: - aurweb.config.replace_key(args.section, args.option, args.value) - aurweb.config.save() - except configparser.NoSectionError: - print("error: no section found", file=sys.stderr) + +def action_unset(args): + do_action(aurweb.config.unset_option, args.section, args.option) def action_get(args): - try: - value = aurweb.config.get(args.section, args.option) - print(value) - except (configparser.NoSectionError): - print("error: no section found", file=sys.stderr) - except (configparser.NoOptionError): - print("error: no option found", file=sys.stderr) + val = do_action(aurweb.config.get, args.section, args.option, save=False) + print(val) def parse_args(): fmt_cls = argparse.RawDescriptionHelpFormatter - actions = ["get", "set"] + actions = ["get", "set", "unset"] parser = argparse.ArgumentParser( description="aurweb configuration tool", formatter_class=lambda prog: fmt_cls(prog=prog, max_help_position=80)) diff --git a/test/test_config.py b/test/test_config.py index 7e9d24b5..b78f477c 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -5,6 +5,8 @@ import re from unittest import mock +import py + from aurweb import config from aurweb.scripts.config import main @@ -77,32 +79,69 @@ def test_config_main_get_unknown_option(get: str): def test_config_main_set(save: None): data = None - def mock_replace_key(section: str, option: str, value: str) -> None: + def set_option(section: str, option: str, value: str) -> None: nonlocal data data = value args = ["aurweb-config", "set", "options", "salt_rounds", "666"] with mock.patch("sys.argv", args): - with mock.patch("aurweb.config.replace_key", - side_effect=mock_replace_key): + with mock.patch("aurweb.config.set_option", side_effect=set_option): main() expected = "666" assert data == expected +def test_config_main_set_real(tmpdir: py.path.local): + """ + Test a real set_option path. + """ + + # Copy AUR_CONFIG to {tmpdir}/aur.config. + aur_config = os.environ.get("AUR_CONFIG") + tmp_aur_config = os.path.join(str(tmpdir), "aur.config") + with open(aur_config) as f: + with open(tmp_aur_config, "w") as o: + o.write(f.read()) + + # Force reset the parser. This should NOT be done publicly. + config._parser = None + + value = 666 + args = ["aurweb-config", "set", "options", "fake-key", str(value)] + with mock.patch.dict("os.environ", {"AUR_CONFIG": tmp_aur_config}): + with mock.patch("sys.argv", args): + # Run aurweb.config.main(). + main() + + # Update the config; fake-key should be set. + config.rehash() + assert config.getint("options", "fake-key") == 666 + + # Restore config back to normal. + args = ["aurweb-config", "unset", "options", "fake-key"] + with mock.patch("sys.argv", args): + main() + + # Return the config back to normal. + config.rehash() + + # fake-key should no longer exist. + assert config.getint("options", "fake-key") is None + + def test_config_main_set_immutable(): data = None - def mock_replace_key(section: str, option: str, value: str) -> None: + def mock_set_option(section: str, option: str, value: str) -> None: nonlocal data data = value args = ["aurweb-config", "set", "options", "salt_rounds", "666"] with mock.patch.dict(os.environ, {"AUR_CONFIG_IMMUTABLE": "1"}): with mock.patch("sys.argv", args): - with mock.patch("aurweb.config.replace_key", - side_effect=mock_replace_key): + with mock.patch("aurweb.config.set_option", + side_effect=mock_set_option): main() expected = None @@ -121,18 +160,18 @@ def test_config_main_set_invalid_value(): assert stderr.getvalue().strip() == expected -@mock.patch("aurweb.config.save", side_effect=noop) +@ mock.patch("aurweb.config.save", side_effect=noop) def test_config_main_set_unknown_section(save: None): stderr = io.StringIO() - def mock_replace_key(section: str, option: str, value: str) -> None: + def mock_set_option(section: str, option: str, value: str) -> None: raise configparser.NoSectionError(section=section) args = ["aurweb-config", "set", "options", "salt_rounds", "666"] with mock.patch("sys.argv", args): with mock.patch("sys.stderr", stderr): - with mock.patch("aurweb.config.replace_key", - side_effect=mock_replace_key): + with mock.patch("aurweb.config.set_option", + side_effect=mock_set_option): main() assert stderr.getvalue().strip() == "error: no section found" From f3efc18b508d505f242426911ce22231dc182e05 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 22:42:12 -0800 Subject: [PATCH 627/844] feat(docker): force test db configuration Signed-off-by: Kevin Morris --- docker/test-mysql-entrypoint.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docker/test-mysql-entrypoint.sh b/docker/test-mysql-entrypoint.sh index a46b2572..262577a6 100755 --- a/docker/test-mysql-entrypoint.sh +++ b/docker/test-mysql-entrypoint.sh @@ -5,4 +5,16 @@ set -eou pipefail cp -vf conf/config.dev conf/config sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config +# We use the root user for testing in Docker. +# The test user must be able to create databases and drop them. +aurweb-config set database user 'root' +aurweb-config set database host 'localhost' +aurweb-config set database socket '/var/run/mysqld/mysqld.sock' + +# Remove possibly problematic configuration options. +# We depend on the database socket within Docker and +# being run as the root user. +aurweb-config unset database password +aurweb-config unset database port + exec "$@" From 0726a08677b589136dbfce1d59990c9c744e56b0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 17:42:04 -0800 Subject: [PATCH 628/844] fix(docker): remove sqlite scripts Signed-off-by: Kevin Morris --- docker/scripts/setup-sqlite.sh | 7 ------- docker/test-sqlite-entrypoint.sh | 16 ---------------- 2 files changed, 23 deletions(-) delete mode 100755 docker/scripts/setup-sqlite.sh delete mode 100755 docker/test-sqlite-entrypoint.sh diff --git a/docker/scripts/setup-sqlite.sh b/docker/scripts/setup-sqlite.sh deleted file mode 100755 index e0b8de50..00000000 --- a/docker/scripts/setup-sqlite.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# Run an sqlite test. This script really just prepares sqlite -# tests by deleting any existing databases so the test can -# initialize cleanly. -DB_NAME="$(grep 'name =' conf/config.sqlite | sed -r 's/^name = (.+)$/\1/')" -rm -vf $DB_NAME -exec "$@" diff --git a/docker/test-sqlite-entrypoint.sh b/docker/test-sqlite-entrypoint.sh deleted file mode 100755 index c26f6735..00000000 --- a/docker/test-sqlite-entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -eou pipefail - -DB_BACKEND="sqlite" -DB_NAME="aurweb.sqlite3" - -# Create an SQLite config from the default dev config. -cp -vf conf/config.dev conf/config.sqlite -cp -vf conf/config.defaults conf/config.sqlite.defaults - -# Modify it for SQLite. -sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config.sqlite -sed -ri "s/^(backend) = .+/\1 = ${DB_BACKEND}/" conf/config.sqlite -sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" conf/config.sqlite - -exec "$@" From 5b350bc3614f29794bdfb710893b76b3d40ba96d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 18:46:42 -0800 Subject: [PATCH 629/844] change(docker): use aurweb-config to update AUR_CONFIG Signed-off-by: Kevin Morris --- docker/fastapi-entrypoint.sh | 26 ++++++++++++++------------ docker/git-entrypoint.sh | 22 +++++++++------------- docker/mariadb-init-entrypoint.sh | 5 +++-- docker/php-entrypoint.sh | 21 +++++++++++---------- 4 files changed, 37 insertions(+), 37 deletions(-) diff --git a/docker/fastapi-entrypoint.sh b/docker/fastapi-entrypoint.sh index 9df6382d..d1519bf8 100755 --- a/docker/fastapi-entrypoint.sh +++ b/docker/fastapi-entrypoint.sh @@ -5,23 +5,25 @@ set -eou pipefail cp -vf conf/config.dev conf/config sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config -# Change database user/password. -sed -ri "s/^;?(user) = .*$/\1 = aur/" conf/config -sed -ri "s/^;?(password) = .*$/\1 = aur/" conf/config +# Setup database. +aurweb-config set database user 'aur' +aurweb-config set database password 'aur' +aurweb-config set database host 'localhost' +aurweb-config set database socket '/var/lib/mysqld/mysqld.sock' +aurweb-config unset database port -sed -ri "s;^(aur_location) = .+;\1 = ${AURWEB_FASTAPI_PREFIX};" conf/config - -# Setup Redis for FastAPI. -sed -ri 's/^(cache) = .+/\1 = redis/' conf/config -sed -ri 's|^(redis_address) = .+|\1 = redis://redis|' conf/config +# Setup some other options. +aurweb-config set options cache 'redis' +aurweb-config set options redis_address 'redis://redis' +aurweb-config set options aur_location "$AURWEB_FASTAPI_PREFIX" +aurweb-config set options git_clone_uri_anon "${AURWEB_FASTAPI_PREFIX}/%s.git" +aurweb-config set options git_clone_uri_priv "${AURWEB_SSHD_PREFIX}/%s.git" if [ ! -z ${COMMIT_HASH+x} ]; then - sed -ri "s/^;?(commit_hash) =.*$/\1 = $COMMIT_HASH/" conf/config + aurweb-config set devel commit_hash "$COMMIT_HASH" fi -sed -ri "s|^(git_clone_uri_anon) = .+|\1 = ${AURWEB_FASTAPI_PREFIX}/%s.git|" conf/config.defaults -sed -ri "s|^(git_clone_uri_priv) = .+|\1 = ${AURWEB_SSHD_PREFIX}/%s.git|" conf/config.defaults - +# Setup prometheus directory. rm -rf $PROMETHEUS_MULTIPROC_DIR mkdir -p $PROMETHEUS_MULTIPROC_DIR diff --git a/docker/git-entrypoint.sh b/docker/git-entrypoint.sh index cfa1879b..96f4d112 100755 --- a/docker/git-entrypoint.sh +++ b/docker/git-entrypoint.sh @@ -42,20 +42,16 @@ EOF cp -vf conf/config.dev $AUR_CONFIG sed -i "s;YOUR_AUR_ROOT;$(pwd);g" $AUR_CONFIG -sed -ri "s/^;?(user) = .*$/\1 = aur/" $AUR_CONFIG -sed -ri "s/^;?(password) = .*$/\1 = aur/" $AUR_CONFIG +# Setup database. +aurweb-config set database user 'aur' +aurweb-config set database password 'aur' +aurweb-config set database host 'localhost' +aurweb-config set database socket '/var/lib/mysqld/mysqld.sock' +aurweb-config unset database port -AUR_CONFIG_DEFAULTS="${AUR_CONFIG}.defaults" - -if [[ "$AUR_CONFIG_DEFAULTS" != "/aurweb/conf/config.defaults" ]]; then - cp -vf conf/config.defaults $AUR_CONFIG_DEFAULTS -fi - -# Set some defaults needed for pathing and ssh uris. -sed -ri "s|^(repo-path) = .+|\1 = /aurweb/aur.git/|" $AUR_CONFIG_DEFAULTS - -# SSH_CMDLINE can be provided via override in docker-compose.aur-dev.yml. -sed -ri "s|^(ssh-cmdline) = .+$|\1 = ${SSH_CMDLINE}|" $AUR_CONFIG_DEFAULTS +# Setup some other options. +aurweb-config set serve repo-path '/aurweb/aur.git/' +aurweb-config set serve ssh-cmdline "$SSH_CMDLINE" # Setup SSH Keys. ssh-keygen -A diff --git a/docker/mariadb-init-entrypoint.sh b/docker/mariadb-init-entrypoint.sh index 6df98e4f..64e66a0f 100755 --- a/docker/mariadb-init-entrypoint.sh +++ b/docker/mariadb-init-entrypoint.sh @@ -4,8 +4,9 @@ set -eou pipefail # Setup a config for our mysql db. cp -vf conf/config.dev conf/config sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config -sed -ri "s/^;?(user) = .*$/\1 = aur/g" conf/config -sed -ri "s/^;?(password) = .*$/\1 = aur/g" conf/config + +aurweb-config set database user 'aur' +aurweb-config set database password 'aur' python -m aurweb.initdb 2>/dev/null || /bin/true diff --git a/docker/php-entrypoint.sh b/docker/php-entrypoint.sh index 05b76408..1756718d 100755 --- a/docker/php-entrypoint.sh +++ b/docker/php-entrypoint.sh @@ -9,17 +9,18 @@ done cp -vf conf/config.dev conf/config sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config -# Change database user/password. -sed -ri "s/^;?(user) = .*$/\1 = aur/" conf/config -sed -ri "s/^;?(password) = .*$/\1 = aur/" conf/config +# Setup database. +aurweb-config set database user 'aur' +aurweb-config set database password 'aur' +aurweb-config set database host 'localhost' +aurweb-config set database socket '/var/lib/mysqld/mysqld.sock' +aurweb-config unset database port -# Enable memcached. -sed -ri 's/^(cache) = .+$/\1 = memcache/' conf/config - -# Setup various location configurations. -sed -ri "s;^(aur_location) = .+;\1 = ${AURWEB_PHP_PREFIX};" conf/config -sed -ri "s|^(git_clone_uri_anon) = .+|\1 = ${AURWEB_PHP_PREFIX}/%s.git|" conf/config.defaults -sed -ri "s|^(git_clone_uri_priv) = .+|\1 = ${AURWEB_SSHD_PREFIX}/%s.git|" conf/config.defaults +# Setup some other options. +aurweb-config set options cache 'memcache' +aurweb-config set options aur_location "$AURWEB_PHP_PREFIX" +aurweb-config set options git_clone_uri_anon "${AURWEB_PHP_PREFIX}/%s.git" +aurweb-config set options git_clone_uri_priv "${AURWEB_SSHD_PREFIX}/%s.git" # Listen on :9000. sed -ri 's/^(listen).*/\1 = 0.0.0.0:9000/' /etc/php/php-fpm.d/www.conf From 84beacd4274d27bf039b691a25cae7758f7d9ac2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 18:57:08 -0800 Subject: [PATCH 630/844] fix(docker): supply AUR_CONFIG_IMMUTABLE for docker-compose Signed-off-by: Kevin Morris --- docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 5d8f7d78..401193d5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,6 +72,8 @@ services: mariadb_init: image: aurweb:latest init: true + environment: + - AUR_CONFIG_IMMUTABLE=${AUR_CONFIG_IMMUTABLE:-0} entrypoint: /docker/mariadb-init-entrypoint.sh command: echo "MariaDB tables initialized." volumes: @@ -104,6 +106,7 @@ services: environment: - AUR_CONFIG=/aurweb/conf/config - SSH_CMDLINE=${SSH_CMDLINE:-ssh ssh://aur@localhost:2222} + - AUR_CONFIG_IMMUTABLE=${AUR_CONFIG_IMMUTABLE:-0} entrypoint: /docker/git-entrypoint.sh command: /docker/scripts/run-sshd.sh ports: @@ -190,6 +193,7 @@ services: - AUR_CONFIG=/aurweb/conf/config - AURWEB_PHP_PREFIX=${AURWEB_PHP_PREFIX} - AURWEB_SSHD_PREFIX=${AURWEB_SSHD_PREFIX} + - AUR_CONFIG_IMMUTABLE=${AUR_CONFIG_IMMUTABLE:-0} entrypoint: /docker/php-entrypoint.sh command: /docker/scripts/run-php.sh healthcheck: @@ -220,6 +224,7 @@ services: - AURWEB_FASTAPI_PREFIX=${AURWEB_FASTAPI_PREFIX} - AURWEB_SSHD_PREFIX=${AURWEB_SSHD_PREFIX} - PROMETHEUS_MULTIPROC_DIR=/tmp_prometheus + - AUR_CONFIG_IMMUTABLE=${AUR_CONFIG_IMMUTABLE:-0} entrypoint: /docker/fastapi-entrypoint.sh command: /docker/scripts/run-fastapi.sh "${FASTAPI_BACKEND}" healthcheck: From 343a306bb8dff96e3d7ab3227646f09f4125255d Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 23:14:39 -0800 Subject: [PATCH 631/844] change(docker): setup AUR_CONFIG in Dockerfile Signed-off-by: Kevin Morris --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 9af78c3e..38d3ca0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,10 @@ COPY . /aurweb # Working directory is aurweb root @ /aurweb. WORKDIR /aurweb +# Copy initial config to conf/config. +RUN cp -vf conf/config.dev conf/config +RUN sed -i "s;YOUR_AUR_ROOT;/aurweb;g" conf/config + # Install Python dependencies. RUN /docker/scripts/install-python-deps.sh From dbeebd3b01044d508531476dc99571890e150065 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 23:15:19 -0800 Subject: [PATCH 632/844] change(fastapi): setup live database in mariadb-init-entrypoint.sh Centralize database setup there and remove all copying of config.dev from the entrypoint scripts (the Dockerfile now does it). Signed-off-by: Kevin Morris --- docker/cron-entrypoint.sh | 28 +++++++++++++++++++++++----- docker/fastapi-entrypoint.sh | 10 +--------- docker/git-entrypoint.sh | 10 +--------- docker/mariadb-init-entrypoint.sh | 12 ++++++++---- docker/php-entrypoint.sh | 10 +--------- docker/test-mysql-entrypoint.sh | 4 ---- 6 files changed, 34 insertions(+), 40 deletions(-) diff --git a/docker/cron-entrypoint.sh b/docker/cron-entrypoint.sh index d4173eaf..5b69ab19 100755 --- a/docker/cron-entrypoint.sh +++ b/docker/cron-entrypoint.sh @@ -1,12 +1,30 @@ #!/bin/bash set -eou pipefail -# Prepare AUR_CONFIG. -cp -vf conf/config.dev conf/config -sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config +# Setup the DB. +NO_INITDB=1 /docker/mariadb-init-entrypoint.sh -# Create directories we need. -mkdir -p /aurweb/aurblup +# Create aurblup's directory. +AURBLUP_DIR="/aurweb/aurblup/" +mkdir -p $AURBLUP_DIR + +# Setup aurblup config for Docker. +AURBLUP_DBS='core extra community multilib testing community-testing' +AURBLUP_SERVER='https://mirrors.kernel.org/archlinux/%s/os/x86_64' +aurweb-config set aurblup db-path "$AURBLUP_DIR" +aurweb-config set aurblup sync-dbs "$AURBLUP_DBS" +aurweb-config set aurblup server "$AURBLUP_SERVER" + +# Setup mkpkglists config for Docker. +ARCHIVE_DIR='/var/lib/aurweb/archives' +aurweb-config set mkpkglists archivedir "$ARCHIVE_DIR" +aurweb-config set mkpkglists packagesfile "$ARCHIVE_DIR/packages.gz" +aurweb-config set mkpkglists packagesmetafile \ + "$ARCHIVE_DIR/packages-meta-v1.json.gz" +aurweb-config set mkpkglists packagesmetaextfile \ + "$ARCHIVE_DIR/packages-meta-ext-v1.json.gz" +aurweb-config set mkpkglists pkgbasefile "$ARCHIVE_DIR/pkgbase.gz" +aurweb-config set mkpkglists userfile "$ARCHIVE_DIR/users.gz" # Install the cron configuration. cp /docker/config/aurweb-cron /etc/cron.d/aurweb-cron diff --git a/docker/fastapi-entrypoint.sh b/docker/fastapi-entrypoint.sh index d1519bf8..c6597313 100755 --- a/docker/fastapi-entrypoint.sh +++ b/docker/fastapi-entrypoint.sh @@ -1,16 +1,8 @@ #!/bin/bash set -eou pipefail -# Setup a config for our mysql db. -cp -vf conf/config.dev conf/config -sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config - # Setup database. -aurweb-config set database user 'aur' -aurweb-config set database password 'aur' -aurweb-config set database host 'localhost' -aurweb-config set database socket '/var/lib/mysqld/mysqld.sock' -aurweb-config unset database port +NO_INITDB=1 /docker/mariadb-init-entrypoint.sh # Setup some other options. aurweb-config set options cache 'redis' diff --git a/docker/git-entrypoint.sh b/docker/git-entrypoint.sh index 96f4d112..c9f1ec30 100755 --- a/docker/git-entrypoint.sh +++ b/docker/git-entrypoint.sh @@ -38,16 +38,8 @@ Match User aur AcceptEnv AUR_OVERWRITE EOF -# Setup a config for our mysql db. -cp -vf conf/config.dev $AUR_CONFIG -sed -i "s;YOUR_AUR_ROOT;$(pwd);g" $AUR_CONFIG - # Setup database. -aurweb-config set database user 'aur' -aurweb-config set database password 'aur' -aurweb-config set database host 'localhost' -aurweb-config set database socket '/var/lib/mysqld/mysqld.sock' -aurweb-config unset database port +NO_INITDB=1 /docker/mariadb-init-entrypoint.sh # Setup some other options. aurweb-config set serve repo-path '/aurweb/aur.git/' diff --git a/docker/mariadb-init-entrypoint.sh b/docker/mariadb-init-entrypoint.sh index 64e66a0f..74980031 100755 --- a/docker/mariadb-init-entrypoint.sh +++ b/docker/mariadb-init-entrypoint.sh @@ -2,12 +2,16 @@ set -eou pipefail # Setup a config for our mysql db. -cp -vf conf/config.dev conf/config -sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config - +aurweb-config set database name 'aurweb' aurweb-config set database user 'aur' aurweb-config set database password 'aur' +aurweb-config set database host 'localhost' +aurweb-config set database socket '/var/run/mysqld/mysqld.sock' +aurweb-config unset database port + +if [ ! -z ${NO_INITDB+x} ]; then + exec "$@" +fi python -m aurweb.initdb 2>/dev/null || /bin/true - exec "$@" diff --git a/docker/php-entrypoint.sh b/docker/php-entrypoint.sh index 1756718d..dc1a91de 100755 --- a/docker/php-entrypoint.sh +++ b/docker/php-entrypoint.sh @@ -5,16 +5,8 @@ for archive in packages pkgbase users packages-meta-v1.json packages-meta-ext-v1 ln -vsf /var/lib/aurweb/archives/${archive}.gz /aurweb/web/html/${archive}.gz done -# Setup a config for our mysql db. -cp -vf conf/config.dev conf/config -sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config - # Setup database. -aurweb-config set database user 'aur' -aurweb-config set database password 'aur' -aurweb-config set database host 'localhost' -aurweb-config set database socket '/var/lib/mysqld/mysqld.sock' -aurweb-config unset database port +NO_INITDB=1 /docker/mariadb-init-entrypoint.sh # Setup some other options. aurweb-config set options cache 'memcache' diff --git a/docker/test-mysql-entrypoint.sh b/docker/test-mysql-entrypoint.sh index 262577a6..1bf85b54 100755 --- a/docker/test-mysql-entrypoint.sh +++ b/docker/test-mysql-entrypoint.sh @@ -1,10 +1,6 @@ #!/bin/bash set -eou pipefail -# Setup a config for our mysql db. -cp -vf conf/config.dev conf/config -sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config - # We use the root user for testing in Docker. # The test user must be able to create databases and drop them. aurweb-config set database user 'root' From 3a65e33abe01e6ef6ae1a246062b0fe5ed2c8f09 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 27 Nov 2021 23:34:05 -0800 Subject: [PATCH 633/844] fix(gitlab-ci): prepare conf/config for setup Signed-off-by: Kevin Morris --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8980fa78..d6d49a55 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ variables: AUR_CONFIG: conf/config # Default MySQL config setup in before_script. DB_HOST: localhost TEST_RECURSION_LIMIT: 10000 + CURRENT_DIR: "$(pwd)" test: stage: test @@ -22,6 +23,8 @@ test: - ./docker/mariadb-entrypoint.sh - (cd '/usr' && /usr/bin/mysqld_safe --datadir='/var/lib/mysql') & - 'until : > /dev/tcp/127.0.0.1/3306; do sleep 1s; done' + - cp -v conf/config.dev conf/config + - sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config - ./docker/test-mysql-entrypoint.sh # Create mysql AUR_CONFIG. - make -C po all install - make -C test clean From 3efb9a57b59297fb844c75306351c2817ca2f4b0 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 07:54:52 -0800 Subject: [PATCH 634/844] change(popupdate): converted to use aurweb.db ORM Signed-off-by: Kevin Morris --- aurweb/routers/packages.py | 9 ++-- aurweb/scripts/popupdate.py | 84 +++++++++++++++++++++++-------------- test/test_rpc.py | 5 +-- 3 files changed, 57 insertions(+), 41 deletions(-) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 23f44ee3..eab75e5a 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -908,8 +908,7 @@ async def pkgbase_vote(request: Request, name: str): VoteTS=now) # Update NumVotes/Popularity. - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) - popupdate.run_single(conn, pkgbase) + popupdate.run_single(pkgbase) return RedirectResponse(f"/pkgbase/{name}", status_code=HTTPStatus.SEE_OTHER) @@ -929,8 +928,7 @@ async def pkgbase_unvote(request: Request, name: str): db.delete(vote) # Update NumVotes/Popularity. - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) - popupdate.run_single(conn, pkgbase) + popupdate.run_single(pkgbase) return RedirectResponse(f"/pkgbase/{name}", status_code=HTTPStatus.SEE_OTHER) @@ -1473,8 +1471,7 @@ async def pkgbase_merge_post(request: Request, name: str, pkgbase_merge_instance(request, pkgbase, target) # Run popupdate on the target. - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) - popupdate.run_single(conn, target) + popupdate.run_single(target) if not next: next = f"/pkgbase/{target.Name}" diff --git a/aurweb/scripts/popupdate.py b/aurweb/scripts/popupdate.py index db4ba170..e2d008f2 100755 --- a/aurweb/scripts/popupdate.py +++ b/aurweb/scripts/popupdate.py @@ -1,51 +1,71 @@ #!/usr/bin/env python3 from datetime import datetime +from typing import List -import aurweb.db +from sqlalchemy import and_, func +from sqlalchemy.sql.functions import coalesce +from sqlalchemy.sql.functions import sum as _sum + +from aurweb import db +from aurweb.models import PackageBase, PackageVote -def run_single(conn, pkgbase): +def run_variable(pkgbases: List[PackageBase] = []) -> None: + """ + Update popularity on a list of PackageBases. + + If no PackageBase is included, we update the popularity + of every PackageBase in the database. + + :param pkgbases: List of PackageBase instances + """ + now = int(datetime.utcnow().timestamp()) + + # NumVotes subquery. + votes_subq = db.get_session().query( + func.count("*") + ).select_from(PackageVote).filter( + PackageVote.PackageBaseID == PackageBase.ID + ) + + # Popularity subquery. + pop_subq = db.get_session().query( + coalesce(_sum(func.pow(0.98, (now - PackageVote.VoteTS) / 86400)), 0.0), + ).select_from(PackageVote).filter( + and_(PackageVote.PackageBaseID == PackageBase.ID, + PackageVote.VoteTS.isnot(None)) + ) + + with db.begin(): + query = db.query(PackageBase) + + ids = set() + if pkgbases: + ids = {pkgbase.ID for pkgbase in pkgbases} + query = query.filter(PackageBase.ID.in_(ids)) + + query.update({ + "NumVotes": votes_subq.scalar_subquery(), + "Popularity": pop_subq.scalar_subquery() + }) + + +def run_single(pkgbase: PackageBase) -> None: """ A single popupdate. The given pkgbase instance will be refreshed after the database update is done. NOTE: This function is compatible only with aurweb FastAPI. - :param conn: db.Connection[Executor] :param pkgbase: Instance of db.PackageBase """ - - conn.execute("UPDATE PackageBases SET NumVotes = (" - "SELECT COUNT(*) FROM PackageVotes " - "WHERE PackageVotes.PackageBaseID = PackageBases.ID) " - "WHERE PackageBases.ID = ?", [pkgbase.ID]) - - now = int(datetime.utcnow().timestamp()) - conn.execute("UPDATE PackageBases SET Popularity = (" - "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " - "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " - "PackageBases.ID AND NOT VoteTS IS NULL) WHERE " - "PackageBases.ID = ?", [now, pkgbase.ID]) - - conn.commit() - conn.close() - aurweb.db.refresh(pkgbase) + run_variable([pkgbase]) + db.refresh(pkgbase) def main(): - conn = aurweb.db.Connection() - conn.execute("UPDATE PackageBases SET NumVotes = (" - "SELECT COUNT(*) FROM PackageVotes " - "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") - - now = int(datetime.utcnow().timestamp()) - conn.execute("UPDATE PackageBases SET Popularity = (" - "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " - "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " - "PackageBases.ID AND NOT VoteTS IS NULL)", [now]) - - conn.commit() - conn.close() + db.get_engine() + run_variable() if __name__ == '__main__': diff --git a/test/test_rpc.py b/test/test_rpc.py index a4cdb5da..b61a7e4e 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -9,7 +9,7 @@ import pytest from fastapi.testclient import TestClient from redis.client import Pipeline -from aurweb import asgi, config, db, scripts +from aurweb import asgi, config, scripts from aurweb.db import begin, create, query from aurweb.models.account_type import AccountType from aurweb.models.dependency_type import DependencyType @@ -187,8 +187,7 @@ def setup(db_test): PackageBase=pkgbase1, VoteTS=5000) - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) - scripts.popupdate.run_single(conn, pkgbase1) + scripts.popupdate.run_single(pkgbase1) @pytest.fixture From 29989b7fdbb6f8a5bacfd6edef48cb66b483b722 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 08:04:33 -0800 Subject: [PATCH 635/844] change(aurblup): converted to use aurweb.db ORM Introduces: - aurweb.testing.alpm.AlpmDatabase - Used to mock up and manage a remote repository. - templates/testing/alpm_package.j2 - Used to generate a single ALPM package desc. - Removed aurblup sharness test Signed-off-by: Kevin Morris --- aurweb/scripts/aurblup.py | 50 ++++++++++------- aurweb/templates.py | 5 ++ aurweb/testing/alpm.py | 87 ++++++++++++++++++++++++++++++ aurweb/util.py | 17 ++++++ templates/testing/alpm_package.j2 | 16 ++++++ test/t2400-aurblup.t | 53 ------------------ test/test_aurblup.py | 90 +++++++++++++++++++++++++++++++ 7 files changed, 246 insertions(+), 72 deletions(-) create mode 100644 aurweb/testing/alpm.py create mode 100644 templates/testing/alpm_package.j2 delete mode 100755 test/t2400-aurblup.t create mode 100644 test/test_aurblup.py diff --git a/aurweb/scripts/aurblup.py b/aurweb/scripts/aurblup.py index e32937ce..9c9059ec 100755 --- a/aurweb/scripts/aurblup.py +++ b/aurweb/scripts/aurblup.py @@ -4,30 +4,34 @@ import re import pyalpm +from sqlalchemy import and_ + import aurweb.config -import aurweb.db -db_path = aurweb.config.get('aurblup', 'db-path') -sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ') -server = aurweb.config.get('aurblup', 'server') +from aurweb import db, util +from aurweb.models import OfficialProvider -def main(): +def _main(force: bool = False): blacklist = set() providers = set() repomap = dict() + db_path = aurweb.config.get("aurblup", "db-path") + sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ') + server = aurweb.config.get('aurblup', 'server') + h = pyalpm.Handle("/", db_path) for sync_db in sync_dbs: repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) repo.servers = [server.replace("%s", sync_db)] t = h.init_transaction() - repo.update(False) + repo.update(force) t.release() for pkg in repo.pkgcache: blacklist.add(pkg.name) - [blacklist.add(x) for x in pkg.replaces] + util.apply_all(pkg.replaces, blacklist.add) providers.add((pkg.name, pkg.name)) repomap[(pkg.name, pkg.name)] = repo.name for provision in pkg.provides: @@ -35,21 +39,29 @@ def main(): providers.add((pkg.name, provisionname)) repomap[(pkg.name, provisionname)] = repo.name - conn = aurweb.db.Connection() + with db.begin(): + old_providers = set( + db.query(OfficialProvider).with_entities( + OfficialProvider.Name.label("Name"), + OfficialProvider.Provides.label("Provides") + ).distinct().order_by("Name").all() + ) - cur = conn.execute("SELECT Name, Provides FROM OfficialProviders") - oldproviders = set(cur.fetchall()) + for name, provides in old_providers.difference(providers): + db.delete_all(db.query(OfficialProvider).filter( + and_(OfficialProvider.Name == name, + OfficialProvider.Provides == provides) + )) - for pkg, provides in oldproviders.difference(providers): - conn.execute("DELETE FROM OfficialProviders " - "WHERE Name = ? AND Provides = ?", [pkg, provides]) - for pkg, provides in providers.difference(oldproviders): - repo = repomap[(pkg, provides)] - conn.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " - "VALUES (?, ?, ?)", [pkg, repo, provides]) + for name, provides in providers.difference(old_providers): + repo = repomap.get((name, provides)) + db.create(OfficialProvider, Name=name, + Repo=repo, Provides=provides) - conn.commit() - conn.close() + +def main(force: bool = False): + db.get_engine() + _main(force) if __name__ == '__main__': diff --git a/aurweb/templates.py b/aurweb/templates.py index 0039535d..a7102ae1 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -125,6 +125,11 @@ async def make_variable_context(request: Request, title: str, next: str = None): return context +def base_template(path: str): + templates = copy.copy(_env) + return templates.get_template(path) + + def render_raw_template(request: Request, path: str, context: dict): """ Render a Jinja2 multi-lingual template with some context. """ # Create a deep copy of our jinja2 _environment. The _environment in diff --git a/aurweb/testing/alpm.py b/aurweb/testing/alpm.py new file mode 100644 index 00000000..6015d859 --- /dev/null +++ b/aurweb/testing/alpm.py @@ -0,0 +1,87 @@ +import hashlib +import os +import re +import shutil +import subprocess + +from typing import List + +from aurweb import logging, util +from aurweb.templates import base_template + +logger = logging.get_logger(__name__) + + +class AlpmDatabase: + """ + Fake libalpm database management class. + + This class can be used to add or remove packages from a + test repository. + """ + repo = "test" + + def __init__(self, database_root: str): + self.root = database_root + self.local = os.path.join(self.root, "local") + self.remote = os.path.join(self.root, "remote") + self.repopath = os.path.join(self.remote, self.repo) + + # Make directories. + os.makedirs(self.local) + os.makedirs(self.remote) + + def _get_pkgdir(self, pkgname: str, pkgver: str, repo: str) -> str: + pkgfile = f"{pkgname}-{pkgver}-1" + pkgdir = os.path.join(self.remote, repo, pkgfile) + os.makedirs(pkgdir) + return pkgdir + + def add(self, pkgname: str, pkgver: str, arch: str, + provides: List[str] = []) -> None: + context = { + "pkgname": pkgname, + "pkgver": pkgver, + "arch": arch, + "provides": provides + } + template = base_template("testing/alpm_package.j2") + pkgdir = self._get_pkgdir(pkgname, pkgver, self.repo) + desc = os.path.join(pkgdir, "desc") + with open(desc, "w") as f: + f.write(template.render(context)) + + self.compile() + + def remove(self, pkgname: str): + files = os.listdir(self.repopath) + logger.info(f"Files: {files}") + expr = "^" + pkgname + r"-[0-9.]+-1$" + logger.info(f"Expression: {expr}") + to_delete = filter(lambda e: re.match(expr, e), files) + + for target in to_delete: + logger.info(f"Deleting {target}") + path = os.path.join(self.repopath, target) + shutil.rmtree(path) + + self.compile() + + def clean(self) -> None: + db_file = os.path.join(self.remote, "test.db") + try: + os.remove(db_file) + except Exception: + pass + + def compile(self) -> None: + self.clean() + cmdline = ["bash", "-c", "bsdtar -czvf ../test.db *"] + proc = subprocess.run(cmdline, cwd=self.repopath) + assert proc.returncode == 0, \ + f"Bad return code while creating alpm database: {proc.returncode}" + + # Print out the md5 hash value of the new test.db. + test_db = os.path.join(self.remote, "test.db") + db_hash = util.file_hash(test_db, hashlib.md5) + logger.debug(f"{test_db}: {db_hash}") diff --git a/aurweb/util.py b/aurweb/util.py index 62575c71..bf2d6e4b 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -176,3 +176,20 @@ def strtobool(value: str) -> bool: if isinstance(value, str): return _strtobool(value) return value + + +def file_hash(filepath: str, hash_function: Callable) -> str: + """ + Return a hash of filepath contents using `hash_function`. + + `hash_function` can be any one of the hashlib module's hash + functions which implement the `hexdigest()` method -- e.g. + hashlib.sha1, hashlib.md5, etc. + + :param filepath: Path to file you want to hash + :param hash_function: hashlib hash function + :return: hash_function(filepath_content).hexdigest() + """ + with open(filepath, "rb") as f: + hash_ = hash_function(f.read()) + return hash_.hexdigest() diff --git a/templates/testing/alpm_package.j2 b/templates/testing/alpm_package.j2 new file mode 100644 index 00000000..0e741729 --- /dev/null +++ b/templates/testing/alpm_package.j2 @@ -0,0 +1,16 @@ +%FILENAME% +{{ pkgname }}-{{ pkgver }}-{{ arch }}.pkg.tar.xz + +%NAME% +{{ pkgname }} + +%VERSION% +{{ pkgver }}-1 + +%ARCH% +{{ arch }} + +{% if provides %} +%PROVIDES% +{{ provides | join("\n") }} +{% endif %} diff --git a/test/t2400-aurblup.t b/test/t2400-aurblup.t deleted file mode 100755 index 42da6791..00000000 --- a/test/t2400-aurblup.t +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh - -test_description='aurblup tests' - -. "$(dirname "$0")/setup.sh" - -test_expect_success 'Test official provider update script.' ' - mkdir -p remote/test/foobar-1.0-1 && - cat <<-EOD >remote/test/foobar-1.0-1/desc && - %FILENAME% - foobar-1.0-any.pkg.tar.xz - - %NAME% - foobar - - %VERSION% - 1.0-1 - - %ARCH% - any - EOD - mkdir -p remote/test/foobar2-1.0-1 && - cat <<-EOD >remote/test/foobar2-1.0-1/desc && - %FILENAME% - foobar2-1.0-any.pkg.tar.xz - - %NAME% - foobar2 - - %VERSION% - 1.0-1 - - %ARCH% - any - - %PROVIDES% - foobar3 - foobar4 - EOD - ( cd remote/test && bsdtar -czf ../test.db * ) && - mkdir sync && - cover "$AURBLUP" && - cat <<-EOD >expected && - foobar|test|foobar - foobar2|test|foobar2 - foobar2|test|foobar3 - foobar2|test|foobar4 - EOD - echo "SELECT Name, Repo, Provides FROM OfficialProviders ORDER BY Provides;" | sqlite3 aur.db >actual && - test_cmp actual expected -' - -test_done diff --git a/test/test_aurblup.py b/test/test_aurblup.py new file mode 100644 index 00000000..7eaae556 --- /dev/null +++ b/test/test_aurblup.py @@ -0,0 +1,90 @@ +import tempfile + +from unittest import mock + +import pytest + +from aurweb import config, db +from aurweb.models import OfficialProvider +from aurweb.scripts import aurblup +from aurweb.testing.alpm import AlpmDatabase + + +@pytest.fixture +def tempdir() -> str: + with tempfile.TemporaryDirectory() as name: + yield name + + +@pytest.fixture +def alpm_db(tempdir: str) -> AlpmDatabase: + yield AlpmDatabase(tempdir) + + +@pytest.fixture(autouse=True) +def setup(db_test, alpm_db: AlpmDatabase, tempdir: str) -> None: + config_get = config.get + + def mock_config_get(section: str, key: str) -> str: + value = config_get(section, key) + if section == "aurblup": + if key == "db-path": + return alpm_db.local + elif key == "server": + return f'file://{alpm_db.remote}' + elif key == "sync-dbs": + return alpm_db.repo + return value + + with mock.patch("aurweb.config.get", side_effect=mock_config_get): + config.rehash() + yield + config.rehash() + + +def test_aurblup(alpm_db: AlpmDatabase): + # Test that we can add a package. + alpm_db.add("pkg", "1.0", "x86_64", provides=["pkg2", "pkg3"]) + alpm_db.add("pkg2", "2.0", "x86_64") + aurblup.main() + + # Test that the package got added to the database. + for name in ("pkg", "pkg2"): + pkg = db.query(OfficialProvider).filter( + OfficialProvider.Name == name).first() + assert pkg is not None + + # Test that we can remove the package. + alpm_db.remove("pkg") + + # Run aurblup again with forced repository update. + aurblup.main(True) + + # Expect that the database got updated accordingly. + pkg = db.query(OfficialProvider).filter( + OfficialProvider.Name == "pkg").first() + assert pkg is None + pkg2 = db.query(OfficialProvider).filter( + OfficialProvider.Name == "pkg2").first() + assert pkg2 is not None + + +def test_aurblup_cleanup(alpm_db: AlpmDatabase): + # Add a package and sync up the database. + alpm_db.add("pkg", "1.0", "x86_64", provides=["pkg2", "pkg3"]) + aurblup.main() + + # Now, let's insert an OfficialPackage that doesn't exist, + # then exercise the old provider deletion path. + with db.begin(): + db.create(OfficialProvider, Name="fake package", + Repo="test", Provides="package") + + # Run aurblup again. + aurblup.main() + + # Expect that the fake package got deleted because it's + # not in alpm_db anymore. + providers = db.query(OfficialProvider).filter( + OfficialProvider.Name == "fake package").all() + assert len(providers) == 0 From c59acbf6d6594971b3953807256e102dd2e740e9 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 21 Nov 2021 22:37:31 -0800 Subject: [PATCH 636/844] add noop testing utility Signed-off-by: Kevin Morris --- aurweb/testing/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aurweb/testing/__init__.py b/aurweb/testing/__init__.py index 8261051d..99671d69 100644 --- a/aurweb/testing/__init__.py +++ b/aurweb/testing/__init__.py @@ -62,3 +62,7 @@ def setup_test_db(*args): aurweb.db.get_session().execute(f"DELETE FROM {table}") aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 1") aurweb.db.get_session().expunge_all() + + +def noop(*args, **kwargs) -> None: + return From 29c2d0de6b83a2287ffdb885d9b55aa63b1d4792 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sun, 21 Nov 2021 00:47:48 -0800 Subject: [PATCH 637/844] change(mkpkglists): converted to use aurweb.db ORM - Improved speed dramatically - Removed mkpkglists sharness Signed-off-by: Kevin Morris --- aurweb/benchmark.py | 21 +++ aurweb/scripts/mkpkglists.py | 282 +++++++++++++++++++++-------------- test/t2100-mkpkglists.t | 65 -------- test/test_mkpkglists.py | 215 ++++++++++++++++++++++++++ 4 files changed, 403 insertions(+), 180 deletions(-) create mode 100644 aurweb/benchmark.py delete mode 100755 test/t2100-mkpkglists.t create mode 100644 test/test_mkpkglists.py diff --git a/aurweb/benchmark.py b/aurweb/benchmark.py new file mode 100644 index 00000000..7086fb08 --- /dev/null +++ b/aurweb/benchmark.py @@ -0,0 +1,21 @@ +from datetime import datetime + + +class Benchmark: + def __init__(self): + self.start() + + def _timestamp(self) -> float: + """ Generate a timestamp. """ + return float(datetime.utcnow().timestamp()) + + def start(self) -> int: + """ Start a benchmark. """ + self.current = self._timestamp() + return self.current + + def end(self): + """ Return the diff between now - start(). """ + n = self._timestamp() - self.current + self.current = float(0) + return n diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py index 307b2b12..92de7931 100755 --- a/aurweb/scripts/mkpkglists.py +++ b/aurweb/scripts/mkpkglists.py @@ -23,23 +23,28 @@ import os import sys from collections import defaultdict -from decimal import Decimal +from typing import Any, Dict import orjson +from sqlalchemy import literal, orm + import aurweb.config -import aurweb.db + +from aurweb import db, logging, models, util +from aurweb.benchmark import Benchmark +from aurweb.models import Package, PackageBase, User + +logger = logging.get_logger("aurweb.scripts.mkpkglists") archivedir = aurweb.config.get("mkpkglists", "archivedir") os.makedirs(archivedir, exist_ok=True) -packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') -packagesmetafile = aurweb.config.get('mkpkglists', 'packagesmetafile') -packagesmetaextfile = aurweb.config.get('mkpkglists', 'packagesmetaextfile') - -pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') - -userfile = aurweb.config.get('mkpkglists', 'userfile') +PACKAGES = aurweb.config.get('mkpkglists', 'packagesfile') +META = aurweb.config.get('mkpkglists', 'packagesmetafile') +META_EXT = aurweb.config.get('mkpkglists', 'packagesmetaextfile') +PKGBASE = aurweb.config.get('mkpkglists', 'pkgbasefile') +USERS = aurweb.config.get('mkpkglists', 'userfile') TYPE_MAP = { @@ -53,7 +58,7 @@ TYPE_MAP = { } -def get_extended_dict(query: str): +def get_extended_dict(query: orm.Query): """ Produce data in the form in a single bulk SQL query: @@ -74,61 +79,75 @@ def get_extended_dict(query: str): output[i].update(data.get(package_id)) """ - conn = aurweb.db.Connection() - - cursor = conn.execute(query) - data = defaultdict(lambda: defaultdict(list)) - for result in cursor.fetchall(): - + for result in query: pkgid = result[0] key = TYPE_MAP.get(result[1], result[1]) output = result[2] if result[3]: output += result[3] - - # In all cases, we have at least an empty License list. - if "License" not in data[pkgid]: - data[pkgid]["License"] = [] - - # In all cases, we have at least an empty Keywords list. - if "Keywords" not in data[pkgid]: - data[pkgid]["Keywords"] = [] - data[pkgid][key].append(output) - conn.close() return data def get_extended_fields(): - # Returns: [ID, Type, Name, Cond] - query = """ - SELECT PackageDepends.PackageID AS ID, DependencyTypes.Name AS Type, - PackageDepends.DepName AS Name, PackageDepends.DepCondition AS Cond - FROM PackageDepends - LEFT JOIN DependencyTypes - ON DependencyTypes.ID = PackageDepends.DepTypeID - UNION SELECT PackageRelations.PackageID AS ID, RelationTypes.Name AS Type, - PackageRelations.RelName AS Name, - PackageRelations.RelCondition AS Cond - FROM PackageRelations - LEFT JOIN RelationTypes - ON RelationTypes.ID = PackageRelations.RelTypeID - UNION SELECT PackageGroups.PackageID AS ID, 'Groups' AS Type, - Groups.Name, '' AS Cond - FROM Groups - INNER JOIN PackageGroups ON PackageGroups.GroupID = Groups.ID - UNION SELECT PackageLicenses.PackageID AS ID, 'License' AS Type, - Licenses.Name, '' as Cond - FROM Licenses - INNER JOIN PackageLicenses ON PackageLicenses.LicenseID = Licenses.ID - UNION SELECT Packages.ID AS ID, 'Keywords' AS Type, - PackageKeywords.Keyword AS Name, '' as Cond - FROM PackageKeywords - INNER JOIN Packages ON Packages.PackageBaseID = PackageKeywords.PackageBaseID - """ + subqueries = [ + # PackageDependency + db.query( + models.PackageDependency + ).join(models.DependencyType).with_entities( + models.PackageDependency.PackageID.label("ID"), + models.DependencyType.Name.label("Type"), + models.PackageDependency.DepName.label("Name"), + models.PackageDependency.DepCondition.label("Cond") + ).distinct().order_by("Name"), + + # PackageRelation + db.query( + models.PackageRelation + ).join(models.RelationType).with_entities( + models.PackageRelation.PackageID.label("ID"), + models.RelationType.Name.label("Type"), + models.PackageRelation.RelName.label("Name"), + models.PackageRelation.RelCondition.label("Cond") + ).distinct().order_by("Name"), + + # Groups + db.query(models.PackageGroup).join( + models.Group, + models.PackageGroup.GroupID == models.Group.ID + ).with_entities( + models.PackageGroup.PackageID.label("ID"), + literal("Groups").label("Type"), + models.Group.Name.label("Name"), + literal(str()).label("Cond") + ).distinct().order_by("Name"), + + # Licenses + db.query(models.PackageLicense).join( + models.License, + models.PackageLicense.LicenseID == models.License.ID + ).with_entities( + models.PackageLicense.PackageID.label("ID"), + literal("License").label("Type"), + models.License.Name.label("Name"), + literal(str()).label("Cond") + ).distinct().order_by("Name"), + + # Keywords + db.query(models.PackageKeyword).join( + models.Package, + Package.PackageBaseID == models.PackageKeyword.PackageBaseID + ).with_entities( + models.Package.ID.label("ID"), + literal("Keywords").label("Type"), + models.PackageKeyword.Keyword.label("Name"), + literal(str()).label("Cond") + ).distinct().order_by("Name") + ] + query = subqueries[0].union_all(*subqueries[1:]) return get_extended_dict(query) @@ -137,89 +156,122 @@ EXTENDED_FIELD_HANDLERS = { } -def is_decimal(column): - """ Check if an SQL column is of decimal.Decimal type. """ - if isinstance(column, Decimal): - return float(column) - return column +def as_dict(package: Package) -> Dict[str, Any]: + return { + "ID": package.ID, + "Name": package.Name, + "PackageBaseID": package.PackageBaseID, + "PackageBase": package.PackageBase, + "Version": package.Version, + "Description": package.Description, + "NumVotes": package.NumVotes, + "Popularity": float(package.Popularity), + "OutOfDate": package.OutOfDate, + "Maintainer": package.Maintainer, + "FirstSubmitted": package.FirstSubmitted, + "LastModified": package.LastModified, + } -def write_archive(archive: str, output: list): - with gzip.open(archive, "wb") as f: - f.write(b"[\n") - for i, item in enumerate(output): - f.write(orjson.dumps(item)) - if i < len(output) - 1: - f.write(b",") - f.write(b"\n") - f.write(b"]") +def _main(): + bench = Benchmark() + logger.info("Started re-creating archives, wait a while...") - -def main(): - conn = aurweb.db.Connection() - - # Query columns; copied from RPC. - columns = ("Packages.ID, Packages.Name, " - "PackageBases.ID AS PackageBaseID, " - "PackageBases.Name AS PackageBase, " - "Version, Description, URL, NumVotes, " - "Popularity, OutOfDateTS AS OutOfDate, " - "Users.UserName AS Maintainer, " - "SubmittedTS AS FirstSubmitted, " - "ModifiedTS AS LastModified") - - # Perform query. - cur = conn.execute(f"SELECT {columns} FROM Packages " - "LEFT JOIN PackageBases " - "ON PackageBases.ID = Packages.PackageBaseID " - "LEFT JOIN Users " - "ON PackageBases.MaintainerUID = Users.ID " - "WHERE PackageBases.PackagerUID IS NOT NULL") + query = db.query(Package).join( + PackageBase, + PackageBase.ID == Package.PackageBaseID + ).join( + User, + PackageBase.MaintainerUID == User.ID, + isouter=True + ).filter(PackageBase.PackagerUID.isnot(None)).with_entities( + Package.ID, + Package.Name, + PackageBase.ID.label("PackageBaseID"), + PackageBase.Name.label("PackageBase"), + Package.Version, + Package.Description, + PackageBase.NumVotes, + PackageBase.Popularity, + PackageBase.OutOfDateTS.label("OutOfDate"), + User.Username.label("Maintainer"), + PackageBase.SubmittedTS.label("FirstSubmitted"), + PackageBase.ModifiedTS.label("LastModified") + ).distinct().order_by("Name") # Produce packages-meta-v1.json.gz output = list() snapshot_uri = aurweb.config.get("options", "snapshot_uri") - for result in cur.fetchall(): - item = { - column[0]: is_decimal(result[i]) - for i, column in enumerate(cur.description) - } - item["URLPath"] = snapshot_uri % item.get("Name") - output.append(item) + gzips = { + "packages": gzip.open(PACKAGES, "wt"), + "meta": gzip.open(META, "wb"), + } - write_archive(packagesmetafile, output) + # Append list opening to the metafile. + gzips["meta"].write(b"[\n") - # Produce packages-meta-ext-v1.json.gz + # Produce packages.gz + packages-meta-ext-v1.json.gz + extended = False if len(sys.argv) > 1 and sys.argv[1] in EXTENDED_FIELD_HANDLERS: + gzips["meta_ext"] = gzip.open(META_EXT, "wb") + # Append list opening to the meta_ext file. + gzips.get("meta_ext").write(b"[\n") f = EXTENDED_FIELD_HANDLERS.get(sys.argv[1]) data = f() + extended = True - default_ = {"Groups": [], "License": [], "Keywords": []} - for i in range(len(output)): - data_ = data.get(output[i].get("ID"), default_) - output[i].update(data_) + results = query.all() + n = len(results) - 1 + for i, result in enumerate(results): + # Append to packages.gz. + gzips.get("packages").write(f"{result.Name}\n") - write_archive(packagesmetaextfile, output) + # Construct our result JSON dictionary. + item = as_dict(result) + item["URLPath"] = snapshot_uri % result.Name - # Produce packages.gz - with gzip.open(packagesfile, "wb") as f: - f.writelines([ - bytes(x.get("Name") + "\n", "UTF-8") - for x in output - ]) + # We stream out package json objects line per line, so + # we also need to include the ',' character at the end + # of package lines (excluding the last package). + suffix = b",\n" if i < n else b'\n' + + # Write out to packagesmetafile + output.append(item) + gzips.get("meta").write(orjson.dumps(output[-1]) + suffix) + + if extended: + # Write out to packagesmetaextfile. + data_ = data.get(result.ID, {}) + output[-1].update(data_) + gzips.get("meta_ext").write(orjson.dumps(output[-1]) + suffix) + + # Append the list closing to meta/meta_ext. + gzips.get("meta").write(b"]") + if extended: + gzips.get("meta_ext").write(b"]") + + # Close gzip files. + util.apply_all(gzips.values(), lambda gz: gz.close()) # Produce pkgbase.gz - with gzip.open(pkgbasefile, "w") as f: - cur = conn.execute("SELECT Name FROM PackageBases " + - "WHERE PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + query = db.query(PackageBase.Name).filter( + PackageBase.PackagerUID.isnot(None)).all() + with gzip.open(PKGBASE, "wt") as f: + f.writelines([f"{base.Name}\n" for i, base in enumerate(query)]) # Produce users.gz - with gzip.open(userfile, "w") as f: - cur = conn.execute("SELECT UserName FROM Users") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + query = db.query(User.Username).all() + with gzip.open(USERS, "wt") as f: + f.writelines([f"{user.Username}\n" for i, user in enumerate(query)]) - conn.close() + seconds = util.number_format(bench.end(), 4) + logger.info(f"Completed in {seconds} seconds.") + + +def main(): + db.get_engine() + with db.begin(): + _main() if __name__ == '__main__': diff --git a/test/t2100-mkpkglists.t b/test/t2100-mkpkglists.t deleted file mode 100755 index d217c4f6..00000000 --- a/test/t2100-mkpkglists.t +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/sh - -test_description='mkpkglists tests' - -. "$(dirname "$0")/setup.sh" - -test_expect_success 'Test package list generation with no packages.' ' - echo "DELETE FROM Packages;" | sqlite3 aur.db && - echo "DELETE FROM PackageBases;" | sqlite3 aur.db && - cover "$MKPKGLISTS" && - test $(zcat packages.gz | wc -l) -eq 0 && - test $(zcat pkgbase.gz | wc -l) -eq 0 -' - -test_expect_success 'Test package list generation.' ' - cat <<-EOD | sqlite3 aur.db && - INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (1, "foobar", 1, 0, 0, ""); - INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (2, "foobar2", 2, 0, 0, ""); - INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (3, "foobar3", NULL, 0, 0, ""); - INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (4, "foobar4", 1, 0, 0, ""); - INSERT INTO Packages (ID, PackageBaseID, Name) VALUES (1, 1, "pkg1"); - INSERT INTO Packages (ID, PackageBaseID, Name) VALUES (2, 1, "pkg2"); - INSERT INTO Packages (ID, PackageBaseID, Name) VALUES (3, 1, "pkg3"); - INSERT INTO Packages (ID, PackageBaseID, Name) VALUES (4, 2, "pkg4"); - INSERT INTO Packages (ID, PackageBaseID, Name) VALUES (5, 3, "pkg5"); - EOD - cover "$MKPKGLISTS" && - cat <<-EOD >expected && - foobar - foobar2 - foobar4 - EOD - gunzip pkgbase.gz && - sed "/^#/d" pkgbase >actual && - test_cmp actual expected && - cat <<-EOD >expected && - pkg1 - pkg2 - pkg3 - pkg4 - EOD - gunzip packages.gz && - sed "/^#/d" packages >actual && - test_cmp actual expected -' - -test_expect_success 'Test user list generation.' ' - cover "$MKPKGLISTS" && - cat <<-EOD >expected && - dev - tu - tu2 - tu3 - tu4 - user - user2 - user3 - user4 - EOD - gunzip users.gz && - sed "/^#/d" users >actual && - test_cmp actual expected -' - -test_done diff --git a/test/test_mkpkglists.py b/test/test_mkpkglists.py new file mode 100644 index 00000000..ee66e4e1 --- /dev/null +++ b/test/test_mkpkglists.py @@ -0,0 +1,215 @@ +import json + +from typing import List, Union +from unittest import mock + +import pytest + +from aurweb import config, db, util +from aurweb.models import License, Package, PackageBase, PackageDependency, PackageLicense, User +from aurweb.models.account_type import USER_ID +from aurweb.models.dependency_type import DEPENDS_ID +from aurweb.testing import noop + + +class FakeFile: + data = str() + __exit__ = noop + + def __init__(self, modes: str) -> "FakeFile": + self.modes = modes + + def __enter__(self, *args, **kwargs) -> "FakeFile": + return self + + def write(self, data: Union[str, bytes]) -> None: + if isinstance(data, bytes): + data = data.decode() + self.data += data + + def writelines(self, dataset: List[Union[str, bytes]]) -> None: + util.apply_all(dataset, self.write) + + def close(self) -> None: + return + + +class MockGzipOpen: + def __init__(self): + self.gzips = dict() + + def open(self, archive: str, modes: str): + self.gzips[archive] = FakeFile(modes) + return self.gzips.get(archive) + + def get(self, key: str) -> FakeFile: + return self.gzips.get(key) + + def __getitem__(self, key: str) -> FakeFile: + return self.get(key) + + def __contains__(self, key: str) -> bool: + return key in self.gzips + + def data(self, archive: str): + return self.get(archive).data + + +@pytest.fixture(autouse=True) +def setup(db_test): + config.rehash() + + +@pytest.fixture +def user() -> User: + with db.begin(): + user = db.create(User, Username="test", + Email="test@example.org", + Passwd="testPassword", + AccountTypeID=USER_ID) + yield user + + +@pytest.fixture +def packages(user: User) -> List[Package]: + output = [] + with db.begin(): + lic = db.create(License, Name="GPL") + for i in range(5): + # Create the package. + pkgbase = db.create(PackageBase, Name=f"pkgbase_{i}", + Packager=user) + pkg = db.create(Package, PackageBase=pkgbase, + Name=f"pkg_{i}") + + # Create some related records. + db.create(PackageLicense, Package=pkg, License=lic) + db.create(PackageDependency, DepTypeID=DEPENDS_ID, + Package=pkg, DepName=f"dep_{i}", + DepCondition=">=1.0") + + # Add the package to our output list. + output.append(pkg) + + # Sort output by the package name and return it. + yield sorted(output, key=lambda k: k.Name) + + +@mock.patch("os.makedirs", side_effect=noop) +def test_mkpkglists_empty(makedirs: mock.MagicMock): + gzips = MockGzipOpen() + with mock.patch("gzip.open", side_effect=gzips.open): + from aurweb.scripts import mkpkglists + mkpkglists.main() + + archives = config.get_section("mkpkglists") + archives.pop("archivedir") + archives.pop("packagesmetaextfile") + + for archive in archives.values(): + assert archive in gzips + + # Expect that packagesfile got created, but is empty because + # we have no DB records. + packages_file = archives.get("packagesfile") + assert gzips.data(packages_file) == str() + + # Expect that pkgbasefile got created, but is empty because + # we have no DB records. + users_file = archives.get("pkgbasefile") + assert gzips.data(users_file) == str() + + # Expect that userfile got created, but is empty because + # we have no DB records. + users_file = archives.get("userfile") + assert gzips.data(users_file) == str() + + # Expect that packagesmetafile got created, but is empty because + # we have no DB records; it's still a valid empty JSON list. + meta_file = archives.get("packagesmetafile") + assert gzips.data(meta_file) == "[\n]" + + +@mock.patch("sys.argv", ["mkpkglists", "--extended"]) +@mock.patch("os.makedirs", side_effect=noop) +def test_mkpkglists_extended_empty(makedirs: mock.MagicMock): + gzips = MockGzipOpen() + with mock.patch("gzip.open", side_effect=gzips.open): + from aurweb.scripts import mkpkglists + mkpkglists.main() + + archives = config.get_section("mkpkglists") + archives.pop("archivedir") + + for archive in archives.values(): + assert archive in gzips + + # Expect that packagesfile got created, but is empty because + # we have no DB records. + packages_file = archives.get("packagesfile") + assert gzips.data(packages_file) == str() + + # Expect that pkgbasefile got created, but is empty because + # we have no DB records. + users_file = archives.get("pkgbasefile") + assert gzips.data(users_file) == str() + + # Expect that userfile got created, but is empty because + # we have no DB records. + users_file = archives.get("userfile") + assert gzips.data(users_file) == str() + + # Expect that packagesmetafile got created, but is empty because + # we have no DB records; it's still a valid empty JSON list. + meta_file = archives.get("packagesmetafile") + assert gzips.data(meta_file) == "[\n]" + + # Expect that packagesmetafile got created, but is empty because + # we have no DB records; it's still a valid empty JSON list. + meta_file = archives.get("packagesmetaextfile") + assert gzips.data(meta_file) == "[\n]" + + +@mock.patch("sys.argv", ["mkpkglists", "--extended"]) +@mock.patch("os.makedirs", side_effect=noop) +def test_mkpkglists_extended(makedirs: mock.MagicMock, user: User, + packages: List[Package]): + gzips = MockGzipOpen() + with mock.patch("gzip.open", side_effect=gzips.open): + from aurweb.scripts import mkpkglists + mkpkglists.main() + + archives = config.get_section("mkpkglists") + archives.pop("archivedir") + + for archive in archives.values(): + assert archive in gzips + + # Expect that packagesfile got created, but is empty because + # we have no DB records. + packages_file = archives.get("packagesfile") + expected = "\n".join([p.Name for p in packages]) + "\n" + assert gzips.data(packages_file) == expected + + # Expect that pkgbasefile got created, but is empty because + # we have no DB records. + users_file = archives.get("pkgbasefile") + expected = "\n".join([p.PackageBase.Name for p in packages]) + "\n" + assert gzips.data(users_file) == expected + + # Expect that userfile got created, but is empty because + # we have no DB records. + users_file = archives.get("userfile") + assert gzips.data(users_file) == "test\n" + + # Expect that packagesmetafile got created, but is empty because + # we have no DB records; it's still a valid empty JSON list. + meta_file = archives.get("packagesmetafile") + data = json.loads(gzips.data(meta_file)) + assert len(data) == 5 + + # Expect that packagesmetafile got created, but is empty because + # we have no DB records; it's still a valid empty JSON list. + meta_file = archives.get("packagesmetaextfile") + data = json.loads(gzips.data(meta_file)) + assert len(data) == 5 From 8d5683d3f18004d44cb4277a67da12c0a88e7698 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 22 Nov 2021 10:28:58 -0800 Subject: [PATCH 638/844] change(tuvotereminder): converted to use aurweb.db ORM - Removed tuvotereminder sharness test. - Added [tuvotereminder] section to config.defaults. - Added `range_start` option to config.defaults [tuvotereminder]. - Added `range_end` option to config.defaults [tuvotereminder]. Signed-off-by: Kevin Morris --- aurweb/scripts/tuvotereminder.py | 33 ++++++---- conf/config.defaults | 8 +++ test/t2200-tuvotereminder.t | 53 ---------------- test/test_tuvotereminder.py | 102 +++++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 65 deletions(-) delete mode 100755 test/t2200-tuvotereminder.t create mode 100644 test/test_tuvotereminder.py diff --git a/aurweb/scripts/tuvotereminder.py b/aurweb/scripts/tuvotereminder.py index eb3874e1..5e860725 100755 --- a/aurweb/scripts/tuvotereminder.py +++ b/aurweb/scripts/tuvotereminder.py @@ -1,27 +1,36 @@ #!/usr/bin/env python3 -import subprocess -import time +from datetime import datetime + +from sqlalchemy import and_ import aurweb.config -import aurweb.db + +from aurweb import db +from aurweb.models import TUVoteInfo +from aurweb.scripts import notify notify_cmd = aurweb.config.get('notifications', 'notify-cmd') def main(): - conn = aurweb.db.Connection() + db.get_engine() - now = int(time.time()) - filter_from = now + 500 - filter_to = now + 172800 + now = int(datetime.utcnow().timestamp()) - cur = conn.execute("SELECT ID FROM TU_VoteInfo " + - "WHERE End >= ? AND End <= ?", - [filter_from, filter_to]) + start = aurweb.config.getint("tuvotereminder", "range_start") + filter_from = now + start - for vote_id in [row[0] for row in cur.fetchall()]: - subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))).wait() + end = aurweb.config.getint("tuvotereminder", "range_end") + filter_to = now + end + + query = db.query(TUVoteInfo.ID).filter( + and_(TUVoteInfo.End >= filter_from, + TUVoteInfo.End <= filter_to) + ) + for voteinfo in query: + notif = notify.TUVoteReminderNotification(voteinfo.ID) + notif.send() if __name__ == '__main__': diff --git a/conf/config.defaults b/conf/config.defaults index a589997b..082d51a5 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -121,3 +121,11 @@ commit_url = https://gitlab.archlinux.org/archlinux/aurweb/-/commits/%s ; Example deployment configuration step: ; sed -r "s/^;?(commit_hash) =.*$/\1 = $(git rev-parse HEAD)/" config ;commit_hash = 1234567 + +[tuvotereminder] +; Offsets used to determine when TUs should be reminded about +; votes that they should make. +; Reminders will be sent out for all votes that a TU has not yet +; voted on based on `now + range_start <= End <= now + range_end`. +range_start = 500 +range_end = 172800 diff --git a/test/t2200-tuvotereminder.t b/test/t2200-tuvotereminder.t deleted file mode 100755 index 2f3836de..00000000 --- a/test/t2200-tuvotereminder.t +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh - -test_description='tuvotereminder tests' - -. "$(dirname "$0")/setup.sh" - -test_expect_success 'Test Trusted User vote reminders.' ' - now=$(date -d now +%s) && - tomorrow=$(date -d tomorrow +%s) && - threedays=$(date -d "3 days" +%s) && - cat <<-EOD | sqlite3 aur.db && - INSERT INTO TU_VoteInfo (ID, Agenda, User, Submitted, End, Quorum, SubmitterID) VALUES (1, "Lorem ipsum.", "user", 0, $now, 0.00, 2); - INSERT INTO TU_VoteInfo (ID, Agenda, User, Submitted, End, Quorum, SubmitterID) VALUES (2, "Lorem ipsum.", "user", 0, $tomorrow, 0.00, 2); - INSERT INTO TU_VoteInfo (ID, Agenda, User, Submitted, End, Quorum, SubmitterID) VALUES (3, "Lorem ipsum.", "user", 0, $tomorrow, 0.00, 2); - INSERT INTO TU_VoteInfo (ID, Agenda, User, Submitted, End, Quorum, SubmitterID) VALUES (4, "Lorem ipsum.", "user", 0, $threedays, 0.00, 2); - EOD - >sendmail.out && - cover "$TUVOTEREMINDER" && - grep -q "Proposal 2" sendmail.out && - grep -q "Proposal 3" sendmail.out && - test_must_fail grep -q "Proposal 1" sendmail.out && - test_must_fail grep -q "Proposal 4" sendmail.out -' - -test_expect_success 'Check that only TUs who did not vote receive reminders.' ' - cat <<-EOD | sqlite3 aur.db && - INSERT INTO TU_Votes (VoteID, UserID) VALUES (1, 2); - INSERT INTO TU_Votes (VoteID, UserID) VALUES (2, 2); - INSERT INTO TU_Votes (VoteID, UserID) VALUES (3, 2); - INSERT INTO TU_Votes (VoteID, UserID) VALUES (4, 2); - INSERT INTO TU_Votes (VoteID, UserID) VALUES (1, 7); - INSERT INTO TU_Votes (VoteID, UserID) VALUES (3, 7); - INSERT INTO TU_Votes (VoteID, UserID) VALUES (2, 8); - INSERT INTO TU_Votes (VoteID, UserID) VALUES (4, 8); - INSERT INTO TU_Votes (VoteID, UserID) VALUES (1, 9); - EOD - >sendmail.out && - cover "$TUVOTEREMINDER" && - cat <<-EOD >expected && - Subject: TU Vote Reminder: Proposal 2 - To: tu2@localhost - Subject: TU Vote Reminder: Proposal 2 - To: tu4@localhost - Subject: TU Vote Reminder: Proposal 3 - To: tu3@localhost - Subject: TU Vote Reminder: Proposal 3 - To: tu4@localhost - EOD - grep "^\(Subject\|To\)" sendmail.out >sendmail.parts && - test_cmp sendmail.parts expected -' - -test_done diff --git a/test/test_tuvotereminder.py b/test/test_tuvotereminder.py new file mode 100644 index 00000000..bb898e3a --- /dev/null +++ b/test/test_tuvotereminder.py @@ -0,0 +1,102 @@ +from datetime import datetime +from typing import Tuple + +import pytest + +from aurweb import config, db +from aurweb.models import TUVote, TUVoteInfo, User +from aurweb.models.account_type import TRUSTED_USER_ID +from aurweb.scripts import tuvotereminder as reminder +from aurweb.testing.email import Email + +aur_location = config.get("options", "aur_location") + + +def create_vote(user: User, voteinfo: TUVoteInfo) -> TUVote: + with db.begin(): + vote = db.create(TUVote, User=user, VoteID=voteinfo.ID) + return vote + + +def create_user(username: str, type_id: int): + with db.begin(): + user = db.create(User, AccountTypeID=type_id, Username=username, + Email=f"{username}@example.org", Passwd=str()) + return user + + +def email_pieces(voteinfo: TUVoteInfo) -> Tuple[str, str]: + """ + Return a (subject, content) tuple based on voteinfo.ID + + :param voteinfo: TUVoteInfo instance + :return: tuple(subject, content) + """ + subject = f"TU Vote Reminder: Proposal {voteinfo.ID}" + content = (f"Please remember to cast your vote on proposal {voteinfo.ID} " + f"[1]. The voting period\nends in less than 48 hours.\n\n" + f"[1] {aur_location}/tu/?id={voteinfo.ID}") + return (subject, content) + + +@pytest.fixture +def user(db_test) -> User: + yield create_user("test", TRUSTED_USER_ID) + + +@pytest.fixture +def user2() -> User: + yield create_user("test2", TRUSTED_USER_ID) + + +@pytest.fixture +def user3() -> User: + yield create_user("test3", TRUSTED_USER_ID) + + +@pytest.fixture +def voteinfo(user: User) -> TUVoteInfo: + now = int(datetime.utcnow().timestamp()) + start = config.getint("tuvotereminder", "range_start") + with db.begin(): + voteinfo = db.create(TUVoteInfo, Agenda="Lorem ipsum.", + User=user.Username, End=(now + start + 1), + Quorum=0.00, Submitter=user, Submitted=0) + yield voteinfo + + +def test_tu_vote_reminders(user: User, user2: User, user3: User, + voteinfo: TUVoteInfo): + reminder.main() + assert Email.count() == 3 + + emails = [Email(i).parse() for i in range(1, 4)] + subject, content = email_pieces(voteinfo) + expectations = [ + # (to, content) + (user.Email, subject, content), + (user2.Email, subject, content), + (user3.Email, subject, content) + ] + for i, element in enumerate(expectations): + email, subject, content = element + assert emails[i].headers.get("To") == email + assert emails[i].headers.get("Subject") == subject + assert emails[i].body == content + + +def test_tu_vote_reminders_only_unvoted(user: User, user2: User, user3: User, + voteinfo: TUVoteInfo): + # Vote with user2 and user3; leaving only user to be notified. + create_vote(user2, voteinfo) + create_vote(user3, voteinfo) + + reminder.main() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == user.Email + + subject, content = email_pieces(voteinfo) + assert email.headers.get("Subject") == subject + assert email.body == content From d097799b34e5392f577db28920f8fb27b35602f3 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 22 Nov 2021 10:51:44 -0800 Subject: [PATCH 639/844] change(usermaint): converted to use aurweb.db ORM - Removed usermaint sharness test Signed-off-by: Kevin Morris --- aurweb/scripts/usermaint.py | 34 ++++++++++++------- test/t2700-usermaint.t | 49 --------------------------- test/test_usermaint.py | 67 +++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 61 deletions(-) delete mode 100755 test/t2700-usermaint.t create mode 100644 test/test_usermaint.py diff --git a/aurweb/scripts/usermaint.py b/aurweb/scripts/usermaint.py index 1621d410..aad3e8de 100755 --- a/aurweb/scripts/usermaint.py +++ b/aurweb/scripts/usermaint.py @@ -1,21 +1,31 @@ #!/usr/bin/env python3 -import time +from datetime import datetime -import aurweb.db +from sqlalchemy import update + +from aurweb import db +from aurweb.models import User + + +def _main(): + limit_to = int(datetime.utcnow().timestamp()) - 86400 * 7 + + update_ = update(User).where( + User.LastLogin < limit_to + ).values(LastLoginIPAddress=None) + db.get_session().execute(update_) + + update_ = update(User).where( + User.LastSSHLogin < limit_to + ).values(LastSSHLoginIPAddress=None) + db.get_session().execute(update_) def main(): - conn = aurweb.db.Connection() - - limit_to = int(time.time()) - 86400 * 7 - conn.execute("UPDATE Users SET LastLoginIPAddress = NULL " + - "WHERE LastLogin < ?", [limit_to]) - conn.execute("UPDATE Users SET LastSSHLoginIPAddress = NULL " + - "WHERE LastSSHLogin < ?", [limit_to]) - - conn.commit() - conn.close() + db.get_engine() + with db.begin(): + _main() if __name__ == '__main__': diff --git a/test/t2700-usermaint.t b/test/t2700-usermaint.t deleted file mode 100755 index c119e3f4..00000000 --- a/test/t2700-usermaint.t +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh - -test_description='usermaint tests' - -. "$(dirname "$0")/setup.sh" - -test_expect_success 'Test removal of login IP addresses.' ' - now=$(date -d now +%s) && - threedaysago=$(date -d "3 days ago" +%s) && - tendaysago=$(date -d "10 days ago" +%s) && - cat <<-EOD | sqlite3 aur.db && - UPDATE Users SET LastLogin = $threedaysago, LastLoginIPAddress = "1.2.3.4" WHERE ID = 1; - UPDATE Users SET LastLogin = $tendaysago, LastLoginIPAddress = "2.3.4.5" WHERE ID = 2; - UPDATE Users SET LastLogin = $now, LastLoginIPAddress = "3.4.5.6" WHERE ID = 3; - UPDATE Users SET LastLogin = 0, LastLoginIPAddress = "4.5.6.7" WHERE ID = 4; - UPDATE Users SET LastLogin = 0, LastLoginIPAddress = "5.6.7.8" WHERE ID = 5; - UPDATE Users SET LastLogin = $tendaysago, LastLoginIPAddress = "6.7.8.9" WHERE ID = 6; - EOD - cover "$USERMAINT" && - cat <<-EOD >expected && - 1.2.3.4 - 3.4.5.6 - EOD - echo "SELECT LastLoginIPAddress FROM Users WHERE LastLoginIPAddress IS NOT NULL;" | sqlite3 aur.db >actual && - test_cmp actual expected -' - -test_expect_success 'Test removal of SSH login IP addresses.' ' - now=$(date -d now +%s) && - threedaysago=$(date -d "3 days ago" +%s) && - tendaysago=$(date -d "10 days ago" +%s) && - cat <<-EOD | sqlite3 aur.db && - UPDATE Users SET LastSSHLogin = $now, LastSSHLoginIPAddress = "1.2.3.4" WHERE ID = 1; - UPDATE Users SET LastSSHLogin = $threedaysago, LastSSHLoginIPAddress = "2.3.4.5" WHERE ID = 2; - UPDATE Users SET LastSSHLogin = $tendaysago, LastSSHLoginIPAddress = "3.4.5.6" WHERE ID = 3; - UPDATE Users SET LastSSHLogin = 0, LastSSHLoginIPAddress = "4.5.6.7" WHERE ID = 4; - UPDATE Users SET LastSSHLogin = 0, LastSSHLoginIPAddress = "5.6.7.8" WHERE ID = 5; - UPDATE Users SET LastSSHLogin = $tendaysago, LastSSHLoginIPAddress = "6.7.8.9" WHERE ID = 6; - EOD - cover "$USERMAINT" && - cat <<-EOD >expected && - 1.2.3.4 - 2.3.4.5 - EOD - echo "SELECT LastSSHLoginIPAddress FROM Users WHERE LastSSHLoginIPAddress IS NOT NULL;" | sqlite3 aur.db >actual && - test_cmp actual expected -' - -test_done diff --git a/test/test_usermaint.py b/test/test_usermaint.py new file mode 100644 index 00000000..f1af59e1 --- /dev/null +++ b/test/test_usermaint.py @@ -0,0 +1,67 @@ +from datetime import datetime + +import pytest + +from aurweb import db +from aurweb.models import User +from aurweb.models.account_type import USER_ID +from aurweb.scripts import usermaint + + +@pytest.fixture(autouse=True) +def setup(db_test): + return + + +@pytest.fixture +def user() -> User: + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + Passwd="testPassword", AccountTypeID=USER_ID) + yield user + + +def test_usermaint_noop(user: User): + """ Last[SSH]Login isn't expired in this test: usermaint is noop. """ + + now = int(datetime.utcnow().timestamp()) + with db.begin(): + user.LastLoginIPAddress = "127.0.0.1" + user.LastLogin = now - 10 + user.LastSSHLoginIPAddress = "127.0.0.1" + user.LastSSHLogin = now - 10 + + usermaint.main() + + assert user.LastLoginIPAddress == "127.0.0.1" + assert user.LastSSHLoginIPAddress == "127.0.0.1" + + +def test_usermaint(user: User): + """ + In this case, we first test that only the expired record gets + updated, but the non-expired record remains untouched. After, + we update the login time on the non-expired record and exercise + its code path. + """ + + now = int(datetime.utcnow().timestamp()) + limit_to = now - 86400 * 7 + with db.begin(): + user.LastLoginIPAddress = "127.0.0.1" + user.LastLogin = limit_to - 666 + user.LastSSHLoginIPAddress = "127.0.0.1" + user.LastSSHLogin = now - 10 + + usermaint.main() + + assert user.LastLoginIPAddress is None + assert user.LastSSHLoginIPAddress == "127.0.0.1" + + with db.begin(): + user.LastSSHLogin = limit_to - 666 + + usermaint.main() + + assert user.LastLoginIPAddress is None + assert user.LastSSHLoginIPAddress is None From f4ef02fa5b744598e366930e63adec4cdbac6706 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 22 Nov 2021 11:07:31 -0800 Subject: [PATCH 640/844] fix(fastapi): fix Package's PackageBase backref cascade Signed-off-by: Kevin Morris --- aurweb/models/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aurweb/models/package.py b/aurweb/models/package.py index 8f82dadd..cfb27634 100644 --- a/aurweb/models/package.py +++ b/aurweb/models/package.py @@ -12,7 +12,8 @@ class Package(Base): __mapper_args__ = {"primary_key": [__table__.c.ID]} PackageBase = relationship( - _PackageBase, backref=backref("packages", lazy="dynamic"), + _PackageBase, backref=backref("packages", lazy="dynamic", + cascade="all, delete"), foreign_keys=[__table__.c.PackageBaseID]) def __init__(self, **kwargs): From b72bd38f76ed290a4e5607dfca177dcf0c1d9864 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 22 Nov 2021 11:08:15 -0800 Subject: [PATCH 641/844] change(pkgmaint): converted to use aurweb.db ORM - Replaced time.time() usage with datetime.utcnow().timestamp() - Removed pkgmaint sharness test Signed-off-by: Kevin Morris --- aurweb/scripts/pkgmaint.py | 28 ++++++++++------ test/t2300-pkgmaint.t | 26 --------------- test/test_pkgmaint.py | 65 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 36 deletions(-) delete mode 100755 test/t2300-pkgmaint.t create mode 100644 test/test_pkgmaint.py diff --git a/aurweb/scripts/pkgmaint.py b/aurweb/scripts/pkgmaint.py index 36da126f..b3992e5c 100755 --- a/aurweb/scripts/pkgmaint.py +++ b/aurweb/scripts/pkgmaint.py @@ -1,19 +1,27 @@ #!/usr/bin/env python3 -import time +from datetime import datetime -import aurweb.db +from sqlalchemy import and_ + +from aurweb import db +from aurweb.models import PackageBase + + +def _main(): + # One day behind. + limit_to = int(datetime.utcnow().timestamp()) - 86400 + + query = db.query(PackageBase).filter( + and_(PackageBase.SubmittedTS < limit_to, + PackageBase.PackagerUID.is_(None))) + db.delete_all(query) def main(): - conn = aurweb.db.Connection() - - limit_to = int(time.time()) - 86400 - conn.execute("DELETE FROM PackageBases WHERE " + - "SubmittedTS < ? AND PackagerUID IS NULL", [limit_to]) - - conn.commit() - conn.close() + db.get_engine() + with db.begin(): + _main() if __name__ == '__main__': diff --git a/test/t2300-pkgmaint.t b/test/t2300-pkgmaint.t deleted file mode 100755 index 997f95b0..00000000 --- a/test/t2300-pkgmaint.t +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -test_description='pkgmaint tests' - -. "$(dirname "$0")/setup.sh" - -test_expect_success 'Test package base cleanup script.' ' - now=$(date -d now +%s) && - threedaysago=$(date -d "3 days ago" +%s) && - cat <<-EOD | sqlite3 aur.db && - INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (1, "foobar", 1, $now, 0, ""); - INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (2, "foobar2", 2, $threedaysago, 0, ""); - INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (3, "foobar3", NULL, $now, 0, ""); - INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (4, "foobar4", NULL, $threedaysago, 0, ""); - EOD - cover "$PKGMAINT" && - cat <<-EOD >expected && - foobar - foobar2 - foobar3 - EOD - echo "SELECT Name FROM PackageBases;" | sqlite3 aur.db >actual && - test_cmp actual expected -' - -test_done diff --git a/test/test_pkgmaint.py b/test/test_pkgmaint.py new file mode 100644 index 00000000..921a6330 --- /dev/null +++ b/test/test_pkgmaint.py @@ -0,0 +1,65 @@ +from datetime import datetime +from typing import List + +import pytest + +from aurweb import db +from aurweb.models import Package, PackageBase, User +from aurweb.models.account_type import USER_ID +from aurweb.scripts import pkgmaint + + +@pytest.fixture(autouse=True) +def setup(db_test): + return + + +@pytest.fixture +def user() -> User: + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + Passwd="testPassword", AccountTypeID=USER_ID) + yield user + + +@pytest.fixture +def packages(user: User) -> List[Package]: + output = [] + + now = int(datetime.utcnow().timestamp()) + with db.begin(): + for i in range(5): + pkgbase = db.create(PackageBase, Name=f"pkg_{i}", + SubmittedTS=now, + ModifiedTS=now) + pkg = db.create(Package, PackageBase=pkgbase, + Name=f"pkg_{i}", Version=f"{i}.0") + output.append(pkg) + yield output + + +def test_pkgmaint_noop(packages: List[Package]): + assert len(packages) == 5 + pkgmaint.main() + packages = db.query(Package).all() + assert len(packages) == 5 + + +def test_pkgmaint(packages: List[Package]): + assert len(packages) == 5 + + # Modify the first package so it's out of date and gets deleted. + with db.begin(): + # Reduce SubmittedTS by a day + 10 seconds. + packages[0].PackageBase.SubmittedTS -= (86400 + 10) + + # Run pkgmaint. + pkgmaint.main() + + # Query package objects again and assert that the + # first package was deleted but all others are intact. + packages = db.query(Package).all() + assert len(packages) == 4 + expected = ["pkg_1", "pkg_2", "pkg_3", "pkg_4"] + for i, pkgname in enumerate(expected): + assert packages[i].Name == pkgname From 9fb1fbe32cd2d7652f4dee9df29002a36b9ff38c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 22 Nov 2021 15:00:22 -0800 Subject: [PATCH 642/844] feat(testing): add email testing utilities Changes: - util/sendmail now populates email files in the 'test-emails' directory. - util/sendmail does this in a serialized fashion based off of the test suite and name retrieved from PYTEST_CURRENT_TEST in the format: `_.n.txt` where n is increased by one every time sendmail is run. - pytest conftest fixtures have been added for test email setup; it wipes out old emails for the particular test function being run. - New aurweb.testing.email.Email class allows developers to test against emails stored by util/sendmail. Simple pass the serial you want to test against, starting at serial = 1; e.g. Email(serial). Signed-off-by: Kevin Morris --- aurweb/testing/email.py | 120 ++++++++++++++++++++++++++++++++++++++++ test/conftest.py | 20 ++++++- util/sendmail | 23 ++++++++ 3 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 aurweb/testing/email.py diff --git a/aurweb/testing/email.py b/aurweb/testing/email.py new file mode 100644 index 00000000..6ff9df99 --- /dev/null +++ b/aurweb/testing/email.py @@ -0,0 +1,120 @@ +import base64 +import copy +import email +import os +import re + + +class Email: + """ + An email class used for testing. + + This class targets a specific serial of emails for PYTEST_CURRENT_TEST. + As emails are sent out with util/sendmail, the serial number increases, + starting at 1. + + Email content sent out by aurweb is always base64-encoded. Email.parse() + decodes that for us and puts it into Email.body. + + Example: + + # Get the {test_suite}_{test_function}.1.txt email. + email = Email(1).parse() + print(email.body) + print(email.headers) + + """ + TEST_DIR = "test-emails" + + def __init__(self, serial: int = 1): + self.serial = serial + self.content = self._get() + + @staticmethod + def email_prefix(suite: bool = False) -> str: + """ + Get the email prefix. + + We find the email prefix by reducing PYTEST_CURRENT_TEST to + either {test_suite}_{test_function}. If `suite` is set, we + reduce it to {test_suite} only. + + :param suite: Reduce PYTEST_CURRENT_TEST to {test_suite} + :return: Email prefix with '/', '.', ',', and ':' chars replaced by '_' + """ + value = os.environ.get("PYTEST_CURRENT_TEST", "email").split(" ")[0] + if suite: + value = value.split(":")[0] + return re.sub(r'(\/|\.|,|:)', "_", value) + + @staticmethod + def count() -> int: + """ + Count the current number of emails sent from the test. + + This function is **only** supported inside of pytest functions. + Do not use it elsewhere as data races will occur. + + :return: Number of emails sent by the current test + """ + files = os.listdir(Email.TEST_DIR) + prefix = Email.email_prefix() + expr = "^" + prefix + r"\.\d+\.txt$" + subset = filter(lambda e: re.match(expr, e), files) + return len(list(subset)) + + def _email_path(self) -> str: + filename = self.email_prefix() + f".{self.serial}.txt" + return os.path.join(Email.TEST_DIR, filename) + + def _get(self) -> str: + """ + Get this email's content by reading its file. + + :return: Email content + """ + path = self._email_path() + with open(path) as f: + return f.read() + + def parse(self) -> "Email": + """ + Parse this email and base64-decode the body. + + This function populates Email.message, Email.headers and Email.body. + + Additionally, after parsing, we write over our email file with + self.glue()'d content (base64-decoded). This is done for ease + of inspection by users. + + :return: self + """ + self.message = email.message_from_string(self.content) + self.headers = dict(self.message) + + # aurweb email notifications always have base64 encoded content. + # Decode it here so self.body is human readable. + self.body = base64.b64decode(self.message.get_payload()).decode() + + path = self._email_path() + with open(path, "w") as f: + f.write(self.glue()) + + return self + + def glue(self) -> str: + """ + Glue parsed content back into a complete email document, but + base64-decoded this time. + + :return: Email document as a string + """ + headers = copy.copy(self.headers) + del headers["Content-Transfer-Encoding"] + + output = [] + for k, v in headers.items(): + output.append(f"{k}: {v}") + output.append("") + output.append(self.body) + return "\n".join(output) diff --git a/test/conftest.py b/test/conftest.py index db2e5997..01131109 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -37,6 +37,8 @@ then clears the database for each test function run in that module. It is done this way because migration has a large cost; migrating ahead of each function takes too long when compared to this method. """ +import os + import pytest from filelock import FileLock @@ -50,6 +52,7 @@ import aurweb.config import aurweb.db from aurweb import initdb, logging, testing +from aurweb.testing.email import Email logger = logging.get_logger(__name__) @@ -120,6 +123,18 @@ def _drop_database(engine: Engine, dbname: str) -> None: conn.close() +def setup_email(): + if not os.path.exists(Email.TEST_DIR): + os.makedirs(Email.TEST_DIR) + + # Cleanup all email files for this test suite. + prefix = Email.email_prefix(suite=True) + files = os.listdir(Email.TEST_DIR) + for file in files: + if file.startswith(prefix): + os.remove(os.path.join(Email.TEST_DIR, file)) + + @pytest.fixture(scope="module") def setup_database(tmp_path_factory: pytest.fixture, worker_id: pytest.fixture) -> None: @@ -129,6 +144,7 @@ def setup_database(tmp_path_factory: pytest.fixture, if worker_id == "master": # pragma: no cover # If we're not running tests through multiproc pytest-xdist. + setup_email() yield _create_database(engine, dbname) _drop_database(engine, dbname) return @@ -143,12 +159,13 @@ def setup_database(tmp_path_factory: pytest.fixture, else: # Otherwise, create the data file and create the database. fn.write_text("1") + setup_email() yield _create_database(engine, dbname) _drop_database(engine, dbname) @pytest.fixture(scope="module") -def db_session(setup_database: pytest.fixture) -> scoped_session: +def db_session(setup_database: None) -> scoped_session: """ Yield a database session based on aurweb.db.name(). @@ -158,6 +175,7 @@ def db_session(setup_database: pytest.fixture) -> scoped_session: # configured database, because PYTEST_CURRENT_TEST is removed. dbname = aurweb.db.name() session = aurweb.db.get_session() + yield session # Close the session and pop it. diff --git a/util/sendmail b/util/sendmail index 06bd9865..9356851a 100755 --- a/util/sendmail +++ b/util/sendmail @@ -1,2 +1,25 @@ #!/bin/bash +# Send email to temporary filesystem for tests. +dir='test-emails' +filename='email.txt' +if [ ! -z ${PYTEST_CURRENT_TEST+x} ]; then + filename="$(echo $PYTEST_CURRENT_TEST | cut -d ' ' -f 1 | sed -r 's/(\/|\.|,|:)/_/g')" +fi +mkdir -p "$dir" + +path="${dir}/${filename}" +serial_file="${path}.serial" +if [ ! -f $serial_file ]; then + echo 0 > $serial_file +fi + +# Increment and update $serial_file. +serial=$(($(cat $serial_file) + 1)) +echo $serial > $serial_file + +# Use the serial we're on to mark the email file. +# Emails have the format: PYTEST_CURRENT_TEST.s.txt +# where s is the current serial for PYTEST_CURRENT_TEST. +cat > "${path}.${serial}.txt" + exit 0 From d8e3ca1abbac6a897f262b1eba0695f4323a13d8 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 22 Nov 2021 12:03:53 -0800 Subject: [PATCH 643/844] change(notify): converted to use aurweb.db ORM - Removed notify sharness test Signed-off-by: Kevin Morris --- aurweb/packages/util.py | 7 +- aurweb/routers/accounts.py | 6 +- aurweb/routers/packages.py | 30 +- aurweb/scripts/notify.py | 402 +++++++++++++---------- aurweb/testing/smtp.py | 42 +++ test/t2500-notify.t | 431 ------------------------- test/test_notify.py | 643 +++++++++++++++++++++++++++++++++++++ 7 files changed, 934 insertions(+), 627 deletions(-) create mode 100644 aurweb/testing/smtp.py delete mode 100755 test/t2500-notify.t create mode 100644 test/test_notify.py diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index 55af3a34..3bb3ae5f 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -239,7 +239,6 @@ def remove_comaintainers(pkgbase: models.PackageBase, :param usernames: Iterable of username strings :return: None """ - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) notifications = [] with db.begin(): for username in usernames: @@ -250,8 +249,7 @@ def remove_comaintainers(pkgbase: models.PackageBase, ).first() notifications.append( notify.ComaintainerRemoveNotification( - conn, comaintainer.User.ID, pkgbase.ID - ) + comaintainer.User.ID, pkgbase.ID) ) db.delete(comaintainer) @@ -283,7 +281,6 @@ def add_comaintainers(request: Request, pkgbase: models.PackageBase, memo[username] = user # Alright, now that we got past the check, add them all to the DB. - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) notifications = [] with db.begin(): for username in usernames: @@ -302,7 +299,7 @@ def add_comaintainers(request: Request, pkgbase: models.PackageBase, notifications.append( notify.ComaintainerAddNotification( - conn, comaintainer.User.ID, pkgbase.ID) + comaintainer.User.ID, pkgbase.ID) ) # Send out notifications. diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index ddee1764..545811f0 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -97,8 +97,7 @@ async def passreset_post(request: Request, with db.begin(): user.ResetKey = resetkey - executor = db.ConnectionExecutor(db.get_engine().raw_connection()) - ResetKeyNotification(executor, user.ID).send() + ResetKeyNotification(user.ID).send() # Render ?step=confirm. return RedirectResponse(url="/passreset?step=confirm", @@ -323,8 +322,7 @@ async def account_register_post(request: Request, Fingerprint=fingerprint) # Send a reset key notification to the new user. - executor = db.ConnectionExecutor(db.get_engine().raw_connection()) - WelcomeNotification(executor, user.ID).send() + WelcomeNotification(user.ID).send() context["complete"] = True context["user"] = user diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index eab75e5a..b5f8478e 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -146,7 +146,6 @@ def delete_package(deleter: models.User, package: models.Package): requests = [] bases_to_delete = [] - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) # In all cases, though, just delete the Package in question. if package.PackageBase.packages.count() == 1: reqtype = db.query(models.RequestType).filter( @@ -162,7 +161,7 @@ def delete_package(deleter: models.User, package: models.Package): # Prepare DeleteNotification. notifications.append( - notify.DeleteNotification(conn, deleter.ID, package.PackageBase.ID) + notify.DeleteNotification(deleter.ID, package.PackageBase.ID) ) # For each PackageRequest created, mock up an open and close notification. @@ -170,12 +169,12 @@ def delete_package(deleter: models.User, package: models.Package): for pkgreq in requests: notifications.append( notify.RequestOpenNotification( - conn, deleter.ID, pkgreq.ID, reqtype.Name, + deleter.ID, pkgreq.ID, reqtype.Name, pkgreq.PackageBase.ID, merge_into=basename or None) ) notifications.append( notify.RequestCloseNotification( - conn, deleter.ID, pkgreq.ID, pkgreq.status_display()) + deleter.ID, pkgreq.ID, pkgreq.status_display()) ) # Perform all the deletions. @@ -666,10 +665,9 @@ async def pkgbase_request_post(request: Request, name: str, Comments=comments, ClosureComment=str()) - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) # Prepare notification object. notif = notify.RequestOpenNotification( - conn, request.user.ID, pkgreq.ID, reqtype.Name, + request.user.ID, pkgreq.ID, reqtype.Name, pkgreq.PackageBase.ID, merge_into=merge_into or None) # Send the notification now that we're out of the DB scope. @@ -688,7 +686,7 @@ async def pkgbase_request_post(request: Request, name: str, pkgreq.Status = ACCEPTED_ID db.refresh(pkgreq) notif = notify.RequestCloseNotification( - conn, request.user.ID, pkgreq.ID, pkgreq.status_display()) + request.user.ID, pkgreq.ID, pkgreq.status_display()) notif.send() elif type == "deletion" and is_maintainer and outdated: packages = pkgbase.packages.all() @@ -742,9 +740,8 @@ async def requests_close_post(request: Request, id: int, pkgreq.Status = reason pkgreq.ClosureComment = comments - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) notify_ = notify.RequestCloseNotification( - conn, request.user.ID, pkgreq.ID, pkgreq.status_display()) + request.user.ID, pkgreq.ID, pkgreq.status_display()) notify_.send() return RedirectResponse("/requests", status_code=HTTPStatus.SEE_OTHER) @@ -936,9 +933,7 @@ async def pkgbase_unvote(request: Request, name: str): def pkgbase_disown_instance(request: Request, pkgbase: models.PackageBase): disowner = request.user - - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) - notif = notify.DisownNotification(conn, disowner.ID, pkgbase.ID) + notif = notify.DisownNotification(disowner.ID, pkgbase.ID) if disowner != pkgbase.Maintainer: with db.begin(): @@ -1003,8 +998,7 @@ def pkgbase_adopt_instance(request: Request, pkgbase: models.PackageBase): with db.begin(): pkgbase.Maintainer = request.user - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) - notif = notify.AdoptNotification(conn, request.user.ID, pkgbase.ID) + notif = notify.AdoptNotification(request.user.ID, pkgbase.ID) notif.send() @@ -1366,7 +1360,7 @@ def pkgbase_merge_instance(request: Request, pkgbase: models.PackageBase, f"{request.user.Username}.") rejected_closure_comment = ("Rejected because another merge request " "for the same package base was accepted.") - conn = db.ConnectionExecutor(db.get_engine().raw_connection()) + if not requests: # If there are no requests, create one owned by request.user. with db.begin(): @@ -1383,7 +1377,7 @@ def pkgbase_merge_instance(request: Request, pkgbase: models.PackageBase, # Add a notification about the opening to our notifs array. notif = notify.RequestOpenNotification( - conn, request.user.ID, pkgreq.ID, MERGE, + request.user.ID, pkgreq.ID, MERGE, pkgbase.ID, merge_into=target.Name) notifs.append(notif) @@ -1417,11 +1411,9 @@ def pkgbase_merge_instance(request: Request, pkgbase: models.PackageBase, for pkgreq in all_requests: # Create notifications for request closure. notif = notify.RequestCloseNotification( - conn, request.user.ID, pkgreq.ID, pkgreq.status_display()) + request.user.ID, pkgreq.ID, pkgreq.status_display()) notifs.append(notif) - conn.close() - # Log this out for accountability purposes. logger.info(f"Trusted User '{request.user.Username}' merged " f"'{pkgbasename}' into '{target.Name}'.") diff --git a/aurweb/scripts/notify.py b/aurweb/scripts/notify.py index ba4ec9eb..e49024d9 100755 --- a/aurweb/scripts/notify.py +++ b/aurweb/scripts/notify.py @@ -7,10 +7,16 @@ import subprocess import sys import textwrap +from sqlalchemy import and_, or_ + import aurweb.config import aurweb.db import aurweb.l10n +from aurweb import db +from aurweb.models import (PackageBase, PackageComaintainer, PackageComment, PackageNotification, PackageRequest, RequestType, + TUVote, User) + aur_location = aurweb.config.get('options', 'aur_location') @@ -22,23 +28,6 @@ def headers_reply(thread_id): return {'In-Reply-To': thread_id, 'References': thread_id} -def username_from_id(conn, uid): - cur = conn.execute('SELECT UserName FROM Users WHERE ID = ?', [uid]) - return cur.fetchone()[0] - - -def pkgbase_from_id(conn, pkgbase_id): - cur = conn.execute('SELECT Name FROM PackageBases WHERE ID = ?', - [pkgbase_id]) - return cur.fetchone()[0] - - -def pkgbase_from_pkgreq(conn, reqid): - cur = conn.execute('SELECT PackageBaseID FROM PackageRequests ' + - 'WHERE ID = ?', [reqid]) - return cur.fetchone()[0] - - class Notification: def get_refs(self): return () @@ -52,8 +41,8 @@ class Notification: def get_body_fmt(self, lang): body = '' for line in self.get_body(lang).splitlines(): - if line == '-- ': - body += '-- \n' + if line == '--': + body += '--\n' continue body += textwrap.fill(line, break_long_words=False) + '\n' for i, ref in enumerate(self.get_refs()): @@ -103,10 +92,11 @@ class Notification: user = aurweb.config.get('notifications', 'smtp-user') passwd = aurweb.config.get('notifications', 'smtp-password') - if use_ssl: - server = smtplib.SMTP_SSL(server_addr, server_port) - else: - server = smtplib.SMTP(server_addr, server_port) + classes = { + False: smtplib.SMTP, + True: smtplib.SMTP_SSL, + } + server = classes[use_ssl](server_addr, server_port) if use_starttls: server.ehlo() @@ -123,12 +113,24 @@ class Notification: class ResetKeyNotification(Notification): - def __init__(self, conn, uid): - cur = conn.execute('SELECT UserName, Email, BackupEmail, ' + - 'LangPreference, ResetKey ' + - 'FROM Users WHERE ID = ? AND Suspended = 0', [uid]) - self._username, self._to, self._backup, self._lang, self._resetkey = \ - cur.fetchone() + def __init__(self, uid): + + user = db.query(User).filter( + and_(User.ID == uid, User.Suspended == 0) + ).with_entities( + User.Username, + User.Email, + User.BackupEmail, + User.LangPreference, + User.ResetKey + ).order_by(User.Username.asc()).first() + + self._username = user.Username + self._to = user.Email + self._backup = user.BackupEmail + self._lang = user.LangPreference + self._resetkey = user.ResetKey + super().__init__() def get_recipients(self): @@ -167,21 +169,28 @@ class WelcomeNotification(ResetKeyNotification): class CommentNotification(Notification): - def __init__(self, conn, uid, pkgbase_id, comment_id): - self._user = username_from_id(conn, uid) - self._pkgbase = pkgbase_from_id(conn, pkgbase_id) - cur = conn.execute('SELECT DISTINCT Users.Email, Users.LangPreference ' - 'FROM Users INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.CommentNotify = 1 AND ' + - 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ? AND ' + - 'Users.Suspended = 0', - [uid, pkgbase_id]) - self._recipients = cur.fetchall() - cur = conn.execute('SELECT Comments FROM PackageComments WHERE ID = ?', - [comment_id]) - self._text = cur.fetchone()[0] + def __init__(self, uid, pkgbase_id, comment_id): + + self._user = db.query(User.Username).filter( + User.ID == uid).first().Username + self._pkgbase = db.query(PackageBase.Name).filter( + PackageBase.ID == pkgbase_id).first().Name + + query = db.query(User).join(PackageNotification).filter( + and_(User.CommentNotify == 1, + PackageNotification.UserID != uid, + PackageNotification.PackageBaseID == pkgbase_id, + User.Suspended == 0) + ).with_entities( + User.Email, + User.LangPreference + ).distinct() + self._recipients = [(u.Email, u.LangPreference) for u in query] + + pkgcomment = db.query(PackageComment.Comments).filter( + PackageComment.ID == comment_id).first() + self._text = pkgcomment.Comments + super().__init__() def get_recipients(self): @@ -196,7 +205,7 @@ class CommentNotification(Notification): body = aurweb.l10n.translator.translate( '{user} [1] added the following comment to {pkgbase} [2]:', lang).format(user=self._user, pkgbase=self._pkgbase) - body += '\n\n' + self._text + '\n\n-- \n' + body += '\n\n' + self._text + '\n\n--\n' dnlabel = aurweb.l10n.translator.translate( 'Disable notifications', lang) body += aurweb.l10n.translator.translate( @@ -216,19 +225,24 @@ class CommentNotification(Notification): class UpdateNotification(Notification): - def __init__(self, conn, uid, pkgbase_id): - self._user = username_from_id(conn, uid) - self._pkgbase = pkgbase_from_id(conn, pkgbase_id) - cur = conn.execute('SELECT DISTINCT Users.Email, ' + - 'Users.LangPreference FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.UpdateNotify = 1 AND ' + - 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ? AND ' + - 'Users.Suspended = 0', - [uid, pkgbase_id]) - self._recipients = cur.fetchall() + def __init__(self, uid, pkgbase_id): + + self._user = db.query(User.Username).filter( + User.ID == uid).first().Username + self._pkgbase = db.query(PackageBase.Name).filter( + PackageBase.ID == pkgbase_id).first().Name + + query = db.query(User).join(PackageNotification).filter( + and_(User.UpdateNotify == 1, + PackageNotification.UserID != uid, + PackageNotification.PackageBaseID == pkgbase_id, + User.Suspended == 0) + ).with_entities( + User.Email, + User.LangPreference + ).distinct() + self._recipients = [(u.Email, u.LangPreference) for u in query] + super().__init__() def get_recipients(self): @@ -243,7 +257,7 @@ class UpdateNotification(Notification): body = aurweb.l10n.translator.translate( '{user} [1] pushed a new commit to {pkgbase} [2].', lang).format(user=self._user, pkgbase=self._pkgbase) - body += '\n\n-- \n' + body += '\n\n--\n' dnlabel = aurweb.l10n.translator.translate( 'Disable notifications', lang) body += aurweb.l10n.translator.translate( @@ -263,23 +277,30 @@ class UpdateNotification(Notification): class FlagNotification(Notification): - def __init__(self, conn, uid, pkgbase_id): - self._user = username_from_id(conn, uid) - self._pkgbase = pkgbase_from_id(conn, pkgbase_id) - cur = conn.execute( - 'SELECT DISTINCT Users.Email, ' + - 'Users.LangPreference FROM Users ' + - 'LEFT JOIN PackageComaintainers ' + - 'ON PackageComaintainers.UsersID = Users.ID ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.MaintainerUID = Users.ID OR ' + - 'PackageBases.ID = PackageComaintainers.PackageBaseID ' + - 'WHERE PackageBases.ID = ? AND ' + - 'Users.Suspended = 0', [pkgbase_id]) - self._recipients = cur.fetchall() - cur = conn.execute('SELECT FlaggerComment FROM PackageBases WHERE ' + - 'ID = ?', [pkgbase_id]) - self._text = cur.fetchone()[0] + def __init__(self, uid, pkgbase_id): + + self._user = db.query(User.Username).filter( + User.ID == uid).first().Username + self._pkgbase = db.query(PackageBase.Name).filter( + PackageBase.ID == pkgbase_id).first().Name + + query = db.query(User).join(PackageComaintainer, isouter=True).join( + PackageBase, + or_(PackageBase.MaintainerUID == User.ID, + PackageBase.ID == PackageComaintainer.PackageBaseID) + ).filter( + and_(PackageBase.ID == pkgbase_id, + User.Suspended == 0) + ).with_entities( + User.Email, + User.LangPreference + ).distinct() + self._recipients = [(u.Email, u.LangPreference) for u in query] + + pkgbase = db.query(PackageBase.FlaggerComment).filter( + PackageBase.ID == pkgbase_id).first() + self._text = pkgbase.FlaggerComment + super().__init__() def get_recipients(self): @@ -304,22 +325,28 @@ class FlagNotification(Notification): class OwnershipEventNotification(Notification): - def __init__(self, conn, uid, pkgbase_id): - self._user = username_from_id(conn, uid) - self._pkgbase = pkgbase_from_id(conn, pkgbase_id) - cur = conn.execute('SELECT DISTINCT Users.Email, ' + - 'Users.LangPreference FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.OwnershipNotify = 1 AND ' + - 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ? AND ' + - 'Users.Suspended = 0', - [uid, pkgbase_id]) - self._recipients = cur.fetchall() - cur = conn.execute('SELECT FlaggerComment FROM PackageBases WHERE ' + - 'ID = ?', [pkgbase_id]) - self._text = cur.fetchone()[0] + def __init__(self, uid, pkgbase_id): + + self._user = db.query(User.Username).filter( + User.ID == uid).first().Username + self._pkgbase = db.query(PackageBase.Name).filter( + PackageBase.ID == pkgbase_id).first().Name + + query = db.query(User).join(PackageNotification).filter( + and_(User.OwnershipNotify == 1, + PackageNotification.UserID != uid, + PackageNotification.PackageBaseID == pkgbase_id, + User.Suspended == 0) + ).with_entities( + User.Email, + User.LangPreference + ).distinct() + self._recipients = [(u.Email, u.LangPreference) for u in query] + + pkgbase = db.query(PackageBase.FlaggerComment).filter( + PackageBase.ID == pkgbase_id).first() + self._text = pkgbase.FlaggerComment + super().__init__() def get_recipients(self): @@ -351,11 +378,22 @@ class DisownNotification(OwnershipEventNotification): class ComaintainershipEventNotification(Notification): - def __init__(self, conn, uid, pkgbase_id): - self._pkgbase = pkgbase_from_id(conn, pkgbase_id) - cur = conn.execute('SELECT Email, LangPreference FROM Users ' + - 'WHERE ID = ? AND Suspended = 0', [uid]) - self._to, self._lang = cur.fetchone() + def __init__(self, uid, pkgbase_id): + + self._pkgbase = db.query(PackageBase.Name).filter( + PackageBase.ID == pkgbase_id).first().Name + + user = db.query(User).filter( + and_(User.ID == uid, + User.Suspended == 0) + ).with_entities( + User.Email, + User.LangPreference + ).first() + + self._to = user.Email + self._lang = user.LangPreference + super().__init__() def get_recipients(self): @@ -385,22 +423,28 @@ class ComaintainerRemoveNotification(ComaintainershipEventNotification): class DeleteNotification(Notification): - def __init__(self, conn, uid, old_pkgbase_id, new_pkgbase_id=None): - self._user = username_from_id(conn, uid) - self._old_pkgbase = pkgbase_from_id(conn, old_pkgbase_id) + def __init__(self, uid, old_pkgbase_id, new_pkgbase_id=None): + + self._user = db.query(User.Username).filter( + User.ID == uid).first().Username + self._old_pkgbase = db.query(PackageBase.Name).filter( + PackageBase.ID == old_pkgbase_id).first().Name + + self._new_pkgbase = None if new_pkgbase_id: - self._new_pkgbase = pkgbase_from_id(conn, new_pkgbase_id) - else: - self._new_pkgbase = None - cur = conn.execute('SELECT DISTINCT Users.Email, ' + - 'Users.LangPreference FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ? AND ' + - 'Users.Suspended = 0', - [uid, old_pkgbase_id]) - self._recipients = cur.fetchall() + self._new_pkgbase = db.query(PackageBase.Name).filter( + PackageBase.ID == new_pkgbase_id).first().Name + + query = db.query(User).join(PackageNotification).filter( + and_(PackageNotification.UserID != uid, + PackageNotification.PackageBaseID == old_pkgbase_id, + User.Suspended == 0) + ).with_entities( + User.Email, + User.LangPreference + ).distinct() + self._recipients = [(u.Email, u.LangPreference) for u in query] + super().__init__() def get_recipients(self): @@ -417,7 +461,7 @@ class DeleteNotification(Notification): 'Disable notifications', lang) return aurweb.l10n.translator.translate( '{user} [1] merged {old} [2] into {new} [3].\n\n' - '-- \n' + '--\n' 'If you no longer wish receive notifications about the ' 'new package, please go to [3] and click "{label}".', lang).format(user=self._user, old=self._old_pkgbase, @@ -438,26 +482,36 @@ class DeleteNotification(Notification): class RequestOpenNotification(Notification): - def __init__(self, conn, uid, reqid, reqtype, pkgbase_id, merge_into=None): - self._user = username_from_id(conn, uid) - self._pkgbase = pkgbase_from_id(conn, pkgbase_id) - cur = conn.execute( - 'SELECT DISTINCT Users.Email FROM PackageRequests ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + - 'LEFT JOIN PackageComaintainers ' + - 'ON PackageComaintainers.PackageBaseID = PackageRequests.PackageBaseID ' + - 'INNER JOIN Users ' + - 'ON Users.ID = PackageRequests.UsersID ' + - 'OR Users.ID = PackageBases.MaintainerUID ' + - 'OR Users.ID = PackageComaintainers.UsersID ' + - 'WHERE PackageRequests.ID = ? AND ' + - 'Users.Suspended = 0', [reqid]) + def __init__(self, uid, reqid, reqtype, pkgbase_id, merge_into=None): + + self._user = db.query(User.Username).filter( + User.ID == uid).first().Username + self._pkgbase = db.query(PackageBase.Name).filter( + PackageBase.ID == pkgbase_id).first().Name + self._to = aurweb.config.get('options', 'aur_request_ml') - self._cc = [row[0] for row in cur.fetchall()] - cur = conn.execute('SELECT Comments FROM PackageRequests WHERE ID = ?', - [reqid]) - self._text = cur.fetchone()[0] + + query = db.query(PackageRequest).join(PackageBase).join( + PackageComaintainer, + PackageComaintainer.PackageBaseID == PackageRequest.PackageBaseID, + isouter=True + ).join( + User, + or_(User.ID == PackageRequest.UsersID, + User.ID == PackageBase.MaintainerUID, + User.ID == PackageComaintainer.UsersID) + ).filter( + and_(PackageRequest.ID == reqid, + User.Suspended == 0) + ).with_entities( + User.Email + ).distinct() + self._cc = [u.Email for u in query] + + pkgreq = db.query(PackageRequest.Comments).filter( + PackageRequest.ID == reqid).first() + + self._text = pkgreq.Comments self._reqid = int(reqid) self._reqtype = reqtype self._merge_into = merge_into @@ -500,31 +554,41 @@ class RequestOpenNotification(Notification): class RequestCloseNotification(Notification): - def __init__(self, conn, uid, reqid, reason): - self._user = username_from_id(conn, uid) if int(uid) else None + def __init__(self, uid, reqid, reason): + user = db.query(User.Username).filter(User.ID == uid).first() + self._user = user.Username if user else None - cur = conn.execute( - 'SELECT DISTINCT Users.Email FROM PackageRequests ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + - 'LEFT JOIN PackageComaintainers ' + - 'ON PackageComaintainers.PackageBaseID = PackageRequests.PackageBaseID ' + - 'INNER JOIN Users ' + - 'ON Users.ID = PackageRequests.UsersID ' + - 'OR Users.ID = PackageBases.MaintainerUID ' + - 'OR Users.ID = PackageComaintainers.UsersID ' + - 'WHERE PackageRequests.ID = ? AND ' + - 'Users.Suspended = 0', [reqid]) self._to = aurweb.config.get('options', 'aur_request_ml') - self._cc = [row[0] for row in cur.fetchall()] - cur = conn.execute('SELECT PackageRequests.ClosureComment, ' + - 'RequestTypes.Name, ' + - 'PackageRequests.PackageBaseName ' + - 'FROM PackageRequests ' + - 'INNER JOIN RequestTypes ' + - 'ON RequestTypes.ID = PackageRequests.ReqTypeID ' + - 'WHERE PackageRequests.ID = ?', [reqid]) - self._text, self._reqtype, self._pkgbase = cur.fetchone() + + query = db.query(PackageRequest).join(PackageBase).join( + PackageComaintainer, + PackageComaintainer.PackageBaseID == PackageRequest.PackageBaseID, + isouter=True + ).join( + User, + or_(User.ID == PackageRequest.UsersID, + User.ID == PackageBase.MaintainerUID, + User.ID == PackageComaintainer.UsersID) + ).filter( + and_(PackageRequest.ID == reqid, + User.Suspended == 0) + ).with_entities( + User.Email + ).distinct() + self._cc = [u.Email for u in query] + + pkgreq = db.query(PackageRequest).join(RequestType).filter( + PackageRequest.ID == reqid + ).with_entities( + PackageRequest.ClosureComment, + RequestType.Name, + PackageRequest.PackageBaseName + ).first() + + self._text = pkgreq.ClosureComment + self._reqtype = pkgreq.Name + self._pkgbase = pkgreq.PackageBaseName + self._reqid = int(reqid) self._reason = reason @@ -567,14 +631,19 @@ class RequestCloseNotification(Notification): class TUVoteReminderNotification(Notification): - def __init__(self, conn, vote_id): + def __init__(self, vote_id): self._vote_id = int(vote_id) - cur = conn.execute('SELECT Email, LangPreference FROM Users ' + - 'WHERE AccountTypeID IN (2, 4) AND ID NOT IN ' + - '(SELECT UserID FROM TU_Votes ' + - 'WHERE TU_Votes.VoteID = ?) AND ' + - 'Users.Suspended = 0', [vote_id]) - self._recipients = cur.fetchall() + + subquery = db.query(TUVote.UserID).filter(TUVote.VoteID == vote_id) + query = db.query(User).filter( + and_(User.AccountTypeID.in_((2, 4)), + ~User.ID.in_(subquery), + User.Suspended == 0) + ).with_entities( + User.Email, User.LangPreference + ) + self._recipients = [(u.Email, u.LangPreference) for u in query] + super().__init__() def get_recipients(self): @@ -596,6 +665,7 @@ class TUVoteReminderNotification(Notification): def main(): + db.get_engine() action = sys.argv[1] action_map = { 'send-resetkey': ResetKeyNotification, @@ -613,14 +683,10 @@ def main(): 'tu-vote-reminder': TUVoteReminderNotification, } - conn = aurweb.db.Connection() - - notification = action_map[action](conn, *sys.argv[2:]) + with db.begin(): + notification = action_map[action](*sys.argv[2:]) notification.send() - conn.commit() - conn.close() - if __name__ == '__main__': main() diff --git a/aurweb/testing/smtp.py b/aurweb/testing/smtp.py new file mode 100644 index 00000000..da64c93f --- /dev/null +++ b/aurweb/testing/smtp.py @@ -0,0 +1,42 @@ +""" Fake SMTP clients that can be used for testing. """ + + +class FakeSMTP: + """ A fake version of smtplib.SMTP used for testing. """ + + starttls_enabled = False + use_ssl = False + + def __init__(self): + self.emails = [] + self.count = 0 + self.ehlo_count = 0 + self.quit_count = 0 + self.set_debuglevel_count = 0 + self.user = None + self.passwd = None + + def ehlo(self) -> None: + self.ehlo_count += 1 + + def starttls(self) -> None: + self.starttls_enabled = True + + def set_debuglevel(self, level: int = 0) -> None: + self.set_debuglevel_count += 1 + + def login(self, user: str, passwd: str) -> None: + self.user = user + self.passwd = passwd + + def sendmail(self, sender: str, to: str, msg: bytes) -> None: + self.emails.append((sender, to, msg.decode())) + self.count += 1 + + def quit(self) -> None: + self.quit_count += 1 + + +class FakeSMTP_SSL(FakeSMTP): + """ A fake version of smtplib.SMTP_SSL used for testing. """ + use_ssl = True diff --git a/test/t2500-notify.t b/test/t2500-notify.t deleted file mode 100755 index a908f125..00000000 --- a/test/t2500-notify.t +++ /dev/null @@ -1,431 +0,0 @@ -#!/bin/sh - -test_description='notify tests' - -. "$(dirname "$0")/setup.sh" - -test_expect_success 'Test out-of-date notifications.' ' - cat <<-EOD | sqlite3 aur.db && - /* Use package base IDs which can be distinguished from user IDs. */ - INSERT INTO PackageBases (ID, Name, MaintainerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (1001, "foobar", 1, 0, 0, "This is a test OOD comment."); - INSERT INTO PackageBases (ID, Name, MaintainerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (1002, "foobar2", 2, 0, 0, ""); - INSERT INTO PackageBases (ID, Name, MaintainerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (1003, "foobar3", NULL, 0, 0, ""); - INSERT INTO PackageBases (ID, Name, MaintainerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (1004, "foobar4", 1, 0, 0, ""); - INSERT INTO PackageComaintainers (PackageBaseID, UsersID, Priority) VALUES (1001, 2, 1); - INSERT INTO PackageComaintainers (PackageBaseID, UsersID, Priority) VALUES (1001, 4, 2); - INSERT INTO PackageComaintainers (PackageBaseID, UsersID, Priority) VALUES (1002, 3, 1); - INSERT INTO PackageComaintainers (PackageBaseID, UsersID, Priority) VALUES (1002, 5, 2); - INSERT INTO PackageComaintainers (PackageBaseID, UsersID, Priority) VALUES (1003, 4, 1); - EOD - >sendmail.out && - cover "$NOTIFY" flag 1 1001 && - cat <<-EOD >expected && - Subject: AUR Out-of-date Notification for foobar - To: tu@localhost - Subject: AUR Out-of-date Notification for foobar - To: user2@localhost - Subject: AUR Out-of-date Notification for foobar - To: user@localhost - EOD - grep "^\(Subject\|To\)" sendmail.out >sendmail.parts && - test_cmp sendmail.parts expected && - cat <<-EOD | sqlite3 aur.db - DELETE FROM PackageComaintainers; - EOD -' - -test_expect_success 'Test subject and body of reset key notifications.' ' - cat <<-EOD | sqlite3 aur.db && - UPDATE Users SET ResetKey = "12345678901234567890123456789012" WHERE ID = 1; - EOD - >sendmail.out && - cover "$NOTIFY" send-resetkey 1 && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: AUR Password Reset - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - A password reset request was submitted for the account user associated - with your email address. If you wish to reset your password follow the - link [1] below, otherwise ignore this message and nothing will happen. - - [1] https://aur.archlinux.org/passreset/?resetkey=12345678901234567890123456789012 - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of welcome notifications.' ' - cat <<-EOD | sqlite3 aur.db && - UPDATE Users SET ResetKey = "12345678901234567890123456789012" WHERE ID = 1; - EOD - >sendmail.out && - cover "$NOTIFY" welcome 1 && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: Welcome to the Arch User Repository - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - Welcome to the Arch User Repository! In order to set an initial - password for your new account, please click the link [1] below. If the - link does not work, try copying and pasting it into your browser. - - [1] https://aur.archlinux.org/passreset/?resetkey=12345678901234567890123456789012 - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of comment notifications.' ' - cat <<-EOD | sqlite3 aur.db && - /* Use package comments IDs which can be distinguished from other IDs. */ - INSERT INTO PackageComments (ID, PackageBaseID, UsersID, Comments, RenderedComment) VALUES (2001, 1001, 1, "This is a test comment.", "This is a test comment."); - INSERT INTO PackageNotifications (PackageBaseID, UserID) VALUES (1001, 2); - UPDATE Users SET CommentNotify = 1 WHERE ID = 2; - EOD - >sendmail.out && - cover "$NOTIFY" comment 1 1001 2001 && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: AUR Comment for foobar - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - user [1] added the following comment to foobar [2]: - - This is a test comment. - - -- - If you no longer wish to receive notifications about this package, - please go to the package page [2] and select "Disable notifications". - - [1] https://aur.archlinux.org/account/user/ - [2] https://aur.archlinux.org/pkgbase/foobar/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of update notifications.' ' - cat <<-EOD | sqlite3 aur.db && - UPDATE Users SET UpdateNotify = 1 WHERE ID = 2; - EOD - >sendmail.out && - cover "$NOTIFY" update 1 1001 && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: AUR Package Update: foobar - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - user [1] pushed a new commit to foobar [2]. - - -- - If you no longer wish to receive notifications about this package, - please go to the package page [2] and select "Disable notifications". - - [1] https://aur.archlinux.org/account/user/ - [2] https://aur.archlinux.org/pkgbase/foobar/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of out-of-date notifications.' ' - >sendmail.out && - cover "$NOTIFY" flag 1 1001 && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: AUR Out-of-date Notification for foobar - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - Your package foobar [1] has been flagged out-of-date by user [2]: - - This is a test OOD comment. - - [1] https://aur.archlinux.org/pkgbase/foobar/ - [2] https://aur.archlinux.org/account/user/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of adopt notifications.' ' - >sendmail.out && - cover "$NOTIFY" adopt 1 1001 && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: AUR Ownership Notification for foobar - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - The package foobar [1] was adopted by user [2]. - - [1] https://aur.archlinux.org/pkgbase/foobar/ - [2] https://aur.archlinux.org/account/user/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of disown notifications.' ' - >sendmail.out && - cover "$NOTIFY" disown 1 1001 && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: AUR Ownership Notification for foobar - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - The package foobar [1] was disowned by user [2]. - - [1] https://aur.archlinux.org/pkgbase/foobar/ - [2] https://aur.archlinux.org/account/user/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of co-maintainer addition notifications.' ' - >sendmail.out && - cover "$NOTIFY" comaintainer-add 1 1001 && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: AUR Co-Maintainer Notification for foobar - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - You were added to the co-maintainer list of foobar [1]. - - [1] https://aur.archlinux.org/pkgbase/foobar/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of co-maintainer removal notifications.' ' - >sendmail.out && - cover "$NOTIFY" comaintainer-remove 1 1001 && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: AUR Co-Maintainer Notification for foobar - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - You were removed from the co-maintainer list of foobar [1]. - - [1] https://aur.archlinux.org/pkgbase/foobar/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of delete notifications.' ' - >sendmail.out && - cover "$NOTIFY" delete 1 1001 && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: AUR Package deleted: foobar - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - user [1] deleted foobar [2]. - - You will no longer receive notifications about this package. - - [1] https://aur.archlinux.org/account/user/ - [2] https://aur.archlinux.org/pkgbase/foobar/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of merge notifications.' ' - >sendmail.out && - cover "$NOTIFY" delete 1 1001 1002 && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: AUR Package deleted: foobar - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - user [1] merged foobar [2] into foobar2 [3]. - - -- - If you no longer wish receive notifications about the new package, - please go to [3] and click "Disable notifications". - - [1] https://aur.archlinux.org/account/user/ - [2] https://aur.archlinux.org/pkgbase/foobar/ - [3] https://aur.archlinux.org/pkgbase/foobar2/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test Cc, subject and body of request open notifications.' ' - cat <<-EOD | sqlite3 aur.db && - /* Use package request IDs which can be distinguished from other IDs. */ - INSERT INTO PackageRequests (ID, PackageBaseID, PackageBaseName, UsersID, ReqTypeID, Comments, ClosureComment) VALUES (3001, 1001, "foobar", 2, 1, "This is a request test comment.", ""); - EOD - >sendmail.out && - cover "$NOTIFY" request-open 1 3001 orphan 1001 && - grep ^Cc: sendmail.out >actual && - cat <<-EOD >expected && - Cc: user@localhost, tu@localhost - EOD - test_cmp actual expected && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: [PRQ#3001] Orphan Request for foobar - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - user [1] filed an orphan request for foobar [2]: - - This is a request test comment. - - [1] https://aur.archlinux.org/account/user/ - [2] https://aur.archlinux.org/pkgbase/foobar/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of request open notifications for merge requests.' ' - >sendmail.out && - cover "$NOTIFY" request-open 1 3001 merge 1001 foobar2 && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: [PRQ#3001] Merge Request for foobar - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - user [1] filed a request to merge foobar [2] into foobar2 [3]: - - This is a request test comment. - - [1] https://aur.archlinux.org/account/user/ - [2] https://aur.archlinux.org/pkgbase/foobar/ - [3] https://aur.archlinux.org/pkgbase/foobar2/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test Cc, subject and body of request close notifications.' ' - >sendmail.out && - cover "$NOTIFY" request-close 1 3001 accepted && - grep ^Cc: sendmail.out >actual && - cat <<-EOD >expected && - Cc: user@localhost, tu@localhost - EOD - test_cmp actual expected && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: [PRQ#3001] Deletion Request for foobar Accepted - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - Request #3001 has been accepted by user [1]. - - [1] https://aur.archlinux.org/account/user/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of request close notifications (auto-accept).' ' - >sendmail.out && - cover "$NOTIFY" request-close 0 3001 accepted && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: [PRQ#3001] Deletion Request for foobar Accepted - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - Request #3001 has been accepted automatically by the Arch User - Repository package request system. - EOD - test_cmp actual expected -' - -test_expect_success 'Test Cc of request close notification with co-maintainer.' ' - cat <<-EOD | sqlite3 aur.db && - /* Use package base IDs which can be distinguished from user IDs. */ - INSERT INTO PackageComaintainers (PackageBaseID, UsersID, Priority) VALUES (1001, 3, 1); - EOD - >sendmail.out && - "$NOTIFY" request-close 0 3001 accepted && - grep ^Cc: sendmail.out >actual && - cat <<-EOD >expected && - Cc: user@localhost, tu@localhost, dev@localhost - EOD - test_cmp actual expected && - cat <<-EOD | sqlite3 aur.db - DELETE FROM PackageComaintainers; - EOD -' - -test_expect_success 'Test subject and body of request close notifications with closure comment.' ' - cat <<-EOD | sqlite3 aur.db && - UPDATE PackageRequests SET ClosureComment = "This is a test closure comment." WHERE ID = 3001; - EOD - >sendmail.out && - cover "$NOTIFY" request-close 1 3001 accepted && - grep ^Subject: sendmail.out >actual && - cat <<-EOD >expected && - Subject: [PRQ#3001] Deletion Request for foobar Accepted - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - Request #3001 has been accepted by user [1]: - - This is a test closure comment. - - [1] https://aur.archlinux.org/account/user/ - EOD - test_cmp actual expected -' - -test_expect_success 'Test subject and body of TU vote reminders.' ' - >sendmail.out && - cover "$NOTIFY" tu-vote-reminder 1 && - grep ^Subject: sendmail.out | head -1 >actual && - cat <<-EOD >expected && - Subject: TU Vote Reminder: Proposal 1 - EOD - test_cmp actual expected && - sed -n "/^\$/,\$p" sendmail.out | head -4 | base64 -d >actual && - echo >>actual && - cat <<-EOD >expected && - Please remember to cast your vote on proposal 1 [1]. The voting period - ends in less than 48 hours. - - [1] https://aur.archlinux.org/tu/?id=1 - EOD - test_cmp actual expected -' - -test_done diff --git a/test/test_notify.py b/test/test_notify.py new file mode 100644 index 00000000..45a1df31 --- /dev/null +++ b/test/test_notify.py @@ -0,0 +1,643 @@ +from datetime import datetime +from typing import List +from unittest import mock + +import pytest + +from aurweb import config, db, models +from aurweb.models import Package, PackageBase, PackageRequest, User +from aurweb.models.account_type import TRUSTED_USER_ID, USER_ID +from aurweb.models.request_type import ORPHAN_ID +from aurweb.scripts import notify, rendercomment +from aurweb.testing.email import Email +from aurweb.testing.smtp import FakeSMTP, FakeSMTP_SSL + +aur_location = config.get("options", "aur_location") +aur_request_ml = config.get("options", "aur_request_ml") + + +@pytest.fixture(autouse=True) +def setup(db_test): + return + + +@pytest.fixture +def user() -> User: + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + Passwd=str(), AccountTypeID=USER_ID) + yield user + + +@pytest.fixture +def user1() -> User: + with db.begin(): + user1 = db.create(User, Username="user1", Email="user1@example.org", + Passwd=str(), AccountTypeID=USER_ID) + yield user1 + + +@pytest.fixture +def user2() -> User: + with db.begin(): + user2 = db.create(User, Username="user2", Email="user2@example.org", + Passwd=str(), AccountTypeID=USER_ID) + yield user2 + + +@pytest.fixture +def pkgbases(user: User) -> List[PackageBase]: + now = int(datetime.utcnow().timestamp()) + + output = [] + with db.begin(): + for i in range(5): + output.append( + db.create(PackageBase, Name=f"pkgbase_{i}", + Maintainer=user, SubmittedTS=now, + ModifiedTS=now)) + db.create(models.PackageNotification, PackageBase=output[-1], + User=user) + yield output + + +@pytest.fixture +def pkgreq(user2: User, pkgbases: List[PackageBase]): + pkgbase = pkgbases[0] + with db.begin(): + pkgreq_ = db.create(PackageRequest, PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, User=user2, + ReqTypeID=ORPHAN_ID, + Comments="This is a request test comment.", + ClosureComment=str()) + yield pkgreq_ + + +@pytest.fixture +def packages(pkgbases: List[PackageBase]) -> List[Package]: + output = [] + with db.begin(): + for i, pkgbase in enumerate(pkgbases): + output.append( + db.create(Package, PackageBase=pkgbase, + Name=f"pkg_{i}", Version=f"{i}.0")) + yield output + + +def test_out_of_date(user: User, user1: User, user2: User, + pkgbases: List[PackageBase]): + pkgbase = pkgbases[0] + # Create two comaintainers. We'll pass the maintainer uid to + # FlagNotification, so we should expect to get two emails. + with db.begin(): + db.create(models.PackageComaintainer, + PackageBase=pkgbase, User=user1, Priority=1) + db.create(models.PackageComaintainer, + PackageBase=pkgbase, User=user2, Priority=2) + + # Send the notification for pkgbases[0]. + notif = notify.FlagNotification(user.ID, pkgbases[0].ID) + notif.send() + + # Should've gotten three emails: maintainer + the two comaintainers. + assert Email.count() == 3 + + # Comaintainer 1. + first = Email(1).parse() + assert first.headers.get("To") == user1.Email + + expected = f"AUR Out-of-date Notification for {pkgbase.Name}" + assert first.headers.get("Subject") == expected + + # Comaintainer 2. + second = Email(2).parse() + assert second.headers.get("To") == user2.Email + + # Maintainer. + third = Email(3).parse() + assert third.headers.get("To") == user.Email + + +def test_reset(user: User): + with db.begin(): + user.ResetKey = "12345678901234567890123456789012" + + notif = notify.ResetKeyNotification(user.ID) + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + expected = "AUR Password Reset" + assert email.headers.get("Subject") == expected + + expected = f"""\ +A password reset request was submitted for the account test associated +with your email address. If you wish to reset your password follow the +link [1] below, otherwise ignore this message and nothing will happen. + +[1] {aur_location}/passreset/?resetkey=12345678901234567890123456789012\ +""" + assert email.body == expected + + +def test_welcome(user: User): + with db.begin(): + user.ResetKey = "12345678901234567890123456789012" + + notif = notify.WelcomeNotification(user.ID) + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + expected = "Welcome to the Arch User Repository" + assert email.headers.get("Subject") == expected + + expected = f"""\ +Welcome to the Arch User Repository! In order to set an initial +password for your new account, please click the link [1] below. If the +link does not work, try copying and pasting it into your browser. + +[1] {aur_location}/passreset/?resetkey=12345678901234567890123456789012\ +""" + assert email.body == expected + + +def test_comment(user: User, user2: User, pkgbases: List[PackageBase]): + pkgbase = pkgbases[0] + + with db.begin(): + comment = db.create(models.PackageComment, PackageBase=pkgbase, + User=user2, Comments="This is a test comment.") + rendercomment.update_comment_render_fastapi(comment) + + notif = notify.CommentNotification(user2.ID, pkgbase.ID, comment.ID) + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == user.Email + expected = f"AUR Comment for {pkgbase.Name}" + assert email.headers.get("Subject") == expected + + expected = f"""\ +{user2.Username} [1] added the following comment to {pkgbase.Name} [2]: + +This is a test comment. + +-- +If you no longer wish to receive notifications about this package, +please go to the package page [2] and select "Disable notifications". + +[1] {aur_location}/account/{user2.Username}/ +[2] {aur_location}/pkgbase/{pkgbase.Name}/\ +""" + assert expected == email.body + + +def test_update(user: User, user2: User, pkgbases: List[PackageBase]): + pkgbase = pkgbases[0] + with db.begin(): + user.UpdateNotify = 1 + + notif = notify.UpdateNotification(user2.ID, pkgbase.ID) + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == user.Email + expected = f"AUR Package Update: {pkgbase.Name}" + assert email.headers.get("Subject") == expected + + expected = f"""\ +{user2.Username} [1] pushed a new commit to {pkgbase.Name} [2]. + +-- +If you no longer wish to receive notifications about this package, +please go to the package page [2] and select "Disable notifications". + +[1] {aur_location}/account/{user2.Username}/ +[2] {aur_location}/pkgbase/{pkgbase.Name}/\ +""" + assert expected == email.body + + +def test_adopt(user: User, user2: User, pkgbases: List[PackageBase]): + pkgbase = pkgbases[0] + notif = notify.AdoptNotification(user2.ID, pkgbase.ID) + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == user.Email + expected = f"AUR Ownership Notification for {pkgbase.Name}" + assert email.headers.get("Subject") == expected + + expected = f"""\ +The package {pkgbase.Name} [1] was adopted by {user2.Username} [2]. + +[1] {aur_location}/pkgbase/{pkgbase.Name}/ +[2] {aur_location}/account/{user2.Username}/\ +""" + assert email.body == expected + + +def test_disown(user: User, user2: User, pkgbases: List[PackageBase]): + pkgbase = pkgbases[0] + notif = notify.DisownNotification(user2.ID, pkgbase.ID) + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == user.Email + expected = f"AUR Ownership Notification for {pkgbase.Name}" + assert email.headers.get("Subject") == expected + + expected = f"""\ +The package {pkgbase.Name} [1] was disowned by {user2.Username} [2]. + +[1] {aur_location}/pkgbase/{pkgbase.Name}/ +[2] {aur_location}/account/{user2.Username}/\ +""" + assert email.body == expected + + +def test_comaintainer_addition(user: User, pkgbases: List[PackageBase]): + # TODO: Add this in fastapi code! + pkgbase = pkgbases[0] + notif = notify.ComaintainerAddNotification(user.ID, pkgbase.ID) + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == user.Email + expected = f"AUR Co-Maintainer Notification for {pkgbase.Name}" + assert email.headers.get("Subject") == expected + + expected = f"""\ +You were added to the co-maintainer list of {pkgbase.Name} [1]. + +[1] {aur_location}/pkgbase/{pkgbase.Name}/\ +""" + assert email.body == expected + + +def test_comaintainer_removal(user: User, pkgbases: List[PackageBase]): + # TODO: Add this in fastapi code! + pkgbase = pkgbases[0] + notif = notify.ComaintainerRemoveNotification(user.ID, pkgbase.ID) + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == user.Email + expected = f"AUR Co-Maintainer Notification for {pkgbase.Name}" + assert email.headers.get("Subject") == expected + + expected = f"""\ +You were removed from the co-maintainer list of {pkgbase.Name} [1]. + +[1] {aur_location}/pkgbase/{pkgbase.Name}/\ +""" + assert email.body == expected + + +def test_delete(user: User, user2: User, pkgbases: List[PackageBase]): + pkgbase = pkgbases[0] + notif = notify.DeleteNotification(user2.ID, pkgbase.ID) + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == user.Email + expected = f"AUR Package deleted: {pkgbase.Name}" + assert email.headers.get("Subject") == expected + + expected = f"""\ +{user2.Username} [1] deleted {pkgbase.Name} [2]. + +You will no longer receive notifications about this package. + +[1] {aur_location}/account/{user2.Username}/ +[2] {aur_location}/pkgbase/{pkgbase.Name}/\ +""" + assert email.body == expected + + +def test_merge(user: User, user2: User, pkgbases: List[PackageBase]): + source, target = pkgbases[:2] + notif = notify.DeleteNotification(user2.ID, source.ID, target.ID) + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == user.Email + expected = f"AUR Package deleted: {source.Name}" + assert email.headers.get("Subject") == expected + + expected = f"""\ +{user2.Username} [1] merged {source.Name} [2] into {target.Name} [3]. + +-- +If you no longer wish receive notifications about the new package, +please go to [3] and click "Disable notifications". + +[1] {aur_location}/account/{user2.Username}/ +[2] {aur_location}/pkgbase/{source.Name}/ +[3] {aur_location}/pkgbase/{target.Name}/\ +""" + assert email.body == expected + + +def set_tu(users: List[User]) -> User: + with db.begin(): + for user in users: + user.AccountTypeID = TRUSTED_USER_ID + + +def test_open_close_request(user: User, user2: User, + pkgreq: PackageRequest, + pkgbases: List[PackageBase]): + set_tu([user]) + pkgbase = pkgbases[0] + + # Send an open request notification. + notif = notify.RequestOpenNotification( + user2.ID, pkgreq.ID, pkgreq.RequestType.Name, pkgbase.ID) + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == aur_request_ml + assert email.headers.get("Cc") == ", ".join([user.Email, user2.Email]) + expected = f"[PRQ#{pkgreq.ID}] Orphan Request for {pkgbase.Name}" + assert email.headers.get("Subject") == expected + + expected = f"""\ +{user2.Username} [1] filed an orphan request for {pkgbase.Name} [2]: + +This is a request test comment. + +[1] {aur_location}/account/{user2.Username}/ +[2] {aur_location}/pkgbase/{pkgbase.Name}/\ +""" + assert email.body == expected + + # Now send a closure notification on the pkgbase we just opened. + notif = notify.RequestCloseNotification(user2.ID, pkgreq.ID, "rejected") + notif.send() + assert Email.count() == 2 + + email = Email(2).parse() + assert email.headers.get("To") == aur_request_ml + assert email.headers.get("Cc") == ", ".join([user.Email, user2.Email]) + expected = f"[PRQ#{pkgreq.ID}] Orphan Request for {pkgbase.Name} Rejected" + assert email.headers.get("Subject") == expected + + expected = f"""\ +Request #{pkgreq.ID} has been rejected by {user2.Username} [1]. + +[1] {aur_location}/account/{user2.Username}/\ +""" + assert email.body == expected + + # Test auto-accept. + notif = notify.RequestCloseNotification(0, pkgreq.ID, "accepted") + notif.send() + assert Email.count() == 3 + + email = Email(3).parse() + assert email.headers.get("To") == aur_request_ml + assert email.headers.get("Cc") == ", ".join([user.Email, user2.Email]) + expected = (f"[PRQ#{pkgreq.ID}] Orphan Request for " + f"{pkgbase.Name} Accepted") + assert email.headers.get("Subject") == expected + + expected = (f"Request #{pkgreq.ID} has been accepted automatically " + "by the Arch User Repository\npackage request system.") + assert email.body == expected + + +def test_close_request_auto_accept(): + pass + + +def test_close_request_comaintainer_cc(user: User, user2: User, + pkgreq: PackageRequest, + pkgbases: List[PackageBase]): + # TODO: Check this in fastapi code! + pkgbase = pkgbases[0] + with db.begin(): + db.create(models.PackageComaintainer, PackageBase=pkgbase, + User=user2, Priority=1) + + notif = notify.RequestCloseNotification(0, pkgreq.ID, "accepted") + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == aur_request_ml + assert email.headers.get("Cc") == ", ".join([user.Email, user2.Email]) + + +def test_close_request_closure_comment(user: User, user2: User, + pkgreq: PackageRequest, + pkgbases: List[PackageBase]): + pkgbase = pkgbases[0] + with db.begin(): + pkgreq.ClosureComment = "This is a test closure comment." + + notif = notify.RequestCloseNotification(user2.ID, pkgreq.ID, "accepted") + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == aur_request_ml + assert email.headers.get("Cc") == ", ".join([user.Email, user2.Email]) + expected = f"[PRQ#{pkgreq.ID}] Orphan Request for {pkgbase.Name} Accepted" + assert email.headers.get("Subject") == expected + + expected = f"""\ +Request #{pkgreq.ID} has been accepted by {user2.Username} [1]: + +This is a test closure comment. + +[1] {aur_location}/account/{user2.Username}/\ +""" + assert email.body == expected + + +def test_tu_vote_reminders(user: User): + set_tu([user]) + + vote_id = 1 + notif = notify.TUVoteReminderNotification(vote_id) + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == user.Email + expected = f"TU Vote Reminder: Proposal {vote_id}" + assert email.headers.get("Subject") == expected + + expected = f"""\ +Please remember to cast your vote on proposal {vote_id} [1]. The voting period +ends in less than 48 hours. + +[1] {aur_location}/tu/?id={vote_id}\ +""" + assert email.body == expected + + +def test_notify_main(user: User): + """ Test TU vote reminder through aurweb.notify.main(). """ + set_tu([user]) + + vote_id = 1 + args = ["aurweb-notify", "tu-vote-reminder", str(vote_id)] + with mock.patch("sys.argv", args): + notify.main() + + assert Email.count() == 1 + + email = Email(1).parse() + assert email.headers.get("To") == user.Email + expected = f"TU Vote Reminder: Proposal {vote_id}" + assert email.headers.get("Subject") == expected + + expected = f"""\ +Please remember to cast your vote on proposal {vote_id} [1]. The voting period +ends in less than 48 hours. + +[1] {aur_location}/tu/?id={vote_id}\ +""" + assert email.body == expected + + +# Save original config.get; we're going to mock it and need +# to be able to fallback when we are not overriding. +config_get = config.get + + +def mock_smtp_config(cls): + def _mock_smtp_config(section: str, key: str): + if section == "notifications": + if key == "sendmail": + return cls() + elif key == "smtp-use-ssl": + return cls(0) + elif key == "smtp-use-starttls": + return cls(0) + elif key == "smtp-user": + return cls() + elif key == "smtp-password": + return cls() + return cls(config_get(section, key)) + return _mock_smtp_config + + +def test_smtp(user: User): + with db.begin(): + user.ResetKey = "12345678901234567890123456789012" + + SMTP = FakeSMTP() + + get = "aurweb.config.get" + getboolean = "aurweb.config.getboolean" + with mock.patch(get, side_effect=mock_smtp_config(str)): + with mock.patch(getboolean, side_effect=mock_smtp_config(bool)): + with mock.patch("smtplib.SMTP", side_effect=lambda a, b: SMTP): + config.rehash() + notif = notify.WelcomeNotification(user.ID) + notif.send() + config.rehash() + assert len(SMTP.emails) == 1 + + +def mock_smtp_starttls_config(cls): + def _mock_smtp_starttls_config(section: str, key: str): + if section == "notifications": + if key == "sendmail": + return cls() + elif key == "smtp-use-ssl": + return cls(0) + elif key == "smtp-use-starttls": + return cls(1) + elif key == "smtp-user": + return cls("test") + elif key == "smtp-password": + return cls("password") + return cls(config_get(section, key)) + return _mock_smtp_starttls_config + + +def test_smtp_starttls(user: User): + # This test does two things: test starttls path and test + # path where we have a backup email. + + with db.begin(): + user.ResetKey = "12345678901234567890123456789012" + user.BackupEmail = "backup@example.org" + + SMTP = FakeSMTP() + + get = "aurweb.config.get" + getboolean = "aurweb.config.getboolean" + with mock.patch(get, side_effect=mock_smtp_starttls_config(str)): + with mock.patch( + getboolean, side_effect=mock_smtp_starttls_config(bool)): + with mock.patch("smtplib.SMTP", side_effect=lambda a, b: SMTP): + notif = notify.WelcomeNotification(user.ID) + notif.send() + assert SMTP.starttls_enabled + assert SMTP.user + assert SMTP.passwd + + assert len(SMTP.emails) == 2 + to = SMTP.emails[0][1] + assert to == [user.Email] + + to = SMTP.emails[1][1] + assert to == [user.BackupEmail] + + +def mock_smtp_ssl_config(cls): + def _mock_smtp_ssl_config(section: str, key: str): + if section == "notifications": + if key == "sendmail": + return cls() + elif key == "smtp-use-ssl": + return cls(1) + elif key == "smtp-use-starttls": + return cls(0) + elif key == "smtp-user": + return cls("test") + elif key == "smtp-password": + return cls("password") + return cls(config_get(section, key)) + return _mock_smtp_ssl_config + + +def test_smtp_ssl(user: User): + with db.begin(): + user.ResetKey = "12345678901234567890123456789012" + + SMTP = FakeSMTP_SSL() + + get = "aurweb.config.get" + getboolean = "aurweb.config.getboolean" + with mock.patch(get, side_effect=mock_smtp_ssl_config(str)): + with mock.patch(getboolean, side_effect=mock_smtp_ssl_config(bool)): + with mock.patch("smtplib.SMTP_SSL", side_effect=lambda a, b: SMTP): + notif = notify.WelcomeNotification(user.ID) + notif.send() + assert len(SMTP.emails) == 1 + assert SMTP.use_ssl + assert SMTP.user + assert SMTP.passwd + + +def test_notification_defaults(): + notif = notify.Notification() + assert notif.get_refs() == tuple() + assert notif.get_headers() == dict() + assert notif.get_cc() == list() From 155aa47a1acf9143f8bce49a81f05f56f7e61042 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 25 Nov 2021 12:03:25 -0800 Subject: [PATCH 644/844] feat(poetry): add posix_ipc Signed-off-by: Kevin Morris --- poetry.lock | 18 +++++++++++++++++- pyproject.toml | 1 + 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index ab559b77..d56ce458 100644 --- a/poetry.lock +++ b/poetry.lock @@ -619,6 +619,14 @@ dunamai = ">=1.5,<2.0" jinja2 = {version = ">=2.11.1,<4", markers = "python_version >= \"3.6\" and python_version < \"4.0\""} tomlkit = ">=0.4" +[[package]] +name = "posix-ipc" +version = "1.0.5" +description = "POSIX IPC primitives (semaphores, shared memory and message queues) for Python" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "priority" version = "2.0.0" @@ -1056,7 +1064,7 @@ h11 = ">=0.9.0,<1" [metadata] lock-version = "1.1" python-versions = ">=3.9,<3.10" -content-hash = "ca42bd35717062d6784025ed3956423502ac66adba059ccc080bcaaa666651cd" +content-hash = "43b3dd494890ef9d419260a06ccd0190638573fdf0b92a226016f1dd5ee87579" [metadata.files] aiofiles = [ @@ -1555,6 +1563,14 @@ poetry-dynamic-versioning = [ {file = "poetry-dynamic-versioning-0.13.1.tar.gz", hash = "sha256:5c0e7b22560db76812057ef95dadad662ecc63eb270145787eabe73da7c222f9"}, {file = "poetry_dynamic_versioning-0.13.1-py3-none-any.whl", hash = "sha256:6d79f76436c624653fc06eb9bb54fb4f39b1d54362bc366ad2496855711d3a78"}, ] +posix-ipc = [ + {file = "posix_ipc-1.0.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ccb36ba90efec56a1796f1566eee9561f355a4f45babbc4d18ac46fb2d0b246b"}, + {file = "posix_ipc-1.0.5-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:613bf1afe90e84c06255ec1a6f52c9b24062492de66e5f0dbe068adf67fc3454"}, + {file = "posix_ipc-1.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6095bb4faa2bba8b8d0e833b804e0aedc352d5ed921edeb715010cbcd361e038"}, + {file = "posix_ipc-1.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:621918abe7ec68591c5839b0771d163a9809bc232bf413b9a681bf986ab68d4d"}, + {file = "posix_ipc-1.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f71587ad3a50e82583987f62bfd4ac2343ab6a206d1032e3fc560e8d55fe0346"}, + {file = "posix_ipc-1.0.5.tar.gz", hash = "sha256:6cddb1ce2cf4aae383f2a0079c26c69bee257fe2720f372201ef047f8ceb8b97"}, +] priority = [ {file = "priority-2.0.0-py3-none-any.whl", hash = "sha256:6f8eefce5f3ad59baf2c080a664037bb4725cd0a790d53d59ab4059288faf6aa"}, {file = "priority-2.0.0.tar.gz", hash = "sha256:c965d54f1b8d0d0b19479db3924c7c36cf672dbf2aec92d43fbdaf4492ba18c0"}, diff --git a/pyproject.toml b/pyproject.toml index 82c439bc..f164967c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ mysql-connector = "^2.2.9" prometheus-fastapi-instrumentator = "^5.7.1" pytest-xdist = "^2.4.0" filelock = "^3.3.2" +posix-ipc = "^1.0.5" [tool.poetry.dev-dependencies] flake8 = "^4.0.1" From 4b0cb0721d8fc4fe2414e37a8a8fbf78949481a5 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 22 Nov 2021 20:06:50 -0800 Subject: [PATCH 645/844] fix(conftest): use synchronization locks for setup_database We were running into data race issues where the `fn.is_file()` check would occur twice before writing the file in the `else` clause. For this reason, a new aurweb.lock.Lock class has been added which doubles as a thread and process lock. We can use this elsewhere in the future, but we are also able to use it to solve this kind of data race issue. That being said, we still need the lock file state to tell us when the first caller acquired the lock. Signed-off-by: Kevin Morris --- aurweb/testing/filelock.py | 32 ++++++++++++++++++++ test/conftest.py | 61 +++++++++++++++++++++++--------------- test/test_filelock.py | 26 ++++++++++++++++ 3 files changed, 95 insertions(+), 24 deletions(-) create mode 100644 aurweb/testing/filelock.py create mode 100644 test/test_filelock.py diff --git a/aurweb/testing/filelock.py b/aurweb/testing/filelock.py new file mode 100644 index 00000000..3a18c153 --- /dev/null +++ b/aurweb/testing/filelock.py @@ -0,0 +1,32 @@ +import hashlib +import os + +from typing import Callable + +from posix_ipc import O_CREAT, Semaphore + +from aurweb import logging + +logger = logging.get_logger(__name__) + + +def default_on_create(path): + logger.info(f"Filelock at {path} acquired.") + + +class FileLock: + def __init__(self, tmpdir, name: str): + self.root = tmpdir + self.path = str(self.root / name) + self._file = str(self.root / (f"{name}.1")) + + def lock(self, on_create: Callable = default_on_create): + hash = hashlib.sha1(self.path.encode()).hexdigest() + with Semaphore(f"/{hash}-lock", flags=O_CREAT, initial_value=1): + retval = os.path.exists(self._file) + if not retval: + with open(self._file, "w") as f: + f.write("1") + on_create(self.path) + + return retval diff --git a/test/conftest.py b/test/conftest.py index 01131109..80f77c9a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -38,10 +38,14 @@ It is done this way because migration has a large cost; migrating ahead of each function takes too long when compared to this method. """ import os +import pathlib +from multiprocessing import Lock + +import py import pytest -from filelock import FileLock +from posix_ipc import O_CREAT, Semaphore from sqlalchemy import create_engine from sqlalchemy.engine import URL from sqlalchemy.engine.base import Engine @@ -53,9 +57,13 @@ import aurweb.db from aurweb import initdb, logging, testing from aurweb.testing.email import Email +from aurweb.testing.filelock import FileLock logger = logging.get_logger(__name__) +# Synchronization lock for database setup. +setup_lock = Lock() + def test_engine() -> Engine: """ @@ -105,7 +113,12 @@ def _create_database(engine: Engine, dbname: str) -> None: try: conn.execute(f"CREATE DATABASE {dbname}") except ProgrammingError: # pragma: no cover - pass + # The database most likely already existed if we hit + # a ProgrammingError. Just drop the database and try + # again. If at that point things still fail, any + # exception will be propogated up to the caller. + conn.execute(f"DROP DATABASE {dbname}") + conn.execute(f"CREATE DATABASE {dbname}") conn.close() initdb.run(AlembicArgs) @@ -124,20 +137,24 @@ def _drop_database(engine: Engine, dbname: str) -> None: def setup_email(): - if not os.path.exists(Email.TEST_DIR): - os.makedirs(Email.TEST_DIR) + # TODO: Fix this data race! This try/catch is ugly; why is it even + # racing here? Perhaps we need to multiproc + multithread lock + # inside of setup_database to block the check? + with Semaphore("/test-emails", flags=O_CREAT, initial_value=1): + if not os.path.exists(Email.TEST_DIR): + # Create the directory. + os.makedirs(Email.TEST_DIR) - # Cleanup all email files for this test suite. - prefix = Email.email_prefix(suite=True) - files = os.listdir(Email.TEST_DIR) - for file in files: - if file.startswith(prefix): - os.remove(os.path.join(Email.TEST_DIR, file)) + # Cleanup all email files for this test suite. + prefix = Email.email_prefix(suite=True) + files = os.listdir(Email.TEST_DIR) + for file in files: + if file.startswith(prefix): + os.remove(os.path.join(Email.TEST_DIR, file)) @pytest.fixture(scope="module") -def setup_database(tmp_path_factory: pytest.fixture, - worker_id: pytest.fixture) -> None: +def setup_database(tmp_path_factory: pathlib.Path, worker_id: str) -> None: """ Create and drop a database for the suite this fixture is used in. """ engine = test_engine() dbname = aurweb.db.name() @@ -149,19 +166,15 @@ def setup_database(tmp_path_factory: pytest.fixture, _drop_database(engine, dbname) return - root_tmp_dir = tmp_path_factory.getbasetemp().parent - fn = root_tmp_dir / dbname + def setup(path): + setup_email() + _create_database(engine, dbname) - with FileLock(str(fn) + ".lock"): - if fn.is_file(): - # If the data file exists, skip database creation. - yield - else: - # Otherwise, create the data file and create the database. - fn.write_text("1") - setup_email() - yield _create_database(engine, dbname) - _drop_database(engine, dbname) + tmpdir = tmp_path_factory.getbasetemp().parent + file_lock = FileLock(tmpdir, dbname) + file_lock.lock(on_create=setup) + yield # Run the test function depending on this fixture. + _drop_database(engine, dbname) # Cleanup the database. @pytest.fixture(scope="module") diff --git a/test/test_filelock.py b/test/test_filelock.py new file mode 100644 index 00000000..70aa7580 --- /dev/null +++ b/test/test_filelock.py @@ -0,0 +1,26 @@ +import py + +from _pytest.logging import LogCaptureFixture + +from aurweb.testing.filelock import FileLock + + +def test_filelock(tmpdir: py.path.local): + cb_path = None + + def setup(path: str): + nonlocal cb_path + cb_path = str(path) + + flock = FileLock(tmpdir, "test") + assert not flock.lock(on_create=setup) + assert cb_path == str(tmpdir / "test") + assert flock.lock() + + +def test_filelock_default(caplog: LogCaptureFixture, tmpdir: py.path.local): + # Test default_on_create here. + flock = FileLock(tmpdir, "test") + assert not flock.lock() + assert caplog.messages[0] == f"Filelock at {flock.path} acquired." + assert flock.lock() From 2d0e09cd63bc2a1c0baee0727aa5ede60e3475b1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 06:20:50 -0800 Subject: [PATCH 646/844] change(rendercomment): converted to use aurweb.db ORM - Added aurweb.util.git_search. - Decoupled away from rendercomment for easier testability. - Added aurweb.testing.git.GitRepository. - Added templates/testing/{PKGBUILD,SRCINFO}.j2. - Added aurweb.testing.git.GitRepository + `git` pytest fixture Signed-off-by: Kevin Morris --- aurweb/scripts/rendercomment.py | 57 ++++----- aurweb/testing/git.py | 110 +++++++++++++++++ aurweb/util.py | 17 +++ templates/testing/PKGBUILD.j2 | 14 +++ templates/testing/SRCINFO.j2 | 10 ++ test/conftest.py | 6 + test/t2600-rendercomment.t | 160 ------------------------- test/test_rendercomment.py | 202 ++++++++++++++++++++++++++++++++ test/test_util.py | 17 +++ 9 files changed, 398 insertions(+), 195 deletions(-) create mode 100644 aurweb/testing/git.py create mode 100644 templates/testing/PKGBUILD.j2 create mode 100644 templates/testing/SRCINFO.j2 delete mode 100755 test/t2600-rendercomment.t create mode 100644 test/test_rendercomment.py diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index efa5357f..33349432 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -7,13 +7,11 @@ import markdown import pygit2 import aurweb.config -import aurweb.db -from aurweb import logging +from aurweb import db, logging, util +from aurweb.models import PackageComment logger = logging.get_logger(__name__) -repo_path = aurweb.config.get('serve', 'repo-path') -commit_uri = aurweb.config.get('options', 'commit_uri') class LinkifyExtension(markdown.extensions.Extension): @@ -64,6 +62,7 @@ class GitCommitsInlineProcessor(markdown.inlinepatterns.InlineProcessor): """ def __init__(self, md, head): + repo_path = aurweb.config.get('serve', 'repo-path') self._repo = pygit2.Repository(repo_path) self._head = head super().__init__(r'\b([0-9a-f]{7,40})\b', md) @@ -74,13 +73,9 @@ class GitCommitsInlineProcessor(markdown.inlinepatterns.InlineProcessor): # Unkwown OID; preserve the orginal text. return (None, None, None) - prefixlen = 12 - while prefixlen < 40: - if oid[:prefixlen] in self._repo: - break - prefixlen += 1 - el = markdown.util.etree.Element('a') + commit_uri = aurweb.config.get("options", "commit_uri") + prefixlen = util.git_search(self._repo, oid) el.set('href', commit_uri % (self._head, oid[:prefixlen])) el.text = markdown.util.AtomicString(oid[:prefixlen]) return (el, m.start(0), m.end(0)) @@ -116,49 +111,41 @@ class HeadingExtension(markdown.extensions.Extension): md.treeprocessors.register(HeadingTreeprocessor(md), 'heading', 30) -def get_comment(conn, commentid): - cur = conn.execute('SELECT PackageComments.Comments, PackageBases.Name ' - 'FROM PackageComments INNER JOIN PackageBases ' - 'ON PackageBases.ID = PackageComments.PackageBaseID ' - 'WHERE PackageComments.ID = ?', [commentid]) - return cur.fetchone() +def save_rendered_comment(comment: PackageComment, html: str): + with db.begin(): + comment.RenderedComment = html -def save_rendered_comment(conn, commentid, html): - conn.execute('UPDATE PackageComments SET RenderedComment = ? WHERE ID = ?', - [html, commentid]) +def update_comment_render_fastapi(comment: PackageComment) -> None: + update_comment_render(comment) -def update_comment_render_fastapi(comment): - conn = aurweb.db.ConnectionExecutor( - aurweb.db.get_engine().raw_connection()) - update_comment_render(conn, comment.ID) - aurweb.db.refresh(comment) +def update_comment_render(comment: PackageComment) -> None: + text = comment.Comments + pkgbasename = comment.PackageBase.Name - -def update_comment_render(conn, commentid): - text, pkgbase = get_comment(conn, commentid) html = markdown.markdown(text, extensions=[ 'fenced_code', LinkifyExtension(), FlysprayLinksExtension(), - GitCommitsExtension(pkgbase), + GitCommitsExtension(pkgbasename), HeadingExtension() ]) allowed_tags = (bleach.sanitizer.ALLOWED_TAGS + ['p', 'pre', 'h4', 'h5', 'h6', 'br', 'hr']) html = bleach.clean(html, tags=allowed_tags) - save_rendered_comment(conn, commentid, html) - - conn.commit() - conn.close() + save_rendered_comment(comment, html) + db.refresh(comment) def main(): - commentid = int(sys.argv[1]) - conn = aurweb.db.Connection() - update_comment_render(conn, commentid) + db.get_engine() + comment_id = int(sys.argv[1]) + comment = db.query(PackageComment).filter( + PackageComment.ID == comment_id + ).first() + update_comment_render(comment) if __name__ == '__main__': diff --git a/aurweb/testing/git.py b/aurweb/testing/git.py new file mode 100644 index 00000000..019d870f --- /dev/null +++ b/aurweb/testing/git.py @@ -0,0 +1,110 @@ +import os +import shlex + +from subprocess import PIPE, Popen +from typing import Tuple + +import py + +from aurweb.models import Package +from aurweb.templates import base_template +from aurweb.testing.filelock import FileLock + + +class GitRepository: + """ + A Git repository class to be used for testing. + + Expects a `tmpdir` fixture on construction, which an 'aur.git' + git repository will be created in. After this class is constructed, + users can call GitRepository.exec for git repository operations. + """ + + def __init__(self, tmpdir: py.path.local): + self.file_lock = FileLock(tmpdir, "aur.git") + self.file_lock.lock(on_create=self._setup) + + def _exec(self, cmdline: str, cwd: str) -> Tuple[int, str, str]: + args = shlex.split(cmdline) + proc = Popen(args, cwd=cwd, stdout=PIPE, stderr=PIPE) + out, err = proc.communicate() + return (proc.returncode, out.decode().strip(), err.decode().strip()) + + def _exec_repository(self, cmdline: str) -> Tuple[int, str, str]: + return self._exec(cmdline, cwd=str(self.file_lock.path)) + + def exec(self, cmdline: str) -> Tuple[int, str, str]: + return self._exec_repository(cmdline) + + def _setup(self, path: str) -> None: + """ + Setup the git repository from scratch. + + Create the `path` directory and run the INSTALL recommended + git initialization commands inside of it. Additionally, install + aurweb.git.update to {path}/hooks/update. + + :param path: Repository path not yet created + """ + + os.makedirs(path) + + commands = [ + "git init -q", + "git config --local transfer.hideRefs '^refs/'", + "git config --local --add transfer.hideRefs '!refs/'", + "git config --local --add transfer.hideRefs '!HEAD'", + "git config --local commit.gpgsign false", + "git config --local user.name 'Test User'", + "git config --local user.email 'test@example.org'", + ] + for cmdline in commands: + return_code, out, err = self.exec(cmdline) + assert return_code == 0 + + # This is also done in the INSTALL script to give the `aur` + # ssh user permissions on the repository. We don't need it + # during testing, since our testing user will be controlling + # the repository. It is left here as a note. + # self.exec("chown -R aur .") + + def commit(self, pkg: Package, message: str): + """ + Commit a Package record to the git repository. + + This function generates a PKGBUILD and .SRCINFO based on + `pkg`, then commits them to the repository with the + `message` commit message. + + :param pkg: Package instance + :param message: Commit message + :return: Output of `git rev-parse HEAD` after committing + """ + ref = f"refs/namespaces/{pkg.Name}/refs/heads/master" + rc, out, err = self.exec(f"git checkout -q --orphan {ref}") + assert rc == 0, f"{(rc, out, err)}" + + # Path to aur.git repository. + repo = os.path.join(self.file_lock.path) + + licenses = [f"'{p.License.Name}'" for p in pkg.package_licenses] + depends = [f"'{p.DepName}'" for p in pkg.package_dependencies] + pkgbuild = base_template("testing/PKGBUILD.j2") + pkgbuild_path = os.path.join(repo, "PKGBUILD") + with open(pkgbuild_path, "w") as f: + data = pkgbuild.render(pkg=pkg, licenses=licenses, depends=depends) + f.write(data) + + srcinfo = base_template("testing/SRCINFO.j2") + srcinfo_path = os.path.join(repo, ".SRCINFO") + with open(srcinfo_path, "w") as f: + f.write(srcinfo.render(pkg=pkg)) + + rc, out, err = self.exec("git add PKGBUILD .SRCINFO") + assert rc == 0, f"{(rc, out, err)}" + + rc, out, err = self.exec(f"git commit -q -m '{message}'") + assert rc == 0, f"{(rc, out, err)}" + + # Return stdout of `git rev-parse HEAD`, which is the new commit hash. + return self.exec("git rev-parse HEAD")[1] diff --git a/aurweb/util.py b/aurweb/util.py index bf2d6e4b..f5ced259 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -13,6 +13,7 @@ from urllib.parse import urlencode, urlparse from zoneinfo import ZoneInfo import fastapi +import pygit2 from email_validator import EmailNotValidError, EmailUndeliverableError, validate_email from jinja2 import pass_context @@ -193,3 +194,19 @@ def file_hash(filepath: str, hash_function: Callable) -> str: with open(filepath, "rb") as f: hash_ = hash_function(f.read()) return hash_.hexdigest() + + +def git_search(repo: pygit2.Repository, commit_hash: str) -> int: + """ + Return the shortest prefix length matching `commit_hash` found. + + :param repo: pygit2.Repository instance + :param commit_hash: Full length commit hash + :return: Shortest unique prefix length found + """ + prefixlen = 12 + while prefixlen < len(commit_hash): + if commit_hash[:prefixlen] in repo: + break + prefixlen += 1 + return prefixlen diff --git a/templates/testing/PKGBUILD.j2 b/templates/testing/PKGBUILD.j2 new file mode 100644 index 00000000..29d3a1d9 --- /dev/null +++ b/templates/testing/PKGBUILD.j2 @@ -0,0 +1,14 @@ +pkgname={{ pkg.PackageBase.Name }} +pkgver={{ pkg.Version }} +pkgrel=1 +pkgdesc='{{ pkg.Description }}' +url='{{ pkg.URL }}' +arch='any' +license=({{ licenses | join(" ") }}) +depends=({{ depends | join(" ") }}) +source=() +md5sums=() + +package() { + {{ body }} +} diff --git a/templates/testing/SRCINFO.j2 b/templates/testing/SRCINFO.j2 new file mode 100644 index 00000000..873b9c1b --- /dev/null +++ b/templates/testing/SRCINFO.j2 @@ -0,0 +1,10 @@ +pkgbase = {{ pkg.PackageBase.name }} + pkgver = {{ pkg.Version }} + pkgrel = 1 + pkgdesc = {{ pkg.Description }} + url = {{ pkg.URL }} + arch='any' + license = {{ pkg.package_licenses | join(", ", attribute="License.Name") }} + depends = {{ pkg.package_dependencies | join(", ", attribute="DepName") }} + +pkgname = {{ pkg.Name }} diff --git a/test/conftest.py b/test/conftest.py index 80f77c9a..fc7f77dc 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -58,6 +58,7 @@ import aurweb.db from aurweb import initdb, logging, testing from aurweb.testing.email import Email from aurweb.testing.filelock import FileLock +from aurweb.testing.git import GitRepository logger = logging.get_logger(__name__) @@ -211,3 +212,8 @@ def db_test(db_session: scoped_session) -> None: session via aurweb.db.get_session(). """ testing.setup_test_db() + + +@pytest.fixture +def git(tmpdir: py.path.local) -> GitRepository: + yield GitRepository(tmpdir) diff --git a/test/t2600-rendercomment.t b/test/t2600-rendercomment.t deleted file mode 100755 index bb84fcfe..00000000 --- a/test/t2600-rendercomment.t +++ /dev/null @@ -1,160 +0,0 @@ -#!/bin/sh - -test_description='rendercomment tests' - -. "$(dirname "$0")/setup.sh" - -test_expect_success 'Test comment rendering.' ' - cat <<-EOD | sqlite3 aur.db && - INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (1, "foobar", 1, 0, 0, ""); - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (1, 1, "Hello world! - This is a comment.", ""); - EOD - cover "$RENDERCOMMENT" 1 && - cat <<-EOD >expected && -

    Hello world! - This is a comment.

    - EOD - cat <<-EOD | sqlite3 aur.db >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 1; - EOD - test_cmp actual expected -' - -test_expect_success 'Test Markdown conversion.' ' - cat <<-EOD | sqlite3 aur.db && - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (2, 1, "*Hello* [world](https://www.archlinux.org/)!", ""); - EOD - cover "$RENDERCOMMENT" 2 && - cat <<-EOD >expected && -

    Hello world!

    - EOD - cat <<-EOD | sqlite3 aur.db >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 2; - EOD - test_cmp actual expected -' - -test_expect_success 'Test HTML sanitizing.' ' - cat <<-EOD | sqlite3 aur.db && - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (3, 1, "", ""); - EOD - cover "$RENDERCOMMENT" 3 && - cat <<-EOD >expected && - <script>alert("XSS!");</script> - EOD - cat <<-EOD | sqlite3 aur.db >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 3; - EOD - test_cmp actual expected -' - -test_expect_success 'Test link conversion.' ' - cat <<-EOD | sqlite3 aur.db && - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (4, 1, " - Visit https://www.archlinux.org/#_test_. - Visit *https://www.archlinux.org/*. - Visit . - Visit \`https://www.archlinux.org/\`. - Visit [Arch Linux](https://www.archlinux.org/). - Visit [Arch Linux][arch]. - [arch]: https://www.archlinux.org/ - ", ""); - EOD - cover "$RENDERCOMMENT" 4 && - cat <<-EOD >expected && -

    Visit https://www.archlinux.org/#_test_. - Visit https://www.archlinux.org/. - Visit https://www.archlinux.org/. - Visit https://www.archlinux.org/. - Visit Arch Linux. - Visit Arch Linux.

    - EOD - cat <<-EOD | sqlite3 aur.db >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 4; - EOD - test_cmp actual expected -' - -test_expect_success 'Test Git commit linkification.' ' - local oid=`git -C aur.git rev-parse --verify HEAD` - cat <<-EOD | sqlite3 aur.db && - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (5, 1, " - $oid - ${oid:0:7} - x.$oid.x - ${oid}x - 0123456789abcdef - \`$oid\` - http://example.com/$oid - ", ""); - EOD - cover "$RENDERCOMMENT" 5 && - cat <<-EOD >expected && -

    ${oid:0:12} - ${oid:0:7} - x.${oid:0:12}.x - ${oid}x - 0123456789abcdef - $oid - http://example.com/$oid

    - EOD - cat <<-EOD | sqlite3 aur.db >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 5; - EOD - test_cmp actual expected -' - -test_expect_success 'Test Flyspray issue linkification.' ' - sqlite3 aur.db <<-EOD && - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (6, 1, " - FS#1234567. - *FS#1234* - FS# - XFS#1 - \`FS#1234\` - https://archlinux.org/?test=FS#1234 - ", ""); - EOD - cover "$RENDERCOMMENT" 6 && - cat <<-EOD >expected && -

    FS#1234567. - FS#1234 - FS# - XFS#1 - FS#1234 - https://archlinux.org/?test=FS#1234

    - EOD - sqlite3 aur.db <<-EOD >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 6; - EOD - test_cmp actual expected -' - -test_expect_success 'Test headings lowering.' ' - sqlite3 aur.db <<-EOD && - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (7, 1, " - # One - ## Two - ### Three - #### Four - ##### Five - ###### Six - ", ""); - EOD - cover "$RENDERCOMMENT" 7 && - cat <<-EOD >expected && -
    One
    -
    Two
    -
    Three
    -
    Four
    -
    Five
    -
    Six
    - EOD - sqlite3 aur.db <<-EOD >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 7; - EOD - test_cmp actual expected -' - -test_done diff --git a/test/test_rendercomment.py b/test/test_rendercomment.py new file mode 100644 index 00000000..c45d4235 --- /dev/null +++ b/test/test_rendercomment.py @@ -0,0 +1,202 @@ +from datetime import datetime +from unittest import mock + +import pytest + +from aurweb import config, db, logging +from aurweb.models import Package, PackageBase, PackageComment, User +from aurweb.models.account_type import USER_ID +from aurweb.scripts import rendercomment +from aurweb.scripts.rendercomment import update_comment_render +from aurweb.testing.git import GitRepository + +logger = logging.get_logger(__name__) +aur_location = config.get("options", "aur_location") + + +@pytest.fixture(autouse=True) +def setup(db_test, git: GitRepository): + config_get = config.get + + def mock_config_get(section: str, key: str) -> str: + if section == "serve" and key == "repo-path": + return git.file_lock.path + elif section == "options" and key == "commit_uri": + return "/cgit/aur.git/log/?h=%s&id=%s" + return config_get(section, key) + + with mock.patch("aurweb.config.get", side_effect=mock_config_get): + yield + + +@pytest.fixture +def user() -> User: + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + Passwd=str(), AccountTypeID=USER_ID) + yield user + + +@pytest.fixture +def pkgbase(user: User) -> PackageBase: + now = int(datetime.utcnow().timestamp()) + with db.begin(): + pkgbase = db.create(PackageBase, Packager=user, Name="pkgbase_0", + SubmittedTS=now, ModifiedTS=now) + yield pkgbase + + +@pytest.fixture +def package(pkgbase: PackageBase) -> Package: + with db.begin(): + package = db.create(Package, PackageBase=pkgbase, + Name=pkgbase.Name, Version="1.0") + yield package + + +def create_comment(user: User, pkgbase: PackageBase, comments: str, + render: bool = True): + with db.begin(): + comment = db.create(PackageComment, User=user, + PackageBase=pkgbase, Comments=comments) + if render: + update_comment_render(comment) + return comment + + +def test_comment_rendering(user: User, pkgbase: PackageBase): + text = "Hello world! This is a comment." + comment = create_comment(user, pkgbase, text) + expected = f"

    {text}

    " + assert comment.RenderedComment == expected + + +def test_rendercomment_main(user: User, pkgbase: PackageBase): + text = "Hello world! This is a comment." + comment = create_comment(user, pkgbase, text, False) + + args = ["aurweb-rendercomment", str(comment.ID)] + with mock.patch("sys.argv", args): + rendercomment.main() + db.refresh(comment) + + expected = f"

    {text}

    " + assert comment.RenderedComment == expected + + +def test_markdown_conversion(user: User, pkgbase: PackageBase): + text = "*Hello* [world](https://aur.archlinux.org)!" + comment = create_comment(user, pkgbase, text) + expected = ('

    Hello ' + 'world!

    ') + assert comment.RenderedComment == expected + + +def test_html_sanitization(user: User, pkgbase: PackageBase): + text = '' + comment = create_comment(user, pkgbase, text) + expected = '<script>alert("XSS!")</script>' + assert comment.RenderedComment == expected + + +def test_link_conversion(user: User, pkgbase: PackageBase): + text = """\ +Visit https://www.archlinux.org/#_test_. +Visit *https://www.archlinux.org/*. +Visit . +Visit `https://www.archlinux.org/`. +Visit [Arch Linux](https://www.archlinux.org/). +Visit [Arch Linux][arch]. +[arch]: https://www.archlinux.org/\ +""" + comment = create_comment(user, pkgbase, text) + expected = '''\ +

    Visit \ +https://www.archlinux.org/#_test_. +Visit https://www.archlinux.org/. +Visit https://www.archlinux.org/. +Visit https://www.archlinux.org/. +Visit Arch Linux. +Visit Arch Linux.

    \ +''' + assert comment.RenderedComment == expected + + +def test_git_commit_link(git: GitRepository, user: User, package: Package): + commit_hash = git.commit(package, "Initial commit.") + logger.info(f"Created commit: {commit_hash}") + logger.info(f"Short hash: {commit_hash[:7]}") + + text = f"""\ +{commit_hash} +{commit_hash[:7]} +x.{commit_hash}.x +{commit_hash}x +0123456789abcdef +`{commit_hash}` +http://example.com/{commit_hash}\ +""" + comment = create_comment(user, package.PackageBase, text) + + pkgname = package.PackageBase.Name + cgit_path = f"/cgit/aur.git/log/?h={pkgname}&" + expected = f"""\ +

    {commit_hash[:12]} +{commit_hash[:7]} +x.{commit_hash[:12]}.x +{commit_hash}x +0123456789abcdef +{commit_hash} +\ +http://example.com/{commit_hash}\ +\ +

    \ +""" + assert comment.RenderedComment == expected + + +def test_flyspray_issue_link(user: User, pkgbase: PackageBase): + text = """\ +FS#1234567. +*FS#1234* +FS# +XFS#1 +`FS#1234` +https://archlinux.org/?test=FS#1234\ +""" + comment = create_comment(user, pkgbase, text) + + expected = """\ +

    FS#1234567. +FS#1234 +FS# +XFS#1 +FS#1234 +\ +https://archlinux.org/?test=FS#1234\ +\ +

    \ +""" + assert comment.RenderedComment == expected + + +def test_lower_headings(user: User, pkgbase: PackageBase): + text = """\ +# One +## Two +### Three +#### Four +##### Five +###### Six\ +""" + comment = create_comment(user, pkgbase, text) + + expected = """\ +
    One
    +
    Two
    +
    Three
    +
    Four
    +
    Five
    +
    Six
    \ +""" + assert comment.RenderedComment == expected diff --git a/test/test_util.py b/test/test_util.py index 99b77a78..2529ed1f 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -40,3 +40,20 @@ def test_round(): assert filters.do_round(1.3) == 1 assert filters.do_round(1.5) == 2 assert filters.do_round(2.0) == 2 + + +def test_git_search(): + """ Test that git_search matches the full commit if necessary. """ + commit_hash = "0123456789abcdef" + repo = {commit_hash} + prefixlen = util.git_search(repo, commit_hash) + assert prefixlen == 16 + + +def test_git_search_double_commit(): + """ Test that git_search matches a shorter prefix length. """ + commit_hash = "0123456789abcdef" + repo = {commit_hash[:13]} + # Locate the shortest prefix length that matches commit_hash. + prefixlen = util.git_search(repo, commit_hash) + assert prefixlen == 13 From bc1cf8b1f6089d0776dc753bc9255032aaf83a8f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 24 Nov 2021 20:30:16 -0800 Subject: [PATCH 647/844] fix(rendercomment): markdown.util.etree -> xml.etree.ElementTree This removes a deprecation warning. Signed-off-by: Kevin Morris --- aurweb/scripts/rendercomment.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index 33349432..2af5384e 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -2,6 +2,8 @@ import sys +from xml.etree.ElementTree import Element + import bleach import markdown import pygit2 @@ -40,7 +42,7 @@ class FlysprayLinksInlineProcessor(markdown.inlinepatterns.InlineProcessor): """ def handleMatch(self, m, data): - el = markdown.util.etree.Element('a') + el = Element('a') el.set('href', f'https://bugs.archlinux.org/task/{m.group(1)}') el.text = markdown.util.AtomicString(m.group(0)) return (el, m.start(0), m.end(0)) @@ -73,7 +75,7 @@ class GitCommitsInlineProcessor(markdown.inlinepatterns.InlineProcessor): # Unkwown OID; preserve the orginal text. return (None, None, None) - el = markdown.util.etree.Element('a') + el = Element('a') commit_uri = aurweb.config.get("options", "commit_uri") prefixlen = util.git_search(self._repo, oid) el.set('href', commit_uri % (self._head, oid[:prefixlen])) From 67a6b8360eccb02ac4a65940fde7fa6801ac2398 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 24 Nov 2021 20:33:42 -0800 Subject: [PATCH 648/844] fix(docker): remove update and build steps from poetry `install` includes dependencies present in poetry.lock and we must stick to them if we wish to pin dependencies. Signed-off-by: Kevin Morris --- docker/scripts/install-python-deps.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/scripts/install-python-deps.sh b/docker/scripts/install-python-deps.sh index 3ab87742..3d5f28f0 100755 --- a/docker/scripts/install-python-deps.sh +++ b/docker/scripts/install-python-deps.sh @@ -6,8 +6,6 @@ pip install --upgrade pip # Install the aurweb package and deps system-wide via poetry. poetry config virtualenvs.create false -poetry update -poetry build poetry install --no-interaction --no-ansi exec "$@" From 4426c639ceb59ea5d6414b70157f2c0e7cea4f6b Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 24 Nov 2021 20:34:45 -0800 Subject: [PATCH 649/844] fix(logging): remove `test` logger definition Like the `aurweb` logger definiton was previously, the `test` logger is being redundant with the root logger. Use root for all aurweb-local logging. Signed-off-by: Kevin Morris --- logging.conf | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/logging.conf b/logging.conf index 3b96e827..deb79cf5 100644 --- a/logging.conf +++ b/logging.conf @@ -1,5 +1,5 @@ [loggers] -keys=root,test,uvicorn,hypercorn,alembic +keys=root,uvicorn,hypercorn,alembic [handlers] keys=simpleHandler,detailedHandler @@ -10,12 +10,7 @@ keys=simpleFormatter,detailedFormatter [logger_root] level=INFO handlers=detailedHandler - -[logger_test] -level=DEBUG -handlers=detailedHandler -qualname=test -propagate=1 +propogate=1 [logger_uvicorn] level=DEBUG From 436d7420179043baf4814b2d10bc1cb460132d97 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Nov 2021 14:02:47 -0800 Subject: [PATCH 650/844] fix(fastapi): use CRED_TU_LIST_VOTES for "Trusted User" navigation item Closes #189 Signed-off-by: Kevin Morris --- aurweb/auth.py | 2 +- templates/partials/archdev-navbar.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index 98a43fd5..4d6dafc6 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -329,7 +329,7 @@ cred_filters = { CRED_PKGREQ_CLOSE: trusted_user_or_dev, CRED_PKGREQ_LIST: trusted_user_or_dev, CRED_TU_ADD_VOTE: trusted_user, - CRED_TU_LIST_VOTES: trusted_user, + CRED_TU_LIST_VOTES: trusted_user_or_dev, CRED_TU_VOTE: trusted_user, CRED_ACCOUNT_EDIT_DEV: developer, CRED_PKGBASE_MERGE: trusted_user_or_dev, diff --git a/templates/partials/archdev-navbar.html b/templates/partials/archdev-navbar.html index ead0d8e2..81695951 100644 --- a/templates/partials/archdev-navbar.html +++ b/templates/partials/archdev-navbar.html @@ -36,8 +36,8 @@ - {# Only CRED_TU_VOTE privileged users see Trusted User #} - {% if request.user.has_credential("CRED_TU_VOTE") %} + {# Only CRED_TU_LIST_VOTES privileged users see Trusted User #} + {% if request.user.has_credential("CRED_TU_LIST_VOTES") %}
  • {% trans %}Trusted User{% endtrans %}
  • From 44f2366675f0ca81af443dfa293d5251b1f2eb02 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Nov 2021 16:20:36 -0800 Subject: [PATCH 651/844] fix: remove TODO comments and noop tests from test_notify Signed-off-by: Kevin Morris --- test/test_notify.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/test_notify.py b/test/test_notify.py index 45a1df31..dc6e7e3e 100644 --- a/test/test_notify.py +++ b/test/test_notify.py @@ -262,7 +262,6 @@ The package {pkgbase.Name} [1] was disowned by {user2.Username} [2]. def test_comaintainer_addition(user: User, pkgbases: List[PackageBase]): - # TODO: Add this in fastapi code! pkgbase = pkgbases[0] notif = notify.ComaintainerAddNotification(user.ID, pkgbase.ID) notif.send() @@ -282,7 +281,6 @@ You were added to the co-maintainer list of {pkgbase.Name} [1]. def test_comaintainer_removal(user: User, pkgbases: List[PackageBase]): - # TODO: Add this in fastapi code! pkgbase = pkgbases[0] notif = notify.ComaintainerRemoveNotification(user.ID, pkgbase.ID) notif.send() @@ -417,14 +415,9 @@ Request #{pkgreq.ID} has been rejected by {user2.Username} [1]. assert email.body == expected -def test_close_request_auto_accept(): - pass - - def test_close_request_comaintainer_cc(user: User, user2: User, pkgreq: PackageRequest, pkgbases: List[PackageBase]): - # TODO: Check this in fastapi code! pkgbase = pkgbases[0] with db.begin(): db.create(models.PackageComaintainer, PackageBase=pkgbase, From 69eb17cb0d659d7de7a51715d2113e806663eb44 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Nov 2021 16:51:16 -0800 Subject: [PATCH 652/844] change(fastapi): remove the GET /logout route; replaced with POST Had to add some additional CSS in to style a form button the same as links are styled. Closes #188 Signed-off-by: Kevin Morris --- aurweb/routers/auth.py | 15 ++------------- templates/partials/archdev-navbar.html | 9 ++++++--- test/test_auth_routes.py | 3 ++- web/html/css/aurweb.css | 15 +++++++++++++++ 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index c5a99419..fdc421f5 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -77,14 +77,9 @@ async def login_post(request: Request, return response -@router.get("/logout") +@router.post("/logout") @auth_required() -async def logout(request: Request, next: str = "/"): - """ A GET and POST route for logging out. - - @param request FastAPI request - @param next Route to redirect to - """ +async def logout(request: Request, next: str = Form(default="/")): if request.user.is_authenticated(): request.user.logout(request) @@ -95,9 +90,3 @@ async def logout(request: Request, next: str = "/"): response.delete_cookie("AURSID") response.delete_cookie("AURTZ") return response - - -@router.post("/logout") -@auth_required() -async def logout_post(request: Request, next: str = "/"): - return await logout(request=request, next=next) diff --git a/templates/partials/archdev-navbar.html b/templates/partials/archdev-navbar.html index 81695951..2e01eeab 100644 --- a/templates/partials/archdev-navbar.html +++ b/templates/partials/archdev-navbar.html @@ -45,9 +45,12 @@ {# All logged in users see Logout #}
  • - - {% trans %}Logout{% endtrans %} - +
  • {% else %} {# All guest users see Register #} diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index a0bb8a7c..dffd1b94 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -154,8 +154,9 @@ def test_unauthenticated_logout_unauthorized(): with client as request: # Alright, let's verify that attempting to /logout when not # authenticated returns 401 Unauthorized. - response = request.get("/logout", allow_redirects=False) + response = request.post("/logout", allow_redirects=False) assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.headers.get("location").startswith("/login") def test_login_missing_username(): diff --git a/web/html/css/aurweb.css b/web/html/css/aurweb.css index 62179769..cd81160d 100644 --- a/web/html/css/aurweb.css +++ b/web/html/css/aurweb.css @@ -229,3 +229,18 @@ input#search-action-submit { .success { color: green; } + +/* Styling used to clone styles for a form.link button. */ +form.link, form.link > button { + display: inline-block; +} +form.link > button { + padding: 0 0.5em; + color: #07b; + background: none; + border: none; +} +form.link > button:hover { + cursor: pointer; + text-decoration: underline; +} From fd8d23a37937f1bab5dfffea393ea30f0656df03 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Nov 2021 19:04:55 -0800 Subject: [PATCH 653/844] fix(fastapi): fix new Logout nav item css Signed-off-by: Kevin Morris --- web/html/css/aurweb.css | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/web/html/css/aurweb.css b/web/html/css/aurweb.css index cd81160d..dafa8c91 100644 --- a/web/html/css/aurweb.css +++ b/web/html/css/aurweb.css @@ -231,16 +231,19 @@ input#search-action-submit { } /* Styling used to clone styles for a form.link button. */ -form.link, form.link > button { +form.link, form.link button { display: inline-block; + font-family: sans-serif; } -form.link > button { +form.link button { padding: 0 0.5em; color: #07b; background: none; border: none; + font-family: inherit; + font-size: inherit; } -form.link > button:hover { +form.link button:hover { cursor: pointer; text-decoration: underline; } From 9bfe2b07ba712f1667f93a4527b6a76b0b3cfcc1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Nov 2021 19:39:27 -0800 Subject: [PATCH 654/844] fix(fastapi): render Logged-in as page on authenticated /login This was missed during the initial porting of the /login route. Modifications: ------------- - A form is now used for the [Logout] link and some css was needed to deal with positioning. Closes #186 Signed-off-by: Kevin Morris --- aurweb/routers/auth.py | 1 - templates/login.html | 162 +++++++++++++++++++++------------------ test/test_auth_routes.py | 10 ++- web/html/css/aurweb.css | 10 ++- 4 files changed, 104 insertions(+), 79 deletions(-) diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index fdc421f5..1e0b026a 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -24,7 +24,6 @@ async def login_template(request: Request, next: str, errors: list = None): @router.get("/login", response_class=HTMLResponse) -@auth_required(False, login=False) async def login_get(request: Request, next: str = "/"): return await login_template(request, next) diff --git a/templates/login.html b/templates/login.html index 2c028936..c62de43e 100644 --- a/templates/login.html +++ b/templates/login.html @@ -5,81 +5,95 @@

    AUR {% trans %}Login{% endtrans %}

    - {% if request.url.scheme == "http" and config.getboolean("options", "disable_http_login") %} - {% set https_login = url_base.replace("http://", "https://") + "/login" %} -

    - {{ "HTTP login is disabled. Please %sswitch to HTTPs%s if you want to login." - | tr - | format( - '' | format(https_login), - "") - | safe - }} -

    - {% elif request.user.is_authenticated() %} -

    - {{ "Logged-in as: %s" - | tr - | format("%s" | format(request.user.Username)) - | safe - }} - [{% trans %}Logout{% endtrans %}] -

    - {% else %} -
    -
    - {% trans %}Enter login credentials{% endtrans %} - - {% if errors %} -
      - {% for error in errors %} -
    • {{ error }}
    • - {% endfor %} -
    - {% endif %} - -

    - - - -

    - -

    - - -

    - -

    - - -

    - -

    - - - [{% trans %}Forgot Password{% endtrans %}] - - - -

    - -
    + {% if request.user.is_authenticated() %} + +

    + {{ + "Logged-in as: %s" | tr + | format("%s" | format(request.user.Username)) + | safe + }} + + +

    + {% else %} + {% if request.url.scheme == "http" and config.getboolean("options", "disable_http_login") %} + {% set https_login = url_base.replace("http://", "https://") + "/login" %} +

    + {{ "HTTP login is disabled. Please %sswitch to HTTPs%s if you want to login." + | tr + | format( + '' | format(https_login), + "") + | safe + }} +

    + {% elif request.user.is_authenticated() %} +

    + {{ "Logged-in as: %s" + | tr + | format("%s" | format(request.user.Username)) + | safe + }} + [{% trans %}Logout{% endtrans %}] +

    + {% else %} +
    +
    + {% trans %}Enter login credentials{% endtrans %} + + {% if errors %} +
      + {% for error in errors %} +
    • {{ error }}
    • + {% endfor %} +
    + {% endif %} + +

    + + + +

    + +

    + + +

    + +

    + + +

    + +

    + + + [{% trans %}Forgot Password{% endtrans %}] + + + +

    + +
    +
    + {% endif %} {% endif %}
    diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index dffd1b94..0157fcc8 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -131,7 +131,7 @@ def test_secure_login(mock): assert user.session == record -def test_authenticated_login_forbidden(): +def test_authenticated_login(): post_data = { "user": "test", "passwd": "testPassword", @@ -139,15 +139,19 @@ def test_authenticated_login_forbidden(): } with client as request: - # Login. + # Try to login. response = request.post("/login", data=post_data, allow_redirects=False) assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.headers.get("location") == "/" + # Now, let's verify that we get the logged in rendering + # when requesting GET /login as an authenticated user. # Now, let's verify that we receive 403 Forbidden when we # try to get /login as an authenticated user. response = request.get("/login", allow_redirects=False) - assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.status_code == int(HTTPStatus.OK) + assert "Logged-in as: test" in response.text def test_unauthenticated_logout_unauthorized(): diff --git a/web/html/css/aurweb.css b/web/html/css/aurweb.css index dafa8c91..739ac7b7 100644 --- a/web/html/css/aurweb.css +++ b/web/html/css/aurweb.css @@ -232,7 +232,7 @@ input#search-action-submit { /* Styling used to clone styles for a form.link button. */ form.link, form.link button { - display: inline-block; + display: inline; font-family: sans-serif; } form.link button { @@ -247,3 +247,11 @@ form.link button:hover { cursor: pointer; text-decoration: underline; } + +/* Customize form.link when used inside of a page. */ +div.box form.link p { + margin: .33em 0 1em; +} +div.box form.link button { + padding: 0; +} From 001e86317fd19b68491e61e8fad811a89a17ad3e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Nov 2021 19:44:18 -0800 Subject: [PATCH 655/844] fix(rpc): fix ordering of related records They were being ordered by IDs; they should be ordered by Names. Signed-off-by: Kevin Morris --- aurweb/rpc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index c70ddf1a..7bdae638 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -193,7 +193,7 @@ class RPC: models.DependencyType.Name.label("Type"), models.PackageDependency.DepName.label("Name"), models.PackageDependency.DepCondition.label("Cond") - ).distinct().order_by("ID"), + ).distinct().order_by("Name"), # PackageRelation db.query( @@ -205,7 +205,7 @@ class RPC: models.RelationType.Name.label("Type"), models.PackageRelation.RelName.label("Name"), models.PackageRelation.RelCondition.label("Cond") - ).distinct().order_by("ID"), + ).distinct().order_by("Name"), # Groups db.query(models.PackageGroup).join( @@ -217,7 +217,7 @@ class RPC: literal("Groups").label("Type"), models.Group.Name.label("Name"), literal(str()).label("Cond") - ).distinct().order_by("ID"), + ).distinct().order_by("Name"), # Licenses db.query(models.PackageLicense).join( @@ -230,7 +230,7 @@ class RPC: literal("License").label("Type"), models.License.Name.label("Name"), literal(str()).label("Cond") - ).distinct().order_by("ID"), + ).distinct().order_by("Name"), # Keywords db.query(models.PackageKeyword).join( @@ -242,7 +242,7 @@ class RPC: literal("Keywords").label("Type"), models.PackageKeyword.Keyword.label("Name"), literal(str()).label("Cond") - ).distinct().order_by("ID") + ).distinct().order_by("Name") ] # Union all subqueries together. From a6ac5f0dbf5b2e8d0d0c55a58e7a41ee7d5ad5dc Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 29 Nov 2021 19:44:18 -0800 Subject: [PATCH 656/844] fix(rpc): fix ordering of related records They were being ordered by IDs; they should be ordered by Names. Signed-off-by: Kevin Morris --- aurweb/rpc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index c70ddf1a..7bdae638 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -193,7 +193,7 @@ class RPC: models.DependencyType.Name.label("Type"), models.PackageDependency.DepName.label("Name"), models.PackageDependency.DepCondition.label("Cond") - ).distinct().order_by("ID"), + ).distinct().order_by("Name"), # PackageRelation db.query( @@ -205,7 +205,7 @@ class RPC: models.RelationType.Name.label("Type"), models.PackageRelation.RelName.label("Name"), models.PackageRelation.RelCondition.label("Cond") - ).distinct().order_by("ID"), + ).distinct().order_by("Name"), # Groups db.query(models.PackageGroup).join( @@ -217,7 +217,7 @@ class RPC: literal("Groups").label("Type"), models.Group.Name.label("Name"), literal(str()).label("Cond") - ).distinct().order_by("ID"), + ).distinct().order_by("Name"), # Licenses db.query(models.PackageLicense).join( @@ -230,7 +230,7 @@ class RPC: literal("License").label("Type"), models.License.Name.label("Name"), literal(str()).label("Cond") - ).distinct().order_by("ID"), + ).distinct().order_by("Name"), # Keywords db.query(models.PackageKeyword).join( @@ -242,7 +242,7 @@ class RPC: literal("Keywords").label("Type"), models.PackageKeyword.Keyword.label("Name"), literal(str()).label("Cond") - ).distinct().order_by("ID") + ).distinct().order_by("Name") ] # Union all subqueries together. From ecbab8546b68574c939b269dac1ac2d77e53766b Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Thu, 21 Oct 2021 17:48:29 -0400 Subject: [PATCH 657/844] fix(FastAPI): access AccountType ID directly Signed-off-by: Steven Guikal --- aurweb/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aurweb/auth.py b/aurweb/auth.py index 4d6dafc6..1527f0a3 100644 --- a/aurweb/auth.py +++ b/aurweb/auth.py @@ -244,7 +244,7 @@ def account_type_required(one_of: set): def decorator(func): @functools.wraps(func) async def wrapper(request: fastapi.Request, *args, **kwargs): - if request.user.AccountType.ID not in one_of: + if request.user.AccountTypeID not in one_of: return RedirectResponse("/", status_code=int(HTTPStatus.SEE_OTHER)) return await func(request, *args, **kwargs) From 125b244f4478146ae293f76ca4a53a77160feda3 Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Thu, 21 Oct 2021 17:49:10 -0400 Subject: [PATCH 658/844] fix(FastAPI): use account type vars instead of strings Signed-off-by: Steven Guikal --- aurweb/routers/trusted_user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aurweb/routers/trusted_user.py b/aurweb/routers/trusted_user.py index 7c0a0404..f0cea61e 100644 --- a/aurweb/routers/trusted_user.py +++ b/aurweb/routers/trusted_user.py @@ -228,7 +228,7 @@ async def trusted_user_proposal_post(request: Request, @router.get("/addvote") @auth_required(True, redirect="/addvote") -@account_type_required({"Trusted User", "Trusted User & Developer"}) +@account_type_required({TRUSTED_USER, TRUSTED_USER_AND_DEV}) async def trusted_user_addvote(request: Request, user: str = str(), type: str = "add_tu", From a10f8663fd9bc4ddb990f1dfba31cb3331f4c9e9 Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Tue, 30 Nov 2021 15:44:18 -0500 Subject: [PATCH 659/844] fix(FastAPI): reorganize credential checkin into dedicated file Signed-off-by: Steven Guikal --- aurweb/{auth.py => auth/__init__.py} | 98 ------------------------ aurweb/auth/creds.py | 76 ++++++++++++++++++ aurweb/models/user.py | 9 ++- aurweb/routers/accounts.py | 6 +- aurweb/routers/packages.py | 56 +++++++------- aurweb/templates.py | 3 +- templates/partials/account/comment.html | 2 +- templates/partials/account_form.html | 2 +- templates/partials/archdev-navbar.html | 4 +- templates/partials/comment_actions.html | 8 +- templates/partials/packages/actions.html | 8 +- templates/partials/packages/comment.html | 2 +- templates/partials/packages/details.html | 4 +- test/test_auth.py | 12 +-- test/test_user.py | 25 +++--- 15 files changed, 143 insertions(+), 172 deletions(-) rename aurweb/{auth.py => auth/__init__.py} (75%) create mode 100644 aurweb/auth/creds.py diff --git a/aurweb/auth.py b/aurweb/auth/__init__.py similarity index 75% rename from aurweb/auth.py rename to aurweb/auth/__init__.py index 1527f0a3..82192cc2 100644 --- a/aurweb/auth.py +++ b/aurweb/auth/__init__.py @@ -250,101 +250,3 @@ def account_type_required(one_of: set): return await func(request, *args, **kwargs) return wrapper return decorator - - -CRED_ACCOUNT_CHANGE_TYPE = 1 -CRED_ACCOUNT_EDIT = 2 -CRED_ACCOUNT_EDIT_DEV = 3 -CRED_ACCOUNT_LAST_LOGIN = 4 -CRED_ACCOUNT_SEARCH = 5 -CRED_ACCOUNT_LIST_COMMENTS = 28 -CRED_COMMENT_DELETE = 6 -CRED_COMMENT_UNDELETE = 27 -CRED_COMMENT_VIEW_DELETED = 22 -CRED_COMMENT_EDIT = 25 -CRED_COMMENT_PIN = 26 -CRED_PKGBASE_ADOPT = 7 -CRED_PKGBASE_SET_KEYWORDS = 8 -CRED_PKGBASE_DELETE = 9 -CRED_PKGBASE_DISOWN = 10 -CRED_PKGBASE_EDIT_COMAINTAINERS = 24 -CRED_PKGBASE_FLAG = 11 -CRED_PKGBASE_LIST_VOTERS = 12 -CRED_PKGBASE_NOTIFY = 13 -CRED_PKGBASE_UNFLAG = 15 -CRED_PKGBASE_VOTE = 16 -CRED_PKGREQ_FILE = 23 -CRED_PKGREQ_CLOSE = 17 -CRED_PKGREQ_LIST = 18 -CRED_TU_ADD_VOTE = 19 -CRED_TU_LIST_VOTES = 20 -CRED_TU_VOTE = 21 -CRED_PKGBASE_MERGE = 29 - - -def has_any(user, *account_types): - return str(user.AccountType) in set(account_types) - - -def user_developer_or_trusted_user(user): - return True - - -def trusted_user(user): - return has_any(user, "Trusted User", "Trusted User & Developer") - - -def developer(user): - return has_any(user, "Developer", "Trusted User & Developer") - - -def trusted_user_or_dev(user): - return has_any(user, "Trusted User", "Developer", - "Trusted User & Developer") - - -# A mapping of functions that users must pass to have credentials. -cred_filters = { - CRED_PKGBASE_FLAG: user_developer_or_trusted_user, - CRED_PKGBASE_NOTIFY: user_developer_or_trusted_user, - CRED_PKGBASE_VOTE: user_developer_or_trusted_user, - CRED_PKGREQ_FILE: user_developer_or_trusted_user, - CRED_ACCOUNT_CHANGE_TYPE: trusted_user_or_dev, - CRED_ACCOUNT_EDIT: trusted_user_or_dev, - CRED_ACCOUNT_LAST_LOGIN: trusted_user_or_dev, - CRED_ACCOUNT_LIST_COMMENTS: trusted_user_or_dev, - CRED_ACCOUNT_SEARCH: trusted_user_or_dev, - CRED_COMMENT_DELETE: trusted_user_or_dev, - CRED_COMMENT_UNDELETE: trusted_user_or_dev, - CRED_COMMENT_VIEW_DELETED: trusted_user_or_dev, - CRED_COMMENT_EDIT: trusted_user_or_dev, - CRED_COMMENT_PIN: trusted_user_or_dev, - CRED_PKGBASE_ADOPT: trusted_user_or_dev, - CRED_PKGBASE_SET_KEYWORDS: trusted_user_or_dev, - CRED_PKGBASE_DELETE: trusted_user_or_dev, - CRED_PKGBASE_EDIT_COMAINTAINERS: trusted_user_or_dev, - CRED_PKGBASE_DISOWN: trusted_user_or_dev, - CRED_PKGBASE_LIST_VOTERS: trusted_user_or_dev, - CRED_PKGBASE_UNFLAG: trusted_user_or_dev, - CRED_PKGREQ_CLOSE: trusted_user_or_dev, - CRED_PKGREQ_LIST: trusted_user_or_dev, - CRED_TU_ADD_VOTE: trusted_user, - CRED_TU_LIST_VOTES: trusted_user_or_dev, - CRED_TU_VOTE: trusted_user, - CRED_ACCOUNT_EDIT_DEV: developer, - CRED_PKGBASE_MERGE: trusted_user_or_dev, -} - - -def has_credential(user: User, - credential: int, - approved_users: list = tuple()): - - if user in approved_users: - return True - - if credential in cred_filters: - cred_filter = cred_filters.get(credential) - return cred_filter(user) - - return False diff --git a/aurweb/auth/creds.py b/aurweb/auth/creds.py new file mode 100644 index 00000000..100aad8c --- /dev/null +++ b/aurweb/auth/creds.py @@ -0,0 +1,76 @@ +from aurweb.models.account_type import DEVELOPER_ID, TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID, USER_ID +from aurweb.models.user import User + +ACCOUNT_CHANGE_TYPE = 1 +ACCOUNT_EDIT = 2 +ACCOUNT_EDIT_DEV = 3 +ACCOUNT_LAST_LOGIN = 4 +ACCOUNT_SEARCH = 5 +ACCOUNT_LIST_COMMENTS = 28 +COMMENT_DELETE = 6 +COMMENT_UNDELETE = 27 +COMMENT_VIEW_DELETED = 22 +COMMENT_EDIT = 25 +COMMENT_PIN = 26 +PKGBASE_ADOPT = 7 +PKGBASE_SET_KEYWORDS = 8 +PKGBASE_DELETE = 9 +PKGBASE_DISOWN = 10 +PKGBASE_EDIT_COMAINTAINERS = 24 +PKGBASE_FLAG = 11 +PKGBASE_LIST_VOTERS = 12 +PKGBASE_NOTIFY = 13 +PKGBASE_UNFLAG = 15 +PKGBASE_VOTE = 16 +PKGREQ_FILE = 23 +PKGREQ_CLOSE = 17 +PKGREQ_LIST = 18 +TU_ADD_VOTE = 19 +TU_LIST_VOTES = 20 +TU_VOTE = 21 +PKGBASE_MERGE = 29 + +user_developer_or_trusted_user = set([USER_ID, TRUSTED_USER_ID, DEVELOPER_ID, TRUSTED_USER_AND_DEV_ID]) +trusted_user_or_dev = set([TRUSTED_USER_ID, DEVELOPER_ID, TRUSTED_USER_AND_DEV_ID]) +developer = set([DEVELOPER_ID, TRUSTED_USER_AND_DEV_ID]) +trusted_user = set([TRUSTED_USER_ID, TRUSTED_USER_AND_DEV_ID]) + +cred_filters = { + PKGBASE_FLAG: user_developer_or_trusted_user, + PKGBASE_NOTIFY: user_developer_or_trusted_user, + PKGBASE_VOTE: user_developer_or_trusted_user, + PKGREQ_FILE: user_developer_or_trusted_user, + ACCOUNT_CHANGE_TYPE: trusted_user_or_dev, + ACCOUNT_EDIT: trusted_user_or_dev, + ACCOUNT_LAST_LOGIN: trusted_user_or_dev, + ACCOUNT_LIST_COMMENTS: trusted_user_or_dev, + ACCOUNT_SEARCH: trusted_user_or_dev, + COMMENT_DELETE: trusted_user_or_dev, + COMMENT_UNDELETE: trusted_user_or_dev, + COMMENT_VIEW_DELETED: trusted_user_or_dev, + COMMENT_EDIT: trusted_user_or_dev, + COMMENT_PIN: trusted_user_or_dev, + PKGBASE_ADOPT: trusted_user_or_dev, + PKGBASE_SET_KEYWORDS: trusted_user_or_dev, + PKGBASE_DELETE: trusted_user_or_dev, + PKGBASE_EDIT_COMAINTAINERS: trusted_user_or_dev, + PKGBASE_DISOWN: trusted_user_or_dev, + PKGBASE_LIST_VOTERS: trusted_user_or_dev, + PKGBASE_UNFLAG: trusted_user_or_dev, + PKGREQ_CLOSE: trusted_user_or_dev, + PKGREQ_LIST: trusted_user_or_dev, + TU_ADD_VOTE: trusted_user, + TU_LIST_VOTES: trusted_user_or_dev, + TU_VOTE: trusted_user, + ACCOUNT_EDIT_DEV: developer, + PKGBASE_MERGE: trusted_user_or_dev, +} + + +def has_credential(user: User, + credential: int, + approved_users: list = tuple()): + + if user in approved_users: + return True + return user.AccountTypeID in cred_filters[credential] diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 03634a36..f0724202 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -1,6 +1,7 @@ import hashlib from datetime import datetime +from typing import List, Set import bcrypt @@ -136,10 +137,10 @@ class User(Base): request.cookies["AURSID"] = self.session.SessionID return self.session.SessionID - def has_credential(self, credential: str, approved: list = tuple()): - import aurweb.auth - cred = getattr(aurweb.auth, credential) - return aurweb.auth.has_credential(self, cred, approved) + def has_credential(self, credential: Set[int], + approved: List["User"] = list()): + from aurweb.auth.creds import has_credential + return has_credential(self, credential, approved) def logout(self, request): del request.cookies["AURSID"] diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 545811f0..360857e8 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -10,7 +10,7 @@ from sqlalchemy import and_, or_ import aurweb.config from aurweb import cookies, db, l10n, logging, models, util -from aurweb.auth import account_type_required, auth_required +from aurweb.auth import account_type_required, auth_required, creds from aurweb.captcha import get_captcha_salts from aurweb.exceptions import ValidationError from aurweb.l10n import get_translator_for_request @@ -176,7 +176,7 @@ def make_account_form_context(context: dict, user_account_type_id = context.get("account_types")[0][0] - if request.user.has_credential("CRED_ACCOUNT_EDIT_DEV"): + if request.user.has_credential(creds.ACCOUNT_EDIT_DEV): context["account_types"].append((at.DEVELOPER_ID, at.DEVELOPER)) context["account_types"].append((at.TRUSTED_USER_AND_DEV_ID, at.TRUSTED_USER_AND_DEV)) @@ -332,7 +332,7 @@ async def account_register_post(request: Request, def cannot_edit(request, user): """ Return a 401 HTMLResponse if the request user doesn't have authorization, otherwise None. """ - has_dev_cred = request.user.has_credential("CRED_ACCOUNT_EDIT_DEV", + has_dev_cred = request.user.has_credential(creds.ACCOUNT_EDIT_DEV, approved=[user]) if not has_dev_cred: return HTMLResponse(status_code=HTTPStatus.UNAUTHORIZED) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index b5f8478e..2bf04949 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -10,7 +10,7 @@ import aurweb.filters import aurweb.packages.util from aurweb import db, defaults, l10n, logging, models, util -from aurweb.auth import auth_required +from aurweb.auth import auth_required, creds from aurweb.exceptions import ValidationError from aurweb.models.package_request import ACCEPTED_ID, PENDING_ID, REJECTED_ID from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID @@ -413,7 +413,7 @@ async def pkgbase_comment_delete(request: Request, name: str, id: int, pkgbase = get_pkg_or_base(name, models.PackageBase) comment = get_pkgbase_comment(pkgbase, id) - authorized = request.user.has_credential("CRED_COMMENT_DELETE", + authorized = request.user.has_credential(creds.COMMENT_DELETE, [comment.User]) if not authorized: _ = l10n.get_translator_for_request(request) @@ -439,7 +439,7 @@ async def pkgbase_comment_undelete(request: Request, name: str, id: int, pkgbase = get_pkg_or_base(name, models.PackageBase) comment = get_pkgbase_comment(pkgbase, id) - has_cred = request.user.has_credential("CRED_COMMENT_UNDELETE", + has_cred = request.user.has_credential(creds.COMMENT_UNDELETE, approved=[comment.User]) if not has_cred: _ = l10n.get_translator_for_request(request) @@ -464,7 +464,7 @@ async def pkgbase_comment_pin(request: Request, name: str, id: int, pkgbase = get_pkg_or_base(name, models.PackageBase) comment = get_pkgbase_comment(pkgbase, id) - has_cred = request.user.has_credential("CRED_COMMENT_PIN", + has_cred = request.user.has_credential(creds.COMMENT_PIN, approved=[pkgbase.Maintainer]) if not has_cred: _ = l10n.get_translator_for_request(request) @@ -489,7 +489,7 @@ async def pkgbase_comment_unpin(request: Request, name: str, id: int, pkgbase = get_pkg_or_base(name, models.PackageBase) comment = get_pkgbase_comment(pkgbase, id) - has_cred = request.user.has_credential("CRED_COMMENT_PIN", + has_cred = request.user.has_credential(creds.COMMENT_PIN, approved=[pkgbase.Maintainer]) if not has_cred: _ = l10n.get_translator_for_request(request) @@ -514,7 +514,7 @@ async def package_base_comaintainers(request: Request, name: str) -> Response: # Unauthorized users (Non-TU/Dev and not the pkgbase maintainer) # get redirected to the package base's page. - has_creds = request.user.has_credential("CRED_PKGBASE_EDIT_COMAINTAINERS", + has_creds = request.user.has_credential(creds.PKGBASE_EDIT_COMAINTAINERS, approved=[pkgbase.Maintainer]) if not has_creds: return RedirectResponse(f"/pkgbase/{name}", @@ -541,7 +541,7 @@ async def package_base_comaintainers_post( # Unauthorized users (Non-TU/Dev and not the pkgbase maintainer) # get redirected to the package base's page. - has_creds = request.user.has_credential("CRED_PKGBASE_EDIT_COMAINTAINERS", + has_creds = request.user.has_credential(creds.PKGBASE_EDIT_COMAINTAINERS, approved=[pkgbase.Maintainer]) if not has_creds: return RedirectResponse(f"/pkgbase/{name}", @@ -779,7 +779,7 @@ async def pkgbase_keywords(request: Request, name: str, async def pkgbase_flag_get(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) - has_cred = request.user.has_credential("CRED_PKGBASE_FLAG") + has_cred = request.user.has_credential(creds.PKGBASE_FLAG) if not has_cred or pkgbase.Flagger is not None: return RedirectResponse(f"/pkgbase/{name}", status_code=HTTPStatus.SEE_OTHER) @@ -803,7 +803,7 @@ async def pkgbase_flag_post(request: Request, name: str, return render_template(request, "packages/flag.html", context, status_code=HTTPStatus.BAD_REQUEST) - has_cred = request.user.has_credential("CRED_PKGBASE_FLAG") + has_cred = request.user.has_credential(creds.PKGBASE_FLAG) if has_cred and not pkgbase.Flagger: now = int(datetime.utcnow().timestamp()) with db.begin(): @@ -830,7 +830,7 @@ async def pkgbase_flag_comment(request: Request, name: str): def pkgbase_unflag_instance(request: Request, pkgbase: models.PackageBase): has_cred = request.user.has_credential( - "CRED_PKGBASE_UNFLAG", approved=[pkgbase.Flagger, pkgbase.Maintainer]) + creds.PKGBASE_UNFLAG, approved=[pkgbase.Flagger, pkgbase.Maintainer]) if has_cred: with db.begin(): pkgbase.OutOfDateTS = None @@ -851,7 +851,7 @@ def pkgbase_notify_instance(request: Request, pkgbase: models.PackageBase): notif = db.query(pkgbase.notifications.filter( models.PackageNotification.UserID == request.user.ID ).exists()).scalar() - has_cred = request.user.has_credential("CRED_PKGBASE_NOTIFY") + has_cred = request.user.has_credential(creds.PKGBASE_NOTIFY) if has_cred and not notif: with db.begin(): db.create(models.PackageNotification, @@ -872,7 +872,7 @@ def pkgbase_unnotify_instance(request: Request, pkgbase: models.PackageBase): notif = pkgbase.notifications.filter( models.PackageNotification.UserID == request.user.ID ).first() - has_cred = request.user.has_credential("CRED_PKGBASE_NOTIFY") + has_cred = request.user.has_credential(creds.PKGBASE_NOTIFY) if has_cred and notif: with db.begin(): db.delete(notif) @@ -895,7 +895,7 @@ async def pkgbase_vote(request: Request, name: str): vote = pkgbase.package_votes.filter( models.PackageVote.UsersID == request.user.ID ).first() - has_cred = request.user.has_credential("CRED_PKGBASE_VOTE") + has_cred = request.user.has_credential(creds.PKGBASE_VOTE) if has_cred and not vote: now = int(datetime.utcnow().timestamp()) with db.begin(): @@ -919,7 +919,7 @@ async def pkgbase_unvote(request: Request, name: str): vote = pkgbase.package_votes.filter( models.PackageVote.UsersID == request.user.ID ).first() - has_cred = request.user.has_credential("CRED_PKGBASE_VOTE") + has_cred = request.user.has_credential(creds.PKGBASE_VOTE) if has_cred and vote: with db.begin(): db.delete(vote) @@ -958,7 +958,7 @@ def pkgbase_disown_instance(request: Request, pkgbase: models.PackageBase): async def pkgbase_disown_get(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) - has_cred = request.user.has_credential("CRED_PKGBASE_DISOWN", + has_cred = request.user.has_credential(creds.PKGBASE_DISOWN, approved=[pkgbase.Maintainer]) if not has_cred: return RedirectResponse(f"/pkgbase/{name}", @@ -975,7 +975,7 @@ async def pkgbase_disown_post(request: Request, name: str, confirm: bool = Form(default=False)): pkgbase = get_pkg_or_base(name, models.PackageBase) - has_cred = request.user.has_credential("CRED_PKGBASE_DISOWN", + has_cred = request.user.has_credential(creds.PKGBASE_DISOWN, approved=[pkgbase.Maintainer]) if not has_cred: return RedirectResponse(f"/pkgbase/{name}", @@ -1007,7 +1007,7 @@ def pkgbase_adopt_instance(request: Request, pkgbase: models.PackageBase): async def pkgbase_adopt_post(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) - has_cred = request.user.has_credential("CRED_PKGBASE_ADOPT") + has_cred = request.user.has_credential(creds.PKGBASE_ADOPT) if has_cred or not pkgbase.Maintainer: # If the user has credentials, they'll adopt the package regardless # of maintainership. Otherwise, we'll promote the user to maintainer @@ -1021,7 +1021,7 @@ async def pkgbase_adopt_post(request: Request, name: str): @router.get("/pkgbase/{name}/delete") @auth_required(True, redirect="/pkgbase/{name}/delete") async def pkgbase_delete_get(request: Request, name: str): - if not request.user.has_credential("CRED_PKGBASE_DELETE"): + if not request.user.has_credential(creds.PKGBASE_DELETE): return RedirectResponse(f"/pkgbase/{name}", status_code=HTTPStatus.SEE_OTHER) @@ -1036,7 +1036,7 @@ async def pkgbase_delete_post(request: Request, name: str, confirm: bool = Form(default=False)): pkgbase = get_pkg_or_base(name, models.PackageBase) - if not request.user.has_credential("CRED_PKGBASE_DELETE"): + if not request.user.has_credential(creds.PKGBASE_DELETE): return RedirectResponse(f"/pkgbase/{name}", status_code=HTTPStatus.SEE_OTHER) @@ -1070,7 +1070,7 @@ async def packages_unflag(request: Request, package_ids: List[int] = [], models.Package.ID.in_(package_ids)).all() for pkg in packages: has_cred = request.user.has_credential( - "CRED_PKGBASE_UNFLAG", approved=[pkg.PackageBase.Flagger]) + creds.PKGBASE_UNFLAG, approved=[pkg.PackageBase.Flagger]) if not has_cred: return (False, ["You did not select any packages to unflag."]) @@ -1106,7 +1106,7 @@ async def packages_notify(request: Request, package_ids: List[int] = [], notif = db.query(pkgbase.notifications.filter( models.PackageNotification.UserID == request.user.ID ).exists()).scalar() - has_cred = request.user.has_credential("CRED_PKGBASE_NOTIFY") + has_cred = request.user.has_credential(creds.PKGBASE_NOTIFY) # If the request user either does not have credentials # or the notification already exists: @@ -1178,7 +1178,7 @@ async def packages_adopt(request: Request, package_ids: List[int] = [], # Check that the user has credentials for every package they selected. for pkgbase in bases: - has_cred = request.user.has_credential("CRED_PKGBASE_ADOPT") + has_cred = request.user.has_credential(creds.PKGBASE_ADOPT) if not (has_cred or not pkgbase.Maintainer): # TODO: This error needs to be translated. return (False, ["You are not allowed to adopt one of the " @@ -1211,7 +1211,7 @@ async def packages_disown(request: Request, package_ids: List[int] = [], # Check that the user has credentials for every package they selected. for pkgbase in bases: - has_cred = request.user.has_credential("CRED_PKGBASE_DISOWN", + has_cred = request.user.has_credential(creds.PKGBASE_DISOWN, approved=[pkgbase.Maintainer]) if not has_cred: # TODO: This error needs to be translated. @@ -1235,7 +1235,7 @@ async def packages_delete(request: Request, package_ids: List[int] = [], return (False, ["The selected packages have not been deleted, " "check the confirmation checkbox."]) - if not request.user.has_credential("CRED_PKGBASE_DELETE"): + if not request.user.has_credential(creds.PKGBASE_DELETE): return (False, ["You do not have permission to delete packages."]) # A "memo" used to store names of packages that we delete. @@ -1329,10 +1329,10 @@ async def pkgbase_merge_get(request: Request, name: str, status_code = HTTPStatus.OK # TODO: Lookup errors from credential instead of hardcoding them. - # Idea: Something like credential_errors("CRED_PKGBASE_MERGE"). - # Perhaps additionally: bad_credential_status_code("CRED_PKGBASE_MERGE"). + # Idea: Something like credential_errors(creds.PKGBASE_MERGE). + # Perhaps additionally: bad_credential_status_code(creds.PKGBASE_MERGE). # Don't take these examples verbatim. We should find good naming. - if not request.user.has_credential("CRED_PKGBASE_MERGE"): + if not request.user.has_credential(creds.PKGBASE_MERGE): context["errors"] = [ "Only Trusted Users and Developers can merge packages."] status_code = HTTPStatus.UNAUTHORIZED @@ -1434,7 +1434,7 @@ async def pkgbase_merge_post(request: Request, name: str, context["pkgbase"] = pkgbase # TODO: Lookup errors from credential instead of hardcoding them. - if not request.user.has_credential("CRED_PKGBASE_MERGE"): + if not request.user.has_credential(creds.PKGBASE_MERGE): context["errors"] = [ "Only Trusted Users and Developers can merge packages."] return render_template(request, "pkgbase/merge.html", context, diff --git a/aurweb/templates.py b/aurweb/templates.py index a7102ae1..635b22b4 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -16,7 +16,7 @@ from fastapi.responses import HTMLResponse import aurweb.config -from aurweb import captcha, cookies, l10n, time, util +from aurweb import auth, captcha, cookies, l10n, time, util # Prepare jinja2 objects. _loader = jinja2.FileSystemLoader(os.path.join( @@ -107,6 +107,7 @@ def make_context(request: Request, title: str, next: str = None): "now": datetime.now(tz=zoneinfo.ZoneInfo(timezone)), "utcnow": int(datetime.utcnow().timestamp()), "config": aurweb.config, + "creds": auth.creds, "next": next if next else request.url.path } diff --git a/templates/partials/account/comment.html b/templates/partials/account/comment.html index bc167cf7..8c310738 100644 --- a/templates/partials/account/comment.html +++ b/templates/partials/account/comment.html @@ -3,7 +3,7 @@ {% set header_cls = "%s %s" | format(header_cls, "comment-deleted") %} {% endif %} -{% if not comment.Deleter or request.user.has_credential("CRED_COMMENT_VIEW_DELETED", approved=[comment.Deleter]) %} +{% if not comment.Deleter or request.user.has_credential(creds.COMMENT_VIEW_DELETED, approved=[comment.Deleter]) %} {% set commented_at = comment.CommentTS | dt | as_timezone(timezone) %}

    diff --git a/templates/partials/account_form.html b/templates/partials/account_form.html index 2e47a932..f3c293d8 100644 --- a/templates/partials/account_form.html +++ b/templates/partials/account_form.html @@ -53,7 +53,7 @@

    {% endif %} - {% if request.user.has_credential("CRED_ACCOUNT_CHANGE_TYPE") %} + {% if request.user.has_credential(creds.ACCOUNT_CHANGE_TYPE) %}

  • {% trans %}Accounts{% endtrans %} @@ -37,7 +37,7 @@
  • {# Only CRED_TU_LIST_VOTES privileged users see Trusted User #} - {% if request.user.has_credential("CRED_TU_LIST_VOTES") %} + {% if request.user.has_credential(creds.TU_LIST_VOTES) %}
  • {% trans %}Trusted User{% endtrans %}
  • diff --git a/templates/partials/comment_actions.html b/templates/partials/comment_actions.html index b8ccf945..78c4cc22 100644 --- a/templates/partials/comment_actions.html +++ b/templates/partials/comment_actions.html @@ -1,7 +1,7 @@ {% set pkgbasename = comment.PackageBase.Name %} {% if not comment.Deleter %} - {% if request.user.has_credential('CRED_COMMENT_DELETE', approved=[comment.User]) %} + {% if request.user.has_credential(creds.COMMENT_DELETE, approved=[comment.User]) %}
    {% endif %} - {% if request.user.has_credential('CRED_COMMENT_EDIT', approved=[comment.User]) %} + {% if request.user.has_credential(creds.COMMENT_EDIT, approved=[comment.User]) %} {% endif %} {% endif %} -{% elif request.user.has_credential("CRED_COMMENT_UNDELETE", approved=[comment.User]) %} +{% elif request.user.has_credential(creds.COMMENT_UNDELETE, approved=[comment.User]) %} {% endif %} - {% if request.user.has_credential('CRED_PKGBASE_EDIT_COMAINTAINERS', approved=[pkgbase.Maintainer]) %} + {% if request.user.has_credential(creds.PKGBASE_EDIT_COMAINTAINERS, approved=[pkgbase.Maintainer]) %}
  • {{ "Manage Co-Maintainers" | tr }} @@ -107,14 +107,14 @@ {{ "Submit Request" | tr }}
  • - {% if request.user.has_credential("CRED_PKGBASE_DELETE") %} + {% if request.user.has_credential(creds.PKGBASE_DELETE) %}
  • {{ "Delete Package" | tr }}
  • {% endif %} - {% if request.user.has_credential("CRED_PKGBASE_MERGE") %} + {% if request.user.has_credential(creds.PKGBASE_MERGE) %}
  • {{ "Merge Package" | tr }} @@ -130,7 +130,7 @@ />
  • - {% elif request.user.has_credential("CRED_PKGBASE_DISOWN", approved=[pkgbase.Maintainer]) %} + {% elif request.user.has_credential(creds.PKGBASE_DISOWN, approved=[pkgbase.Maintainer]) %}
  • {{ "Disown Package" | tr }} diff --git a/templates/partials/packages/comment.html b/templates/partials/packages/comment.html index 676a7a73..1427e0a0 100644 --- a/templates/partials/packages/comment.html +++ b/templates/partials/packages/comment.html @@ -5,7 +5,7 @@ {% set article_cls = "%s %s" | format(article_cls, "comment-deleted") %} {% endif %} -{% if not comment.Deleter or request.user.has_credential("CRED_COMMENT_VIEW_DELETED", approved=[comment.Deleter]) %} +{% if not comment.Deleter or request.user.has_credential(creds.COMMENT_VIEW_DELETED, approved=[comment.Deleter]) %}

    {% set commented_at = comment.CommentTS | dt | as_timezone(timezone) %} {% set view_account_info = 'View account information for %s' | tr | format(comment.User.Username) %} diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index dbb81c19..78e0ad1c 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -33,10 +33,10 @@ {% endif %} - {% if pkgbase.keywords.count() or request.user.has_credential("CRED_PKGBASE_SET_KEYWORDS", approved=[pkgbase.Maintainer]) %} + {% if pkgbase.keywords.count() or request.user.has_credential(creds.PKGBASE_SET_KEYWORDS, approved=[pkgbase.Maintainer]) %} {{ "Keywords" | tr }}: - {% if request.user.has_credential("CRED_PKGBASE_SET_KEYWORDS", approved=[pkgbase.Maintainer]) %} + {% if request.user.has_credential(creds.PKGBASE_SET_KEYWORDS, approved=[pkgbase.Maintainer]) %}
    Date: Thu, 18 Nov 2021 14:17:46 -0500 Subject: [PATCH 660/844] fix(FastAPI): remove login and redirect parameters from auth_required Signed-off-by: Steven Guikal --- aurweb/auth/__init__.py | 27 ++++++-------- aurweb/routers/accounts.py | 22 ++++++------ aurweb/routers/auth.py | 2 +- aurweb/routers/packages.py | 60 ++++++++++++++++---------------- aurweb/routers/trusted_user.py | 10 +++--- test/test_trusted_user_routes.py | 5 +-- 6 files changed, 61 insertions(+), 65 deletions(-) diff --git a/aurweb/auth/__init__.py b/aurweb/auth/__init__.py index 82192cc2..7aa4b526 100644 --- a/aurweb/auth/__init__.py +++ b/aurweb/auth/__init__.py @@ -1,5 +1,4 @@ import functools -import re from datetime import datetime from http import HTTPStatus @@ -122,17 +121,12 @@ class BasicAuthBackend(AuthenticationBackend): def auth_required(is_required: bool = True, - login: bool = True, - redirect: str = "/", template: tuple = None, status_code: HTTPStatus = HTTPStatus.UNAUTHORIZED): """ Authentication route decorator. - If redirect is given, the user will be redirected if the auth state - does not match is_required. - If template is given, it will be rendered with Unauthorized if - is_required does not match and take priority over redirect. + is_required does not match. A precondition of this function is that, if template is provided, it **must** match the following format: @@ -152,8 +146,6 @@ def auth_required(is_required: bool = True, applying any format operations. :param is_required: A boolean indicating whether the function requires auth - :param login: Redirect to `/login`, passing `next=` - :param redirect: Path to redirect to if is_required isn't True :param template: A three-element template tuple: (path, title_iterable, variable_iterable) :param status_code: An optional status_code for template render. @@ -166,14 +158,17 @@ def auth_required(is_required: bool = True, if request.user.is_authenticated() != is_required: url = "/" - if redirect: - path_params_expr = re.compile(r'\{(\w+)\}') - match = re.findall(path_params_expr, redirect) - args = {k: request.path_params.get(k) for k in match} - url = redirect.format(**args) + if is_required: + if request.method == "GET": + url = request.url.path + elif request.method == "POST" and (referer := request.headers.get("Referer")): + aur = aurweb.config.get("options", "aur_location") + "/" + if not referer.startswith(aur): + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, + detail=_("Bad Referer header.")) + url = referer[len(aur) - 1:] - if login: - url = "/login?" + util.urlencode({"next": url}) + url = "/login?" + util.urlencode({"next": url}) if template: # template=("template.html", diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 360857e8..dade92bb 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -27,14 +27,14 @@ logger = logging.get_logger(__name__) @router.get("/passreset", response_class=HTMLResponse) -@auth_required(False, login=False) +@auth_required(False) async def passreset(request: Request): context = await make_variable_context(request, "Password Reset") return render_template(request, "passreset.html", context) @router.post("/passreset", response_class=HTMLResponse) -@auth_required(False, login=False) +@auth_required(False) async def passreset_post(request: Request, user: str = Form(...), resetkey: str = Form(default=None), @@ -226,7 +226,7 @@ def make_account_form_context(context: dict, @router.get("/register", response_class=HTMLResponse) -@auth_required(False, login=False) +@auth_required(False) async def account_register(request: Request, U: str = Form(default=str()), # Username E: str = Form(default=str()), # Email @@ -252,7 +252,7 @@ async def account_register(request: Request, @router.post("/register", response_class=HTMLResponse) -@auth_required(False, login=False) +@auth_required(False) async def account_register_post(request: Request, U: str = Form(default=str()), # Username E: str = Form(default=str()), # Email @@ -340,7 +340,7 @@ def cannot_edit(request, user): @router.get("/account/{username}/edit", response_class=HTMLResponse) -@auth_required(True, redirect="/account/{username}") +@auth_required(True) async def account_edit(request: Request, username: str): user = db.query(models.User, models.User.Username == username).first() @@ -356,7 +356,7 @@ async def account_edit(request: Request, username: str): @router.post("/account/{username}/edit", response_class=HTMLResponse) -@auth_required(True, redirect="/account/{username}") +@auth_required(True) async def account_edit_post(request: Request, username: str, U: str = Form(default=str()), # Username @@ -443,7 +443,7 @@ async def account(request: Request, username: str): @router.get("/account/{username}/comments") -@auth_required(redirect="/account/{username}/comments") +@auth_required() async def account_comments(request: Request, username: str): user = get_user_by_name(username) context = make_context(request, "Accounts") @@ -454,7 +454,7 @@ async def account_comments(request: Request, username: str): @router.get("/accounts") -@auth_required(True, redirect="/accounts") +@auth_required(True) @account_type_required({at.TRUSTED_USER, at.DEVELOPER, at.TRUSTED_USER_AND_DEV}) @@ -464,7 +464,7 @@ async def accounts(request: Request): @router.post("/accounts") -@auth_required(True, redirect="/accounts") +@auth_required(True) @account_type_required({at.TRUSTED_USER, at.DEVELOPER, at.TRUSTED_USER_AND_DEV}) @@ -548,7 +548,7 @@ def render_terms_of_service(request: Request, @router.get("/tos") -@auth_required(True, redirect="/tos") +@auth_required(True) async def terms_of_service(request: Request): # Query the database for terms that were previously accepted, # but now have a bumped Revision that needs to be accepted. @@ -572,7 +572,7 @@ async def terms_of_service(request: Request): @router.post("/tos") -@auth_required(True, redirect="/tos") +@auth_required(True) async def terms_of_service_post(request: Request, accept: bool = Form(default=False)): # Query the database for terms that were previously accepted, diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index 1e0b026a..74763667 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -29,7 +29,7 @@ async def login_get(request: Request, next: str = "/"): @router.post("/login", response_class=HTMLResponse) -@auth_required(False, login=False) +@auth_required(False) async def login_post(request: Request, next: str = Form(...), user: str = Form(default=str()), diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 2bf04949..4a2cdce3 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -295,7 +295,7 @@ async def package_base_voters(request: Request, name: str) -> Response: @router.post("/pkgbase/{name}/comments") -@auth_required(True, redirect="/pkgbase/{name}/comments") +@auth_required(True) async def pkgbase_comments_post( request: Request, name: str, comment: str = Form(default=str()), @@ -327,7 +327,7 @@ async def pkgbase_comments_post( @router.get("/pkgbase/{name}/comments/{id}/form") -@auth_required(True, login=False) +@auth_required(True) async def pkgbase_comment_form(request: Request, name: str, id: int, next: str = Query(default=None)): """ Produce a comment form for comment {id}. """ @@ -353,7 +353,7 @@ async def pkgbase_comment_form(request: Request, name: str, id: int, @router.post("/pkgbase/{name}/comments/{id}") -@auth_required(True, redirect="/pkgbase/{name}/comments/{id}") +@auth_required(True) async def pkgbase_comment_post( request: Request, name: str, id: int, comment: str = Form(default=str()), @@ -392,7 +392,7 @@ async def pkgbase_comment_post( @router.get("/pkgbase/{name}/comments/{id}/edit") -@auth_required(True, redirect="/pkgbase/{name}/comments/{id}/edit") +@auth_required(True) async def pkgbase_comment_edit(request: Request, name: str, id: int, next: str = Form(default=None)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -407,7 +407,7 @@ async def pkgbase_comment_edit(request: Request, name: str, id: int, @router.post("/pkgbase/{name}/comments/{id}/delete") -@auth_required(True, redirect="/pkgbase/{name}/comments/{id}/delete") +@auth_required(True) async def pkgbase_comment_delete(request: Request, name: str, id: int, next: str = Form(default=None)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -433,7 +433,7 @@ async def pkgbase_comment_delete(request: Request, name: str, id: int, @router.post("/pkgbase/{name}/comments/{id}/undelete") -@auth_required(True, redirect="/pkgbase/{name}/comments/{id}/undelete") +@auth_required(True) async def pkgbase_comment_undelete(request: Request, name: str, id: int, next: str = Form(default=None)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -458,7 +458,7 @@ async def pkgbase_comment_undelete(request: Request, name: str, id: int, @router.post("/pkgbase/{name}/comments/{id}/pin") -@auth_required(True, redirect="/pkgbase/{name}/comments/{id}/pin") +@auth_required(True) async def pkgbase_comment_pin(request: Request, name: str, id: int, next: str = Form(default=None)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -483,7 +483,7 @@ async def pkgbase_comment_pin(request: Request, name: str, id: int, @router.post("/pkgbase/{name}/comments/{id}/unpin") -@auth_required(True, redirect="/pkgbase/{name}/comments/{id}/unpin") +@auth_required(True) async def pkgbase_comment_unpin(request: Request, name: str, id: int, next: str = Form(default=None)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -507,7 +507,7 @@ async def pkgbase_comment_unpin(request: Request, name: str, id: int, @router.get("/pkgbase/{name}/comaintainers") -@auth_required(True, redirect="/pkgbase/{name}/comaintainers") +@auth_required(True) async def package_base_comaintainers(request: Request, name: str) -> Response: # Get the PackageBase. pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -532,7 +532,7 @@ async def package_base_comaintainers(request: Request, name: str) -> Response: @router.post("/pkgbase/{name}/comaintainers") -@auth_required(True, redirect="/pkgbase/{name}/comaintainers") +@auth_required(True) async def package_base_comaintainers_post( request: Request, name: str, users: str = Form(default=str())) -> Response: @@ -584,7 +584,7 @@ async def package_base_comaintainers_post( @router.get("/requests") -@auth_required(True, redirect="/requests") +@auth_required(True) async def requests(request: Request, O: int = Query(default=defaults.O), PP: int = Query(default=defaults.PP)): @@ -618,7 +618,7 @@ async def requests(request: Request, @router.get("/pkgbase/{name}/request") -@auth_required(True, redirect="/pkgbase/{name}/request") +@auth_required(True) async def package_request(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) context = await make_variable_context(request, "Submit Request") @@ -627,7 +627,7 @@ async def package_request(request: Request, name: str): @router.post("/pkgbase/{name}/request") -@auth_required(True, redirect="/pkgbase/{name}/request") +@auth_required(True) async def pkgbase_request_post(request: Request, name: str, type: str = Form(...), merge_into: str = Form(default=None), @@ -699,7 +699,7 @@ async def pkgbase_request_post(request: Request, name: str, @router.get("/requests/{id}/close") -@auth_required(True, redirect="/requests/{id}/close") +@auth_required(True) async def requests_close(request: Request, id: int): pkgreq = get_pkgreq_by_id(id) if not request.user.is_elevated() and request.user != pkgreq.User: @@ -712,7 +712,7 @@ async def requests_close(request: Request, id: int): @router.post("/requests/{id}/close") -@auth_required(True, redirect="/requests/{id}/close") +@auth_required(True) async def requests_close_post(request: Request, id: int, reason: int = Form(default=0), comments: str = Form(default=str())): @@ -775,7 +775,7 @@ async def pkgbase_keywords(request: Request, name: str, @router.get("/pkgbase/{name}/flag") -@auth_required(True, redirect="/pkgbase/{name}/flag") +@auth_required(True) async def pkgbase_flag_get(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -790,7 +790,7 @@ async def pkgbase_flag_get(request: Request, name: str): @router.post("/pkgbase/{name}/flag") -@auth_required(True, redirect="/pkgbase/{name}/flag") +@auth_required(True) async def pkgbase_flag_post(request: Request, name: str, comments: str = Form(default=str())): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -839,7 +839,7 @@ def pkgbase_unflag_instance(request: Request, pkgbase: models.PackageBase): @router.post("/pkgbase/{name}/unflag") -@auth_required(True, redirect="/pkgbase/{name}") +@auth_required(True) async def pkgbase_unflag(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) pkgbase_unflag_instance(request, pkgbase) @@ -860,7 +860,7 @@ def pkgbase_notify_instance(request: Request, pkgbase: models.PackageBase): @router.post("/pkgbase/{name}/notify") -@auth_required(True, redirect="/pkgbase/{name}") +@auth_required(True) async def pkgbase_notify(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) pkgbase_notify_instance(request, pkgbase) @@ -879,7 +879,7 @@ def pkgbase_unnotify_instance(request: Request, pkgbase: models.PackageBase): @router.post("/pkgbase/{name}/unnotify") -@auth_required(True, redirect="/pkgbase/{name}") +@auth_required(True) async def pkgbase_unnotify(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) pkgbase_unnotify_instance(request, pkgbase) @@ -888,7 +888,7 @@ async def pkgbase_unnotify(request: Request, name: str): @router.post("/pkgbase/{name}/vote") -@auth_required(True, redirect="/pkgbase/{name}") +@auth_required(True) async def pkgbase_vote(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -912,7 +912,7 @@ async def pkgbase_vote(request: Request, name: str): @router.post("/pkgbase/{name}/unvote") -@auth_required(True, redirect="/pkgbase/{name}") +@auth_required(True) async def pkgbase_unvote(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -954,7 +954,7 @@ def pkgbase_disown_instance(request: Request, pkgbase: models.PackageBase): @router.get("/pkgbase/{name}/disown") -@auth_required(True, redirect="/pkgbase/{name}/disown") +@auth_required(True) async def pkgbase_disown_get(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -970,7 +970,7 @@ async def pkgbase_disown_get(request: Request, name: str): @router.post("/pkgbase/{name}/disown") -@auth_required(True, redirect="/pkgbase/{name}/disown") +@auth_required(True) async def pkgbase_disown_post(request: Request, name: str, confirm: bool = Form(default=False)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -1003,7 +1003,7 @@ def pkgbase_adopt_instance(request: Request, pkgbase: models.PackageBase): @router.post("/pkgbase/{name}/adopt") -@auth_required(True, redirect="/pkgbase/{name}") +@auth_required(True) async def pkgbase_adopt_post(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -1019,7 +1019,7 @@ async def pkgbase_adopt_post(request: Request, name: str): @router.get("/pkgbase/{name}/delete") -@auth_required(True, redirect="/pkgbase/{name}/delete") +@auth_required(True) async def pkgbase_delete_get(request: Request, name: str): if not request.user.has_credential(creds.PKGBASE_DELETE): return RedirectResponse(f"/pkgbase/{name}", @@ -1031,7 +1031,7 @@ async def pkgbase_delete_get(request: Request, name: str): @router.post("/pkgbase/{name}/delete") -@auth_required(True, redirect="/pkgbase/{name}/delete") +@auth_required(True) async def pkgbase_delete_post(request: Request, name: str, confirm: bool = Form(default=False)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -1279,7 +1279,7 @@ PACKAGE_ACTIONS = { @router.post("/packages") -@auth_required(redirect="/packages") +@auth_required() async def packages_post(request: Request, IDs: List[int] = Form(default=[]), action: str = Form(default=str()), @@ -1311,7 +1311,7 @@ async def packages_post(request: Request, @router.get("/pkgbase/{name}/merge") -@auth_required(redirect="/pkgbase/{name}/merge") +@auth_required() async def pkgbase_merge_get(request: Request, name: str, into: str = Query(default=str()), next: str = Query(default=str())): @@ -1423,7 +1423,7 @@ def pkgbase_merge_instance(request: Request, pkgbase: models.PackageBase, @router.post("/pkgbase/{name}/merge") -@auth_required(redirect="/pkgbase/{name}/merge") +@auth_required() async def pkgbase_merge_post(request: Request, name: str, into: str = Form(default=str()), confirm: bool = Form(default=False), diff --git a/aurweb/routers/trusted_user.py b/aurweb/routers/trusted_user.py index f0cea61e..09de58fe 100644 --- a/aurweb/routers/trusted_user.py +++ b/aurweb/routers/trusted_user.py @@ -41,7 +41,7 @@ ADDVOTE_SPECIFICS = { @router.get("/tu") -@auth_required(True, redirect="/tu") +@auth_required(True) @account_type_required(REQUIRED_TYPES) async def trusted_user(request: Request, coff: int = 0, # current offset @@ -147,7 +147,7 @@ def render_proposal(request: Request, @router.get("/tu/{proposal}") -@auth_required(True, redirect="/tu/{proposal}") +@auth_required(True) @account_type_required(REQUIRED_TYPES) async def trusted_user_proposal(request: Request, proposal: int): context = await make_variable_context(request, "Trusted User") @@ -176,7 +176,7 @@ async def trusted_user_proposal(request: Request, proposal: int): @router.post("/tu/{proposal}") -@auth_required(True, redirect="/tu/{proposal}") +@auth_required(True) @account_type_required(REQUIRED_TYPES) async def trusted_user_proposal_post(request: Request, proposal: int, @@ -227,7 +227,7 @@ async def trusted_user_proposal_post(request: Request, @router.get("/addvote") -@auth_required(True, redirect="/addvote") +@auth_required(True) @account_type_required({TRUSTED_USER, TRUSTED_USER_AND_DEV}) async def trusted_user_addvote(request: Request, user: str = str(), @@ -247,7 +247,7 @@ async def trusted_user_addvote(request: Request, @router.post("/addvote") -@auth_required(True, redirect="/addvote") +@auth_required(True) @account_type_required({TRUSTED_USER, TRUSTED_USER_AND_DEV}) async def trusted_user_addvote_post(request: Request, user: str = Form(default=str()), diff --git a/test/test_trusted_user_routes.py b/test/test_trusted_user_routes.py index 43a3443b..ac7f82d5 100644 --- a/test/test_trusted_user_routes.py +++ b/test/test_trusted_user_routes.py @@ -9,7 +9,7 @@ import pytest from fastapi.testclient import TestClient -from aurweb import db, util +from aurweb import config, db, util from aurweb.models.account_type import AccountType from aurweb.models.tu_vote import TUVote from aurweb.models.tu_voteinfo import TUVoteInfo @@ -124,8 +124,9 @@ def proposal(user, tu_user): def test_tu_index_guest(client): + headers = {"referer": config.get("options", "aur_location") + "/tu"} with client as request: - response = request.get("/tu", allow_redirects=False) + response = request.get("/tu", allow_redirects=False, headers=headers) assert response.status_code == int(HTTPStatus.SEE_OTHER) params = util.urlencode({"next": "/tu"}) From 0b30216229f561cfdcffd27231f200c3901ce26d Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Thu, 18 Nov 2021 15:18:17 -0500 Subject: [PATCH 661/844] fix(FastAPI): remove unnecessary arguments to auth_required Signed-off-by: Steven Guikal --- aurweb/auth/__init__.py | 59 ++-------------------------------- aurweb/routers/accounts.py | 24 ++++++-------- aurweb/routers/packages.py | 54 +++++++++++++++---------------- aurweb/routers/trusted_user.py | 10 +++--- 4 files changed, 43 insertions(+), 104 deletions(-) diff --git a/aurweb/auth/__init__.py b/aurweb/auth/__init__.py index 7aa4b526..8ceb136c 100644 --- a/aurweb/auth/__init__.py +++ b/aurweb/auth/__init__.py @@ -5,6 +5,7 @@ from http import HTTPStatus import fastapi +from fastapi import HTTPException from fastapi.responses import RedirectResponse from sqlalchemy import and_ from starlette.authentication import AuthCredentials, AuthenticationBackend @@ -15,7 +16,6 @@ import aurweb.config from aurweb import db, l10n, util from aurweb.models import Session, User from aurweb.models.account_type import ACCOUNT_TYPE_ID -from aurweb.templates import make_variable_context, render_template class StubQuery: @@ -125,29 +125,7 @@ def auth_required(is_required: bool = True, status_code: HTTPStatus = HTTPStatus.UNAUTHORIZED): """ Authentication route decorator. - If template is given, it will be rendered with Unauthorized if - is_required does not match. - - A precondition of this function is that, if template is provided, - it **must** match the following format: - - template=("template.html", ["Some Template For", "{}"], ["username"]) - - Where `username` is a FastAPI request path parameter, fitting - a route like: `/some_route/{username}`. - - If you wish to supply a non-formatted template, just omit any Python - format strings (with the '{}' substring). The third tuple element - will not be used, and so anything can be supplied. - - template=("template.html", ["Some Page"], None) - - All title shards and format parameters will be translated before - applying any format operations. - :param is_required: A boolean indicating whether the function requires auth - :param template: A three-element template tuple: - (path, title_iterable, variable_iterable) :param status_code: An optional status_code for template render. Redirects are always SEE_OTHER. """ @@ -164,45 +142,12 @@ def auth_required(is_required: bool = True, elif request.method == "POST" and (referer := request.headers.get("Referer")): aur = aurweb.config.get("options", "aur_location") + "/" if not referer.startswith(aur): + _ = l10n.get_translator_for_request(request) raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=_("Bad Referer header.")) url = referer[len(aur) - 1:] url = "/login?" + util.urlencode({"next": url}) - - if template: - # template=("template.html", - # ["Some Title", "someFormatted {}"], - # ["variable"]) - # => render template.html with title: - # "Some Title someFormatted variables" - path, title_parts, variables = template - _ = l10n.get_translator_for_request(request) - - # Step through title_parts; for each part which contains - # a '{}' in it, apply .format(var) where var = the current - # iteration of variables. - # - # This implies that len(variables) is equal to - # len([part for part in title_parts if '{}' in part]) - # and this must always be true. - # - sanitized = [] - _variables = iter(variables) - for part in title_parts: - if "{}" in part: # If this part is formattable. - key = next(_variables) - var = request.path_params.get(key) - sanitized.append(_(part.format(var))) - else: # Otherwise, just add the translated part. - sanitized.append(_(part)) - - # Glue all title parts together, separated by spaces. - title = " ".join(sanitized) - - context = await make_variable_context(request, title) - return render_template(request, path, context, - status_code=status_code) return RedirectResponse(url, status_code=int(HTTPStatus.SEE_OTHER)) return await func(request, *args, **kwargs) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index dade92bb..388daf84 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -340,7 +340,7 @@ def cannot_edit(request, user): @router.get("/account/{username}/edit", response_class=HTMLResponse) -@auth_required(True) +@auth_required() async def account_edit(request: Request, username: str): user = db.query(models.User, models.User.Username == username).first() @@ -356,7 +356,7 @@ async def account_edit(request: Request, username: str): @router.post("/account/{username}/edit", response_class=HTMLResponse) -@auth_required(True) +@auth_required() async def account_edit_post(request: Request, username: str, U: str = Form(default=str()), # Username @@ -424,20 +424,14 @@ async def account_edit_post(request: Request, aurtz=TZ, aurlang=L) -account_template = ( - "account/show.html", - ["Account", "{}"], - ["username"] # Query parameters to replace in the title string. -) - - @router.get("/account/{username}") -@auth_required(True, template=account_template, - status_code=HTTPStatus.UNAUTHORIZED) async def account(request: Request, username: str): _ = l10n.get_translator_for_request(request) context = await make_variable_context( request, _("Account") + " " + username) + if not request.user.is_authenticated(): + return render_template(request, "account/show.html", context, + status_code=HTTPStatus.UNAUTHORIZED) context["user"] = get_user_by_name(username) return render_template(request, "account/show.html", context) @@ -454,7 +448,7 @@ async def account_comments(request: Request, username: str): @router.get("/accounts") -@auth_required(True) +@auth_required() @account_type_required({at.TRUSTED_USER, at.DEVELOPER, at.TRUSTED_USER_AND_DEV}) @@ -464,7 +458,7 @@ async def accounts(request: Request): @router.post("/accounts") -@auth_required(True) +@auth_required() @account_type_required({at.TRUSTED_USER, at.DEVELOPER, at.TRUSTED_USER_AND_DEV}) @@ -548,7 +542,7 @@ def render_terms_of_service(request: Request, @router.get("/tos") -@auth_required(True) +@auth_required() async def terms_of_service(request: Request): # Query the database for terms that were previously accepted, # but now have a bumped Revision that needs to be accepted. @@ -572,7 +566,7 @@ async def terms_of_service(request: Request): @router.post("/tos") -@auth_required(True) +@auth_required() async def terms_of_service_post(request: Request, accept: bool = Form(default=False)): # Query the database for terms that were previously accepted, diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 4a2cdce3..c06ec51f 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -295,7 +295,7 @@ async def package_base_voters(request: Request, name: str) -> Response: @router.post("/pkgbase/{name}/comments") -@auth_required(True) +@auth_required() async def pkgbase_comments_post( request: Request, name: str, comment: str = Form(default=str()), @@ -327,7 +327,7 @@ async def pkgbase_comments_post( @router.get("/pkgbase/{name}/comments/{id}/form") -@auth_required(True) +@auth_required() async def pkgbase_comment_form(request: Request, name: str, id: int, next: str = Query(default=None)): """ Produce a comment form for comment {id}. """ @@ -353,7 +353,7 @@ async def pkgbase_comment_form(request: Request, name: str, id: int, @router.post("/pkgbase/{name}/comments/{id}") -@auth_required(True) +@auth_required() async def pkgbase_comment_post( request: Request, name: str, id: int, comment: str = Form(default=str()), @@ -392,7 +392,7 @@ async def pkgbase_comment_post( @router.get("/pkgbase/{name}/comments/{id}/edit") -@auth_required(True) +@auth_required() async def pkgbase_comment_edit(request: Request, name: str, id: int, next: str = Form(default=None)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -407,7 +407,7 @@ async def pkgbase_comment_edit(request: Request, name: str, id: int, @router.post("/pkgbase/{name}/comments/{id}/delete") -@auth_required(True) +@auth_required() async def pkgbase_comment_delete(request: Request, name: str, id: int, next: str = Form(default=None)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -433,7 +433,7 @@ async def pkgbase_comment_delete(request: Request, name: str, id: int, @router.post("/pkgbase/{name}/comments/{id}/undelete") -@auth_required(True) +@auth_required() async def pkgbase_comment_undelete(request: Request, name: str, id: int, next: str = Form(default=None)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -458,7 +458,7 @@ async def pkgbase_comment_undelete(request: Request, name: str, id: int, @router.post("/pkgbase/{name}/comments/{id}/pin") -@auth_required(True) +@auth_required() async def pkgbase_comment_pin(request: Request, name: str, id: int, next: str = Form(default=None)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -483,7 +483,7 @@ async def pkgbase_comment_pin(request: Request, name: str, id: int, @router.post("/pkgbase/{name}/comments/{id}/unpin") -@auth_required(True) +@auth_required() async def pkgbase_comment_unpin(request: Request, name: str, id: int, next: str = Form(default=None)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -507,7 +507,7 @@ async def pkgbase_comment_unpin(request: Request, name: str, id: int, @router.get("/pkgbase/{name}/comaintainers") -@auth_required(True) +@auth_required() async def package_base_comaintainers(request: Request, name: str) -> Response: # Get the PackageBase. pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -532,7 +532,7 @@ async def package_base_comaintainers(request: Request, name: str) -> Response: @router.post("/pkgbase/{name}/comaintainers") -@auth_required(True) +@auth_required() async def package_base_comaintainers_post( request: Request, name: str, users: str = Form(default=str())) -> Response: @@ -584,7 +584,7 @@ async def package_base_comaintainers_post( @router.get("/requests") -@auth_required(True) +@auth_required() async def requests(request: Request, O: int = Query(default=defaults.O), PP: int = Query(default=defaults.PP)): @@ -618,7 +618,7 @@ async def requests(request: Request, @router.get("/pkgbase/{name}/request") -@auth_required(True) +@auth_required() async def package_request(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) context = await make_variable_context(request, "Submit Request") @@ -627,7 +627,7 @@ async def package_request(request: Request, name: str): @router.post("/pkgbase/{name}/request") -@auth_required(True) +@auth_required() async def pkgbase_request_post(request: Request, name: str, type: str = Form(...), merge_into: str = Form(default=None), @@ -699,7 +699,7 @@ async def pkgbase_request_post(request: Request, name: str, @router.get("/requests/{id}/close") -@auth_required(True) +@auth_required() async def requests_close(request: Request, id: int): pkgreq = get_pkgreq_by_id(id) if not request.user.is_elevated() and request.user != pkgreq.User: @@ -712,7 +712,7 @@ async def requests_close(request: Request, id: int): @router.post("/requests/{id}/close") -@auth_required(True) +@auth_required() async def requests_close_post(request: Request, id: int, reason: int = Form(default=0), comments: str = Form(default=str())): @@ -775,7 +775,7 @@ async def pkgbase_keywords(request: Request, name: str, @router.get("/pkgbase/{name}/flag") -@auth_required(True) +@auth_required() async def pkgbase_flag_get(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -790,7 +790,7 @@ async def pkgbase_flag_get(request: Request, name: str): @router.post("/pkgbase/{name}/flag") -@auth_required(True) +@auth_required() async def pkgbase_flag_post(request: Request, name: str, comments: str = Form(default=str())): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -839,7 +839,7 @@ def pkgbase_unflag_instance(request: Request, pkgbase: models.PackageBase): @router.post("/pkgbase/{name}/unflag") -@auth_required(True) +@auth_required() async def pkgbase_unflag(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) pkgbase_unflag_instance(request, pkgbase) @@ -860,7 +860,7 @@ def pkgbase_notify_instance(request: Request, pkgbase: models.PackageBase): @router.post("/pkgbase/{name}/notify") -@auth_required(True) +@auth_required() async def pkgbase_notify(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) pkgbase_notify_instance(request, pkgbase) @@ -879,7 +879,7 @@ def pkgbase_unnotify_instance(request: Request, pkgbase: models.PackageBase): @router.post("/pkgbase/{name}/unnotify") -@auth_required(True) +@auth_required() async def pkgbase_unnotify(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) pkgbase_unnotify_instance(request, pkgbase) @@ -888,7 +888,7 @@ async def pkgbase_unnotify(request: Request, name: str): @router.post("/pkgbase/{name}/vote") -@auth_required(True) +@auth_required() async def pkgbase_vote(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -912,7 +912,7 @@ async def pkgbase_vote(request: Request, name: str): @router.post("/pkgbase/{name}/unvote") -@auth_required(True) +@auth_required() async def pkgbase_unvote(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -954,7 +954,7 @@ def pkgbase_disown_instance(request: Request, pkgbase: models.PackageBase): @router.get("/pkgbase/{name}/disown") -@auth_required(True) +@auth_required() async def pkgbase_disown_get(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -970,7 +970,7 @@ async def pkgbase_disown_get(request: Request, name: str): @router.post("/pkgbase/{name}/disown") -@auth_required(True) +@auth_required() async def pkgbase_disown_post(request: Request, name: str, confirm: bool = Form(default=False)): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -1003,7 +1003,7 @@ def pkgbase_adopt_instance(request: Request, pkgbase: models.PackageBase): @router.post("/pkgbase/{name}/adopt") -@auth_required(True) +@auth_required() async def pkgbase_adopt_post(request: Request, name: str): pkgbase = get_pkg_or_base(name, models.PackageBase) @@ -1019,7 +1019,7 @@ async def pkgbase_adopt_post(request: Request, name: str): @router.get("/pkgbase/{name}/delete") -@auth_required(True) +@auth_required() async def pkgbase_delete_get(request: Request, name: str): if not request.user.has_credential(creds.PKGBASE_DELETE): return RedirectResponse(f"/pkgbase/{name}", @@ -1031,7 +1031,7 @@ async def pkgbase_delete_get(request: Request, name: str): @router.post("/pkgbase/{name}/delete") -@auth_required(True) +@auth_required() async def pkgbase_delete_post(request: Request, name: str, confirm: bool = Form(default=False)): pkgbase = get_pkg_or_base(name, models.PackageBase) diff --git a/aurweb/routers/trusted_user.py b/aurweb/routers/trusted_user.py index 09de58fe..fac68f04 100644 --- a/aurweb/routers/trusted_user.py +++ b/aurweb/routers/trusted_user.py @@ -41,7 +41,7 @@ ADDVOTE_SPECIFICS = { @router.get("/tu") -@auth_required(True) +@auth_required() @account_type_required(REQUIRED_TYPES) async def trusted_user(request: Request, coff: int = 0, # current offset @@ -147,7 +147,7 @@ def render_proposal(request: Request, @router.get("/tu/{proposal}") -@auth_required(True) +@auth_required() @account_type_required(REQUIRED_TYPES) async def trusted_user_proposal(request: Request, proposal: int): context = await make_variable_context(request, "Trusted User") @@ -176,7 +176,7 @@ async def trusted_user_proposal(request: Request, proposal: int): @router.post("/tu/{proposal}") -@auth_required(True) +@auth_required() @account_type_required(REQUIRED_TYPES) async def trusted_user_proposal_post(request: Request, proposal: int, @@ -227,7 +227,7 @@ async def trusted_user_proposal_post(request: Request, @router.get("/addvote") -@auth_required(True) +@auth_required() @account_type_required({TRUSTED_USER, TRUSTED_USER_AND_DEV}) async def trusted_user_addvote(request: Request, user: str = str(), @@ -247,7 +247,7 @@ async def trusted_user_addvote(request: Request, @router.post("/addvote") -@auth_required(True) +@auth_required() @account_type_required({TRUSTED_USER, TRUSTED_USER_AND_DEV}) async def trusted_user_addvote_post(request: Request, user: str = Form(default=str()), From 2fee6205a6d11ad6ecae4b003991fc3aea1e992f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 01:42:19 -0800 Subject: [PATCH 662/844] housekeep(fastapi): rewrite test_rpc with fixtures Signed-off-by: Kevin Morris --- test/test_rpc.py | 532 +++++++++++++++++++++++++++-------------------- 1 file changed, 308 insertions(+), 224 deletions(-) diff --git a/test/test_rpc.py b/test/test_rpc.py index b61a7e4e..acb82cad 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -1,6 +1,8 @@ import re +from datetime import datetime from http import HTTPStatus +from typing import List from unittest import mock import orjson @@ -9,10 +11,11 @@ import pytest from fastapi.testclient import TestClient from redis.client import Pipeline -from aurweb import asgi, config, scripts -from aurweb.db import begin, create, query -from aurweb.models.account_type import AccountType -from aurweb.models.dependency_type import DependencyType +import aurweb.models.dependency_type as dt +import aurweb.models.relation_type as rt + +from aurweb import asgi, config, db, scripts +from aurweb.models.account_type import USER_ID from aurweb.models.license import License from aurweb.models.package import Package from aurweb.models.package_base import PackageBase @@ -21,7 +24,6 @@ from aurweb.models.package_keyword import PackageKeyword from aurweb.models.package_license import PackageLicense from aurweb.models.package_relation import PackageRelation from aurweb.models.package_vote import PackageVote -from aurweb.models.relation_type import RelationType from aurweb.models.user import User from aurweb.redis import redis_connection @@ -31,163 +33,172 @@ def client() -> TestClient: yield TestClient(app=asgi.app) -@pytest.fixture(autouse=True) -def setup(db_test): - # TODO: Rework this into organized fixtures. +@pytest.fixture +def user(db_test) -> User: + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User 1", Passwd=str(), + AccountTypeID=USER_ID) + yield user - # Create test package details. - with begin(): - # Get ID types. - account_type = query(AccountType, AccountType.AccountType == "User").first() - dependency_depends = query(DependencyType, DependencyType.Name == "depends").first() - dependency_optdepends = query(DependencyType, DependencyType.Name == "optdepends").first() - dependency_makedepends = query(DependencyType, DependencyType.Name == "makedepends").first() - dependency_checkdepends = query(DependencyType, DependencyType.Name == "checkdepends").first() +@pytest.fixture +def user2() -> User: + with db.begin(): + user = db.create(User, Username="user2", Email="user2@example.org", + RealName="Test User 2", Passwd=str(), + AccountTypeID=USER_ID) + yield user - relation_conflicts = query(RelationType, RelationType.Name == "conflicts").first() - relation_provides = query(RelationType, RelationType.Name == "provides").first() - relation_replaces = query(RelationType, RelationType.Name == "replaces").first() - # Create database info. - user1 = create(User, - Username="user1", - Email="user1@example.com", - RealName="Test User 1", - Passwd="testPassword", - AccountType=account_type) +@pytest.fixture +def user3() -> User: + with db.begin(): + user = db.create(User, Username="user3", Email="user3@example.org", + RealName="Test User 3", Passwd=str(), + AccountTypeID=USER_ID) + yield user - user2 = create(User, - Username="user2", - Email="user2@example.com", - RealName="Test User 2", - Passwd="testPassword", - AccountType=account_type) - user3 = create(User, - Username="user3", - Email="user3@example.com", - RealName="Test User 3", - Passwd="testPassword", - AccountType=account_type) +@pytest.fixture +def packages(user: User, user2: User, user3: User) -> List[Package]: + output = [] - pkgbase1 = create(PackageBase, Name="big-chungus", - Maintainer=user1, - Packager=user1) + # Create package records used in our tests. + with db.begin(): + pkgbase = db.create(PackageBase, Name="big-chungus", + Maintainer=user, Packager=user) + pkg = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name, + Description="Bunny bunny around bunny", + URL="https://example.com/") + output.append(pkg) - pkgname1 = create(Package, - PackageBase=pkgbase1, - Name=pkgbase1.Name, - Description="Bunny bunny around bunny", - URL="https://example.com/") + pkgbase = db.create(PackageBase, Name="chungy-chungus", + Maintainer=user, Packager=user) + pkg = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name, + Description="Wubby wubby on wobba wuubu", + URL="https://example.com/") + output.append(pkg) - pkgbase2 = create(PackageBase, Name="chungy-chungus", - Maintainer=user1, - Packager=user1) + pkgbase = db.create(PackageBase, Name="gluggly-chungus", + Maintainer=user, Packager=user) + pkg = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name, + Description="glurrba glurrba gur globba", + URL="https://example.com/") + output.append(pkg) - pkgname2 = create(Package, - PackageBase=pkgbase2, - Name=pkgbase2.Name, - Description="Wubby wubby on wobba wuubu", - URL="https://example.com/") - - pkgbase3 = create(PackageBase, Name="gluggly-chungus", - Maintainer=user1, - Packager=user1) - - pkgbase4 = create(PackageBase, Name="fugly-chungus", - Maintainer=user1, - Packager=user1) + pkgbase = db.create(PackageBase, Name="fugly-chungus", + Maintainer=user, Packager=user) desc = "A Package belonging to a PackageBase with another name." - create(Package, - PackageBase=pkgbase4, - Name="other-pkg", - Description=desc, - URL="https://example.com") + pkg = db.create(Package, PackageBase=pkgbase, Name="other-pkg", + Description=desc, URL="https://example.com") + output.append(pkg) - create(Package, - PackageBase=pkgbase3, - Name=pkgbase3.Name, - Description="glurrba glurrba gur globba", - URL="https://example.com/") + pkgbase = db.create(PackageBase, Name="woogly-chungus") + pkg = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name, + Description="wuggla woblabeloop shemashmoop", + URL="https://example.com/") + output.append(pkg) - pkgbase4 = create(PackageBase, Name="woogly-chungus") + # Setup a few more related records on the first package: + # a license, some keywords and some votes. + with db.begin(): + lic = db.create(License, Name="GPL") + db.create(PackageLicense, Package=output[0], License=lic) - create(Package, - PackageBase=pkgbase4, - Name=pkgbase4.Name, - Description="wuggla woblabeloop shemashmoop", - URL="https://example.com/") + for keyword in ["big-chungus", "smol-chungus", "sizeable-chungus"]: + db.create(PackageKeyword, + PackageBase=output[0].PackageBase, + Keyword=keyword) - # Dependencies. - create(PackageDependency, - Package=pkgname1, - DependencyType=dependency_depends, - DepName="chungus-depends") + now = int(datetime.utcnow().timestamp()) + for user_ in [user, user2, user3]: + db.create(PackageVote, User=user_, + PackageBase=output[0].PackageBase, VoteTS=now) + scripts.popupdate.run_single(output[0].PackageBase) - create(PackageDependency, - Package=pkgname2, - DependencyType=dependency_depends, - DepName="chungy-depends") + yield output - create(PackageDependency, - Package=pkgname1, - DependencyType=dependency_optdepends, - DepName="chungus-optdepends", - DepCondition="=50") - create(PackageDependency, - Package=pkgname1, - DependencyType=dependency_makedepends, - DepName="chungus-makedepends") +@pytest.fixture +def depends(packages: List[Package]) -> List[PackageDependency]: + output = [] - create(PackageDependency, - Package=pkgname1, - DependencyType=dependency_checkdepends, - DepName="chungus-checkdepends") + with db.begin(): + dep = db.create(PackageDependency, + Package=packages[0], + DepTypeID=dt.DEPENDS_ID, + DepName="chungus-depends") + output.append(dep) - # Relations. - create(PackageRelation, - Package=pkgname1, - RelationType=relation_conflicts, - RelName="chungus-conflicts") + dep = db.create(PackageDependency, + Package=packages[1], + DepTypeID=dt.DEPENDS_ID, + DepName="chungy-depends") + output.append(dep) - create(PackageRelation, - Package=pkgname2, - RelationType=relation_conflicts, - RelName="chungy-conflicts") + dep = db.create(PackageDependency, + Package=packages[0], + DepTypeID=dt.OPTDEPENDS_ID, + DepName="chungus-optdepends", + DepCondition="=50") + output.append(dep) - create(PackageRelation, - Package=pkgname1, - RelationType=relation_provides, - RelName="chungus-provides", - RelCondition="<=200") + dep = db.create(PackageDependency, + Package=packages[0], + DepTypeID=dt.MAKEDEPENDS_ID, + DepName="chungus-makedepends") + output.append(dep) - create(PackageRelation, - Package=pkgname1, - RelationType=relation_replaces, - RelName="chungus-replaces", - RelCondition="<=200") + dep = db.create(PackageDependency, + Package=packages[0], + DepTypeID=dt.CHECKDEPENDS_ID, + DepName="chungus-checkdepends") + output.append(dep) - license = create(License, Name="GPL") + yield output - create(PackageLicense, - Package=pkgname1, - License=license) - for i in ["big-chungus", "smol-chungus", "sizeable-chungus"]: - create(PackageKeyword, - PackageBase=pkgbase1, - Keyword=i) +@pytest.fixture +def relations(user: User, packages: List[Package]) -> List[PackageRelation]: + output = [] - for i in [user1, user2, user3]: - create(PackageVote, - User=i, - PackageBase=pkgbase1, - VoteTS=5000) + with db.begin(): + rel = db.create(PackageRelation, + Package=packages[0], + RelTypeID=rt.CONFLICTS_ID, + RelName="chungus-conflicts") + output.append(rel) - scripts.popupdate.run_single(pkgbase1) + rel = db.create(PackageRelation, + Package=packages[1], + RelTypeID=rt.CONFLICTS_ID, + RelName="chungy-conflicts") + output.append(rel) + + rel = db.create(PackageRelation, + Package=packages[0], + RelTypeID=rt.PROVIDES_ID, + RelName="chungus-provides", + RelCondition="<=200") + output.append(rel) + + rel = db.create(PackageRelation, + Package=packages[0], + RelTypeID=rt.REPLACES_ID, + RelName="chungus-replaces", + RelCondition="<=200") + output.append(rel) + + # Finally, yield the packages. + yield output + + +@pytest.fixture(autouse=True) +def setup(db_test): + # Create some extra package relationships. + pass @pytest.fixture @@ -195,28 +206,35 @@ def pipeline(): redis = redis_connection() pipeline = redis.pipeline() + # The 'testclient' host is used when requesting the app + # via fastapi.testclient.TestClient. pipeline.delete("ratelimit-ws:testclient") pipeline.delete("ratelimit:testclient") - one, two = pipeline.execute() + pipeline.execute() yield pipeline -def test_rpc_singular_info(client: TestClient): +def test_rpc_singular_info(client: TestClient, + user: User, + packages: List[Package], + depends: List[PackageDependency], + relations: List[PackageRelation]): # Define expected response. + pkg = packages[0] expected_data = { "version": 5, "results": [{ - "Name": "big-chungus", - "Version": "", - "Description": "Bunny bunny around bunny", - "URL": "https://example.com/", - "PackageBase": "big-chungus", - "NumVotes": 3, - "Popularity": 0.0, + "Name": pkg.Name, + "Version": pkg.Version, + "Description": pkg.Description, + "URL": pkg.URL, + "PackageBase": pkg.PackageBase.Name, + "NumVotes": pkg.PackageBase.NumVotes, + "Popularity": float(pkg.PackageBase.Popularity), "OutOfDate": None, - "Maintainer": "user1", - "URLPath": "/cgit/aur.git/snapshot/big-chungus.tar.gz", + "Maintainer": user.Username, + "URLPath": f"/cgit/aur.git/snapshot/{pkg.Name}.tar.gz", "Depends": ["chungus-depends"], "OptDepends": ["chungus-optdepends=50"], "MakeDepends": ["chungus-makedepends"], @@ -224,7 +242,7 @@ def test_rpc_singular_info(client: TestClient): "Conflicts": ["chungus-conflicts"], "Provides": ["chungus-provides<=200"], "Replaces": ["chungus-replaces<=200"], - "License": ["GPL"], + "License": [pkg.package_licenses.first().License.Name], "Keywords": [ "big-chungus", "sizeable-chungus", @@ -237,20 +255,23 @@ def test_rpc_singular_info(client: TestClient): # Make dummy request. with client as request: - response_arg = request.get( - "/rpc/?v=5&type=info&arg=chungy-chungus&arg=big-chungus") + resp = request.get("/rpc", params={ + "v": 5, + "type": "info", + "arg": ["chungy-chungus", "big-chungus"], + }) # Load request response into Python dictionary. - response_info_arg = orjson.loads(response_arg.content.decode()) + response_data = orjson.loads(resp.text) # Remove the FirstSubmitted LastModified, ID and PackageBaseID keys from # reponse, as the key's values aren't guaranteed to match between the two # (the keys are already removed from 'expected_data'). for i in ["FirstSubmitted", "LastModified", "ID", "PackageBaseID"]: - response_info_arg["results"][0].pop(i) + response_data["results"][0].pop(i) # Validate that the new dictionaries are the same. - assert response_info_arg == expected_data + assert response_data == expected_data def test_rpc_nonexistent_package(client: TestClient): @@ -265,12 +286,13 @@ def test_rpc_nonexistent_package(client: TestClient): assert response_data["resultcount"] == 0 -def test_rpc_multiinfo(client: TestClient): +def test_rpc_multiinfo(client: TestClient, packages: List[Package]): # Make dummy request. request_packages = ["big-chungus", "chungy-chungus"] with client as request: - response = request.get( - "/rpc/?v=5&type=info&arg[]=big-chungus&arg[]=chungy-chungus") + response = request.get("/rpc", params={ + "v": 5, "type": "info", "arg[]": request_packages + }) # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -282,19 +304,21 @@ def test_rpc_multiinfo(client: TestClient): assert request_packages == [] -def test_rpc_mixedargs(client: TestClient): +def test_rpc_mixedargs(client: TestClient, packages: List[Package]): # Make dummy request. response1_packages = ["gluggly-chungus"] response2_packages = ["gluggly-chungus", "chungy-chungus"] with client as request: + # Supply all of the args in the url to enforce ordering. response1 = request.get( "/rpc?v=5&arg[]=big-chungus&arg=gluggly-chungus&type=info") assert response1.status_code == int(HTTPStatus.OK) with client as request: response2 = request.get( - "/rpc?v=5&arg=big-chungus&arg[]=gluggly-chungus&type=info&arg[]=chungy-chungus") + "/rpc?v=5&arg=big-chungus&arg[]=gluggly-chungus" + "&type=info&arg[]=chungy-chungus") assert response1.status_code == int(HTTPStatus.OK) # Load request response into Python dictionary. @@ -312,22 +336,27 @@ def test_rpc_mixedargs(client: TestClient): assert i == [] -def test_rpc_no_dependencies(client: TestClient): - """This makes sure things like 'MakeDepends' get removed from JSON strings - when they don't have set values.""" - +def test_rpc_no_dependencies_omits_key(client: TestClient, user: User, + packages: List[Package], + depends: List[PackageDependency], + relations: List[PackageRelation]): + """ + This makes sure things like 'MakeDepends' get removed from JSON strings + when they don't have set values. + """ + pkg = packages[1] expected_response = { 'version': 5, 'results': [{ - 'Name': 'chungy-chungus', - 'Version': '', - 'Description': 'Wubby wubby on wobba wuubu', - 'URL': 'https://example.com/', - 'PackageBase': 'chungy-chungus', - 'NumVotes': 0, - 'Popularity': 0.0, + 'Name': pkg.Name, + 'Version': pkg.Version, + 'Description': pkg.Description, + 'URL': pkg.URL, + 'PackageBase': pkg.PackageBase.Name, + 'NumVotes': pkg.PackageBase.NumVotes, + 'Popularity': int(pkg.PackageBase.Popularity), 'OutOfDate': None, - 'Maintainer': 'user1', + 'Maintainer': user.Username, 'URLPath': '/cgit/aur.git/snapshot/chungy-chungus.tar.gz', 'Depends': ['chungy-depends'], 'Conflicts': ['chungy-conflicts'], @@ -340,7 +369,9 @@ def test_rpc_no_dependencies(client: TestClient): # Make dummy request. with client as request: - response = request.get("/rpc/?v=5&type=info&arg=chungy-chungus") + response = request.get("/rpc", params={ + "v": 5, "type": "info", "arg": "chungy-chungus" + }) response_data = orjson.loads(response.content.decode()) # Remove inconsistent keys. @@ -362,7 +393,9 @@ def test_rpc_bad_type(client: TestClient): # Make dummy request. with client as request: - response = request.get("/rpc/?v=5&type=invalid-type&arg=big-chungus") + response = request.get("/rpc", params={ + "v": 5, "type": "invalid-type", "arg": "big-chungus" + }) # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -383,7 +416,9 @@ def test_rpc_bad_version(client: TestClient): # Make dummy request. with client as request: - response = request.get("/rpc/?v=0&type=info&arg=big-chungus") + response = request.get("/rpc", params={ + "v": 0, "type": "info", "arg": "big-chungus" + }) # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -404,7 +439,10 @@ def test_rpc_no_version(client: TestClient): # Make dummy request. with client as request: - response = request.get("/rpc/?type=info&arg=big-chungus") + response = request.get("/rpc", params={ + "type": "info", + "arg": "big-chungus" + }) # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -425,7 +463,7 @@ def test_rpc_no_type(client: TestClient): # Make dummy request. with client as request: - response = request.get("/rpc/?v=5&arg=big-chungus") + response = request.get("/rpc", params={"v": 5, "arg": "big-chungus"}) # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -446,7 +484,7 @@ def test_rpc_no_args(client: TestClient): # Make dummy request. with client as request: - response = request.get("/rpc/?v=5&type=info") + response = request.get("/rpc", params={"v": 5, "type": "info"}) # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -455,10 +493,12 @@ def test_rpc_no_args(client: TestClient): assert expected_data == response_data -def test_rpc_no_maintainer(client: TestClient): +def test_rpc_no_maintainer(client: TestClient, packages: List[Package]): # Make dummy request. with client as request: - response = request.get("/rpc/?v=5&type=info&arg=woogly-chungus") + response = request.get("/rpc", params={ + "v": 5, "type": "info", "arg": "woogly-chungus" + }) # Load request response into Python dictionary. response_data = orjson.loads(response.content.decode()) @@ -467,39 +507,45 @@ def test_rpc_no_maintainer(client: TestClient): assert response_data["results"][0]["Maintainer"] is None -def test_rpc_suggest_pkgbase(client: TestClient): +def test_rpc_suggest_pkgbase(client: TestClient, packages: List[Package]): + params = {"v": 5, "type": "suggest-pkgbase", "arg": "big"} with client as request: - response = request.get("/rpc?v=5&type=suggest-pkgbase&arg=big") + response = request.get("/rpc", params=params) data = response.json() assert data == ["big-chungus"] + params["arg"] = "chungy" with client as request: - response = request.get("/rpc?v=5&type=suggest-pkgbase&arg=chungy") + response = request.get("/rpc", params=params) data = response.json() assert data == ["chungy-chungus"] # Test no arg supplied. + del params["arg"] with client as request: - response = request.get("/rpc?v=5&type=suggest-pkgbase") + response = request.get("/rpc", params=params) data = response.json() assert data == [] -def test_rpc_suggest(client: TestClient): +def test_rpc_suggest(client: TestClient, packages: List[Package]): + params = {"v": 5, "type": "suggest", "arg": "other"} with client as request: - response = request.get("/rpc?v=5&type=suggest&arg=other") + response = request.get("/rpc", params=params) data = response.json() assert data == ["other-pkg"] # Test non-existent Package. + params["arg"] = "nonexistent" with client as request: - response = request.get("/rpc?v=5&type=suggest&arg=nonexistent") + response = request.get("/rpc", params=params) data = response.json() assert data == [] # Test no arg supplied. + del params["arg"] with client as request: - response = request.get("/rpc?v=5&type=suggest") + response = request.get("/rpc", params=params) data = response.json() assert data == [] @@ -514,16 +560,18 @@ def mock_config_getint(section: str, key: str): @mock.patch("aurweb.config.getint", side_effect=mock_config_getint) def test_rpc_ratelimit(getint: mock.MagicMock, client: TestClient, - pipeline: Pipeline): + pipeline: Pipeline, packages: List[Package]): + params = {"v": 5, "type": "suggest-pkgbase", "arg": "big"} + for i in range(4): # The first 4 requests should be good. with client as request: - response = request.get("/rpc?v=5&type=suggest-pkgbase&arg=big") + response = request.get("/rpc", params=params) assert response.status_code == int(HTTPStatus.OK) # The fifth request should be banned. with client as request: - response = request.get("/rpc?v=5&type=suggest-pkgbase&arg=big") + response = request.get("/rpc", params=params) assert response.status_code == int(HTTPStatus.TOO_MANY_REQUESTS) # Delete the cached records. @@ -534,124 +582,155 @@ def test_rpc_ratelimit(getint: mock.MagicMock, client: TestClient, # The new first request should be good. with client as request: - response = request.get("/rpc?v=5&type=suggest-pkgbase&arg=big") + response = request.get("/rpc", params=params) assert response.status_code == int(HTTPStatus.OK) -def test_rpc_etag(client: TestClient): - with client as request: - response1 = request.get("/rpc?v=5&type=suggest-pkgbase&arg=big") +def test_rpc_etag(client: TestClient, packages: List[Package]): + params = {"v": 5, "type": "suggest-pkgbase", "arg": "big"} with client as request: - response2 = request.get("/rpc?v=5&type=suggest-pkgbase&arg=big") + response1 = request.get("/rpc", params=params) + with client as request: + response2 = request.get("/rpc", params=params) + assert response1.headers.get("ETag") is not None assert response1.headers.get("ETag") != str() assert response1.headers.get("ETag") == response2.headers.get("ETag") def test_rpc_search_arg_too_small(client: TestClient): + params = {"v": 5, "type": "search", "arg": "b"} with client as request: - response = request.get("/rpc?v=5&type=search&arg=b") + response = request.get("/rpc", params=params) assert response.status_code == int(HTTPStatus.OK) assert response.json().get("error") == "Query arg too small." -def test_rpc_search(client: TestClient): +def test_rpc_search(client: TestClient, packages: List[Package]): + params = {"v": 5, "type": "search", "arg": "big"} with client as request: - response = request.get("/rpc?v=5&type=search&arg=big") + response = request.get("/rpc", params=params) assert response.status_code == int(HTTPStatus.OK) data = response.json() assert data.get("resultcount") == 1 result = data.get("results")[0] - assert result.get("Name") == "big-chungus" + assert result.get("Name") == packages[0].Name # Test the If-None-Match headers. etag = response.headers.get("ETag").strip('"') headers = {"If-None-Match": etag} - response = request.get("/rpc?v=5&type=search&arg=big", headers=headers) + response = request.get("/rpc", params=params, headers=headers) assert response.status_code == int(HTTPStatus.NOT_MODIFIED) assert response.content == b'' # No args on non-m by types return an error. - response = request.get("/rpc?v=5&type=search") + del params["arg"] + with client as request: + response = request.get("/rpc", params=params) assert response.json().get("error") == "No request type/data specified." -def test_rpc_msearch(client: TestClient): +def test_rpc_msearch(client: TestClient, user: User, packages: List[Package]): + params = {"v": 5, "type": "msearch", "arg": user.Username} with client as request: - response = request.get("/rpc?v=5&type=msearch&arg=user1") + response = request.get("/rpc", params=params) data = response.json() # user1 maintains 4 packages; assert that we got them all. assert data.get("resultcount") == 4 names = list(sorted(r.get("Name") for r in data.get("results"))) - expected_results = list(sorted([ + expected_results = [ "big-chungus", "chungy-chungus", "gluggly-chungus", "other-pkg" - ])) + ] assert names == expected_results # Search for a non-existent maintainer, giving us zero packages. - response = request.get("/rpc?v=5&type=msearch&arg=blah-blah") + params["arg"] = "blah-blah" + response = request.get("/rpc", params=params) data = response.json() assert data.get("resultcount") == 0 # A missing arg still succeeds, but it returns all orphans. # Just verify that we receive no error and the orphaned result. - response = request.get("/rpc?v=5&type=msearch") + params.pop("arg") + response = request.get("/rpc", params=params) data = response.json() assert data.get("resultcount") == 1 result = data.get("results")[0] assert result.get("Name") == "woogly-chungus" -def test_rpc_search_depends(client: TestClient): +def test_rpc_search_depends(client: TestClient, packages: List[Package], + depends: List[PackageDependency]): + params = { + "v": 5, "type": "search", "by": "depends", "arg": "chungus-depends" + } with client as request: - response = request.get( - "/rpc?v=5&type=search&by=depends&arg=chungus-depends") + response = request.get("/rpc", params=params) data = response.json() assert data.get("resultcount") == 1 result = data.get("results")[0] - assert result.get("Name") == "big-chungus" + assert result.get("Name") == packages[0].Name -def test_rpc_search_makedepends(client: TestClient): +def test_rpc_search_makedepends(client: TestClient, packages: List[Package], + depends: List[PackageDependency]): + params = { + "v": 5, + "type": "search", + "by": "makedepends", + "arg": "chungus-makedepends" + } with client as request: - response = request.get( - "/rpc?v=5&type=search&by=makedepends&arg=chungus-makedepends") + response = request.get("/rpc", params=params) data = response.json() assert data.get("resultcount") == 1 result = data.get("results")[0] - assert result.get("Name") == "big-chungus" + assert result.get("Name") == packages[0].Name -def test_rpc_search_optdepends(client: TestClient): +def test_rpc_search_optdepends(client: TestClient, packages: List[Package], + depends: List[PackageDependency]): + params = { + "v": 5, + "type": "search", + "by": "optdepends", + "arg": "chungus-optdepends" + } with client as request: - response = request.get( - "/rpc?v=5&type=search&by=optdepends&arg=chungus-optdepends") + response = request.get("/rpc", params=params) data = response.json() assert data.get("resultcount") == 1 result = data.get("results")[0] - assert result.get("Name") == "big-chungus" + assert result.get("Name") == packages[0].Name -def test_rpc_search_checkdepends(client: TestClient): +def test_rpc_search_checkdepends(client: TestClient, packages: List[Package], + depends: List[PackageDependency]): + params = { + "v": 5, + "type": "search", + "by": "checkdepends", + "arg": "chungus-checkdepends" + } with client as request: - response = request.get( - "/rpc?v=5&type=search&by=checkdepends&arg=chungus-checkdepends") + response = request.get("/rpc", params=params) data = response.json() assert data.get("resultcount") == 1 result = data.get("results")[0] - assert result.get("Name") == "big-chungus" + assert result.get("Name") == packages[0].Name def test_rpc_incorrect_by(client: TestClient): + params = {"v": 5, "type": "search", "by": "fake", "arg": "big"} with client as request: - response = request.get("/rpc?v=5&type=search&by=fake&arg=big") + response = request.get("/rpc", params=params) assert response.json().get("error") == "Incorrect by field specified." @@ -661,15 +740,20 @@ def test_rpc_jsonp_callback(client: TestClient): For end-to-end verification, the `examples/jsonp.html` file can be used to submit jsonp callback requests to the RPC. """ + params = { + "v": 5, + "type": "search", + "arg": "big", + "callback": "jsonCallback" + } with client as request: - response = request.get( - "/rpc?v=5&type=search&arg=big&callback=jsonCallback") + response = request.get("/rpc", params=params) assert response.headers.get("content-type") == "text/javascript" assert re.search(r'^/\*\*/jsonCallback\(.*\)$', response.text) is not None # Test an invalid callback name; we get an application/json error. + params["callback"] = "jsonCallback!" with client as request: - response = request.get( - "/rpc?v=5&type=search&arg=big&callback=jsonCallback!") + response = request.get("/rpc", params=params) assert response.headers.get("content-type") == "application/json" assert response.json().get("error") == "Invalid callback name." From 604df50b88ac6a1b7babfeebb7ee2ae2844fba2a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 01:49:35 -0800 Subject: [PATCH 663/844] housekeep(fastapi): rewrite test_package_comment with fixtures Signed-off-by: Kevin Morris --- test/test_package_comment.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/test/test_package_comment.py b/test/test_package_comment.py index b00e08c3..c89e23af 100644 --- a/test/test_package_comment.py +++ b/test/test_package_comment.py @@ -8,21 +8,29 @@ from aurweb.models.package_base import PackageBase from aurweb.models.package_comment import PackageComment from aurweb.models.user import User -user = pkgbase = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, pkgbase + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", AccountTypeID=USER_ID) + yield user + + +@pytest.fixture +def pkgbase(user: User) -> PackageBase: + with db.begin(): pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) + yield pkgbase -def test_package_comment_creation(): +def test_package_comment_creation(user: User, pkgbase: PackageBase): with db.begin(): package_comment = db.create(PackageComment, PackageBase=pkgbase, User=user, Comments="Test comment.", @@ -30,26 +38,28 @@ def test_package_comment_creation(): assert bool(package_comment.ID) -def test_package_comment_null_package_base_raises_exception(): +def test_package_comment_null_pkgbase_raises(user: User): with pytest.raises(IntegrityError): PackageComment(User=user, Comments="Test comment.", RenderedComment="Test rendered comment.") -def test_package_comment_null_user_raises_exception(): +def test_package_comment_null_user_raises(pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageComment(PackageBase=pkgbase, Comments="Test comment.", RenderedComment="Test rendered comment.") -def test_package_comment_null_comments_raises_exception(): +def test_package_comment_null_comments_raises(user: User, + pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageComment(PackageBase=pkgbase, User=user, RenderedComment="Test rendered comment.") -def test_package_comment_null_renderedcomment_defaults(): +def test_package_comment_null_renderedcomment_defaults(user: User, + pkgbase: PackageBase): with db.begin(): record = db.create(PackageComment, PackageBase=pkgbase, User=user, Comments="Test comment.") From 012dd24fd85d24369556d4a3eb7e0675fa0c5856 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 01:53:02 -0800 Subject: [PATCH 664/844] housekeep(fastapi): rewrite test_tu_vote with fixtures Signed-off-by: Kevin Morris --- test/test_tu_vote.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/test/test_tu_vote.py b/test/test_tu_vote.py index 1dd33387..9bb344b1 100644 --- a/test/test_tu_vote.py +++ b/test/test_tu_vote.py @@ -10,28 +10,33 @@ from aurweb.models.tu_vote import TUVote from aurweb.models.tu_voteinfo import TUVoteInfo from aurweb.models.user import User -user = tu_voteinfo = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, tu_voteinfo + return - ts = int(datetime.utcnow().timestamp()) + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", AccountTypeID=TRUSTED_USER_ID) + yield user - tu_voteinfo = db.create(TUVoteInfo, - Agenda="Blah blah.", + +@pytest.fixture +def tu_voteinfo(user: User) -> TUVoteInfo: + ts = int(datetime.utcnow().timestamp()) + with db.begin(): + tu_voteinfo = db.create(TUVoteInfo, Agenda="Blah blah.", User=user.Username, Submitted=ts, End=ts + 5, - Quorum=0.5, - Submitter=user) + Quorum=0.5, Submitter=user) + yield tu_voteinfo -def test_tu_vote_creation(): +def test_tu_vote_creation(user: User, tu_voteinfo: TUVoteInfo): with db.begin(): tu_vote = db.create(TUVote, User=user, VoteInfo=tu_voteinfo) @@ -41,11 +46,11 @@ def test_tu_vote_creation(): assert tu_vote in tu_voteinfo.tu_votes -def test_tu_vote_null_user_raises_exception(): +def test_tu_vote_null_user_raises_exception(tu_voteinfo: TUVoteInfo): with pytest.raises(IntegrityError): TUVote(VoteInfo=tu_voteinfo) -def test_tu_vote_null_voteinfo_raises_exception(): +def test_tu_vote_null_voteinfo_raises_exception(user: User): with pytest.raises(IntegrityError): TUVote(User=user) From adafa6ebc14769b8ab3d8564b3c59d6ae441b81a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 01:56:49 -0800 Subject: [PATCH 665/844] housekeep(fastapi): rewrite test_package_request with fixtures Signed-off-by: Kevin Morris --- test/test_package_request.py | 37 ++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/test/test_package_request.py b/test/test_package_request.py index 4b5dfb2b..1ba48e09 100644 --- a/test/test_package_request.py +++ b/test/test_package_request.py @@ -12,21 +12,29 @@ from aurweb.models.package_request import (ACCEPTED, ACCEPTED_ID, CLOSED, CLOSED from aurweb.models.request_type import MERGE_ID from aurweb.models.user import User -user = pkgbase = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, pkgbase + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", AccountTypeID=USER_ID) + yield user + + +@pytest.fixture +def pkgbase(user: User) -> PackageBase: + with db.begin(): pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) + yield pkgbase -def test_package_request_creation(): +def test_package_request_creation(user: User, pkgbase: PackageBase): with db.begin(): package_request = db.create(PackageRequest, ReqTypeID=MERGE_ID, User=user, PackageBase=pkgbase, @@ -45,7 +53,7 @@ def test_package_request_creation(): assert package_request in pkgbase.requests -def test_package_request_closed(): +def test_package_request_closed(user: User, pkgbase: PackageBase): ts = int(datetime.utcnow().timestamp()) with db.begin(): package_request = db.create(PackageRequest, ReqTypeID=MERGE_ID, @@ -61,49 +69,54 @@ def test_package_request_closed(): assert package_request in user.closed_requests -def test_package_request_null_request_type_raises_exception(): +def test_package_request_null_request_type_raises(user: User, + pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageRequest(User=user, PackageBase=pkgbase, PackageBaseName=pkgbase.Name, Comments=str(), ClosureComment=str()) -def test_package_request_null_user_raises_exception(): +def test_package_request_null_user_raises(pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageRequest(ReqTypeID=MERGE_ID, PackageBase=pkgbase, PackageBaseName=pkgbase.Name, Comments=str(), ClosureComment=str()) -def test_package_request_null_package_base_raises_exception(): +def test_package_request_null_package_base_raises(user: User, + pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageRequest(ReqTypeID=MERGE_ID, User=user, PackageBaseName=pkgbase.Name, Comments=str(), ClosureComment=str()) -def test_package_request_null_package_base_name_raises_exception(): +def test_package_request_null_package_base_name_raises(user: User, + pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageRequest(ReqTypeID=MERGE_ID, User=user, PackageBase=pkgbase, Comments=str(), ClosureComment=str()) -def test_package_request_null_comments_raises_exception(): +def test_package_request_null_comments_raises(user: User, + pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageRequest(ReqTypeID=MERGE_ID, User=user, PackageBase=pkgbase, PackageBaseName=pkgbase.Name, ClosureComment=str()) -def test_package_request_null_closure_comment_raises_exception(): +def test_package_request_null_closure_comment_raises(user: User, + pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageRequest(ReqTypeID=MERGE_ID, User=user, PackageBase=pkgbase, PackageBaseName=pkgbase.Name, Comments=str()) -def test_package_request_status_display(): +def test_package_request_status_display(user: User, pkgbase: PackageBase): """ Test status_display() based on the Status column value. """ with db.begin(): pkgreq = db.create(PackageRequest, ReqTypeID=MERGE_ID, From 735c5f57cb467174f966c5030c6f13cb5481756b Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 16:25:04 -0800 Subject: [PATCH 666/844] housekeep(fastapi): rewrite test_package_blacklist Signed-off-by: Kevin Morris --- test/test_package_blacklist.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/test/test_package_blacklist.py b/test/test_package_blacklist.py index 6f4c36d7..427c3be4 100644 --- a/test/test_package_blacklist.py +++ b/test/test_package_blacklist.py @@ -3,21 +3,12 @@ import pytest from sqlalchemy.exc import IntegrityError from aurweb import db -from aurweb.models.package_base import PackageBase from aurweb.models.package_blacklist import PackageBlacklist -from aurweb.models.user import User - -user = pkgbase = None @pytest.fixture(autouse=True) def setup(db_test): - global user, pkgbase - - with db.begin(): - user = db.create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword") - pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) + return def test_package_blacklist_creation(): From d6cb3b9fac73d531b399ddcb86710df765260bc5 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 16:30:33 -0800 Subject: [PATCH 667/844] housekeep(fastapi): rewrite test_auth with fixtures Signed-off-by: Kevin Morris --- test/test_auth.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/test/test_auth.py b/test/test_auth.py index 0dc26f86..b63fb96f 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -11,35 +11,40 @@ from aurweb.models.session import Session from aurweb.models.user import User from aurweb.testing.requests import Request -user = backend = request = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, backend, request + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.com", RealName="Test User", Passwd="testPassword", AccountTypeID=USER_ID) + yield user - backend = BasicAuthBackend() - request = Request() + +@pytest.fixture +def backend() -> BasicAuthBackend: + yield BasicAuthBackend() @pytest.mark.asyncio -async def test_auth_backend_missing_sid(): +async def test_auth_backend_missing_sid(backend: BasicAuthBackend): # The request has no AURSID cookie, so authentication fails, and # AnonymousUser is returned. - _, result = await backend.authenticate(request) + _, result = await backend.authenticate(Request()) assert not result.is_authenticated() @pytest.mark.asyncio -async def test_auth_backend_invalid_sid(): +async def test_auth_backend_invalid_sid(backend: BasicAuthBackend): # Provide a fake AURSID that won't be found in the database. # This results in our path going down the invalid sid route, # which gives us an AnonymousUser. + request = Request() request.cookies["AURSID"] = "fake" _, result = await backend.authenticate(request) assert not result.is_authenticated() @@ -55,13 +60,15 @@ async def test_auth_backend_invalid_user_id(): @pytest.mark.asyncio -async def test_basic_auth_backend(): +async def test_basic_auth_backend(user: User, backend: BasicAuthBackend): # This time, everything matches up. We expect the user to # equal the real_user. now_ts = datetime.utcnow().timestamp() with db.begin(): db.create(Session, UsersID=user.ID, SessionID="realSession", LastUpdateTS=now_ts + 5) + + request = Request() request.cookies["AURSID"] = "realSession" _, result = await backend.authenticate(request) assert result == user From 91f65911414423fc9dae8cf509beb57e3fadabea Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 16:35:52 -0800 Subject: [PATCH 668/844] housekeep(fastapi): rewrite test_accepted_term with fixtures Signed-off-by: Kevin Morris --- test/test_accepted_term.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/test/test_accepted_term.py b/test/test_accepted_term.py index de18c61a..2af7127b 100644 --- a/test/test_accepted_term.py +++ b/test/test_accepted_term.py @@ -8,39 +8,48 @@ from aurweb.models.account_type import USER_ID from aurweb.models.term import Term from aurweb.models.user import User -user = term = accepted_term = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, term + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", AccountTypeID=USER_ID) + yield user + +@pytest.fixture +def term() -> Term: + with db.begin(): term = db.create(Term, Description="Test term", URL="https://test.term") - yield term -def test_accepted_term(): +@pytest.fixture +def accepted_term(user: User, term: Term) -> AcceptedTerm: with db.begin(): accepted_term = db.create(AcceptedTerm, User=user, Term=term) + yield accepted_term + +def test_accepted_term(user: User, term: Term, accepted_term: AcceptedTerm): # Make sure our AcceptedTerm relationships got initialized properly. assert accepted_term.User == user assert accepted_term in user.accepted_terms assert accepted_term in term.accepted_terms -def test_accepted_term_null_user_raises_exception(): +def test_accepted_term_null_user_raises_exception(term: Term): with pytest.raises(IntegrityError): AcceptedTerm(Term=term) -def test_accepted_term_null_term_raises_exception(): +def test_accepted_term_null_term_raises_exception(user: User): with pytest.raises(IntegrityError): AcceptedTerm(User=user) From b20ec9925a5f92c020b8705bc733cdc4831ca5e4 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 17:11:10 -0800 Subject: [PATCH 669/844] housekeep(fastapi): rewrite test_ssh_pub_key with fixtures Signed-off-by: Kevin Morris --- test/test_ssh_pub_key.py | 48 ++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/test/test_ssh_pub_key.py b/test/test_ssh_pub_key.py index e17af5a7..68b6e7a0 100644 --- a/test/test_ssh_pub_key.py +++ b/test/test_ssh_pub_key.py @@ -16,47 +16,53 @@ lfv98Kr0NUp51zpf55Arxn9j0Rz9xTA7FiODQgCn6iQ0SDtzUNL0IKTCw26xJY5gzMxbfpvzPQGeul\ x/ioM= kevr@volcano """ -user = ssh_pub_key = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, ssh_pub_key + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", AccountTypeID=USER_ID) + yield user + +@pytest.fixture +def pubkey(user: User) -> SSHPubKey: with db.begin(): - ssh_pub_key = db.create(SSHPubKey, UserID=user.ID, - Fingerprint="testFingerprint", - PubKey="testPubKey") + pubkey = db.create(SSHPubKey, User=user, + Fingerprint="testFingerprint", + PubKey="testPubKey") + yield pubkey -def test_ssh_pub_key(): - assert ssh_pub_key.UserID == user.ID - assert ssh_pub_key.User == user - assert ssh_pub_key.Fingerprint == "testFingerprint" - assert ssh_pub_key.PubKey == "testPubKey" +def test_pubkey(user: User, pubkey: SSHPubKey): + assert pubkey.UserID == user.ID + assert pubkey.User == user + assert pubkey.Fingerprint == "testFingerprint" + assert pubkey.PubKey == "testPubKey" -def test_ssh_pub_key_cs(): +def test_pubkey_cs(user: User): """ Test case sensitivity of the database table. """ with db.begin(): - ssh_pub_key_cs = db.create(SSHPubKey, UserID=user.ID, - Fingerprint="TESTFINGERPRINT", - PubKey="TESTPUBKEY") + pubkey_cs = db.create(SSHPubKey, User=user, + Fingerprint="TESTFINGERPRINT", + PubKey="TESTPUBKEY") - assert ssh_pub_key_cs.Fingerprint == "TESTFINGERPRINT" - assert ssh_pub_key_cs.PubKey == "TESTPUBKEY" - assert ssh_pub_key.Fingerprint == "testFingerprint" - assert ssh_pub_key.PubKey == "testPubKey" + assert pubkey_cs.Fingerprint == "TESTFINGERPRINT" + assert pubkey_cs.Fingerprint != "testFingerprint" + assert pubkey_cs.PubKey == "TESTPUBKEY" + assert pubkey_cs.PubKey != "testPubKey" -def test_ssh_pub_key_fingerprint(): +def test_pubkey_fingerprint(): assert get_fingerprint(TEST_SSH_PUBKEY) is not None -def test_ssh_pub_key_invalid_fingerprint(): +def test_pubkey_invalid_fingerprint(): assert get_fingerprint("ssh-rsa fake and invalid") is None From a082de5244f55aa4098608e2e6b3341463669c01 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 17:21:42 -0800 Subject: [PATCH 670/844] housekeep(fastapi): rewrite test_package_keyword with fixtures Signed-off-by: Kevin Morris --- test/test_package_keyword.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/test/test_package_keyword.py b/test/test_package_keyword.py index 88ccb734..ff466efc 100644 --- a/test/test_package_keyword.py +++ b/test/test_package_keyword.py @@ -8,26 +8,32 @@ from aurweb.models.package_base import PackageBase from aurweb.models.package_keyword import PackageKeyword from aurweb.models.user import User -user = pkgbase = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, pkgbase + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", AccountTypeID=USER_ID) - pkgbase = db.create(PackageBase, - Name="beautiful-package", - Maintainer=user) + yield user -def test_package_keyword(): +@pytest.fixture +def pkgbase(user: User) -> PackageBase: with db.begin(): - pkg_keyword = db.create(PackageKeyword, - PackageBase=pkgbase, + pkgbase = db.create(PackageBase, Name="beautiful-package", + Maintainer=user) + yield pkgbase + + +def test_package_keyword(pkgbase: PackageBase): + with db.begin(): + pkg_keyword = db.create(PackageKeyword, PackageBase=pkgbase, Keyword="test") assert pkg_keyword in pkgbase.keywords assert pkgbase == pkg_keyword.PackageBase From 655b98d19e5e82817146f3f8a00215d10fd0da0a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 17:29:03 -0800 Subject: [PATCH 671/844] housekeep(fastapi): rewrite test_package_license with fixtures Signed-off-by: Kevin Morris --- test/test_package_license.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/test/test_package_license.py b/test/test_package_license.py index 965d0c6f..c43423b8 100644 --- a/test/test_package_license.py +++ b/test/test_package_license.py @@ -10,25 +10,37 @@ from aurweb.models.package_base import PackageBase from aurweb.models.package_license import PackageLicense from aurweb.models.user import User -user = license = pkgbase = package = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, license, pkgbase, package + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", AccountTypeID=USER_ID) - license = db.create(License, Name="Test License") + yield user + +@pytest.fixture +def license() -> License: + with db.begin(): + license = db.create(License, Name="Test License") + yield license + + +@pytest.fixture +def package(user: User, license: License): with db.begin(): pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) package = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + yield package -def test_package_license(): +def test_package_license(license: License, package: Package): with db.begin(): package_license = db.create(PackageLicense, Package=package, License=license) @@ -36,11 +48,11 @@ def test_package_license(): assert package_license.Package == package -def test_package_license_null_package_raises_exception(): +def test_package_license_null_package_raises(license: License): with pytest.raises(IntegrityError): PackageLicense(License=license) -def test_package_license_null_license_raises_exception(): +def test_package_license_null_license_raises(package: Package): with pytest.raises(IntegrityError): PackageLicense(Package=package) From ff3931e43506a466d01dde48fa647c4d13740ce3 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 18:10:40 -0800 Subject: [PATCH 672/844] housekeep(fastapi): rewrite test_package_notification with fixtures Signed-off-by: Kevin Morris --- test/test_package_notification.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/test/test_package_notification.py b/test/test_package_notification.py index 2e505dd8..e7a72a43 100644 --- a/test/test_package_notification.py +++ b/test/test_package_notification.py @@ -7,20 +7,28 @@ from aurweb.models.package_base import PackageBase from aurweb.models.package_notification import PackageNotification from aurweb.models.user import User -user = pkgbase = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, pkgbase + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword") + yield user + + +@pytest.fixture +def pkgbase(user: User) -> PackageBase: + with db.begin(): pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) + yield pkgbase -def test_package_notification_creation(): +def test_package_notification_creation(user: User, pkgbase: PackageBase): with db.begin(): package_notification = db.create( PackageNotification, User=user, PackageBase=pkgbase) @@ -29,11 +37,11 @@ def test_package_notification_creation(): assert package_notification.PackageBase == pkgbase -def test_package_notification_null_user_raises_exception(): +def test_package_notification_null_user_raises(pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageNotification(PackageBase=pkgbase) -def test_package_notification_null_pkgbase_raises_exception(): +def test_package_notification_null_pkgbase_raises(user: User): with pytest.raises(IntegrityError): PackageNotification(User=user) From 14d80d756fe73039d7e5b7836ad1c343668a7e9f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 18:18:48 -0800 Subject: [PATCH 673/844] housekeep(fastapi): rewrite test_package_comaintainer with fixtures Signed-off-by: Kevin Morris --- test/test_package_comaintainer.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/test/test_package_comaintainer.py b/test/test_package_comaintainer.py index ff74cddf..e377edc0 100644 --- a/test/test_package_comaintainer.py +++ b/test/test_package_comaintainer.py @@ -3,24 +3,34 @@ import pytest from sqlalchemy.exc import IntegrityError from aurweb import db +from aurweb.models.account_type import USER_ID from aurweb.models.package_base import PackageBase from aurweb.models.package_comaintainer import PackageComaintainer from aurweb.models.user import User -user = pkgbase = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, pkgbase + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword") + RealName="Test User", Passwd="testPassword", + AccountTypeID=USER_ID) + yield user + + +@pytest.fixture +def pkgbase(user: User) -> PackageBase: + with db.begin(): pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) + yield pkgbase -def test_package_comaintainer_creation(): +def test_package_comaintainer_creation(user: User, pkgbase: PackageBase): with db.begin(): package_comaintainer = db.create(PackageComaintainer, User=user, PackageBase=pkgbase, Priority=5) @@ -30,16 +40,17 @@ def test_package_comaintainer_creation(): assert package_comaintainer.Priority == 5 -def test_package_comaintainer_null_user_raises_exception(): +def test_package_comaintainer_null_user_raises(pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageComaintainer(PackageBase=pkgbase, Priority=1) -def test_package_comaintainer_null_pkgbase_raises_exception(): +def test_package_comaintainer_null_pkgbase_raises(user: User): with pytest.raises(IntegrityError): PackageComaintainer(User=user, Priority=1) -def test_package_comaintainer_null_priority_raises_exception(): +def test_package_comaintainer_null_priority_raises(user: User, + pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageComaintainer(User=user, PackageBase=pkgbase) From 31a093ba063168825595fe82d3c42d34d9d67a15 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 18:51:50 -0800 Subject: [PATCH 674/844] housekeep(fastapi): rewrite test_package_relation with fixtures Signed-off-by: Kevin Morris --- test/test_package_relation.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/test/test_package_relation.py b/test/test_package_relation.py index e5f7f453..6e9a5545 100644 --- a/test/test_package_relation.py +++ b/test/test_package_relation.py @@ -10,28 +10,32 @@ from aurweb.models.package_relation import PackageRelation from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID from aurweb.models.user import User -user = pkgbase = package = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, pkgbase, package + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", AccountTypeID=USER_ID) - pkgbase = db.create(PackageBase, - Name="test-package", - Maintainer=user) - package = db.create(Package, - PackageBase=pkgbase, - Name=pkgbase.Name, + yield user + + +@pytest.fixture +def package(user: User) -> Package: + with db.begin(): + pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) + package = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name, Description="Test description.", URL="https://test.package") + yield package -def test_package_relation(): +def test_package_relation(package: Package): with db.begin(): pkgrel = db.create(PackageRelation, Package=package, RelTypeID=CONFLICTS_ID, @@ -48,16 +52,16 @@ def test_package_relation(): pkgrel.RelTypeID = REPLACES_ID -def test_package_relation_null_package_raises_exception(): +def test_package_relation_null_package_raises(): with pytest.raises(IntegrityError): PackageRelation(RelTypeID=CONFLICTS_ID, RelName="test-relation") -def test_package_relation_null_relation_type_raises_exception(): +def test_package_relation_null_relation_type_raises(package: Package): with pytest.raises(IntegrityError): PackageRelation(Package=package, RelName="test-relation") -def test_package_relation_null_relname_raises_exception(): +def test_package_relation_null_relname_raises(package: Package): with pytest.raises(IntegrityError): PackageRelation(Package=package, RelTypeID=CONFLICTS_ID) From a0e1a1641d08d533b1919ffb60298476f21de237 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 20:12:06 -0800 Subject: [PATCH 675/844] fix(fastapi): support UsersID and User columns in the Session model Signed-off-by: Kevin Morris --- aurweb/models/session.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/aurweb/models/session.py b/aurweb/models/session.py index 7a06eddc..37ab4bce 100644 --- a/aurweb/models/session.py +++ b/aurweb/models/session.py @@ -18,10 +18,16 @@ class Session(Base): def __init__(self, **kwargs): super().__init__(**kwargs) - user_exists = db.query( - db.query(_User).filter(_User.ID == self.UsersID).exists() - ).scalar() - if not user_exists: + # We'll try to either use UsersID or User.ID if we can. + # If neither exist, an AttributeError is raised, in which case + # we set the uid to 0, which triggers IntegrityError below. + try: + uid = self.UsersID or self.User.ID + except AttributeError: + uid = 0 + + user_exists = db.query(_User).filter(_User.ID == uid).exists() + if not db.query(user_exists).scalar(): raise IntegrityError( statement=("Foreign key UsersID cannot be null and " "must be a valid user's ID."), From ca25595022e4a5c525ecce9de0c009bf19c6fd2c Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 20:12:31 -0800 Subject: [PATCH 676/844] housekeep(fastapi): rewrite test_sesion with fixtures Also, added a new test function which tests the IntegrityError exception. Signed-off-by: Kevin Morris --- test/test_session.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/test/test_session.py b/test/test_session.py index 7d3037a1..67b1ada0 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -4,31 +4,37 @@ from unittest import mock import pytest +from sqlalchemy.exc import IntegrityError + from aurweb import db -from aurweb.models.account_type import AccountType +from aurweb.models.account_type import USER_ID from aurweb.models.session import Session, generate_unique_sid from aurweb.models.user import User -account_type = user = session = None - @pytest.fixture(autouse=True) def setup(db_test): - global account_type, user, session + return - account_type = db.query(AccountType, - AccountType.AccountType == "User").first() + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", ResetKey="testReset", Passwd="testPassword", - AccountType=account_type) + AccountTypeID=USER_ID) + yield user + +@pytest.fixture +def session(user: User) -> Session: with db.begin(): - session = db.create(Session, UsersID=user.ID, SessionID="testSession", + session = db.create(Session, User=user, SessionID="testSession", LastUpdateTS=datetime.utcnow().timestamp()) + yield session -def test_session(): +def test_session(user: User, session: Session): assert session.SessionID == "testSession" assert session.UsersID == user.ID @@ -38,22 +44,27 @@ def test_session_cs(): with db.begin(): user2 = db.create(User, Username="test2", Email="test2@example.org", ResetKey="testReset2", Passwd="testPassword", - AccountType=account_type) + AccountTypeID=USER_ID) with db.begin(): - session_cs = db.create(Session, UsersID=user2.ID, - SessionID="TESTSESSION", + session_cs = db.create(Session, User=user2, SessionID="TESTSESSION", LastUpdateTS=datetime.utcnow().timestamp()) + assert session_cs.SessionID == "TESTSESSION" - assert session.SessionID == "testSession" + assert session_cs.SessionID != "testSession" -def test_session_user_association(): +def test_session_user_association(user: User, session: Session): # Make sure that the Session user attribute is correct. assert session.User == user -def test_generate_unique_sid(): +def test_session_null_user_raises(): + with pytest.raises(IntegrityError): + Session() + + +def test_generate_unique_sid(session: Session): # Mock up aurweb.models.session.generate_sid by returning # sids[i % 2] from 0 .. n. This will swap between each sid # between each call. From ae728179506bdd39ac0c929f36324626cea53a91 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 20:17:14 -0800 Subject: [PATCH 677/844] housekeep(fastapi): rewrite test_routes with fixtures Signed-off-by: Kevin Morris --- test/test_routes.py | 46 +++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/test/test_routes.py b/test/test_routes.py index 32f507f3..85d30c02 100644 --- a/test/test_routes.py +++ b/test/test_routes.py @@ -14,30 +14,34 @@ from aurweb.models.account_type import USER_ID from aurweb.models.user import User from aurweb.testing.requests import Request -user = client = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, client + return + +@pytest.fixture +def client() -> TestClient: + yield TestClient(app=app) + + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", AccountTypeID=USER_ID) - - client = TestClient(app) + yield user -def test_index(): +def test_index(client: TestClient): """ Test the index route at '/'. """ - # Use `with` to trigger FastAPI app events. with client as req: response = req.get("/") assert response.status_code == int(HTTPStatus.OK) -def test_index_security_headers(): +def test_index_security_headers(client: TestClient): """ Check for the existence of CSP, XCTO, XFO and RP security headers. CSP: Content-Security-Policy @@ -55,15 +59,16 @@ def test_index_security_headers(): assert response.headers.get("X-Frame-Options") == "SAMEORIGIN" -def test_favicon(): +def test_favicon(client: TestClient): """ Test the favicon route at '/favicon.ico'. """ - response1 = client.get("/static/images/favicon.ico") - response2 = client.get("/favicon.ico") + with client as request: + response1 = request.get("/static/images/favicon.ico") + response2 = request.get("/favicon.ico") assert response1.status_code == int(HTTPStatus.OK) assert response1.content == response2.content -def test_language(): +def test_language(client: TestClient): """ Test the language post route as a guest user. """ post_data = { "set_lang": "de", @@ -74,7 +79,7 @@ def test_language(): assert response.status_code == int(HTTPStatus.SEE_OTHER) -def test_language_invalid_next(): +def test_language_invalid_next(client: TestClient): """ Test an invalid next route at '/language'. """ post_data = { "set_lang": "de", @@ -85,7 +90,7 @@ def test_language_invalid_next(): assert response.status_code == int(HTTPStatus.BAD_REQUEST) -def test_user_language(): +def test_user_language(client: TestClient, user: User): """ Test the language post route as an authenticated user. """ post_data = { "set_lang": "de", @@ -102,7 +107,7 @@ def test_user_language(): assert user.LangPreference == "de" -def test_language_query_params(): +def test_language_query_params(client: TestClient): """ Test the language post route with query params. """ next = urllib.parse.quote_plus("/") post_data = { @@ -117,14 +122,15 @@ def test_language_query_params(): assert response.status_code == int(HTTPStatus.SEE_OTHER) -def test_error_messages(): - response1 = client.get("/thisroutedoesnotexist") - response2 = client.get("/raisefivethree") +def test_error_messages(client: TestClient): + with client as request: + response1 = request.get("/thisroutedoesnotexist") + response2 = request.get("/raisefivethree") assert response1.status_code == int(HTTPStatus.NOT_FOUND) assert response2.status_code == int(HTTPStatus.SERVICE_UNAVAILABLE) -def test_nonce_csp(): +def test_nonce_csp(client: TestClient): with client as request: response = request.get("/") data = response.headers.get("Content-Security-Policy") @@ -146,7 +152,7 @@ def test_nonce_csp(): assert nonce_verified is True -def test_id_redirect(): +def test_id_redirect(client: TestClient): with client as request: response = request.get("/", params={ "id": "test", # This param will be rewritten into Location. From 93bc91cce252ab789493aa5d0bc0c796f5133ae2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 20:25:10 -0800 Subject: [PATCH 678/844] housekeep(fastapi): rewrite test_tu_voteinfo with fixtures Signed-off-by: Kevin Morris --- test/test_tu_voteinfo.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/test/test_tu_voteinfo.py b/test/test_tu_voteinfo.py index 5926fbf9..26fa9522 100644 --- a/test/test_tu_voteinfo.py +++ b/test/test_tu_voteinfo.py @@ -5,27 +5,27 @@ import pytest from sqlalchemy.exc import IntegrityError from aurweb import db -from aurweb.db import create, query, rollback -from aurweb.models.account_type import AccountType +from aurweb.db import create, rollback +from aurweb.models.account_type import TRUSTED_USER_ID from aurweb.models.tu_voteinfo import TUVoteInfo from aurweb.models.user import User -user = None - @pytest.fixture(autouse=True) def setup(db_test): - global user + return - tu_type = query(AccountType, - AccountType.AccountType == "Trusted User").first() + +@pytest.fixture +def user() -> User: with db.begin(): user = create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", - AccountType=tu_type) + AccountTypeID=TRUSTED_USER_ID) + yield user -def test_tu_voteinfo_creation(): +def test_tu_voteinfo_creation(user: User): ts = int(datetime.utcnow().timestamp()) with db.begin(): tu_voteinfo = create(TUVoteInfo, @@ -49,7 +49,7 @@ def test_tu_voteinfo_creation(): assert tu_voteinfo in user.tu_voteinfo_set -def test_tu_voteinfo_is_running(): +def test_tu_voteinfo_is_running(user: User): ts = int(datetime.utcnow().timestamp()) with db.begin(): tu_voteinfo = create(TUVoteInfo, @@ -65,7 +65,7 @@ def test_tu_voteinfo_is_running(): assert tu_voteinfo.is_running() is False -def test_tu_voteinfo_total_votes(): +def test_tu_voteinfo_total_votes(user: User): ts = int(datetime.utcnow().timestamp()) with db.begin(): tu_voteinfo = create(TUVoteInfo, @@ -83,7 +83,7 @@ def test_tu_voteinfo_total_votes(): assert tu_voteinfo.total_votes() == 9 -def test_tu_voteinfo_null_submitter_raises_exception(): +def test_tu_voteinfo_null_submitter_raises(user: User): with pytest.raises(IntegrityError): with db.begin(): create(TUVoteInfo, @@ -94,7 +94,7 @@ def test_tu_voteinfo_null_submitter_raises_exception(): rollback() -def test_tu_voteinfo_null_agenda_raises_exception(): +def test_tu_voteinfo_null_agenda_raises(user: User): with pytest.raises(IntegrityError): with db.begin(): create(TUVoteInfo, @@ -105,7 +105,7 @@ def test_tu_voteinfo_null_agenda_raises_exception(): rollback() -def test_tu_voteinfo_null_user_raises_exception(): +def test_tu_voteinfo_null_user_raises(user: User): with pytest.raises(IntegrityError): with db.begin(): create(TUVoteInfo, @@ -116,7 +116,7 @@ def test_tu_voteinfo_null_user_raises_exception(): rollback() -def test_tu_voteinfo_null_submitted_raises_exception(): +def test_tu_voteinfo_null_submitted_raises(user: User): with pytest.raises(IntegrityError): with db.begin(): create(TUVoteInfo, @@ -128,7 +128,7 @@ def test_tu_voteinfo_null_submitted_raises_exception(): rollback() -def test_tu_voteinfo_null_end_raises_exception(): +def test_tu_voteinfo_null_end_raises(user: User): with pytest.raises(IntegrityError): with db.begin(): create(TUVoteInfo, @@ -140,7 +140,7 @@ def test_tu_voteinfo_null_end_raises_exception(): rollback() -def test_tu_voteinfo_null_quorum_raises_exception(): +def test_tu_voteinfo_null_quorum_raises(user: User): with pytest.raises(IntegrityError): with db.begin(): create(TUVoteInfo, From 171b347dadddf92f90e16f98fab388112174053a Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 20:38:49 -0800 Subject: [PATCH 679/844] housekeep(fastapi): rewrite test_package_base with fixtures Signed-off-by: Kevin Morris --- test/test_package_base.py | 57 +++++++++++++++------------------------ 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/test/test_package_base.py b/test/test_package_base.py index 8e4b2edf..5be7e40b 100644 --- a/test/test_package_base.py +++ b/test/test_package_base.py @@ -2,35 +2,36 @@ import pytest from sqlalchemy.exc import IntegrityError -import aurweb.config - from aurweb import db -from aurweb.models.account_type import AccountType +from aurweb.models.account_type import USER_ID from aurweb.models.package_base import PackageBase from aurweb.models.user import User -user = None - @pytest.fixture(autouse=True) def setup(db_test): - global user + return - account_type = db.query(AccountType, - AccountType.AccountType == "User").first() + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", - AccountType=account_type) + AccountTypeID=USER_ID) + yield user -def test_package_base(): +@pytest.fixture +def pkgbase(user: User) -> PackageBase: with db.begin(): - pkgbase = db.create(PackageBase, - Name="beautiful-package", + pkgbase = db.create(PackageBase, Name="beautiful-package", Maintainer=user) - assert pkgbase in user.maintained_bases + yield pkgbase + +def test_package_base(user: User, pkgbase: PackageBase): + assert pkgbase in user.maintained_bases assert not pkgbase.OutOfDateTS assert pkgbase.SubmittedTS > 0 assert pkgbase.ModifiedTS > 0 @@ -42,33 +43,19 @@ def test_package_base(): assert pkgbase.Popularity == 0.0 -def test_package_base_ci(): +def test_package_base_ci(user: User, pkgbase: PackageBase): """ Test case insensitivity of the database table. """ - if aurweb.config.get("database", "backend") == "sqlite": - return None # SQLite doesn't seem handle this. - - with db.begin(): - pkgbase = db.create(PackageBase, - Name="beautiful-package", - Maintainer=user) - assert bool(pkgbase.ID) - with pytest.raises(IntegrityError): with db.begin(): - db.create(PackageBase, - Name="Beautiful-Package", - Maintainer=user) + db.create(PackageBase, Name=pkgbase.Name.upper(), Maintainer=user) db.rollback() -def test_package_base_relationships(): +def test_package_base_relationships(user: User, pkgbase: PackageBase): with db.begin(): - pkgbase = db.create(PackageBase, - Name="beautiful-package", - Flagger=user, - Maintainer=user, - Submitter=user, - Packager=user) + pkgbase.Flagger = user + pkgbase.Submitter = user + pkgbase.Packager = user assert pkgbase in user.flagged_bases assert pkgbase in user.maintained_bases assert pkgbase in user.submitted_bases @@ -77,6 +64,4 @@ def test_package_base_relationships(): def test_package_base_null_name_raises_exception(): with pytest.raises(IntegrityError): - with db.begin(): - db.create(PackageBase) - db.rollback() + PackageBase() From df530d8a7358f71c6579e6bfd0fea1143dfc89b7 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 20:42:50 -0800 Subject: [PATCH 680/844] housekeep(fastapi): rewrite test_package_source with fixtures Signed-off-by: Kevin Morris --- test/test_package_source.py | 51 ++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/test/test_package_source.py b/test/test_package_source.py index b83c9d48..e5797f90 100644 --- a/test/test_package_source.py +++ b/test/test_package_source.py @@ -2,46 +2,45 @@ import pytest from sqlalchemy.exc import IntegrityError -from aurweb.db import begin, create, query, rollback -from aurweb.models.account_type import AccountType +from aurweb import db +from aurweb.models.account_type import USER_ID from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.package_source import PackageSource from aurweb.models.user import User -from aurweb.testing import setup_test_db - -user = pkgbase = package = None @pytest.fixture(autouse=True) def setup(db_test): - global user, pkgbase, package - - setup_test_db("PackageSources", "Packages", "PackageBases", "Users") - - account_type = query(AccountType, - AccountType.AccountType == "User").first() - with begin(): - user = create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - pkgbase = create(PackageBase, - Name="test-package", - Maintainer=user) - package = create(Package, PackageBase=pkgbase, Name="test-package") + return -def test_package_source(): - with begin(): - pkgsource = create(PackageSource, Package=package) +@pytest.fixture +def user() -> User: + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + RealName="Test User", Passwd="testPassword", + AccountTypeID=USER_ID) + yield user + + +@pytest.fixture +def package(user: User) -> Package: + with db.begin(): + pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) + package = db.create(Package, PackageBase=pkgbase, Name="test-package") + yield package + + +def test_package_source(package: Package): + with db.begin(): + pkgsource = db.create(PackageSource, Package=package) assert pkgsource.Package == package # By default, PackageSources.Source assigns the string '/dev/null'. assert pkgsource.Source == "/dev/null" assert pkgsource.SourceArch is None -def test_package_source_null_package_raises_exception(): +def test_package_source_null_package_raises(): with pytest.raises(IntegrityError): - with begin(): - create(PackageSource) - rollback() + PackageSource() From 150c944758bb67f97625c566cd635500d27ef7ef Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 20:45:08 -0800 Subject: [PATCH 681/844] housekeep(fastapi): rewrite test_package_group with fixtures Signed-off-by: Kevin Morris --- test/test_package_group.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/test/test_package_group.py b/test/test_package_group.py index 2c91e0b1..0cb83ee2 100644 --- a/test/test_package_group.py +++ b/test/test_package_group.py @@ -10,36 +10,48 @@ from aurweb.models.package_base import PackageBase from aurweb.models.package_group import PackageGroup from aurweb.models.user import User -user = group = pkgbase = package = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, group, pkgbase, package + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", AccountTypeID=USER_ID) - group = db.create(Group, Name="Test Group") + yield user + +@pytest.fixture +def group() -> Group: + with db.begin(): + group = db.create(Group, Name="Test Group") + yield group + + +@pytest.fixture +def package(user: User) -> Package: with db.begin(): pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) package = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + yield package -def test_package_group(): +def test_package_group(package: Package, group: Group): with db.begin(): package_group = db.create(PackageGroup, Package=package, Group=group) assert package_group.Group == group assert package_group.Package == package -def test_package_group_null_package_raises_exception(): +def test_package_group_null_package_raises(group: Group): with pytest.raises(IntegrityError): PackageGroup(Group=group) -def test_package_group_null_group_raises_exception(): +def test_package_group_null_group_raises(package: Package): with pytest.raises(IntegrityError): PackageGroup(Package=package) From 05bd6e9076d03006358c6862bfb2d95d78eb93b9 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 20:48:01 -0800 Subject: [PATCH 682/844] housekeep(fastapi): rewrite test_package_vote with fixtures Signed-off-by: Kevin Morris --- test/test_package_vote.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/test/test_package_vote.py b/test/test_package_vote.py index d1ec203b..08edb92d 100644 --- a/test/test_package_vote.py +++ b/test/test_package_vote.py @@ -5,24 +5,34 @@ import pytest from sqlalchemy.exc import IntegrityError from aurweb import db +from aurweb.models.account_type import USER_ID from aurweb.models.package_base import PackageBase from aurweb.models.package_vote import PackageVote from aurweb.models.user import User -user = pkgbase = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, pkgbase + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword") + RealName="Test User", Passwd=str(), + AccountTypeID=USER_ID) + yield user + + +@pytest.fixture +def pkgbase(user: User) -> PackageBase: + with db.begin(): pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) + yield pkgbase -def test_package_vote_creation(): +def test_package_vote_creation(user: User, pkgbase: PackageBase): ts = int(datetime.utcnow().timestamp()) with db.begin(): @@ -34,16 +44,16 @@ def test_package_vote_creation(): assert package_vote.VoteTS == ts -def test_package_vote_null_user_raises_exception(): +def test_package_vote_null_user_raises(pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageVote(PackageBase=pkgbase, VoteTS=1) -def test_package_vote_null_pkgbase_raises_exception(): +def test_package_vote_null_pkgbase_raises(user: User): with pytest.raises(IntegrityError): PackageVote(User=user, VoteTS=1) -def test_package_vote_null_votets_raises_exception(): +def test_package_vote_null_votets_raises(user: User, pkgbase: PackageBase): with pytest.raises(IntegrityError): PackageVote(User=user, PackageBase=pkgbase) From 140f9b1fb225e00163662290de426e8c2a064264 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 20:50:17 -0800 Subject: [PATCH 683/844] housekeep(fastapi): rewrite test_package_dependency with fixtures Signed-off-by: Kevin Morris --- test/test_package_dependency.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/test/test_package_dependency.py b/test/test_package_dependency.py index e6125669..7297abe4 100644 --- a/test/test_package_dependency.py +++ b/test/test_package_dependency.py @@ -10,28 +10,32 @@ from aurweb.models.package_base import PackageBase from aurweb.models.package_dependency import PackageDependency from aurweb.models.user import User -user = pkgbase = package = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, pkgbase, package + return + +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", - RealName="Test User", Passwd="testPassword", + RealName="Test User", Passwd=str(), AccountTypeID=USER_ID) - pkgbase = db.create(PackageBase, - Name="test-package", - Maintainer=user) - package = db.create(Package, - PackageBase=pkgbase, - Name=pkgbase.Name, + yield user + + +@pytest.fixture +def package(user: User) -> Package: + with db.begin(): + pkgbase = db.create(PackageBase, Name="test-package", Maintainer=user) + package = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name, Description="Test description.", URL="https://test.package") + yield package -def test_package_dependencies(): +def test_package_dependencies(user: User, package: Package): with db.begin(): pkgdep = db.create(PackageDependency, Package=package, DepTypeID=DEPENDS_ID, DepName="test-dep") @@ -57,16 +61,16 @@ def test_package_dependencies(): assert pkgdep.is_package() -def test_package_dependencies_null_package_raises_exception(): +def test_package_dependencies_null_package_raises(): with pytest.raises(IntegrityError): PackageDependency(DepTypeID=DEPENDS_ID, DepName="test-dep") -def test_package_dependencies_null_dependency_type_raises_exception(): +def test_package_dependencies_null_dependency_type_raises(package: Package): with pytest.raises(IntegrityError): PackageDependency(Package=package, DepName="test-dep") -def test_package_dependencies_null_depname_raises_exception(): +def test_package_dependencies_null_depname_raises(package: Package): with pytest.raises(IntegrityError): PackageDependency(DepTypeID=DEPENDS_ID, Package=package) From 5b14ad406560b87773b7bf81b0ef0fbb6a362898 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 21:16:49 -0800 Subject: [PATCH 684/844] housekeep(fastapi): rewrite test_user with fixtures Signed-off-by: Kevin Morris --- test/test_user.py | 110 +++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 66 deletions(-) diff --git a/test/test_user.py b/test/test_user.py index dbb45166..52cdc89e 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -8,10 +8,10 @@ import pytest import aurweb.auth import aurweb.config +import aurweb.models.account_type as at from aurweb import db from aurweb.auth import creds -from aurweb.models.account_type import AccountType from aurweb.models.ban import Ban from aurweb.models.package import Package from aurweb.models.package_base import PackageBase @@ -22,23 +22,30 @@ from aurweb.models.ssh_pub_key import SSHPubKey from aurweb.models.user import User from aurweb.testing.requests import Request -account_type = user = None - @pytest.fixture(autouse=True) def setup(db_test): - global account_type, user + return - account_type = db.query(AccountType, - AccountType.AccountType == "User").first() +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", - AccountType=account_type) + AccountTypeID=at.USER_ID) + yield user -def test_user_login_logout(): +@pytest.fixture +def package(user: User) -> Package: + with db.begin(): + pkgbase = db.create(PackageBase, Name="pkg1", Maintainer=user) + pkg = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + yield pkg + + +def test_user_login_logout(user: User): """ Test creating a user and reading its columns. """ # Assert that make_user created a valid user. assert bool(user.ID) @@ -47,8 +54,6 @@ def test_user_login_logout(): assert user.valid_password("testPassword") assert not user.valid_password("badPassword") - assert user in account_type.users - # Make a raw request. request = Request() assert not user.login(request, "badPassword") @@ -81,10 +86,6 @@ def test_user_login_logout(): assert result.valid_password("testPassword") assert result.is_authenticated() - # Ensure we've got the correct account type. - assert user.AccountType.ID == account_type.ID - assert user.AccountType.AccountType == account_type.AccountType - # Test out user string functions. assert repr(user) == f"" @@ -95,13 +96,13 @@ def test_user_login_logout(): assert not user.is_authenticated() -def test_user_login_twice(): +def test_user_login_twice(user: User): request = Request() assert user.login(request, "testPassword") assert user.login(request, "testPassword") -def test_user_login_banned(): +def test_user_login_banned(user: User): # Add ban for the next 30 seconds. banned_timestamp = datetime.utcnow() + timedelta(seconds=30) with db.begin(): @@ -112,13 +113,13 @@ def test_user_login_banned(): assert not user.login(request, "testPassword") -def test_user_login_suspended(): +def test_user_login_suspended(user: User): with db.begin(): user.Suspended = True assert not user.login(Request(), "testPassword") -def test_legacy_user_authentication(): +def test_legacy_user_authentication(user: User): with db.begin(): user.Salt = bcrypt.gensalt().decode() user.Passwd = hashlib.md5( @@ -132,7 +133,7 @@ def test_legacy_user_authentication(): assert not user.valid_password(None) -def test_user_login_with_outdated_sid(): +def test_user_login_with_outdated_sid(user: User): # Make a session with a LastUpdateTS 5 seconds ago, causing # user.login to update it with a new sid. with db.begin(): @@ -143,7 +144,7 @@ def test_user_login_with_outdated_sid(): assert sid != "stub" -def test_user_update_password(): +def test_user_update_password(user: User): user.update_password("secondPassword") assert not user.valid_password("testPassword") assert user.valid_password("secondPassword") @@ -154,11 +155,11 @@ def test_user_minimum_passwd_length(): assert User.minimum_passwd_length() == passwd_min_len -def test_user_has_credential(): - assert not user.has_credential(aurweb.auth.creds.ACCOUNT_CHANGE_TYPE) +def test_user_has_credential(user: User): + assert not user.has_credential(creds.ACCOUNT_CHANGE_TYPE) -def test_user_ssh_pub_key(): +def test_user_ssh_pub_key(user: User): assert user.ssh_pub_key is None with db.begin(): @@ -169,34 +170,26 @@ def test_user_ssh_pub_key(): assert user.ssh_pub_key == ssh_pub_key -def test_user_credential_types(): +def test_user_credential_types(user: User): assert user.AccountTypeID in creds.user_developer_or_trusted_user assert user.AccountTypeID not in creds.trusted_user assert user.AccountTypeID not in creds.developer assert user.AccountTypeID not in creds.trusted_user_or_dev - trusted_user_type = db.query(AccountType).filter( - AccountType.AccountType == "Trusted User" - ).first() with db.begin(): - user.AccountType = trusted_user_type + user.AccountTypeID = at.TRUSTED_USER_ID assert user.AccountTypeID in creds.trusted_user assert user.AccountTypeID in creds.trusted_user_or_dev - developer_type = db.query(AccountType, - AccountType.AccountType == "Developer").first() with db.begin(): - user.AccountType = developer_type + user.AccountTypeID = at.DEVELOPER_ID assert user.AccountTypeID in creds.developer assert user.AccountTypeID in creds.trusted_user_or_dev - type_str = "Trusted User & Developer" - elevated_type = db.query(AccountType, - AccountType.AccountType == type_str).first() with db.begin(): - user.AccountType = elevated_type + user.AccountTypeID = at.TRUSTED_USER_AND_DEV_ID assert user.AccountTypeID in creds.trusted_user assert user.AccountTypeID in creds.developer @@ -208,7 +201,7 @@ def test_user_credential_types(): assert user.is_developer() -def test_user_json(): +def test_user_json(user: User): data = json.loads(user.json()) assert data.get("ID") == user.ID assert data.get("Username") == user.Username @@ -217,7 +210,7 @@ def test_user_json(): assert isinstance(data.get("RegistrationTS"), int) -def test_user_as_dict(): +def test_user_as_dict(user: User): data = user.as_dict() assert data.get("ID") == user.ID assert data.get("Username") == user.Username @@ -226,57 +219,42 @@ def test_user_as_dict(): assert isinstance(data.get("RegistrationTS"), datetime) -def test_user_is_trusted_user(): - tu_type = db.query(AccountType, - AccountType.AccountType == "Trusted User").first() +def test_user_is_trusted_user(user: User): with db.begin(): - user.AccountType = tu_type + user.AccountTypeID = at.TRUSTED_USER_ID assert user.is_trusted_user() is True # Do it again with the combined role. - tu_type = db.query( - AccountType, - AccountType.AccountType == "Trusted User & Developer").first() with db.begin(): - user.AccountType = tu_type + user.AccountTypeID = at.TRUSTED_USER_AND_DEV_ID assert user.is_trusted_user() is True -def test_user_is_developer(): - dev_type = db.query(AccountType, - AccountType.AccountType == "Developer").first() +def test_user_is_developer(user: User): with db.begin(): - user.AccountType = dev_type + user.AccountTypeID = at.DEVELOPER_ID assert user.is_developer() is True # Do it again with the combined role. - dev_type = db.query( - AccountType, - AccountType.AccountType == "Trusted User & Developer").first() with db.begin(): - user.AccountType = dev_type + user.AccountTypeID = at.TRUSTED_USER_AND_DEV_ID assert user.is_developer() is True -def test_user_voted_for(): +def test_user_voted_for(user: User, package: Package): + pkgbase = package.PackageBase now = int(datetime.utcnow().timestamp()) with db.begin(): - pkgbase = db.create(PackageBase, Name="pkg1", Maintainer=user) - pkg = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) db.create(PackageVote, PackageBase=pkgbase, User=user, VoteTS=now) - assert user.voted_for(pkg) + assert user.voted_for(package) -def test_user_notified(): +def test_user_notified(user: User, package: Package): + pkgbase = package.PackageBase with db.begin(): - pkgbase = db.create(PackageBase, Name="pkg1", Maintainer=user) - pkg = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) db.create(PackageNotification, PackageBase=pkgbase, User=user) - assert user.notified(pkg) + assert user.notified(package) -def test_user_packages(): - with db.begin(): - pkgbase = db.create(PackageBase, Name="pkg1", Maintainer=user) - pkg = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) - assert pkg in user.packages() +def test_user_packages(user: User, package: Package): + assert package in user.packages() From eb396813a89c29bfbb96d6d5f7c884d1c83117bc Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 21:27:02 -0800 Subject: [PATCH 685/844] housekeep(fastapi): rewrite test_package with fixtures Signed-off-by: Kevin Morris --- test/test_package.py | 54 ++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/test/test_package.py b/test/test_package.py index c2afa660..1408a182 100644 --- a/test/test_package.py +++ b/test/test_package.py @@ -4,7 +4,7 @@ from sqlalchemy import and_ from sqlalchemy.exc import IntegrityError from aurweb import db -from aurweb.models.account_type import AccountType +from aurweb.models.account_type import USER_ID from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.user import User @@ -14,28 +14,30 @@ user = pkgbase = package = None @pytest.fixture(autouse=True) def setup(db_test): - global user, pkgbase, package + return - account_type = db.query(AccountType, - AccountType.AccountType == "User").first() +@pytest.fixture +def user() -> User: with db.begin(): user = db.create(User, Username="test", Email="test@example.org", RealName="Test User", Passwd="testPassword", - AccountType=account_type) + AccountTypeID=USER_ID) + yield user - pkgbase = db.create(PackageBase, - Name="beautiful-package", + +@pytest.fixture +def package(user: User) -> Package: + with db.begin(): + pkgbase = db.create(PackageBase, Name="beautiful-package", Maintainer=user) - package = db.create(Package, - PackageBase=pkgbase, - Name=pkgbase.Name, + package = db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name, Description="Test description.", URL="https://test.package") + yield package -def test_package(): - assert pkgbase == package.PackageBase +def test_package(package: Package): assert package.Name == "beautiful-package" assert package.Description == "Test description." assert package.Version == str() # Default version. @@ -46,27 +48,21 @@ def test_package(): package.Version = "1.2.3" # Make sure it got updated in the database. - record = db.query(Package, - and_(Package.ID == package.ID, - Package.Version == "1.2.3")).first() + record = db.query(Package).filter( + and_(Package.ID == package.ID, + Package.Version == "1.2.3") + ).first() assert record is not None -def test_package_null_pkgbase_raises_exception(): +def test_package_null_pkgbase_raises(): with pytest.raises(IntegrityError): - with db.begin(): - db.create(Package, - Name="some-package", - Description="Some description.", - URL="https://some.package") - db.rollback() + Package(Name="some-package", Description="Some description.", + URL="https://some.package") -def test_package_null_name_raises_exception(): +def test_package_null_name_raises(package: Package): + pkgbase = package.PackageBase with pytest.raises(IntegrityError): - with db.begin(): - db.create(Package, - PackageBase=pkgbase, - Description="Some description.", - URL="https://some.package") - db.rollback() + Package(PackageBase=pkgbase, Description="Some description.", + URL="https://some.package") From de0f9190778b160fcb05cf4a6945eb4dd56b1aa1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 22:06:32 -0800 Subject: [PATCH 686/844] housekeep(fastapi): rewrite test_ban with fixtures Signed-off-by: Kevin Morris --- test/test_ban.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/test/test_ban.py b/test/test_ban.py index 2c705410..ff49f7e2 100644 --- a/test/test_ban.py +++ b/test/test_ban.py @@ -11,20 +11,21 @@ from aurweb.db import create from aurweb.models.ban import Ban, is_banned from aurweb.testing.requests import Request -ban = request = None - @pytest.fixture(autouse=True) def setup(db_test): - global ban, request + return + +@pytest.fixture +def ban() -> Ban: ts = datetime.utcnow() + timedelta(seconds=30) with db.begin(): ban = create(Ban, IPAddress="127.0.0.1", BanTS=ts) - request = Request() + yield ban -def test_ban(): +def test_ban(ban: Ban): assert ban.IPAddress == "127.0.0.1" assert bool(ban.BanTS) @@ -45,11 +46,13 @@ def test_invalid_ban(): db.rollback() -def test_banned(): +def test_banned(ban: Ban): + request = Request() request.client.host = "127.0.0.1" assert is_banned(request) -def test_not_banned(): +def test_not_banned(ban: Ban): + request = Request() request.client.host = "192.168.0.1" assert not is_banned(request) From 7ef3e3438681b29d6a32aed54c92f3965737c7af Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 22:43:29 -0800 Subject: [PATCH 687/844] housekeep(fastapi): rewrite test_accounts_routes with fixtures Signed-off-by: Kevin Morris --- test/test_accounts_routes.py | 233 ++++++++++++++++++----------------- 1 file changed, 118 insertions(+), 115 deletions(-) diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index be929e97..f08efcd2 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -23,16 +23,12 @@ from aurweb.models.user import User from aurweb.testing.html import get_errors from aurweb.testing.requests import Request +logger = logging.get_logger(__name__) + # Some test global constants. TEST_USERNAME = "test" TEST_EMAIL = "test@example.org" -# Global mutables. -client = TestClient(app) -user = None - -logger = logging.get_logger(__name__) - def make_ssh_pubkey(): # Create a public key with ssh-keygen (this adds ssh-keygen as a @@ -50,29 +46,32 @@ def make_ssh_pubkey(): @pytest.fixture(autouse=True) def setup(db_test): - global user + return - account_type = query(AccountType, - AccountType.AccountType == "User").first() +@pytest.fixture +def client() -> TestClient: + yield TestClient(app=app) + + +@pytest.fixture +def user() -> User: with db.begin(): user = create(User, Username=TEST_USERNAME, Email=TEST_EMAIL, RealName="Test UserZ", Passwd="testPassword", - IRCNick="testZ", AccountType=account_type) + IRCNick="testZ", AccountTypeID=USER_ID) yield user @pytest.fixture -def tu_user(): +def tu_user(user: User): with db.begin(): - user.AccountType = query(AccountType).filter( - AccountType.ID == TRUSTED_USER_AND_DEV_ID - ).first() + user.AccountTypeID = TRUSTED_USER_AND_DEV_ID yield user -def test_get_passreset_authed_redirects(): +def test_get_passreset_authed_redirects(client: TestClient, user: User): sid = user.login(Request(), "testPassword") assert sid is not None @@ -84,39 +83,39 @@ def test_get_passreset_authed_redirects(): assert response.headers.get("location") == "/" -def test_get_passreset(): +def test_get_passreset(client: TestClient): with client as request: response = request.get("/passreset") assert response.status_code == int(HTTPStatus.OK) -def test_get_passreset_translation(): +def test_get_passreset_translation(client: TestClient): # Test that translation works; set it to de. with client as request: response = request.get("/passreset", cookies={"AURLANG": "de"}) # The header title should be translated. - assert "Passwort zurücksetzen".encode("utf-8") in response.content + assert "Passwort zurücksetzen" in response.text # The form input label should be translated. - assert "Benutzername oder primäre E-Mail-Adresse eingeben:".encode( - "utf-8") in response.content + expected = "Benutzername oder primäre E-Mail-Adresse eingeben:" + assert expected in response.text # And the button. - assert "Weiter".encode("utf-8") in response.content + assert "Weiter" in response.text # Restore english. with client as request: response = request.get("/passreset", cookies={"AURLANG": "en"}) -def test_get_passreset_with_resetkey(): +def test_get_passreset_with_resetkey(client: TestClient): with client as request: response = request.get("/passreset", data={"resetkey": "abcd"}) assert response.status_code == int(HTTPStatus.OK) -def test_post_passreset_authed_redirects(): +def test_post_passreset_authed_redirects(client: TestClient, user: User): sid = user.login(Request(), "testPassword") assert sid is not None @@ -130,7 +129,7 @@ def test_post_passreset_authed_redirects(): assert response.headers.get("location") == "/" -def test_post_passreset_user(): +def test_post_passreset_user(client: TestClient, user: User): # With username. with client as request: response = request.post("/passreset", data={"user": TEST_USERNAME}) @@ -144,7 +143,7 @@ def test_post_passreset_user(): assert response.headers.get("location") == "/passreset?step=confirm" -def test_post_passreset_resetkey(): +def test_post_passreset_resetkey(client: TestClient, user: User): with db.begin(): user.session = Session(UsersID=user.ID, SessionID="blah", LastUpdateTS=datetime.utcnow().timestamp()) @@ -171,28 +170,7 @@ def test_post_passreset_resetkey(): assert response.headers.get("location") == "/passreset?step=complete" -def test_post_passreset_error_invalid_email(): - # First, test with a user that doesn't even exist. - with client as request: - response = request.post("/passreset", data={"user": "invalid"}) - assert response.status_code == int(HTTPStatus.NOT_FOUND) - - error = "Invalid e-mail." - assert error in response.content.decode("utf-8") - - # Then, test with an invalid resetkey for a real user. - _ = make_resetkey() - post_data = make_passreset_data("fake") - post_data["password"] = "abcd1234" - post_data["confirm"] = "abcd1234" - - with client as request: - response = request.post("/passreset", data=post_data) - assert response.status_code == int(HTTPStatus.NOT_FOUND) - assert error in response.content.decode("utf-8") - - -def make_resetkey(): +def make_resetkey(client: TestClient, user: User): with client as request: response = request.post("/passreset", data={"user": TEST_USERNAME}) assert response.status_code == int(HTTPStatus.SEE_OTHER) @@ -200,18 +178,37 @@ def make_resetkey(): return user.ResetKey -def make_passreset_data(resetkey): +def make_passreset_data(user: User, resetkey: str): return { "user": user.Username, "resetkey": resetkey } -def test_post_passreset_error_missing_field(): +def test_post_passreset_error_invalid_email(client: TestClient, user: User): + # First, test with a user that doesn't even exist. + with client as request: + response = request.post("/passreset", data={"user": "invalid"}) + assert response.status_code == int(HTTPStatus.NOT_FOUND) + assert "Invalid e-mail." in response.text + + # Then, test with an invalid resetkey for a real user. + _ = make_resetkey(client, user) + post_data = make_passreset_data(user, "fake") + post_data["password"] = "abcd1234" + post_data["confirm"] = "abcd1234" + + with client as request: + response = request.post("/passreset", data=post_data) + assert response.status_code == int(HTTPStatus.NOT_FOUND) + assert "Invalid e-mail." in response.text + + +def test_post_passreset_error_missing_field(client: TestClient, user: User): # Now that we've prepared the password reset, prepare a POST # request with the user's ResetKey. - resetkey = make_resetkey() - post_data = make_passreset_data(resetkey) + resetkey = make_resetkey(client, user) + post_data = make_passreset_data(user, resetkey) with client as request: response = request.post("/passreset", data=post_data) @@ -222,9 +219,10 @@ def test_post_passreset_error_missing_field(): assert error in response.content.decode("utf-8") -def test_post_passreset_error_password_mismatch(): - resetkey = make_resetkey() - post_data = make_passreset_data(resetkey) +def test_post_passreset_error_password_mismatch(client: TestClient, + user: User): + resetkey = make_resetkey(client, user) + post_data = make_passreset_data(user, resetkey) post_data["password"] = "abcd1234" post_data["confirm"] = "mismatched" @@ -238,9 +236,10 @@ def test_post_passreset_error_password_mismatch(): assert error in response.content.decode("utf-8") -def test_post_passreset_error_password_requirements(): - resetkey = make_resetkey() - post_data = make_passreset_data(resetkey) +def test_post_passreset_error_password_requirements(client: TestClient, + user: User): + resetkey = make_resetkey(client, user) + post_data = make_passreset_data(user, resetkey) passwd_min_len = User.minimum_passwd_length() assert passwd_min_len >= 4 @@ -257,7 +256,7 @@ def test_post_passreset_error_password_requirements(): assert error in response.content.decode("utf-8") -def test_get_register(): +def test_get_register(client: TestClient): with client as request: response = request.get("/register") assert response.status_code == int(HTTPStatus.OK) @@ -288,7 +287,7 @@ def post_register(request, **kwargs): return request.post("/register", data=data, allow_redirects=False) -def test_post_register(): +def test_post_register(client: TestClient): with client as request: response = post_register(request) assert response.status_code == int(HTTPStatus.OK) @@ -298,7 +297,7 @@ def test_post_register(): assert expected in response.content.decode() -def test_post_register_rejects_case_insensitive_spoof(): +def test_post_register_rejects_case_insensitive_spoof(client: TestClient): with client as request: response = post_register(request, U="newUser", E="newUser@example.org") assert response.status_code == int(HTTPStatus.OK) @@ -319,7 +318,7 @@ def test_post_register_rejects_case_insensitive_spoof(): assert expected in response.content.decode() -def test_post_register_error_expired_captcha(): +def test_post_register_error_expired_captcha(client: TestClient): with client as request: response = post_register(request, captcha_salt="invalid-salt") @@ -329,7 +328,7 @@ def test_post_register_error_expired_captcha(): assert "This CAPTCHA has expired. Please try again." in content -def test_post_register_error_missing_captcha(): +def test_post_register_error_missing_captcha(client: TestClient): with client as request: response = post_register(request, captcha=None) @@ -339,7 +338,7 @@ def test_post_register_error_missing_captcha(): assert "The CAPTCHA is missing." in content -def test_post_register_error_invalid_captcha(): +def test_post_register_error_invalid_captcha(client: TestClient): with client as request: response = post_register(request, captcha="invalid blah blah") @@ -349,7 +348,7 @@ def test_post_register_error_invalid_captcha(): assert "The entered CAPTCHA answer is invalid." in content -def test_post_register_error_ip_banned(): +def test_post_register_error_ip_banned(client: TestClient): # 'testclient' is used as request.client.host via FastAPI TestClient. with db.begin(): create(Ban, IPAddress="testclient", BanTS=datetime.utcnow()) @@ -365,7 +364,7 @@ def test_post_register_error_ip_banned(): "inconvenience.") in content -def test_post_register_error_missing_username(): +def test_post_register_error_missing_username(client: TestClient): with client as request: response = post_register(request, U="") @@ -375,7 +374,7 @@ def test_post_register_error_missing_username(): assert "Missing a required field." in content -def test_post_register_error_missing_email(): +def test_post_register_error_missing_email(client: TestClient): with client as request: response = post_register(request, E="") @@ -385,7 +384,7 @@ def test_post_register_error_missing_email(): assert "Missing a required field." in content -def test_post_register_error_invalid_username(): +def test_post_register_error_invalid_username(client: TestClient): with client as request: # Our test config requires at least three characters for a # valid username, so test against two characters: 'ba'. @@ -397,7 +396,7 @@ def test_post_register_error_invalid_username(): assert "The username is invalid." in content -def test_post_register_invalid_password(): +def test_post_register_invalid_password(client: TestClient): with client as request: response = post_register(request, P="abc", C="abc") @@ -408,7 +407,7 @@ def test_post_register_invalid_password(): assert re.search(expected, content) -def test_post_register_error_missing_confirm(): +def test_post_register_error_missing_confirm(client: TestClient): with client as request: response = post_register(request, C=None) @@ -418,7 +417,7 @@ def test_post_register_error_missing_confirm(): assert "Please confirm your new password." in content -def test_post_register_error_mismatched_confirm(): +def test_post_register_error_mismatched_confirm(client: TestClient): with client as request: response = post_register(request, C="mismatched") @@ -428,7 +427,7 @@ def test_post_register_error_mismatched_confirm(): assert "Password fields do not match." in content -def test_post_register_error_invalid_email(): +def test_post_register_error_invalid_email(client: TestClient): with client as request: response = post_register(request, E="bad@email") @@ -438,7 +437,7 @@ def test_post_register_error_invalid_email(): assert "The email address is invalid." in content -def test_post_register_error_undeliverable_email(): +def test_post_register_error_undeliverable_email(client: TestClient): with client as request: # At the time of writing, webchat.freenode.net does not contain # mx records; if it ever does, it'll break this test. @@ -450,7 +449,7 @@ def test_post_register_error_undeliverable_email(): assert "The email address is invalid." in content -def test_post_register_invalid_backup_email(): +def test_post_register_invalid_backup_email(client: TestClient): with client as request: response = post_register(request, BE="bad@email") @@ -460,7 +459,7 @@ def test_post_register_invalid_backup_email(): assert "The backup email address is invalid." in content -def test_post_register_error_invalid_homepage(): +def test_post_register_error_invalid_homepage(client: TestClient): with client as request: response = post_register(request, HP="bad") @@ -471,7 +470,7 @@ def test_post_register_error_invalid_homepage(): assert expected in content -def test_post_register_error_invalid_pgp_fingerprints(): +def test_post_register_error_invalid_pgp_fingerprints(client: TestClient): with client as request: response = post_register(request, K="bad") @@ -492,7 +491,7 @@ def test_post_register_error_invalid_pgp_fingerprints(): assert expected in content -def test_post_register_error_invalid_ssh_pubkeys(): +def test_post_register_error_invalid_ssh_pubkeys(client: TestClient): with client as request: response = post_register(request, PK="bad") @@ -510,7 +509,7 @@ def test_post_register_error_invalid_ssh_pubkeys(): assert "The SSH public key is invalid." in content -def test_post_register_error_unsupported_language(): +def test_post_register_error_unsupported_language(client: TestClient): with client as request: response = post_register(request, L="bad") @@ -521,7 +520,7 @@ def test_post_register_error_unsupported_language(): assert expected in content -def test_post_register_error_unsupported_timezone(): +def test_post_register_error_unsupported_timezone(client: TestClient): with client as request: response = post_register(request, TZ="ABCDEFGH") @@ -532,7 +531,7 @@ def test_post_register_error_unsupported_timezone(): assert expected in content -def test_post_register_error_username_taken(): +def test_post_register_error_username_taken(client: TestClient, user: User): with client as request: response = post_register(request, U="test") @@ -543,7 +542,7 @@ def test_post_register_error_username_taken(): assert re.search(expected, content) -def test_post_register_error_email_taken(): +def test_post_register_error_email_taken(client: TestClient, user: User): with client as request: response = post_register(request, E="test@example.org") @@ -554,7 +553,7 @@ def test_post_register_error_email_taken(): assert re.search(expected, content) -def test_post_register_error_ssh_pubkey_taken(): +def test_post_register_error_ssh_pubkey_taken(client: TestClient, user: User): pk = str() # Create a public key with ssh-keygen (this adds ssh-keygen as a @@ -584,7 +583,7 @@ def test_post_register_error_ssh_pubkey_taken(): assert re.search(expected, content) -def test_post_register_with_ssh_pubkey(): +def test_post_register_with_ssh_pubkey(client: TestClient): pk = str() # Create a public key with ssh-keygen (this adds ssh-keygen as a @@ -605,7 +604,7 @@ def test_post_register_with_ssh_pubkey(): assert response.status_code == int(HTTPStatus.OK) -def test_get_account_edit(): +def test_get_account_edit(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -617,7 +616,7 @@ def test_get_account_edit(): assert response.status_code == int(HTTPStatus.OK) -def test_get_account_edit_unauthorized(): +def test_get_account_edit_unauthorized(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -633,7 +632,7 @@ def test_get_account_edit_unauthorized(): assert response.status_code == int(HTTPStatus.UNAUTHORIZED) -def test_post_account_edit(): +def test_post_account_edit(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -655,7 +654,7 @@ def test_post_account_edit(): assert expected in response.content.decode() -def test_post_account_edit_dev(): +def test_post_account_edit_dev(client: TestClient, user: User): # Modify our user to be a "Trusted User & Developer" name = "Trusted User & Developer" tu_or_dev = query(AccountType, AccountType.AccountType == name).first() @@ -683,7 +682,7 @@ def test_post_account_edit_dev(): assert expected in response.content.decode() -def test_post_account_edit_language(): +def test_post_account_edit_language(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -710,7 +709,7 @@ def test_post_account_edit_language(): assert lang_nodes[0] == "selected" -def test_post_account_edit_timezone(): +def test_post_account_edit_timezone(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -729,7 +728,8 @@ def test_post_account_edit_timezone(): assert response.status_code == int(HTTPStatus.OK) -def test_post_account_edit_error_missing_password(): +def test_post_account_edit_error_missing_password(client: TestClient, + user: User): request = Request() sid = user.login(request, "testPassword") @@ -751,7 +751,8 @@ def test_post_account_edit_error_missing_password(): assert "Invalid password." in content -def test_post_account_edit_error_invalid_password(): +def test_post_account_edit_error_invalid_password(client: TestClient, + user: User): request = Request() sid = user.login(request, "testPassword") @@ -773,7 +774,8 @@ def test_post_account_edit_error_invalid_password(): assert "Invalid password." in content -def test_post_account_edit_inactivity_unauthorized(): +def test_post_account_edit_inactivity_unauthorized(client: TestClient, + user: User): cookies = {"AURSID": user.login(Request(), "testPassword")} post_data = { "U": "test", @@ -791,7 +793,7 @@ def test_post_account_edit_inactivity_unauthorized(): assert errors[0].text.strip() == expected -def test_post_account_edit_inactivity(): +def test_post_account_edit_inactivity(client: TestClient, user: User): with db.begin(): user.AccountTypeID = TRUSTED_USER_ID assert not user.Suspended @@ -822,7 +824,7 @@ def test_post_account_edit_inactivity(): assert user.InactivityTS == 0 -def test_post_account_edit_error_unauthorized(): +def test_post_account_edit_error_unauthorized(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -845,7 +847,7 @@ def test_post_account_edit_error_unauthorized(): assert response.status_code == int(HTTPStatus.UNAUTHORIZED) -def test_post_account_edit_ssh_pub_key(): +def test_post_account_edit_ssh_pub_key(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -874,7 +876,7 @@ def test_post_account_edit_ssh_pub_key(): assert response.status_code == int(HTTPStatus.OK) -def test_post_account_edit_missing_ssh_pubkey(): +def test_post_account_edit_missing_ssh_pubkey(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -907,7 +909,7 @@ def test_post_account_edit_missing_ssh_pubkey(): assert response.status_code == int(HTTPStatus.OK) -def test_post_account_edit_invalid_ssh_pubkey(): +def test_post_account_edit_invalid_ssh_pubkey(client: TestClient, user: User): pubkey = "ssh-rsa fake key" request = Request() @@ -930,7 +932,7 @@ def test_post_account_edit_invalid_ssh_pubkey(): assert response.status_code == int(HTTPStatus.BAD_REQUEST) -def test_post_account_edit_password(): +def test_post_account_edit_password(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -952,7 +954,7 @@ def test_post_account_edit_password(): assert user.valid_password("newPassword") -def test_post_account_edit_account_types(): +def test_post_account_edit_account_types(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") cookies = {"AURSID": sid} @@ -1066,7 +1068,7 @@ def test_post_account_edit_account_types(): assert user.AccountTypeID == DEVELOPER_ID -def test_get_account(): +def test_get_account(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -1077,7 +1079,7 @@ def test_get_account(): assert response.status_code == int(HTTPStatus.OK) -def test_get_account_not_found(): +def test_get_account_not_found(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -1088,7 +1090,7 @@ def test_get_account_not_found(): assert response.status_code == int(HTTPStatus.NOT_FOUND) -def test_get_account_unauthenticated(): +def test_get_account_unauthenticated(client: TestClient, user: User): with client as request: response = request.get("/account/test", allow_redirects=False) assert response.status_code == int(HTTPStatus.UNAUTHORIZED) @@ -1097,7 +1099,7 @@ def test_get_account_unauthenticated(): assert "You must log in to view user information." in content -def test_get_accounts(tu_user): +def test_get_accounts(client: TestClient, user: User, tu_user: User): """ Test that we can GET request /accounts and receive a form which can be used to POST /accounts. """ sid = user.login(Request(), "testPassword") @@ -1156,7 +1158,7 @@ def get_rows(html): return root.xpath('//table[contains(@class, "users")]/tbody/tr') -def test_post_accounts(tu_user): +def test_post_accounts(client: TestClient, user: User, tu_user: User): # Set a PGPKey. with db.begin(): user.PGPKey = "5F18B20346188419750745D7335F2CB41F253D30" @@ -1211,7 +1213,7 @@ def test_post_accounts(tu_user): % (_user.ID, _user.Username)) -def test_post_accounts_username(tu_user): +def test_post_accounts_username(client: TestClient, user: User, tu_user: User): # Test the U parameter path. sid = user.login(Request(), "testPassword") cookies = {"AURSID": sid} @@ -1231,7 +1233,8 @@ def test_post_accounts_username(tu_user): assert username.text.strip() == user.Username -def test_post_accounts_account_type(tu_user): +def test_post_accounts_account_type(client: TestClient, user: User, + tu_user: User): # Check the different account type options. sid = user.login(Request(), "testPassword") cookies = {"AURSID": sid} @@ -1324,7 +1327,7 @@ def test_post_accounts_account_type(tu_user): assert type.text.strip() == "Trusted User & Developer" -def test_post_accounts_status(tu_user): +def test_post_accounts_status(client: TestClient, user: User, tu_user: User): # Test the functionality of Suspended. sid = user.login(Request(), "testPassword") cookies = {"AURSID": sid} @@ -1356,7 +1359,7 @@ def test_post_accounts_status(tu_user): assert status.text.strip() == "Suspended" -def test_post_accounts_email(tu_user): +def test_post_accounts_email(client: TestClient, user: User, tu_user: User): sid = user.login(Request(), "testPassword") cookies = {"AURSID": sid} @@ -1370,7 +1373,7 @@ def test_post_accounts_email(tu_user): assert len(rows) == 1 -def test_post_accounts_realname(tu_user): +def test_post_accounts_realname(client: TestClient, user: User, tu_user: User): # Test the R parameter path. sid = user.login(Request(), "testPassword") cookies = {"AURSID": sid} @@ -1384,7 +1387,7 @@ def test_post_accounts_realname(tu_user): assert len(rows) == 1 -def test_post_accounts_irc(tu_user): +def test_post_accounts_irc(client: TestClient, user: User, tu_user: User): # Test the I parameter path. sid = user.login(Request(), "testPassword") cookies = {"AURSID": sid} @@ -1398,7 +1401,7 @@ def test_post_accounts_irc(tu_user): assert len(rows) == 1 -def test_post_accounts_sortby(tu_user): +def test_post_accounts_sortby(client: TestClient, user: User, tu_user: User): # Create a second user so we can compare sorts. account_type = query(AccountType, AccountType.ID == DEVELOPER_ID).first() @@ -1481,7 +1484,7 @@ def test_post_accounts_sortby(tu_user): assert compare_text_values(1, first_rows, reversed(rows)) -def test_post_accounts_pgp_key(tu_user): +def test_post_accounts_pgp_key(client: TestClient, user: User, tu_user: User): with db.begin(): user.PGPKey = "5F18B20346188419750745D7335F2CB41F253D30" @@ -1498,7 +1501,7 @@ def test_post_accounts_pgp_key(tu_user): assert len(rows) == 1 -def test_post_accounts_paged(tu_user): +def test_post_accounts_paged(client: TestClient, user: User, tu_user: User): # Create 150 users. users = [user] account_type = query(AccountType, @@ -1572,7 +1575,7 @@ def test_post_accounts_paged(tu_user): assert page_next.attrib["disabled"] == "disabled" -def test_get_terms_of_service(): +def test_get_terms_of_service(client: TestClient, user: User): with db.begin(): term = create(Term, Description="Test term.", URL="http://localhost", Revision=1) @@ -1624,7 +1627,7 @@ def test_get_terms_of_service(): assert response.status_code == int(HTTPStatus.SEE_OTHER) -def test_post_terms_of_service(): +def test_post_terms_of_service(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -1682,7 +1685,7 @@ def test_post_terms_of_service(): assert response.headers.get("location") == "/" -def test_account_comments_not_found(): +def test_account_comments_not_found(client: TestClient, user: User): cookies = {"AURSID": user.login(Request(), "testPassword")} with client as request: resp = request.get("/account/non-existent/comments", cookies=cookies) From fccd8b63d271785e741cae0ec453b349ee0d6d38 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 23:07:34 -0800 Subject: [PATCH 688/844] housekeep(fastapi): rewrite test_auth_routes with fixtures Signed-off-by: Kevin Morris --- test/test_auth_routes.py | 95 ++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index 0157fcc8..a8d0db11 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -8,11 +8,12 @@ from fastapi.testclient import TestClient import aurweb.config +from aurweb import db from aurweb.asgi import app -from aurweb.db import begin, create, query -from aurweb.models.account_type import AccountType +from aurweb.models.account_type import USER_ID from aurweb.models.session import Session from aurweb.models.user import User +from aurweb.testing.requests import Request # Some test global constants. TEST_USERNAME = "test" @@ -21,30 +22,32 @@ TEST_REFERER = { "referer": aurweb.config.get("options", "aur_location") + "/login", } -# Global mutables. -user = client = None - @pytest.fixture(autouse=True) def setup(db_test): - global user, client + return - account_type = query(AccountType, - AccountType.AccountType == "User").first() - with begin(): - user = create(User, Username=TEST_USERNAME, Email=TEST_EMAIL, - RealName="Test User", Passwd="testPassword", - AccountType=account_type) - - client = TestClient(app) +@pytest.fixture +def client() -> TestClient: + client = TestClient(app=app) # Necessary for forged login CSRF protection on the login route. Set here # instead of only on the necessary requests for convenience. client.headers.update(TEST_REFERER) + yield client -def test_login_logout(): +@pytest.fixture +def user() -> User: + with db.begin(): + user = db.create(User, Username=TEST_USERNAME, Email=TEST_EMAIL, + RealName="Test User", Passwd="testPassword", + AccountTypeID=USER_ID) + yield user + + +def test_login_logout(client: TestClient, user: User): post_data = { "user": "test", "passwd": "testPassword", @@ -83,7 +86,7 @@ def mock_getboolean(a, b): @mock.patch("aurweb.config.getboolean", side_effect=mock_getboolean) -def test_secure_login(mock): +def test_secure_login(getboolean: bool, client: TestClient, user: User): """ In this test, we check to verify the course of action taken by starlette when providing secure=True to a response cookie. This is achieved by mocking aurweb.config.getboolean to return @@ -94,11 +97,11 @@ def test_secure_login(mock): on such a request. """ # Create a local TestClient here since we mocked configuration. - client = TestClient(app) + # client = TestClient(app) # Necessary for forged login CSRF protection on the login route. Set here # instead of only on the necessary requests for convenience. - client.headers.update(TEST_REFERER) + # client.headers.update(TEST_REFERER) # Data used for our upcoming http post request. post_data = { @@ -126,18 +129,19 @@ def test_secure_login(mock): # Let's make sure we actually have a session relationship # with the AURSID we ended up with. - record = query(Session, Session.SessionID == cookie.value).first() + record = db.query(Session, Session.SessionID == cookie.value).first() assert record is not None and record.User == user assert user.session == record -def test_authenticated_login(): +def test_authenticated_login(client: TestClient, user: User): post_data = { "user": "test", "passwd": "testPassword", "next": "/" } + cookies = {"AURSID": user.login(Request(), "testPassword")} with client as request: # Try to login. response = request.post("/login", data=post_data, @@ -149,12 +153,13 @@ def test_authenticated_login(): # when requesting GET /login as an authenticated user. # Now, let's verify that we receive 403 Forbidden when we # try to get /login as an authenticated user. - response = request.get("/login", allow_redirects=False) + response = request.get("/login", cookies=cookies, + allow_redirects=False) assert response.status_code == int(HTTPStatus.OK) assert "Logged-in as: test" in response.text -def test_unauthenticated_logout_unauthorized(): +def test_unauthenticated_logout_unauthorized(client: TestClient): with client as request: # Alright, let's verify that attempting to /logout when not # authenticated returns 401 Unauthorized. @@ -163,7 +168,7 @@ def test_unauthenticated_logout_unauthorized(): assert response.headers.get("location").startswith("/login") -def test_login_missing_username(): +def test_login_missing_username(client: TestClient): post_data = { "passwd": "testPassword", "next": "/" @@ -179,7 +184,7 @@ def test_login_missing_username(): assert "checked" not in content -def test_login_remember_me(): +def test_login_remember_me(client: TestClient, user: User): post_data = { "user": "test", "passwd": "testPassword", @@ -197,16 +202,15 @@ def test_login_remember_me(): "options", "persistent_cookie_timeout") expected_ts = datetime.utcnow().timestamp() + cookie_timeout - _session = query(Session, - Session.UsersID == user.ID).first() + session = db.query(Session).filter(Session.UsersID == user.ID).first() # Expect that LastUpdateTS was within 5 seconds of the expected_ts, # which is equal to the current timestamp + persistent_cookie_timeout. - assert _session.LastUpdateTS > expected_ts - 5 - assert _session.LastUpdateTS < expected_ts + 5 + assert session.LastUpdateTS > expected_ts - 5 + assert session.LastUpdateTS < expected_ts + 5 -def test_login_incorrect_password_remember_me(): +def test_login_incorrect_password_remember_me(client: TestClient, user: User): post_data = { "user": "test", "passwd": "badPassword", @@ -218,15 +222,14 @@ def test_login_incorrect_password_remember_me(): response = request.post("/login", data=post_data) assert "AURSID" not in response.cookies - # Make sure username is prefilled, password isn't prefilled, and remember_me - # is checked. - content = response.content.decode() - assert post_data["user"] in content - assert post_data["passwd"] not in content - assert "checked" in content + # Make sure username is prefilled, password isn't prefilled, + # and remember_me is checked. + assert post_data["user"] in response.text + assert post_data["passwd"] not in response.text + assert "checked" in response.text -def test_login_missing_password(): +def test_login_missing_password(client: TestClient): post_data = { "user": "test", "next": "/" @@ -237,12 +240,11 @@ def test_login_missing_password(): assert "AURSID" not in response.cookies # Make sure username is prefilled and remember_me isn't checked. - content = response.content.decode() - assert post_data["user"] in content - assert "checked" not in content + assert post_data["user"] in response.text + assert "checked" not in response.text -def test_login_incorrect_password(): +def test_login_incorrect_password(client: TestClient): post_data = { "user": "test", "passwd": "badPassword", @@ -253,15 +255,14 @@ def test_login_incorrect_password(): response = request.post("/login", data=post_data) assert "AURSID" not in response.cookies - # Make sure username is prefilled, password isn't prefilled and remember_me - # isn't checked. - content = response.content.decode() - assert post_data["user"] in content - assert post_data["passwd"] not in content - assert "checked" not in content + # Make sure username is prefilled, password isn't prefilled + # and remember_me isn't checked. + assert post_data["user"] in response.text + assert post_data["passwd"] not in response.text + assert "checked" not in response.text -def test_login_bad_referer(): +def test_login_bad_referer(client: TestClient): post_data = { "user": "test", "passwd": "testPassword", From 043ac7fe9211b96222274fcab6797df8a48d55b1 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Tue, 30 Nov 2021 23:24:42 -0800 Subject: [PATCH 689/844] fix(test_aurblup): use correct type hint for tmpdir Signed-off-by: Kevin Morris --- test/test_aurblup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test_aurblup.py b/test/test_aurblup.py index 7eaae556..0b499d57 100644 --- a/test/test_aurblup.py +++ b/test/test_aurblup.py @@ -2,6 +2,7 @@ import tempfile from unittest import mock +import py import pytest from aurweb import config, db @@ -17,12 +18,12 @@ def tempdir() -> str: @pytest.fixture -def alpm_db(tempdir: str) -> AlpmDatabase: +def alpm_db(tempdir: py.path.local) -> AlpmDatabase: yield AlpmDatabase(tempdir) @pytest.fixture(autouse=True) -def setup(db_test, alpm_db: AlpmDatabase, tempdir: str) -> None: +def setup(db_test, alpm_db: AlpmDatabase, tempdir: py.path.local) -> None: config_get = config.get def mock_config_get(section: str, key: str) -> str: From 112837e0e99c3d6c84660ece76240d959760dcdf Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 1 Dec 2021 11:53:43 -0800 Subject: [PATCH 690/844] fix(test_auth): cover mismatched referer situation Signed-off-by: Kevin Morris --- aurweb/testing/requests.py | 12 ++++++++++-- test/test_auth.py | 22 +++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/aurweb/testing/requests.py b/aurweb/testing/requests.py index a8c077db..76f7afca 100644 --- a/aurweb/testing/requests.py +++ b/aurweb/testing/requests.py @@ -1,3 +1,5 @@ +from typing import Dict + import aurweb.config @@ -27,7 +29,13 @@ class URL: class Request: """ A fake Request object which mimics a FastAPI Request for tests. """ client = Client() - cookies = dict() - headers = dict() user = User() url = URL() + + def __init__(self, + method: str = "GET", + headers: Dict[str, str] = dict(), + cookies: Dict[str, str] = dict()) -> "Request": + self.method = method.upper() + self.headers = headers + self.cookies = cookies diff --git a/test/test_auth.py b/test/test_auth.py index b63fb96f..b607a038 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1,11 +1,13 @@ from datetime import datetime +import fastapi import pytest +from fastapi import HTTPException from sqlalchemy.exc import IntegrityError from aurweb import db -from aurweb.auth import AnonymousUser, BasicAuthBackend, account_type_required +from aurweb.auth import AnonymousUser, BasicAuthBackend, account_type_required, auth_required from aurweb.models.account_type import USER, USER_ID from aurweb.models.session import Session from aurweb.models.user import User @@ -74,6 +76,24 @@ async def test_basic_auth_backend(user: User, backend: BasicAuthBackend): assert result == user +@pytest.mark.asyncio +async def test_auth_required_redirection_bad_referrer(): + # Create a fake route function which can be wrapped by auth_required. + def bad_referrer_route(request: fastapi.Request): + pass + + # Get down to the nitty gritty internal wrapper. + bad_referrer_route = auth_required()(bad_referrer_route) + + # Execute the route with a "./blahblahblah" Referer, which does not + # match aur_location; `./` has been used as a prefix to attempt to + # ensure we're providing a fake referer. + with pytest.raises(HTTPException) as exc: + request = Request(method="POST", headers={"Referer": "./blahblahblah"}) + await bad_referrer_route(request) + assert exc.detail == "Bad Referer header." + + def test_account_type_required(): """ This test merely asserts that a few different paths do not raise exceptions. """ From c09784d58f600a249f321e6d2a80f9073cd12d49 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 1 Dec 2021 11:56:44 -0800 Subject: [PATCH 691/844] fix(auth.auth_required): remove unused keyword arguments Signed-off-by: Kevin Morris --- aurweb/auth/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/aurweb/auth/__init__.py b/aurweb/auth/__init__.py index 8ceb136c..18356ac2 100644 --- a/aurweb/auth/__init__.py +++ b/aurweb/auth/__init__.py @@ -120,9 +120,7 @@ class BasicAuthBackend(AuthenticationBackend): return (AuthCredentials(["authenticated"]), user) -def auth_required(is_required: bool = True, - template: tuple = None, - status_code: HTTPStatus = HTTPStatus.UNAUTHORIZED): +def auth_required(is_required: bool = True): """ Authentication route decorator. :param is_required: A boolean indicating whether the function requires auth From 0435c56a41c8e9cf3c286ffbbf4ae21ad33bd6e6 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 1 Dec 2021 12:27:14 -0800 Subject: [PATCH 692/844] update test/README.md to be more aligned with the current state Signed-off-by: Kevin Morris --- test/README.md | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/test/README.md b/test/README.md index 13fb0a0c..0d86a879 100644 --- a/test/README.md +++ b/test/README.md @@ -117,13 +117,12 @@ To run `sharness` shell test suites (requires Arch Linux): To run `pytest` Python test suites: - $ make -C test pytest + $ pytest **Note:** For SQLite tests, users may want to use `eatmydata` to improve speed: $ eatmydata -- make -C test sh - $ eatmydata -- make -C test pytest To produce coverage reports related to Python when running tests manually, use the following method: @@ -147,11 +146,9 @@ Almost all of our `pytest` suites use the database in some way. There are a few particular testing utilities in `aurweb` that one should keep aware of to aid testing code: -- `aurweb.testing.setup_init_db(*tables)` - - Prepares test database tables to be cleared before a test - is run. Be careful not to specify any tables we depend on - for constant records, like `AccountTypes`, `DependencyTypes`, - `RelationTypes` and `RequestTypes`. +- `db_test` pytest fixture + - Prepares test databases for the module and cleans out database + tables for each test function requiring this fixture. - `aurweb.testing.requests.Request` - A fake stripped down version of `fastapi.Request` that can be passed to any functions in our codebase which use @@ -168,14 +165,16 @@ Example code: @pytest.fixture(autouse=True) - def setup(): - setup_test_db(User.__tablename__) + def setup(db_test): + return @pytest.fixture def user(): - yield db.create(User, Passwd="testPassword", ...) + with db.begin(): + user = db.create(User, Passwd="testPassword", ...) + yield user - def test_user_login(user): + def test_user_login(user: User): assert isinstance(user, User) is True fake_request = Request() From 42701514e75ac373c01d787508e092b7ed0d144d Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Wed, 1 Dec 2021 02:16:08 -0500 Subject: [PATCH 693/844] fix(FastAPI): Use HTTPStatus instead of raw number Signed-off-by: Steven Guikal --- aurweb/routers/errors.py | 8 ++++++-- aurweb/routers/html.py | 2 +- aurweb/routers/sso.py | 11 +++++++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/aurweb/routers/errors.py b/aurweb/routers/errors.py index eb935b57..9ed1e80d 100644 --- a/aurweb/routers/errors.py +++ b/aurweb/routers/errors.py @@ -1,14 +1,18 @@ +from http import HTTPStatus + from aurweb.templates import make_context, render_template async def not_found(request, exc): context = make_context(request, "Page Not Found") - return render_template(request, "errors/404.html", context, 404) + return render_template(request, "errors/404.html", context, + HTTPStatus.NOT_FOUND) async def service_unavailable(request, exc): context = make_context(request, "Service Unavailable") - return render_template(request, "errors/503.html", context, 503) + return render_template(request, "errors/503.html", context, + HTTPStatus.SERVICE_UNAVAILABLE) # Maps HTTP errors to functions exceptions = { diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index 525fb626..337acce6 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -221,4 +221,4 @@ async def metrics(request: Request): @router.get("/raisefivethree", response_class=HTMLResponse) async def raise_service_unavailable(request: Request): - raise HTTPException(status_code=503) + raise HTTPException(status_code=HTTPStatus.SERVICE_UNAVAILABLE) diff --git a/aurweb/routers/sso.py b/aurweb/routers/sso.py index edeb7c6b..eff1c63f 100644 --- a/aurweb/routers/sso.py +++ b/aurweb/routers/sso.py @@ -1,6 +1,7 @@ import time import uuid +from http import HTTPStatus from urllib.parse import urlencode import fastapi @@ -59,7 +60,8 @@ def open_session(request, conn, user_id): """ if is_account_suspended(conn, user_id): _ = get_translator_for_request(request) - raise HTTPException(status_code=403, detail=_('Account suspended')) + raise HTTPException(status_code=HTTPStatus.FORBIDDEN, + detail=_('Account suspended')) # TODO This is a terrible message because it could imply the attempt at # logging in just caused the suspension. @@ -104,7 +106,7 @@ async def authenticate(request: Request, redirect: str = None, conn=Depends(aurw if is_ip_banned(conn, request.client.host): _ = get_translator_for_request(request) raise HTTPException( - status_code=403, + status_code=HTTPStatus.FORBIDDEN, detail=_('The login form is currently disabled for your IP address, ' 'probably due to sustained spam attacks. Sorry for the ' 'inconvenience.')) @@ -117,13 +119,14 @@ async def authenticate(request: Request, redirect: str = None, conn=Depends(aurw # Let’s give attackers as little information as possible. _ = get_translator_for_request(request) raise HTTPException( - status_code=400, + status_code=HTTPStatus.BAD_REQUEST, detail=_('Bad OAuth token. Please retry logging in from the start.')) sub = user.get("sub") # this is the SSO account ID in JWT terminology if not sub: _ = get_translator_for_request(request) - raise HTTPException(status_code=400, detail=_("JWT is missing its `sub` field.")) + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, + detail=_("JWT is missing its `sub` field.")) aur_accounts = conn.execute(select([Users.c.ID]).where(Users.c.SSOAccountID == sub)) \ .fetchall() From e1bf6dd56256eddfbed2909fd658406755ce06f2 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 2 Dec 2021 17:09:37 -0800 Subject: [PATCH 694/844] fix(fastapi): restore stripped whitespace in archdev-navbar Signed-off-by: Kevin Morris --- templates/partials/archdev-navbar.html | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/templates/partials/archdev-navbar.html b/templates/partials/archdev-navbar.html index dc2377fe..98bb1841 100644 --- a/templates/partials/archdev-navbar.html +++ b/templates/partials/archdev-navbar.html @@ -54,17 +54,10 @@

  • {% else %} {# All guest users see Register #} -
  • - - {% trans %}Register{% endtrans %} - -
  • +
  • {% trans %}Register{% endtrans %}
  • + {# All guest users see Login #} -
  • - - {% trans %}Login{% endtrans %} - -
  • +
  • {% trans %}Login{% endtrans %}
  • {% endif %} From abfd41f31e76a11619f6aa233058aa0bb25c2dec Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 2 Dec 2021 23:22:31 -0800 Subject: [PATCH 695/844] change(fastapi): centralize HTTPException Signed-off-by: Kevin Morris --- aurweb/asgi.py | 24 +++++++++++++----------- aurweb/routers/errors.py | 21 --------------------- templates/errors/404.html | 8 -------- templates/errors/503.html | 8 -------- templates/errors/detail.html | 8 ++++++++ test/test_asgi.py | 10 +++++++--- 6 files changed, 28 insertions(+), 51 deletions(-) delete mode 100644 aurweb/routers/errors.py delete mode 100644 templates/errors/404.html delete mode 100644 templates/errors/503.html create mode 100644 templates/errors/detail.html diff --git a/aurweb/asgi.py b/aurweb/asgi.py index b399cfb1..ef8d5933 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -6,8 +6,8 @@ import typing from urllib.parse import quote_plus -from fastapi import FastAPI, HTTPException, Request -from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi import FastAPI, HTTPException, Request, Response +from fastapi.responses import RedirectResponse from fastapi.staticfiles import StaticFiles from prometheus_client import multiprocess from sqlalchemy import and_, or_ @@ -21,10 +21,11 @@ from aurweb.auth import BasicAuthBackend from aurweb.db import get_engine, query from aurweb.models import AcceptedTerm, Term from aurweb.prometheus import http_api_requests_total, http_requests_total, instrumentator -from aurweb.routers import accounts, auth, errors, html, packages, rpc, rss, sso, trusted_user +from aurweb.routers import accounts, auth, html, packages, rpc, rss, sso, trusted_user +from aurweb.templates import make_context, render_template # Setup the FastAPI app. -app = FastAPI(exception_handlers=errors.exceptions) +app = FastAPI() # Instrument routes with the prometheus-fastapi-instrumentator # library with custom collectors and expose /metrics. @@ -93,14 +94,15 @@ def child_exit(server, worker): # pragma: no cover @app.exception_handler(HTTPException) -async def http_exception_handler(request, exc): - """ - Dirty HTML error page to replace the default JSON error responses. - In the future this should use a proper Arch-themed HTML template. - """ +async def http_exception_handler(request: Request, exc: HTTPException) \ + -> Response: + """ Handle an HTTPException thrown in a route. """ phrase = http.HTTPStatus(exc.status_code).phrase - return HTMLResponse(f"

    {exc.status_code} {phrase}

    {exc.detail}

    ", - status_code=exc.status_code) + context = make_context(request, phrase) + context["exc"] = exc + context["phrase"] = phrase + return render_template(request, "errors/detail.html", context, + exc.status_code) @app.middleware("http") diff --git a/aurweb/routers/errors.py b/aurweb/routers/errors.py deleted file mode 100644 index 9ed1e80d..00000000 --- a/aurweb/routers/errors.py +++ /dev/null @@ -1,21 +0,0 @@ -from http import HTTPStatus - -from aurweb.templates import make_context, render_template - - -async def not_found(request, exc): - context = make_context(request, "Page Not Found") - return render_template(request, "errors/404.html", context, - HTTPStatus.NOT_FOUND) - - -async def service_unavailable(request, exc): - context = make_context(request, "Service Unavailable") - return render_template(request, "errors/503.html", context, - HTTPStatus.SERVICE_UNAVAILABLE) - -# Maps HTTP errors to functions -exceptions = { - 404: not_found, - 503: service_unavailable -} diff --git a/templates/errors/404.html b/templates/errors/404.html deleted file mode 100644 index 4926aff6..00000000 --- a/templates/errors/404.html +++ /dev/null @@ -1,8 +0,0 @@ -{% extends 'partials/layout.html' %} - -{% block pageContent %} -
    -

    404 - {% trans %}Page Not Found{% endtrans %}

    -

    {% trans %}Sorry, the page you've requested does not exist.{% endtrans %}

    -
    -{% endblock %} diff --git a/templates/errors/503.html b/templates/errors/503.html deleted file mode 100644 index 9a0ed56a..00000000 --- a/templates/errors/503.html +++ /dev/null @@ -1,8 +0,0 @@ -{% extends 'partials/layout.html' %} - -{% block pageContent %} -
    -

    503 - {% trans %}Service Unavailable{% endtrans %}

    -

    {% trans %}Don't panic! This site is down due to maintenance. We will be back soon.{% endtrans %}

    -
    -{% endblock %} diff --git a/templates/errors/detail.html b/templates/errors/detail.html new file mode 100644 index 00000000..f382a9bb --- /dev/null +++ b/templates/errors/detail.html @@ -0,0 +1,8 @@ +{% extends 'partials/layout.html' %} + +{% block pageContent %} +
    +

    {{ "%d" | format(exc.status_code) }} - {{ phrase }}

    +

    {{ exc.detail }}

    +
    +{% endblock %} diff --git a/test/test_asgi.py b/test/test_asgi.py index fa2df5a1..16b07c31 100644 --- a/test/test_asgi.py +++ b/test/test_asgi.py @@ -11,6 +11,8 @@ import aurweb.asgi import aurweb.config import aurweb.redis +from aurweb.testing.requests import Request + @pytest.mark.asyncio async def test_asgi_startup_session_secret_exception(monkeypatch): @@ -42,9 +44,11 @@ async def test_asgi_startup_exception(monkeypatch): async def test_asgi_http_exception_handler(): exc = HTTPException(status_code=422, detail="EXCEPTION!") phrase = http.HTTPStatus(exc.status_code).phrase - response = await aurweb.asgi.http_exception_handler(None, exc) - assert response.body.decode() == \ - f"

    {exc.status_code} {phrase}

    {exc.detail}

    " + response = await aurweb.asgi.http_exception_handler(Request(), exc) + assert response.status_code == 422 + content = response.body.decode() + assert f"{exc.status_code} - {phrase}" in content + assert "EXCEPTION!" in content @pytest.mark.asyncio From 806a19b91a3f2e254e1956243a2fff89b123ff4f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 2 Dec 2021 23:26:42 -0800 Subject: [PATCH 696/844] feat(fastapi): render a 500 html response when unique SID generation fails We've seen a bug in the past where unique SID generation fails and still ends up raising an exception. This commit reworks how we deal with database exceptions internally, tries for 36 iterations to set a fresh unique SID, and raises a 500 HTTPException if we were unable to. Signed-off-by: Kevin Morris --- aurweb/models/user.py | 67 ++++++++++++++++++++++++---------------- test/test_auth_routes.py | 45 +++++++++++++++++++++++++++ test/test_user.py | 2 -- 3 files changed, 85 insertions(+), 29 deletions(-) diff --git a/aurweb/models/user.py b/aurweb/models/user.py index f0724202..dcf5f519 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -1,12 +1,14 @@ import hashlib from datetime import datetime +from http import HTTPStatus from typing import List, Set import bcrypt -from fastapi import Request +from fastapi import HTTPException, Request from sqlalchemy import or_ +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import backref, relationship import aurweb.config @@ -108,33 +110,45 @@ class User(Base): if not self.authenticated: return None - now_ts = datetime.utcnow().timestamp() - session_ts = now_ts + ( - session_time if session_time - else aurweb.config.getint("options", "login_timeout") - ) + # Maximum number of iterations where we attempt to generate + # a unique SID. In cases where the Session table has + # exhausted all possible values, this will catch exceptions + # instead of raising them and include details about failing + # generation in an HTTPException. + tries = 36 - sid = None + exc = None + for i in range(tries): + exc = None + now_ts = datetime.utcnow().timestamp() + session_ts = now_ts + ( + session_time if session_time + else aurweb.config.getint("options", "login_timeout") + ) + try: + with db.begin(): + self.LastLogin = now_ts + self.LastLoginIPAddress = request.client.host + if not self.session: + sid = generate_unique_sid() + self.session = db.create(Session, User=self, + SessionID=sid, + LastUpdateTS=session_ts) + else: + last_updated = self.session.LastUpdateTS + if last_updated and last_updated < now_ts: + self.session.SessionID = generate_unique_sid() + self.session.LastUpdateTS = session_ts + break + except IntegrityError as exc_: + exc = exc_ - with db.begin(): - self.LastLogin = now_ts - self.LastLoginIPAddress = request.client.host - if not self.session: - sid = generate_unique_sid() - self.session = Session(UsersID=self.ID, SessionID=sid, - LastUpdateTS=session_ts) - db.add(self.session) - else: - last_updated = self.session.LastUpdateTS - if last_updated and last_updated < now_ts: - self.session.SessionID = sid = generate_unique_sid() - else: - # Session is still valid; retrieve the current SID. - sid = self.session.SessionID + if exc: + detail = ("Unable to generate a unique session ID in " + f"{tries} iterations.") + raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail=detail) - self.session.LastUpdateTS = session_ts - - request.cookies["AURSID"] = self.session.SessionID return self.session.SessionID def has_credential(self, credential: Set[int], @@ -142,8 +156,7 @@ class User(Base): from aurweb.auth.creds import has_credential return has_credential(self, credential, approved) - def logout(self, request): - del request.cookies["AURSID"] + def logout(self, request: Request): self.authenticated = False if self.session: with db.begin(): diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index a8d0db11..3455a019 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -283,3 +283,48 @@ def test_login_bad_referer(client: TestClient): response = request.post("/login", data=post_data, headers=BAD_REFERER) assert response.status_code == int(HTTPStatus.BAD_REQUEST) assert "AURSID" not in response.cookies + + +def test_generate_unique_sid_exhausted(client: TestClient, user: User): + """ + In this test, we mock up generate_unique_sid() to infinitely return + the same SessionID given to `user`. Within that mocking, we try + to login as `user2` and expect the internal server error rendering + by our error handler. + + This exercises the bad path of /login, where we can't find a unique + SID to assign the user. + """ + now = int(datetime.utcnow().timestamp()) + with db.begin(): + # Create a second user; we'll login with this one. + user2 = db.create(User, Username="test2", Email="test2@example.org", + ResetKey="testReset", Passwd="testPassword", + AccountTypeID=USER_ID) + + # Create a session with ID == "testSession" for `user`. + db.create(Session, User=user, SessionID="testSession", + LastUpdateTS=now) + + # Mock out generate_unique_sid; always return "testSession" which + # causes us to eventually error out and raise an internal error. + def mock_generate_sid(): + return "testSession" + + # Login as `user2`; we expect an internal server error response + # with a relevent detail. + post_data = { + "user": user2.Username, + "passwd": "testPassword", + "next": "/", + } + generate_unique_sid_ = "aurweb.models.session.generate_unique_sid" + with mock.patch(generate_unique_sid_, mock_generate_sid): + with client as request: + # Set cookies = {} to remove any previous login kept by TestClient. + response = request.post("/login", data=post_data, cookies={}) + assert response.status_code == int(HTTPStatus.INTERNAL_SERVER_ERROR) + + expected = "Unable to generate a unique session ID" + assert expected in response.text + assert "500 - Internal Server Error" in response.text diff --git a/test/test_user.py b/test/test_user.py index 52cdc89e..2c8dd847 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -62,7 +62,6 @@ def test_user_login_logout(user: User): sid = user.login(request, "testPassword") assert sid is not None assert user.is_authenticated() - assert "AURSID" in request.cookies # Expect that User session relationships work right. user_session = db.query(Session, @@ -92,7 +91,6 @@ def test_user_login_logout(user: User): # Test logout. user.logout(request) - assert "AURSID" not in request.cookies assert not user.is_authenticated() From 81f8c2326566f993fcbcb2f0ef189c13657b0676 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Thu, 2 Dec 2021 23:42:13 -0800 Subject: [PATCH 697/844] fix(fastapi): log out IntegrityError from failed SID generation Signed-off-by: Kevin Morris --- aurweb/models/user.py | 5 ++++- test/test_auth_routes.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/aurweb/models/user.py b/aurweb/models/user.py index dcf5f519..8e66b490 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -15,11 +15,13 @@ import aurweb.config import aurweb.models.account_type import aurweb.schema -from aurweb import db, schema +from aurweb import db, logging, schema from aurweb.models.account_type import AccountType as _AccountType from aurweb.models.ban import is_banned from aurweb.models.declarative import Base +logger = logging.get_logger(__name__) + SALT_ROUNDS_DEFAULT = 12 @@ -146,6 +148,7 @@ class User(Base): if exc: detail = ("Unable to generate a unique session ID in " f"{tries} iterations.") + logger.error(str(exc)) raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=detail) diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index 3455a019..3ae8a56c 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -285,7 +285,8 @@ def test_login_bad_referer(client: TestClient): assert "AURSID" not in response.cookies -def test_generate_unique_sid_exhausted(client: TestClient, user: User): +def test_generate_unique_sid_exhausted(client: TestClient, user: User, + caplog: pytest.LogCaptureFixture): """ In this test, we mock up generate_unique_sid() to infinitely return the same SessionID given to `user`. Within that mocking, we try @@ -328,3 +329,6 @@ def test_generate_unique_sid_exhausted(client: TestClient, user: User): expected = "Unable to generate a unique session ID" assert expected in response.text assert "500 - Internal Server Error" in response.text + + # Make sure an IntegrityError from the DB got logged out. + assert "IntegrityError" in caplog.text From 75ad2fb53d04e5f85dc32779bfa0e373f9301e74 Mon Sep 17 00:00:00 2001 From: Steven Guikal Date: Wed, 1 Dec 2021 16:35:24 -0500 Subject: [PATCH 698/844] fix(FastAPI): cleanup auth_required decorator Signed-off-by: Steven Guikal --- aurweb/auth/__init__.py | 46 ++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/aurweb/auth/__init__.py b/aurweb/auth/__init__.py index 18356ac2..b6dd6e3f 100644 --- a/aurweb/auth/__init__.py +++ b/aurweb/auth/__init__.py @@ -120,35 +120,39 @@ class BasicAuthBackend(AuthenticationBackend): return (AuthCredentials(["authenticated"]), user) -def auth_required(is_required: bool = True): - """ Authentication route decorator. +def auth_required(auth_goal: bool = True): + """ Enforce a user's authentication status, bringing them to the login page + or homepage if their authentication status does not match the goal. - :param is_required: A boolean indicating whether the function requires auth - :param status_code: An optional status_code for template render. - Redirects are always SEE_OTHER. + :param auth_goal: Whether authentication is required or entirely disallowed + for a user to perform this request. + :return: Return the FastAPI function this decorator wraps. """ def decorator(func): @functools.wraps(func) async def wrapper(request, *args, **kwargs): - if request.user.is_authenticated() != is_required: - url = "/" + if request.user.is_authenticated() == auth_goal: + return await func(request, *args, **kwargs) - if is_required: - if request.method == "GET": - url = request.url.path - elif request.method == "POST" and (referer := request.headers.get("Referer")): - aur = aurweb.config.get("options", "aur_location") + "/" - if not referer.startswith(aur): - _ = l10n.get_translator_for_request(request) - raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, - detail=_("Bad Referer header.")) - url = referer[len(aur) - 1:] + url = "/" + if auth_goal is False: + return RedirectResponse(url, status_code=int(HTTPStatus.SEE_OTHER)) - url = "/login?" + util.urlencode({"next": url}) - return RedirectResponse(url, - status_code=int(HTTPStatus.SEE_OTHER)) - return await func(request, *args, **kwargs) + # Use the request path when the user can visit a page directly but + # is not authenticated and use the Referer header if visiting the + # page itself is not directly possible (e.g. submitting a form). + if request.method in ("GET", "HEAD"): + url = request.url.path + elif (referer := request.headers.get("Referer")): + aur = aurweb.config.get("options", "aur_location") + "/" + if not referer.startswith(aur): + _ = l10n.get_translator_for_request(request) + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, + detail=_("Bad Referer header.")) + url = referer[len(aur) - 1:] + url = "/login?" + util.urlencode({"next": url}) + return RedirectResponse(url, status_code=int(HTTPStatus.SEE_OTHER)) return wrapper return decorator From b0b5e4c9d10ee3c31c7e3a61286c9fdabd3c8ceb Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 3 Dec 2021 15:13:41 -0800 Subject: [PATCH 699/844] fix(fastapi): use `secrets` module to generate random strings Signed-off-by: Kevin Morris --- aurweb/util.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/aurweb/util.py b/aurweb/util.py index f5ced259..542dfc2e 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -1,7 +1,6 @@ import base64 import copy import math -import random import re import secrets import string @@ -25,9 +24,9 @@ from aurweb import defaults, logging logger = logging.get_logger(__name__) -def make_random_string(length): - return ''.join(random.choices(string.ascii_lowercase - + string.digits, k=length)) +def make_random_string(length: int) -> str: + alphanumerics = string.ascii_lowercase + string.digits + return ''.join([secrets.choice(alphanumerics) for i in range(length)]) def make_nonce(length: int = 8): From aa717a4ef9d45d0b2a454a75bdd4f55dd18a2225 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 3 Dec 2021 15:41:54 -0800 Subject: [PATCH 700/844] change(fastapi): no longer care about ResetKey collisions Signed-off-by: Kevin Morris --- aurweb/models/user.py | 6 +++--- aurweb/routers/accounts.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 8e66b490..d0bdea30 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -15,7 +15,7 @@ import aurweb.config import aurweb.models.account_type import aurweb.schema -from aurweb import db, logging, schema +from aurweb import db, logging, schema, util from aurweb.models.account_type import AccountType as _AccountType from aurweb.models.ban import is_banned from aurweb.models.declarative import Base @@ -249,5 +249,5 @@ class User(Base): self.ID, str(self.AccountType), self.Username) -def generate_unique_resetkey(): - return db.make_random_value(User, User.ResetKey, 32) +def generate_resetkey(): + return util.make_random_string(32) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 388daf84..f61ccdd2 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -16,7 +16,7 @@ from aurweb.exceptions import ValidationError from aurweb.l10n import get_translator_for_request from aurweb.models import account_type as at from aurweb.models.ssh_pub_key import get_fingerprint -from aurweb.models.user import generate_unique_resetkey +from aurweb.models.user import generate_resetkey from aurweb.scripts.notify import ResetKeyNotification, WelcomeNotification from aurweb.templates import make_context, make_variable_context, render_template from aurweb.users import update, validate @@ -93,7 +93,7 @@ async def passreset_post(request: Request, status_code=HTTPStatus.SEE_OTHER) # If we got here, we continue with issuing a resetkey for the user. - resetkey = generate_unique_resetkey() + resetkey = generate_resetkey() with db.begin(): user.ResetKey = resetkey @@ -291,7 +291,7 @@ async def account_register_post(request: Request, # Create a user with no password with a resetkey, then send # an email off about it. - resetkey = generate_unique_resetkey() + resetkey = generate_resetkey() # By default, we grab the User account type to associate with. atype = db.query(models.AccountType, From bfa916c7b294fb82f3b935f973b649f42849557b Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 3 Dec 2021 23:40:16 -0800 Subject: [PATCH 701/844] fix(fastapi): fix PGP Key Fingerprint display for account/show.html There's a space between every 4 characters in the fingerprint in PHP; we were missing it in FastAPI. This commit fixes that inconsistency. Signed-off-by: Kevin Morris --- aurweb/routers/accounts.py | 11 ++++++++++- templates/account/show.html | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index f61ccdd2..946ffc31 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -432,7 +432,16 @@ async def account(request: Request, username: str): if not request.user.is_authenticated(): return render_template(request, "account/show.html", context, status_code=HTTPStatus.UNAUTHORIZED) - context["user"] = get_user_by_name(username) + + # Get related User record, if possible. + user = get_user_by_name(username) + context["user"] = user + + # Format PGPKey for display with a space between each 4 characters. + k = user.PGPKey or str() + context["pgp_key"] = " ".join([k[i:i + 4] for i in range(0, len(k), 4)]) + + # Render the template. return render_template(request, "account/show.html", context) diff --git a/templates/account/show.html b/templates/account/show.html index 0c99c99f..23b262b0 100644 --- a/templates/account/show.html +++ b/templates/account/show.html @@ -46,7 +46,7 @@ {% trans %}PGP Key Fingerprint{% endtrans %}: - {{ user.PGPKey or '' }} + {{ pgp_key }} {% trans %}Status{% endtrans %}: From d0fc56d53fa1232aa5d5c1b5a7bca12ad1edb773 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Dec 2021 00:14:55 -0800 Subject: [PATCH 702/844] fix(python): redirect when the request user can't edit target user Signed-off-by: Kevin Morris --- aurweb/routers/accounts.py | 24 +++++++++++++++++------- test/test_accounts_routes.py | 30 ++++++++++++++++++------------ 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index f61ccdd2..ff2c3040 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -329,13 +329,23 @@ async def account_register_post(request: Request, return render_template(request, "register.html", context) -def cannot_edit(request, user): - """ Return a 401 HTMLResponse if the request user doesn't - have authorization, otherwise None. """ - has_dev_cred = request.user.has_credential(creds.ACCOUNT_EDIT_DEV, - approved=[user]) - if not has_dev_cred: - return HTMLResponse(status_code=HTTPStatus.UNAUTHORIZED) +def cannot_edit(request: Request, user: models.User) \ + -> typing.Optional[RedirectResponse]: + """ + Decide if `request.user` cannot edit `user`. + + If the request user can edit the target user, None is returned. + Otherwise, a redirect is returned to /account/{user.Username}. + + :param request: FastAPI request + :param user: Target user to be edited + :return: RedirectResponse if approval != granted else None + """ + approved = request.user.has_credential(creds.ACCOUNT_EDIT, approved=[user]) + if not approved and (to := "/"): + if user: + to = f"/account/{user.Username}" + return RedirectResponse(to, status_code=HTTPStatus.SEE_OTHER) return None diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index f08efcd2..348a6994 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -620,16 +620,19 @@ def test_get_account_edit_unauthorized(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") - create(User, Username="test2", Email="test2@example.org", - Passwd="testPassword") + with db.begin(): + user2 = create(User, Username="test2", Email="test2@example.org", + Passwd="testPassword", AccountTypeID=USER_ID) + endpoint = f"/account/{user2.Username}/edit" with client as request: # Try to edit `test2` while authenticated as `test`. - response = request.get("/account/test2/edit", cookies={ - "AURSID": sid - }, allow_redirects=False) + response = request.get(endpoint, cookies={"AURSID": sid}, + allow_redirects=False) + assert response.status_code == int(HTTPStatus.SEE_OTHER) - assert response.status_code == int(HTTPStatus.UNAUTHORIZED) + expected = f"/account/{user2.Username}" + assert response.headers.get("location") == expected def test_post_account_edit(client: TestClient, user: User): @@ -828,8 +831,9 @@ def test_post_account_edit_error_unauthorized(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") - create(User, Username="test2", - Email="test2@example.org", Passwd="testPassword") + with db.begin(): + user2 = create(User, Username="test2", Email="test2@example.org", + Passwd="testPassword", AccountTypeID=USER_ID) post_data = { "U": "test", @@ -838,13 +842,15 @@ def test_post_account_edit_error_unauthorized(client: TestClient, user: User): "passwd": "testPassword" } + endpoint = f"/account/{user2.Username}/edit" with client as request: # Attempt to edit 'test2' while logged in as 'test'. - response = request.post("/account/test2/edit", cookies={ - "AURSID": sid - }, data=post_data, allow_redirects=False) + response = request.post(endpoint, cookies={"AURSID": sid}, + data=post_data, allow_redirects=False) + assert response.status_code == int(HTTPStatus.SEE_OTHER) - assert response.status_code == int(HTTPStatus.UNAUTHORIZED) + expected = f"/account/{user2.Username}" + assert response.headers.get("location") == expected def test_post_account_edit_ssh_pub_key(client: TestClient, user: User): From 973dbf04828c1b5f475c189d12dd8c099ac8b35e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Dec 2021 00:15:34 -0800 Subject: [PATCH 703/844] fix(python): use creds to determine account links to display Signed-off-by: Kevin Morris --- templates/account/show.html | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/templates/account/show.html b/templates/account/show.html index 0c99c99f..14a4eccf 100644 --- a/templates/account/show.html +++ b/templates/account/show.html @@ -69,20 +69,24 @@ | safe }} -
  • - {{ "%sEdit this user's account%s" - | tr - | format('' | format(user.Username), "") - | safe - }} -
  • -
  • - {{ "%sList this user's comments%s" - | tr - | format('' | format(user.Username), "") - | safe - }} -
  • + {% if request.user.has_credential(creds.ACCOUNT_EDIT, approved=[user]) %} +
  • + {{ "%sEdit this user's account%s" + | tr + | format('' | format(user.Username), "") + | safe + }} +
  • + {% endif %} + {% if request.user.has_credential(creds.ACCOUNT_LIST_COMMENTS, approved=[user]) %} +
  • + {{ "%sList this user's comments%s" + | tr + | format('' | format(user.Username), "") + | safe + }} +
  • + {% endif %} From 2ea4559b60135b38c07b949d5905c99ec98739dc Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Dec 2021 00:50:32 -0800 Subject: [PATCH 704/844] fix(python): use correct Status field in account/show.html Signed-off-by: Kevin Morris --- templates/account/show.html | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/templates/account/show.html b/templates/account/show.html index e1074394..3e36faf0 100644 --- a/templates/account/show.html +++ b/templates/account/show.html @@ -50,6 +50,17 @@ {% trans %}Status{% endtrans %}: + {% if not user.InactivityTS %} + {{ "Active" | tr }} + {% else %} + {% set inactive_ds = user.InactivityTS | dt | as_timezone(timezone) %} + + {{ + "Inactive since %s" | tr + | format(inactive_ds.strftime("%Y-%m-%d %H:%M")) + }} + + {% endif %} {{ "Active" if not user.Suspended else "Suspended" | tr }} From 224a0de784634c1ee5569e12a7f95d1e9bc1bb5f Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Dec 2021 01:16:14 -0800 Subject: [PATCH 705/844] fix(python): add logged in date field to account/show.html Signed-off-by: Kevin Morris --- aurweb/routers/accounts.py | 7 +++++++ templates/account/show.html | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index fc25a7e8..8eecaa31 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -451,6 +451,13 @@ async def account(request: Request, username: str): k = user.PGPKey or str() context["pgp_key"] = " ".join([k[i:i + 4] for i in range(0, len(k), 4)]) + login_ts = None + session = db.query(models.Session).filter( + models.Session.UsersID == user.ID).first() + if session: + login_ts = user.session.LastUpdateTS + context["login_ts"] = login_ts + # Render the template. return render_template(request, "account/show.html", context) diff --git a/templates/account/show.html b/templates/account/show.html index 3e36faf0..c6a53f4a 100644 --- a/templates/account/show.html +++ b/templates/account/show.html @@ -61,7 +61,6 @@ }} {% endif %} - {{ "Active" if not user.Suspended else "Suspended" | tr }} {% trans %}Registration date{% endtrans %}: @@ -69,6 +68,13 @@ {{ user.RegistrationTS.strftime("%Y-%m-%d") }} + {% if login_ts %} + + {% trans %}Last Login{% endtrans %}: + {% set login_ds = login_ts | dt | as_timezone(timezone) %} + {{ login_ds.strftime("%Y-%m-%d") }} + + {% endif %} {% trans %}Links{% endtrans %}: From 8501bba0ac7e6ad03f80d9f370bcd4cbd63db296 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Dec 2021 02:12:20 -0800 Subject: [PATCH 706/844] change(python): rework session timing Previously, we were just relying on the cookie expiration for sessions to expire. We were not cleaning up Session records either. Rework timing to depend on an AURREMEMBER cookie which is now emitted on login during BasicAuthBackend processing. If the SID does still have a session but it's expired, we now delete the session record before returning. Otherwise, we update the session's LastUpdateTS to the current time. In addition, stored the unauthenticated result value in a variable to reduce redundancy. Signed-off-by: Kevin Morris --- aurweb/auth/__init__.py | 22 +++++++++++++++------- aurweb/models/user.py | 8 ++------ aurweb/routers/auth.py | 4 ++++ test/test_auth.py | 24 +++++++++++++++++++++++- test/test_auth_routes.py | 16 ++++++---------- 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/aurweb/auth/__init__.py b/aurweb/auth/__init__.py index b6dd6e3f..5f55e2fb 100644 --- a/aurweb/auth/__init__.py +++ b/aurweb/auth/__init__.py @@ -7,7 +7,6 @@ import fastapi from fastapi import HTTPException from fastapi.responses import RedirectResponse -from sqlalchemy import and_ from starlette.authentication import AuthCredentials, AuthenticationBackend from starlette.requests import HTTPConnection @@ -97,18 +96,27 @@ class AnonymousUser: class BasicAuthBackend(AuthenticationBackend): async def authenticate(self, conn: HTTPConnection): + unauthenticated = (None, AnonymousUser()) sid = conn.cookies.get("AURSID") if not sid: - return (None, AnonymousUser()) + return unauthenticated - now_ts = datetime.utcnow().timestamp() - record = db.query(Session).filter( - and_(Session.SessionID == sid, - Session.LastUpdateTS >= now_ts)).first() + timeout = aurweb.config.getint("options", "login_timeout") + remembered = ("AURREMEMBER" in conn.cookies + and bool(conn.cookies.get("AURREMEMBER"))) + if remembered: + timeout = aurweb.config.getint("options", + "persistent_cookie_timeout") # If no session with sid and a LastUpdateTS now or later exists. + now_ts = int(datetime.utcnow().timestamp()) + record = db.query(Session).filter(Session.SessionID == sid).first() if not record: - return (None, AnonymousUser()) + return unauthenticated + elif record.LastUpdateTS < (now_ts - timeout): + with db.begin(): + db.delete_all([record]) + return unauthenticated # At this point, we cannot have an invalid user if the record # exists, due to ForeignKey constraints in the schema upheld diff --git a/aurweb/models/user.py b/aurweb/models/user.py index d0bdea30..5ead606e 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -123,10 +123,6 @@ class User(Base): for i in range(tries): exc = None now_ts = datetime.utcnow().timestamp() - session_ts = now_ts + ( - session_time if session_time - else aurweb.config.getint("options", "login_timeout") - ) try: with db.begin(): self.LastLogin = now_ts @@ -135,12 +131,12 @@ class User(Base): sid = generate_unique_sid() self.session = db.create(Session, User=self, SessionID=sid, - LastUpdateTS=session_ts) + LastUpdateTS=now_ts) else: last_updated = self.session.LastUpdateTS if last_updated and last_updated < now_ts: self.session.SessionID = generate_unique_sid() - self.session.LastUpdateTS = session_ts + self.session.LastUpdateTS = now_ts break except IntegrityError as exc_: exc = exc_ diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index 74763667..8815c896 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -73,6 +73,10 @@ async def login_post(request: Request, response.set_cookie("AURLANG", user.LangPreference, secure=secure, httponly=secure, samesite=cookies.samesite()) + response.set_cookie("AURREMEMBER", remember_me, + expires=expires_at, + secure=secure, httponly=secure, + samesite=cookies.samesite()) return response diff --git a/test/test_auth.py b/test/test_auth.py index b607a038..0094aa25 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -6,7 +6,7 @@ import pytest from fastapi import HTTPException from sqlalchemy.exc import IntegrityError -from aurweb import db +from aurweb import config, db from aurweb.auth import AnonymousUser, BasicAuthBackend, account_type_required, auth_required from aurweb.models.account_type import USER, USER_ID from aurweb.models.session import Session @@ -76,6 +76,28 @@ async def test_basic_auth_backend(user: User, backend: BasicAuthBackend): assert result == user +@pytest.mark.asyncio +async def test_expired_session(backend: BasicAuthBackend, user: User): + """ Login, expire the session manually, then authenticate. """ + # First, build a Request with a logged in user. + request = Request() + request.user = user + sid = request.user.login(Request(), "testPassword") + request.cookies["AURSID"] = sid + + # Set Session.LastUpdateTS to 20 seconds expired. + timeout = config.getint("options", "login_timeout") + now_ts = int(datetime.utcnow().timestamp()) + with db.begin(): + request.user.session.LastUpdateTS = now_ts - timeout - 20 + + # Run through authentication backend and get the session + # deleted due to its expiration. + await backend.authenticate(request) + session = db.query(Session).filter(Session.SessionID == sid).first() + assert session is None + + @pytest.mark.asyncio async def test_auth_required_redirection_bad_referrer(): # Create a fake route function which can be wrapped by auth_required. diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index 3ae8a56c..f3e2a011 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -13,7 +13,6 @@ from aurweb.asgi import app from aurweb.models.account_type import USER_ID from aurweb.models.session import Session from aurweb.models.user import User -from aurweb.testing.requests import Request # Some test global constants. TEST_USERNAME = "test" @@ -136,12 +135,11 @@ def test_secure_login(getboolean: bool, client: TestClient, user: User): def test_authenticated_login(client: TestClient, user: User): post_data = { - "user": "test", + "user": user.Username, "passwd": "testPassword", "next": "/" } - cookies = {"AURSID": user.login(Request(), "testPassword")} with client as request: # Try to login. response = request.post("/login", data=post_data, @@ -153,7 +151,7 @@ def test_authenticated_login(client: TestClient, user: User): # when requesting GET /login as an authenticated user. # Now, let's verify that we receive 403 Forbidden when we # try to get /login as an authenticated user. - response = request.get("/login", cookies=cookies, + response = request.get("/login", cookies=response.cookies, allow_redirects=False) assert response.status_code == int(HTTPStatus.OK) assert "Logged-in as: test" in response.text @@ -200,14 +198,12 @@ def test_login_remember_me(client: TestClient, user: User): cookie_timeout = aurweb.config.getint( "options", "persistent_cookie_timeout") - expected_ts = datetime.utcnow().timestamp() + cookie_timeout - + now_ts = int(datetime.utcnow().timestamp()) session = db.query(Session).filter(Session.UsersID == user.ID).first() - # Expect that LastUpdateTS was within 5 seconds of the expected_ts, - # which is equal to the current timestamp + persistent_cookie_timeout. - assert session.LastUpdateTS > expected_ts - 5 - assert session.LastUpdateTS < expected_ts + 5 + # Expect that LastUpdateTS is not past the cookie timeout + # for a remembered session. + assert session.LastUpdateTS > (now_ts - cookie_timeout) def test_login_incorrect_password_remember_me(client: TestClient, user: User): From cf978e23aa787a002d66403866140cc4be2617fe Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Dec 2021 00:51:33 -0800 Subject: [PATCH 707/844] fix(python): use S argument to decide Suspended Signed-off-by: Kevin Morris --- aurweb/users/update.py | 4 ++-- test/test_accounts_routes.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/aurweb/users/update.py b/aurweb/users/update.py index 60a6184e..fd42a194 100644 --- a/aurweb/users/update.py +++ b/aurweb/users/update.py @@ -12,7 +12,7 @@ def simple(U: str = str(), E: str = str(), H: bool = False, BE: str = str(), R: str = str(), HP: str = str(), I: str = str(), K: str = str(), J: bool = False, CN: bool = False, UN: bool = False, ON: bool = False, - user: models.User = None, + S: bool = False, user: models.User = None, **kwargs) -> None: now = int(datetime.utcnow().timestamp()) with db.begin(): @@ -24,7 +24,7 @@ def simple(U: str = str(), E: str = str(), H: bool = False, user.Homepage = HP or user.Homepage user.IRCNick = I or user.IRCNick user.PGPKey = K or user.PGPKey - user.Suspended = strtobool(J) + user.Suspended = strtobool(S) user.InactivityTS = now * int(strtobool(J)) user.CommentNotify = strtobool(CN) user.UpdateNotify = strtobool(UN) diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index 348a6994..d3435089 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -814,7 +814,6 @@ def test_post_account_edit_inactivity(client: TestClient, user: User): assert resp.status_code == int(HTTPStatus.OK) # Make sure the user record got updated correctly. - assert user.Suspended assert user.InactivityTS > 0 post_data.update({"J": False}) @@ -823,10 +822,37 @@ def test_post_account_edit_inactivity(client: TestClient, user: User): cookies=cookies) assert resp.status_code == int(HTTPStatus.OK) - assert not user.Suspended assert user.InactivityTS == 0 +def test_post_account_edit_suspended(client: TestClient, user: User): + with db.begin(): + user.AccountTypeID = TRUSTED_USER_ID + assert not user.Suspended + + cookies = {"AURSID": user.login(Request(), "testPassword")} + post_data = { + "U": "test", + "E": "test@example.org", + "S": True, + "passwd": "testPassword" + } + endpoint = f"/account/{user.Username}/edit" + with client as request: + resp = request.post(endpoint, data=post_data, cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + # Make sure the user record got updated correctly. + assert user.Suspended + + post_data.update({"S": False}) + with client as request: + resp = request.post(endpoint, data=post_data, cookies=cookies) + assert resp.status_code == int(HTTPStatus.OK) + + assert not user.Suspended + + def test_post_account_edit_error_unauthorized(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") From 27f8603dc511ad4723c8100829f458b0e5a3c719 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Sat, 4 Dec 2021 00:51:59 -0800 Subject: [PATCH 708/844] fix(python): fix ordering of fields in partials/account_form.html Signed-off-by: Kevin Morris --- templates/partials/account_form.html | 64 ++++++++++++++-------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/templates/partials/account_form.html b/templates/partials/account_form.html index f3c293d8..37bb85c4 100644 --- a/templates/partials/account_form.html +++ b/templates/partials/account_form.html @@ -42,6 +42,38 @@ "account is inactive." | tr }}

    + {% if request.user.has_credential(creds.ACCOUNT_CHANGE_TYPE) %} +

    + + +

    + +

    + + + +

    + {% endif %} + {% if request.user.is_elevated() %}

    @@ -53,38 +85,6 @@

    {% endif %} - {% if request.user.has_credential(creds.ACCOUNT_CHANGE_TYPE) %} -

    - - -

    - -

    - - - -

    - {% endif %} -

    +

    + + +

    +

    +

    + + +

    +

    +

    + + +

    +

    -

    - - -

    -

    +

    +

    -

    - - -

    -

    +

    + {{ + "This action will close any pending package requests " + "related to it. If %sComments%s are omitted, a closure " + "comment will be autogenerated." + | tr | format("", "") | safe + }} +

    +

    {{ "By selecting the checkbox, you confirm that you want to " @@ -38,8 +47,11 @@

    - +

    diff --git a/templates/pkgbase/merge.html b/templates/pkgbase/merge.html index b5129801..981bd649 100644 --- a/templates/pkgbase/merge.html +++ b/templates/pkgbase/merge.html @@ -28,6 +28,15 @@ {% endfor %} +

    + {{ + "This action will close any pending package requests " + "related to it. If %sComments%s are omitted, a closure " + "comment will be autogenerated." + | tr | format("", "") | safe + }} +

    +

    {{ "Once the package has been merged it cannot be reversed. " | tr }} {{ "Enter the package name you wish to merge the package into. " | tr }} @@ -37,6 +46,16 @@

    + +

    + + +

    +

    -

    - - -

    -