From e9cc2fb4373cae8c24bab5043f24c019829ceb99 Mon Sep 17 00:00:00 2001 From: Christian Heusel Date: Fri, 2 Jun 2023 16:19:44 +0200 Subject: [PATCH 001/128] change: only require .SRCINFO in the latest revision This is done in order to relax the constraints so that dropping packages from the official repos can be done with preserving their history. Its sufficient to also have this present in the latest commit of a push. Signed-off-by: Christian Heusel --- aurweb/git/update.py | 163 +++++++++++++++++++--------------------- test/t1300-git-update.t | 4 +- 2 files changed, 80 insertions(+), 87 deletions(-) diff --git a/aurweb/git/update.py b/aurweb/git/update.py index b1256fdb..467b540f 100755 --- a/aurweb/git/update.py +++ b/aurweb/git/update.py @@ -258,6 +258,63 @@ def die_commit(msg, commit): exit(1) +def validate_metadata(metadata, commit): # noqa: C901 + try: + metadata_pkgbase = metadata["pkgbase"] + except KeyError: + die_commit( + "invalid .SRCINFO, does not contain a pkgbase (is the file empty?)", + str(commit.id), + ) + if not re.match(repo_regex, metadata_pkgbase): + die_commit("invalid pkgbase: {:s}".format(metadata_pkgbase), str(commit.id)) + + if not metadata["packages"]: + die_commit("missing pkgname entry", str(commit.id)) + + for pkgname in set(metadata["packages"].keys()): + pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) + + for field in ("pkgver", "pkgrel", "pkgname"): + if field not in pkginfo: + die_commit( + "missing mandatory field: {:s}".format(field), str(commit.id) + ) + + if "epoch" in pkginfo and not pkginfo["epoch"].isdigit(): + die_commit("invalid epoch: {:s}".format(pkginfo["epoch"]), str(commit.id)) + + if not re.match(r"[a-z0-9][a-z0-9\.+_-]*$", pkginfo["pkgname"]): + die_commit( + "invalid package name: {:s}".format(pkginfo["pkgname"]), + str(commit.id), + ) + + max_len = {"pkgname": 255, "pkgdesc": 255, "url": 8000} + for field in max_len.keys(): + if field in pkginfo and len(pkginfo[field]) > max_len[field]: + die_commit( + "{:s} field too long: {:s}".format(field, pkginfo[field]), + str(commit.id), + ) + + for field in ("install", "changelog"): + if field in pkginfo and not pkginfo[field] in commit.tree: + die_commit( + "missing {:s} file: {:s}".format(field, pkginfo[field]), + str(commit.id), + ) + + for field in extract_arch_fields(pkginfo, "source"): + fname = field["value"] + if len(fname) > 8000: + die_commit("source entry too long: {:s}".format(fname), str(commit.id)) + if "://" in fname or "lp:" in fname: + continue + if fname not in commit.tree: + die_commit("missing source file: {:s}".format(fname), str(commit.id)) + + def main(): # noqa: C901 repo = pygit2.Repository(repo_path) @@ -295,12 +352,30 @@ def main(): # noqa: C901 if sha1_old != "0" * 40: walker.hide(sha1_old) + head_commit = repo[sha1_new] + if ".SRCINFO" not in head_commit.tree: + die_commit("missing .SRCINFO", str(head_commit.id)) + + # Read .SRCINFO from the HEAD commit. + metadata_raw = repo[head_commit.tree[".SRCINFO"].id].data.decode() + (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) + if errors: + sys.stderr.write( + "error: The following errors occurred " "when parsing .SRCINFO in commit\n" + ) + sys.stderr.write("error: {:s}:\n".format(str(head_commit.id))) + for error in errors: + for err in error["error"]: + sys.stderr.write("error: line {:d}: {:s}\n".format(error["line"], err)) + exit(1) + + # check if there is a correct .SRCINFO file in the latest revision + validate_metadata(metadata, head_commit) + # Validate all new commits. for commit in walker: - for fname in (".SRCINFO", "PKGBUILD"): - if fname not in commit.tree: - die_commit("missing {:s}".format(fname), str(commit.id)) - + if "PKGBUILD" not in commit.tree: + die_commit("missing PKGBUILD", str(commit.id)) for treeobj in commit.tree: blob = repo[treeobj.id] @@ -320,82 +395,6 @@ def main(): # noqa: C901 str(commit.id), ) - metadata_raw = repo[commit.tree[".SRCINFO"].id].data.decode() - (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) - if errors: - sys.stderr.write( - "error: The following errors occurred " - "when parsing .SRCINFO in commit\n" - ) - sys.stderr.write("error: {:s}:\n".format(str(commit.id))) - for error in errors: - for err in error["error"]: - sys.stderr.write( - "error: line {:d}: {:s}\n".format(error["line"], err) - ) - exit(1) - - try: - metadata_pkgbase = metadata["pkgbase"] - except KeyError: - die_commit( - "invalid .SRCINFO, does not contain a pkgbase (is the file empty?)", - str(commit.id), - ) - if not re.match(repo_regex, metadata_pkgbase): - die_commit("invalid pkgbase: {:s}".format(metadata_pkgbase), str(commit.id)) - - if not metadata["packages"]: - die_commit("missing pkgname entry", str(commit.id)) - - for pkgname in set(metadata["packages"].keys()): - pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) - - for field in ("pkgver", "pkgrel", "pkgname"): - if field not in pkginfo: - die_commit( - "missing mandatory field: {:s}".format(field), str(commit.id) - ) - - if "epoch" in pkginfo and not pkginfo["epoch"].isdigit(): - die_commit( - "invalid epoch: {:s}".format(pkginfo["epoch"]), str(commit.id) - ) - - if not re.match(r"[a-z0-9][a-z0-9\.+_-]*$", pkginfo["pkgname"]): - die_commit( - "invalid package name: {:s}".format(pkginfo["pkgname"]), - str(commit.id), - ) - - max_len = {"pkgname": 255, "pkgdesc": 255, "url": 8000} - for field in max_len.keys(): - if field in pkginfo and len(pkginfo[field]) > max_len[field]: - die_commit( - "{:s} field too long: {:s}".format(field, pkginfo[field]), - str(commit.id), - ) - - for field in ("install", "changelog"): - if field in pkginfo and not pkginfo[field] in commit.tree: - die_commit( - "missing {:s} file: {:s}".format(field, pkginfo[field]), - str(commit.id), - ) - - for field in extract_arch_fields(pkginfo, "source"): - fname = field["value"] - if len(fname) > 8000: - die_commit( - "source entry too long: {:s}".format(fname), str(commit.id) - ) - if "://" in fname or "lp:" in fname: - continue - if fname not in commit.tree: - die_commit( - "missing source file: {:s}".format(fname), str(commit.id) - ) - # Display a warning if .SRCINFO is unchanged. if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new): srcinfo_id_old = repo[sha1_old].tree[".SRCINFO"].id @@ -403,10 +402,6 @@ def main(): # noqa: C901 if srcinfo_id_old == srcinfo_id_new: warn(".SRCINFO unchanged. " "The package database will not be updated!") - # Read .SRCINFO from the HEAD commit. - metadata_raw = repo[repo[sha1_new].tree[".SRCINFO"].id].data.decode() - (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) - # Ensure that the package base name matches the repository name. metadata_pkgbase = metadata["pkgbase"] if metadata_pkgbase != pkgbase: diff --git a/test/t1300-git-update.t b/test/t1300-git-update.t index e9d943c0..a8ea5cab 100755 --- a/test/t1300-git-update.t +++ b/test/t1300-git-update.t @@ -175,10 +175,8 @@ test_expect_success 'Removing .SRCINFO with a follow-up fix.' ' git -C aur.git commit -q -m "Remove .SRCINFO" && git -C aur.git revert --no-edit HEAD && new=$(git -C aur.git rev-parse HEAD) && - test_must_fail \ env AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \ - cover "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 && - grep -q "^error: missing .SRCINFO$" actual + cover "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1 ' test_expect_success 'Removing PKGBUILD.' ' From 26b2566b3fa5fe7165deaedd6e0be6b7da6a3b0f Mon Sep 17 00:00:00 2001 From: Christian Heusel Date: Thu, 8 Jun 2023 12:42:31 +0200 Subject: [PATCH 002/128] change: print the user name if connecting via ssh this is similar to the message that gitlab produces: $ ssh -T aur.archlinux.org Welcome to AUR, gromit! Interactive shell is disabled. Try `ssh ssh://aur@aur.archlinux.org help` for a list of commands. $ ssh -T gitlab.archlinux.org Welcome to GitLab, @gromit! Signed-off-by: Christian Heusel --- aurweb/git/serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aurweb/git/serve.py b/aurweb/git/serve.py index 8dbbf3f7..2ac1f10e 100755 --- a/aurweb/git/serve.py +++ b/aurweb/git/serve.py @@ -648,7 +648,7 @@ def main(): ssh_client = os.environ.get("SSH_CLIENT") if not ssh_cmd: - die_with_help("Interactive shell is disabled.") + die_with_help(f"Welcome to AUR, {user}! Interactive shell is disabled.") cmdargv = shlex.split(ssh_cmd) action = cmdargv[0] remote_addr = ssh_client.split(" ")[0] if ssh_client else None From 1c11c901a2d389bf497e886c990b634a70a4df7a Mon Sep 17 00:00:00 2001 From: moson-mo Date: Sat, 10 Jun 2023 09:40:35 +0200 Subject: [PATCH 003/128] feat: switch requests filter for pkgname to "contains" Use "contains" filtering instead of an exact match when a package name filter is given. This makes it easier to find requests for a "group" of packages. Signed-off-by: moson-mo --- aurweb/routers/requests.py | 4 ++-- test/test_requests.py | 20 +++++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/aurweb/routers/requests.py b/aurweb/routers/requests.py index 585dc157..4cfda269 100644 --- a/aurweb/routers/requests.py +++ b/aurweb/routers/requests.py @@ -99,9 +99,9 @@ async def requests( in_filters.append(REJECTED_ID) filtered = query.filter(PackageRequest.Status.in_(in_filters)) - # Name filter + # Name filter (contains) if filter_pkg_name: - filtered = filtered.filter(PackageBase.Name == filter_pkg_name) + filtered = filtered.filter(PackageBase.Name.like(f"%{filter_pkg_name}%")) # Additionally filter for requests made from package maintainer if filter_maintainer_requests: diff --git a/test/test_requests.py b/test/test_requests.py index 7ddb76a0..eb88cd94 100644 --- a/test/test_requests.py +++ b/test/test_requests.py @@ -925,14 +925,28 @@ def test_requests_with_package_name_filter( request.cookies = cookies resp = request.get( "/requests", - params={"filter_pkg_name": packages[0].PackageBase.Name}, + params={"filter_pkg_name": "kg_1"}, ) assert resp.status_code == int(HTTPStatus.OK) root = parse_root(resp.text) rows = root.xpath('//table[@class="results"]/tbody/tr') - # We only expect 1 request for our first package - assert len(rows) == 1 + # We expect 11 requests for all packages containing "kg_1" + assert len(rows) == 11 + + # test as TU, no results + with client as request: + request.cookies = cookies + resp = request.get( + "/requests", + params={"filter_pkg_name": "x"}, + ) + assert resp.status_code == int(HTTPStatus.OK) + + root = parse_root(resp.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + # We expect 0 requests since we don't have anything containing "x" + assert len(rows) == 0 # test as regular user, not related to our package cookies = {"AURSID": user2.login(Request(), "testPassword")} From ed17486da6ada6c6bb1ca6fb1fddfbd1ccee4708 Mon Sep 17 00:00:00 2001 From: moson-mo Date: Sun, 11 Jun 2023 12:20:02 +0200 Subject: [PATCH 004/128] change(git): allow keys/pgp subdir with .asc files This allows migration of git history for packages dropped from a repo to AUR in case they contain PGP key material Signed-off-by: moson-mo --- aurweb/git/update.py | 53 +++++++++++++++------ test/t1300-git-update.t | 103 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 141 insertions(+), 15 deletions(-) diff --git a/aurweb/git/update.py b/aurweb/git/update.py index 467b540f..cd7813e0 100755 --- a/aurweb/git/update.py +++ b/aurweb/git/update.py @@ -315,6 +315,14 @@ def validate_metadata(metadata, commit): # noqa: C901 die_commit("missing source file: {:s}".format(fname), str(commit.id)) +def validate_blob_size(blob: pygit2.Object, commit: pygit2.Commit): + if isinstance(blob, pygit2.Blob) and blob.size > max_blob_size: + die_commit( + "maximum blob size ({:s}) exceeded".format(size_humanize(max_blob_size)), + str(commit.id), + ) + + def main(): # noqa: C901 repo = pygit2.Repository(repo_path) @@ -376,25 +384,42 @@ def main(): # noqa: C901 for commit in walker: if "PKGBUILD" not in commit.tree: die_commit("missing PKGBUILD", str(commit.id)) + + # Iterate over files in root dir for treeobj in commit.tree: - blob = repo[treeobj.id] - - if isinstance(blob, pygit2.Tree): + # Don't allow any subdirs besides "keys/" + if isinstance(treeobj, pygit2.Tree) and treeobj.name != "keys": die_commit( - "the repository must not contain subdirectories", str(commit.id) - ) - - if not isinstance(blob, pygit2.Blob): - die_commit("not a blob object: {:s}".format(treeobj), str(commit.id)) - - if blob.size > max_blob_size: - die_commit( - "maximum blob size ({:s}) exceeded".format( - size_humanize(max_blob_size) - ), + "the repository must not contain subdirectories", str(commit.id), ) + # Check size of files in root dir + validate_blob_size(treeobj, commit) + + # If we got a subdir keys/, + # make sure it only contains a pgp/ subdir with key files + if "keys" in commit.tree: + # Check for forbidden files/dirs in keys/ + for keyobj in commit.tree["keys"]: + if not isinstance(keyobj, pygit2.Tree) or keyobj.name != "pgp": + die_commit( + "the keys/ subdir may only contain a pgp/ directory", + str(commit.id), + ) + # Check for forbidden files in keys/pgp/ + if "keys/pgp" in commit.tree: + for pgpobj in commit.tree["keys/pgp"]: + if not isinstance(pgpobj, pygit2.Blob) or not pgpobj.name.endswith( + ".asc" + ): + die_commit( + "the subdir may only contain .asc (PGP pub key) files", + str(commit.id), + ) + # Check file size for pgp key files + validate_blob_size(pgpobj, commit) + # Display a warning if .SRCINFO is unchanged. if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new): srcinfo_id_old = repo[sha1_old].tree[".SRCINFO"].id diff --git a/test/t1300-git-update.t b/test/t1300-git-update.t index a8ea5cab..4fdb487b 100755 --- a/test/t1300-git-update.t +++ b/test/t1300-git-update.t @@ -191,7 +191,7 @@ test_expect_success 'Removing PKGBUILD.' ' grep -q "^error: missing PKGBUILD$" actual ' -test_expect_success 'Pushing a tree with a subdirectory.' ' +test_expect_success 'Pushing a tree with a forbidden subdirectory.' ' old=$(git -C aur.git rev-parse HEAD) && test_when_finished "git -C aur.git reset --hard $old" && mkdir aur.git/subdir && @@ -205,6 +205,107 @@ test_expect_success 'Pushing a tree with a subdirectory.' ' grep -q "^error: the repository must not contain subdirectories$" actual ' +test_expect_success 'Pushing a tree with an allowed subdirectory for pgp keys; wrong files.' ' + old=$(git -C aur.git rev-parse HEAD) && + test_when_finished "git -C aur.git reset --hard $old" && + mkdir -p aur.git/keys/pgp/ && + touch aur.git/keys/pgp/nonsense && + git -C aur.git add keys/pgp/nonsense && + git -C aur.git commit -q -m "Add some nonsense" && + new=$(git -C aur.git rev-parse HEAD) && + test_must_fail \ + env AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \ + cover "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 && + grep -q "^error: the subdir may only contain .asc (PGP pub key) files$" actual +' + +test_expect_success 'Pushing a tree with an allowed subdirectory for pgp keys; another subdir.' ' + old=$(git -C aur.git rev-parse HEAD) && + test_when_finished "git -C aur.git reset --hard $old" && + mkdir -p aur.git/keys/pgp/bla/ && + touch aur.git/keys/pgp/bla/x.asc && + git -C aur.git add keys/pgp/bla/x.asc && + git -C aur.git commit -q -m "Add some nonsense" && + new=$(git -C aur.git rev-parse HEAD) && + test_must_fail \ + env AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \ + cover "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 && + grep -q "^error: the subdir may only contain .asc (PGP pub key) files$" actual +' + +test_expect_success 'Pushing a tree with an allowed subdirectory for pgp keys; wrong subdir.' ' + old=$(git -C aur.git rev-parse HEAD) && + test_when_finished "git -C aur.git reset --hard $old" && + mkdir -p aur.git/keys/xyz/ && + touch aur.git/keys/xyz/x.asc && + git -C aur.git add keys/xyz/x.asc && + git -C aur.git commit -q -m "Add some nonsense" && + new=$(git -C aur.git rev-parse HEAD) && + test_must_fail \ + env AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \ + cover "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 && + grep -q "^error: the keys/ subdir may only contain a pgp/ directory$" actual +' + +test_expect_success 'Pushing a tree with an allowed subdirectory with pgp keys; additional files' ' + old=$(git -C aur.git rev-parse HEAD) && + test_when_finished "git -C aur.git reset --hard $old" && + mkdir -p aur.git/keys/pgp/ && + touch aur.git/keys/pgp/x.asc && + touch aur.git/keys/nonsense && + git -C aur.git add keys/pgp/x.asc && + git -C aur.git add keys/nonsense && + git -C aur.git commit -q -m "Add pgp key" && + new=$(git -C aur.git rev-parse HEAD) && + test_must_fail \ + env AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \ + cover "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 && + grep -q "^error: the keys/ subdir may only contain a pgp/ directory$" actual +' + +test_expect_success 'Pushing a tree with an allowed subdirectory with pgp keys; additional subdir' ' + old=$(git -C aur.git rev-parse HEAD) && + test_when_finished "git -C aur.git reset --hard $old" && + mkdir -p aur.git/keys/pgp/ && + mkdir -p aur.git/somedir/ && + touch aur.git/keys/pgp/x.asc && + touch aur.git/somedir/nonsense && + git -C aur.git add keys/pgp/x.asc && + git -C aur.git add somedir/nonsense && + git -C aur.git commit -q -m "Add pgp key" && + new=$(git -C aur.git rev-parse HEAD) && + test_must_fail \ + env AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \ + cover "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 && + grep -q "^error: the repository must not contain subdirectories$" actual +' + +test_expect_success 'Pushing a tree with an allowed subdirectory with pgp keys; keys to large' ' + old=$(git -C aur.git rev-parse HEAD) && + test_when_finished "git -C aur.git reset --hard $old" && + mkdir -p aur.git/keys/pgp/ && + printf "%256001s" x > aur.git/keys/pgp/x.asc && + git -C aur.git add keys/pgp/x.asc && + git -C aur.git commit -q -m "Add pgp key" && + new=$(git -C aur.git rev-parse HEAD) && + test_must_fail \ + env AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \ + cover "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 && + grep -q "^error: maximum blob size (250.00KiB) exceeded$" actual +' + +test_expect_success 'Pushing a tree with an allowed subdirectory with pgp keys.' ' + old=$(git -C aur.git rev-parse HEAD) && + test_when_finished "git -C aur.git reset --hard $old" && + mkdir -p aur.git/keys/pgp/ && + touch aur.git/keys/pgp/x.asc && + git -C aur.git add keys/pgp/x.asc && + git -C aur.git commit -q -m "Add pgp key" && + new=$(git -C aur.git rev-parse HEAD) && + env AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \ + cover "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1 +' + test_expect_success 'Pushing a tree with a large blob.' ' old=$(git -C aur.git rev-parse HEAD) && test_when_finished "git -C aur.git reset --hard $old" && From 58158505b06c1856420c22b1827f42eec450b477 Mon Sep 17 00:00:00 2001 From: moson-mo Date: Sun, 11 Jun 2023 21:04:35 +0200 Subject: [PATCH 005/128] fix: browser hints for password fields Co-authored-by: eNV25 Signed-off-by: moson-mo --- templates/partials/account_form.html | 6 +++--- templates/passreset.html | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/partials/account_form.html b/templates/partials/account_form.html index 4d135a56..28dc0cd5 100644 --- a/templates/partials/account_form.html +++ b/templates/partials/account_form.html @@ -246,7 +246,7 @@ -

@@ -255,7 +255,7 @@ {% trans %}Re-type password{% endtrans %}: -

@@ -333,7 +333,7 @@ -

{% else %} diff --git a/templates/passreset.html b/templates/passreset.html index 6a31109f..08493fe9 100644 --- a/templates/passreset.html +++ b/templates/passreset.html @@ -26,14 +26,14 @@ {% trans %}Enter your new password:{% endtrans %} - {% trans %}Confirm your new password:{% endtrans %} - From 32461f28eaf786b34d9ee3a8a27d97ee1356228a Mon Sep 17 00:00:00 2001 From: moson-mo Date: Thu, 15 Jun 2023 14:16:38 +0200 Subject: [PATCH 006/128] fix(docker): Suppress error PEP-668 When using docker (compose), we don't create a venv and just install python packages system-wide. With python 3.11 (PEP 668) we need to explicitly tell pip to allow this. Signed-off-by: moson-mo --- docker/scripts/install-python-deps.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/scripts/install-python-deps.sh b/docker/scripts/install-python-deps.sh index 01a6eaa7..f1942498 100755 --- a/docker/scripts/install-python-deps.sh +++ b/docker/scripts/install-python-deps.sh @@ -1,10 +1,8 @@ #!/bin/bash set -eou pipefail -# Upgrade PIP; Arch Linux's version of pip is outdated for Poetry. -pip install --upgrade pip - if [ ! -z "${COMPOSE+x}" ]; then + export PIP_BREAK_SYSTEM_PACKAGES=1 poetry config virtualenvs.create false fi poetry install --no-interaction --no-ansi From c6c81f0789e72e2a99dd4474941344350dd246c9 Mon Sep 17 00:00:00 2001 From: moson-mo Date: Fri, 16 Jun 2023 13:33:39 +0200 Subject: [PATCH 007/128] housekeep: Amend .gitignore and .dockerignore Prevent some files/dirs to end up in the repo / docker image: * directories typically used for python virtualenvs * files that are being generated by running tests Signed-off-by: moson-mo --- .dockerignore | 19 ++++++++++++++++++- .gitignore | 8 ++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 6ec5547d..56ac1964 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,23 @@ -*/*.mo +# Config files conf/config conf/config.sqlite conf/config.sqlite.defaults conf/docker conf/docker.defaults + +# Compiled translation files +**/*.mo + +# Typical virtualenv directories +env/ +venv/ +.venv/ + +# Test output +htmlcov/ +test-emails/ +test/__pycache__ +test/test-results +test/trash_directory* +.coverage +.pytest_cache diff --git a/.gitignore b/.gitignore index a3314c27..68de7cd5 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,6 @@ conf/docker conf/docker.defaults data.sql dummy-data.sql* -env/ fastapi_aw/ htmlcov/ po/*.mo @@ -32,7 +31,7 @@ po/*.po~ po/POTFILES schema/aur-schema-sqlite.sql test/test-results/ -test/trash directory* +test/trash_directory* web/locale/*/ web/html/*.gz @@ -53,3 +52,8 @@ report.xml # Ignore test emails test-emails/ + +# Ignore typical virtualenv directories +env/ +venv/ +.venv/ From 143575c9dec9d1126e087dc451417b1910352ed2 Mon Sep 17 00:00:00 2001 From: moson-mo Date: Sun, 11 Jun 2023 20:31:51 +0200 Subject: [PATCH 008/128] fix: restore command, remove premature creation of pkgbase We're currently creating a "PackageBases" when the "restore" command is executed. This is problematic for pkgbases that never existed before. In those cases it will create the record but fail in the update.py script. Thus it leaves an orphan "PackageBases" record in the DB (which does not have any related "Packages" record(s)) Navigating to such a packages /pkgbase/... URL will result in a crash since it is not foreseen to have "orphan" pkgbase records. We can safely remove the early creation of that record because it'll be taken care of in the update.py script that is being called We'll also fix some tests. Before it was executing a dummy script instead of "update.py" which might be a bit misleading since it did not check the real outcome of our "restore" action. Signed-off-by: moson-mo --- aurweb/git/serve.py | 24 +++++------------------- test/setup.sh | 9 +-------- test/t1200-git-serve.t | 23 +++++++++++++++-------- 3 files changed, 21 insertions(+), 35 deletions(-) diff --git a/aurweb/git/serve.py b/aurweb/git/serve.py index 2ac1f10e..333d0394 100755 --- a/aurweb/git/serve.py +++ b/aurweb/git/serve.py @@ -52,7 +52,7 @@ def list_repos(user): conn.close() -def create_pkgbase(pkgbase, user): +def validate_pkgbase(pkgbase, user): if not re.match(repo_regex, pkgbase): raise aurweb.exceptions.InvalidRepositoryNameException(pkgbase) if pkgbase_exists(pkgbase): @@ -62,26 +62,12 @@ def create_pkgbase(pkgbase, user): cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) userid = cur.fetchone()[0] + + conn.close() + if userid == 0: raise aurweb.exceptions.InvalidUserException(user) - now = int(time.time()) - cur = conn.execute( - "INSERT INTO PackageBases (Name, SubmittedTS, " - + "ModifiedTS, SubmitterUID, MaintainerUID, " - + "FlaggerComment) VALUES (?, ?, ?, ?, ?, '')", - [pkgbase, now, now, userid, userid], - ) - pkgbase_id = cur.lastrowid - - cur = conn.execute( - "INSERT INTO PackageNotifications " + "(PackageBaseID, UserID) VALUES (?, ?)", - [pkgbase_id, userid], - ) - - conn.commit() - conn.close() - def pkgbase_adopt(pkgbase, user, privileged): pkgbase_id = pkgbase_from_name(pkgbase) @@ -577,7 +563,7 @@ def serve(action, cmdargv, user, privileged, remote_addr): # noqa: C901 checkarg(cmdargv, "repository name") pkgbase = cmdargv[1] - create_pkgbase(pkgbase, user) + validate_pkgbase(pkgbase, user) os.environ["AUR_USER"] = user os.environ["AUR_PKGBASE"] = pkgbase diff --git a/test/setup.sh b/test/setup.sh index 2db897bf..ccf24086 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -56,7 +56,7 @@ ssh-options = restrict repo-path = ./aur.git/ repo-regex = [a-z0-9][a-z0-9.+_-]*$ git-shell-cmd = ./git-shell.sh -git-update-cmd = ./update.sh +git-update-cmd = $GIT_UPDATE ssh-cmdline = ssh aur@aur.archlinux.org [update] @@ -90,13 +90,6 @@ echo $GIT_NAMESPACE EOF chmod +x git-shell.sh -cat >update.sh <<-\EOF -#!/bin/sh -echo $AUR_USER -echo $AUR_PKGBASE -EOF -chmod +x update.sh - AUR_CONFIG=config export AUR_CONFIG diff --git a/test/t1200-git-serve.t b/test/t1200-git-serve.t index dbb465bc..bb3a004f 100755 --- a/test/t1200-git-serve.t +++ b/test/t1200-git-serve.t @@ -137,14 +137,21 @@ test_expect_success "Try to push to someone else's repository as Trusted User." ' test_expect_success "Test restore." ' + # Delete from DB echo "DELETE FROM PackageBases WHERE Name = '"'"'foobar'"'"';" | \ sqlite3 aur.db && - cat >expected <<-EOF && - user - foobar - EOF + # "Create branch" as if it had been there + new=$(git -C aur.git rev-parse HEAD^) && + echo $new > aur.git/.git/refs/heads/foobar && + # Restore deleted package SSH_ORIGINAL_COMMAND="restore foobar" AUR_USER=user AUR_PRIVILEGED=0 \ - cover "$GIT_SERVE" 2>&1 >actual + cover "$GIT_SERVE" 2>&1 && + # We should find foobar with a new ID (3) in the DB after restore + echo "SELECT ID FROM PackageBases WHERE Name = '"'"'foobar'"'"';" | \ + sqlite3 aur.db >actual && + cat >expected <<-EOF && + 3 + EOF test_cmp expected actual ' @@ -174,7 +181,7 @@ test_expect_success "Adopt a package base as a regular user." ' SSH_ORIGINAL_COMMAND="adopt foobar" AUR_USER=user AUR_PRIVILEGED=0 \ cover "$GIT_SERVE" 2>&1 && cat >expected <<-EOF && - *foobar + foobar EOF SSH_ORIGINAL_COMMAND="list-repos" AUR_USER=user AUR_PRIVILEGED=0 \ cover "$GIT_SERVE" 2>&1 >actual && @@ -252,7 +259,7 @@ test_expect_success "Try to steal another user's package as a Trusted User." ' cover "$GIT_SERVE" 2>&1 >actual && test_cmp expected actual && cat >expected <<-EOF && - *foobar + foobar EOF SSH_ORIGINAL_COMMAND="list-repos" AUR_USER=tu AUR_PRIVILEGED=1 \ cover "$GIT_SERVE" 2>&1 >actual && @@ -340,7 +347,7 @@ test_expect_success "Disown a package base and check (co-)maintainer list." ' SSH_ORIGINAL_COMMAND="disown foobar" AUR_USER=user AUR_PRIVILEGED=0 \ cover "$GIT_SERVE" 2>&1 && cat >expected <<-EOF && - *foobar + foobar EOF SSH_ORIGINAL_COMMAND="list-repos" AUR_USER=user2 AUR_PRIVILEGED=0 \ cover "$GIT_SERVE" 2>&1 >actual && From e2c113caee0f42584d1a25644423a5d9455ffde0 Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Thu, 22 Jun 2023 19:22:56 +0100 Subject: [PATCH 009/128] chore(release): prepare for 6.2.5 Signed-off-by: Leonidas Spyropoulos --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e25fe90a..69f04fab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ combine_as_imports = true # [tool.poetry] name = "aurweb" -version = "v6.2.4" +version = "v6.2.5" license = "GPL-2.0-only" description = "Source code for the Arch User Repository's website" homepage = "https://aur.archlinux.org" From c41f2e854a1aeb4aab963a3756cf0768374a742b Mon Sep 17 00:00:00 2001 From: moson-mo Date: Sun, 2 Jul 2023 13:21:11 +0200 Subject: [PATCH 010/128] perf: tweak some search queries We currently sorting on two columns in different tables which is quite expensive in terms of performance: MariaDB is first merging the data into some temporary table to apply the sorting and record limiting. We can tweak a couple of these queries by changing the "order by" clause such that they refer to columns within the same table (PackageBases). So instead performing the second sorting on "Packages.Name", we do this on "PackageBases.Name" instead. This should still be "good enough" to produce properly sorted results. Signed-off-by: moson-mo --- aurweb/packages/search.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aurweb/packages/search.py b/aurweb/packages/search.py index 62de1ea8..78b27a9a 100644 --- a/aurweb/packages/search.py +++ b/aurweb/packages/search.py @@ -195,13 +195,13 @@ class PackageSearch: def _sort_by_votes(self, order: str): column = getattr(models.PackageBase.NumVotes, order) - name = getattr(models.Package.Name, order) + name = getattr(models.PackageBase.Name, order) self.query = self.query.order_by(column(), name()) return self def _sort_by_popularity(self, order: str): column = getattr(models.PackageBase.Popularity, order) - name = getattr(models.Package.Name, order) + name = getattr(models.PackageBase.Name, order) self.query = self.query.order_by(column(), name()) return self @@ -236,7 +236,7 @@ class PackageSearch: def _sort_by_last_modified(self, order: str): column = getattr(models.PackageBase.ModifiedTS, order) - name = getattr(models.Package.Name, order) + name = getattr(models.PackageBase.Name, order) self.query = self.query.order_by(column(), name()) return self From 7c8b9ba6bcacfe45e416ec37cf16fa1824659825 Mon Sep 17 00:00:00 2001 From: moson-mo Date: Sun, 2 Jul 2023 13:55:21 +0200 Subject: [PATCH 011/128] perf: add index to tweak our default search query Adds an index on PackageBases.Popularity and PackageBases.Name to improve performance of our default search query sorted by "Popularity" Signed-off-by: moson-mo --- ...0_add_index_on_packagebases_popularity_.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 migrations/versions/c5a6a9b661a0_add_index_on_packagebases_popularity_.py diff --git a/migrations/versions/c5a6a9b661a0_add_index_on_packagebases_popularity_.py b/migrations/versions/c5a6a9b661a0_add_index_on_packagebases_popularity_.py new file mode 100644 index 00000000..12f97028 --- /dev/null +++ b/migrations/versions/c5a6a9b661a0_add_index_on_packagebases_popularity_.py @@ -0,0 +1,24 @@ +"""Add index on PackageBases.Popularity and .Name + +Revision ID: c5a6a9b661a0 +Revises: e4e49ffce091 +Create Date: 2023-07-02 13:46:52.522146 + +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c5a6a9b661a0" +down_revision = "e4e49ffce091" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_index( + "BasesPopularityName", "PackageBases", ["Popularity", "Name"], unique=False + ) + + +def downgrade(): + op.drop_index("BasesPopularityName", table_name="PackageBases") From 3acfb08a0f839ce3582d9ce92c01e321e99e69f3 Mon Sep 17 00:00:00 2001 From: moson-mo Date: Sun, 2 Jul 2023 01:06:34 +0200 Subject: [PATCH 012/128] feat: cache package search results with Redis The queries being done on the package search page are quite costly. (Especially the default one ordered by "Popularity" when navigating to /packages) Let's add the search results to the Redis cache: Every result of a search query is being pushed to Redis until we hit our maximum of 50k. An entry expires after 3 minutes before it's evicted from the cache. Lifetime an Max values are configurable. Signed-off-by: moson-mo --- aurweb/cache.py | 38 ++++++++--- aurweb/routers/html.py | 20 +++--- aurweb/routers/packages.py | 15 ++++- aurweb/util.py | 8 +++ conf/config.defaults | 6 ++ test/test_cache.py | 121 ++++++++++++++++++++--------------- test/test_packages_routes.py | 13 +++- test/test_util.py | 26 +++++++- 8 files changed, 173 insertions(+), 74 deletions(-) diff --git a/aurweb/cache.py b/aurweb/cache.py index 1572e2fc..56bb45b7 100644 --- a/aurweb/cache.py +++ b/aurweb/cache.py @@ -1,21 +1,43 @@ -from redis import Redis +import pickle + from sqlalchemy import orm +from aurweb import config +from aurweb.aur_redis import redis_connection -async def db_count_cache( - redis: Redis, key: str, query: orm.Query, expire: int = None -) -> int: +_redis = redis_connection() + + +async def db_count_cache(key: str, query: orm.Query, expire: int = None) -> int: """Store and retrieve a query.count() via redis cache. - :param redis: Redis handle :param key: Redis key :param query: SQLAlchemy ORM query :param expire: Optional expiration in seconds :return: query.count() """ - result = redis.get(key) + result = _redis.get(key) if result is None: - redis.set(key, (result := int(query.count()))) + _redis.set(key, (result := int(query.count()))) if expire: - redis.expire(key, expire) + _redis.expire(key, expire) return int(result) + + +async def db_query_cache(key: str, query: orm.Query, expire: int = None): + """Store and retrieve query results via redis cache. + + :param key: Redis key + :param query: SQLAlchemy ORM query + :param expire: Optional expiration in seconds + :return: query.all() + """ + result = _redis.get(key) + if result is None: + if _redis.dbsize() > config.getint("cache", "max_search_entries", 50000): + return query.all() + _redis.set(key, (result := pickle.dumps(query.all())), ex=expire) + if expire: + _redis.expire(key, expire) + + return pickle.loads(result) diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index 38303837..fc9f3519 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -89,22 +89,20 @@ async def index(request: Request): bases = db.query(models.PackageBase) - redis = aurweb.aur_redis.redis_connection() - cache_expire = 300 # Five minutes. - + cache_expire = aurweb.config.getint("cache", "expiry_time") # Package statistics. context["package_count"] = await db_count_cache( - redis, "package_count", bases, expire=cache_expire + "package_count", bases, expire=cache_expire ) query = bases.filter(models.PackageBase.MaintainerUID.is_(None)) context["orphan_count"] = await db_count_cache( - redis, "orphan_count", query, expire=cache_expire + "orphan_count", query, expire=cache_expire ) query = db.query(models.User) context["user_count"] = await db_count_cache( - redis, "user_count", query, expire=cache_expire + "user_count", query, expire=cache_expire ) query = query.filter( @@ -114,7 +112,7 @@ async def index(request: Request): ) ) context["trusted_user_count"] = await db_count_cache( - redis, "trusted_user_count", query, expire=cache_expire + "trusted_user_count", query, expire=cache_expire ) # Current timestamp. @@ -130,26 +128,26 @@ async def index(request: Request): query = bases.filter(models.PackageBase.SubmittedTS >= seven_days_ago) context["seven_days_old_added"] = await db_count_cache( - redis, "seven_days_old_added", query, expire=cache_expire + "seven_days_old_added", query, expire=cache_expire ) query = updated.filter(models.PackageBase.ModifiedTS >= seven_days_ago) context["seven_days_old_updated"] = await db_count_cache( - redis, "seven_days_old_updated", query, expire=cache_expire + "seven_days_old_updated", query, expire=cache_expire ) year = seven_days * 52 # Fifty two weeks worth: one year. year_ago = now - year query = updated.filter(models.PackageBase.ModifiedTS >= year_ago) context["year_old_updated"] = await db_count_cache( - redis, "year_old_updated", query, expire=cache_expire + "year_old_updated", query, expire=cache_expire ) query = bases.filter( models.PackageBase.ModifiedTS - models.PackageBase.SubmittedTS < 3600 ) context["never_updated"] = await db_count_cache( - redis, "never_updated", query, expire=cache_expire + "never_updated", query, expire=cache_expire ) # Get the 15 most recently updated packages. diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 83bfe6e2..779efb4b 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -7,6 +7,7 @@ from fastapi import APIRouter, Form, Query, Request, Response import aurweb.filters # noqa: F401 from aurweb import aur_logging, config, db, defaults, models, util from aurweb.auth import creds, requires_auth +from aurweb.cache import db_count_cache, db_query_cache from aurweb.exceptions import InvariantError, handle_form_exceptions from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID from aurweb.packages import util as pkgutil @@ -14,6 +15,7 @@ from aurweb.packages.search import PackageSearch from aurweb.packages.util import get_pkg_or_base from aurweb.pkgbase import actions as pkgbase_actions, util as pkgbaseutil from aurweb.templates import make_context, make_variable_context, render_template +from aurweb.util import hash_query logger = aur_logging.get_logger(__name__) router = APIRouter() @@ -87,7 +89,11 @@ async def packages_get( # Collect search result count here; we've applied our keywords. # Including more query operations below, like ordering, will # increase the amount of time required to collect a count. - num_packages = search.count() + # we use redis for caching the results of the query + cache_expire = config.getint("cache", "expiry_time") + num_packages = await db_count_cache( + hash_query(search.query), search.query, cache_expire + ) # Apply user-specified sort column and ordering. search.sort_by(sort_by, sort_order) @@ -108,7 +114,12 @@ async def packages_get( models.PackageNotification.PackageBaseID.label("Notify"), ) - packages = results.limit(per_page).offset(offset) + # paging + results = results.limit(per_page).offset(offset) + + # we use redis for caching the results of the query + packages = await db_query_cache(hash_query(results), results, cache_expire) + context["packages"] = packages context["packages_count"] = num_packages diff --git a/aurweb/util.py b/aurweb/util.py index d80b0311..7050b482 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -4,6 +4,7 @@ import secrets import shlex import string from datetime import datetime +from hashlib import sha1 from http import HTTPStatus from subprocess import PIPE, Popen from typing import Callable, Iterable, Tuple, Union @@ -13,6 +14,7 @@ import fastapi import pygit2 from email_validator import EmailSyntaxError, validate_email from fastapi.responses import JSONResponse +from sqlalchemy.orm import Query import aurweb.config from aurweb import aur_logging, defaults @@ -200,3 +202,9 @@ def shell_exec(cmdline: str, cwd: str) -> Tuple[int, str, str]: proc = Popen(args, cwd=cwd, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() return proc.returncode, out.decode().strip(), err.decode().strip() + + +def hash_query(query: Query): + return sha1( + str(query.statement.compile(compile_kwargs={"literal_binds": True})).encode() + ).hexdigest() diff --git a/conf/config.defaults b/conf/config.defaults index c059444d..4e2415ed 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -165,3 +165,9 @@ commit_url = https://gitlab.archlinux.org/archlinux/aurweb/-/commits/%s ; voted on based on `now + range_start <= End <= now + range_end`. range_start = 500 range_end = 172800 + +[cache] +; maximum number of keys/entries (for search results) in our redis cache, default is 50000 +max_search_entries = 50000 +; number of seconds after a cache entry expires, default is 3 minutes +expiry_time = 180 diff --git a/test/test_cache.py b/test/test_cache.py index 83a9755a..e19fa6a2 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -1,6 +1,8 @@ +from unittest import mock + import pytest -from aurweb import cache, db +from aurweb import cache, config, db from aurweb.models.account_type import USER_ID from aurweb.models.user import User @@ -10,68 +12,85 @@ def setup(db_test): return -class StubRedis: - """A class which acts as a RedisConnection without using Redis.""" - - cache = dict() - expires = dict() - - def get(self, key, *args): - if "key" not in self.cache: - self.cache[key] = None - return self.cache[key] - - def set(self, key, *args): - self.cache[key] = list(args)[0] - - def expire(self, key, *args): - self.expires[key] = list(args)[0] - - async def execute(self, command, key, *args): - f = getattr(self, command) - return f(key, *args) - - @pytest.fixture -def redis(): - yield StubRedis() +def user() -> User: + with db.begin(): + user = db.create( + User, + Username="test", + Email="test@example.org", + RealName="Test User", + Passwd="testPassword", + AccountTypeID=USER_ID, + ) + yield user + + +@pytest.fixture(autouse=True) +def clear_fakeredis_cache(): + cache._redis.flushall() @pytest.mark.asyncio -async def test_db_count_cache(redis): - db.create( - User, - Username="user1", - Email="user1@example.org", - Passwd="testPassword", - AccountTypeID=USER_ID, - ) - +async def test_db_count_cache(user): query = db.query(User) - # Now, perform several checks that db_count_cache matches query.count(). - # We have no cached value yet. - assert await cache.db_count_cache(redis, "key1", query) == query.count() + assert cache._redis.get("key1") is None + + # Add to cache + assert await cache.db_count_cache("key1", query) == query.count() # It's cached now. - assert await cache.db_count_cache(redis, "key1", query) == query.count() + assert cache._redis.get("key1") is not None + + # It does not expire + assert cache._redis.ttl("key1") == -1 + + # Cache a query with an expire. + value = await cache.db_count_cache("key2", query, 100) + assert value == query.count() + + assert cache._redis.ttl("key2") == 100 @pytest.mark.asyncio -async def test_db_count_cache_expires(redis): - db.create( - User, - Username="user1", - Email="user1@example.org", - Passwd="testPassword", - AccountTypeID=USER_ID, - ) - +async def test_db_query_cache(user): query = db.query(User) - # Cache a query with an expire. - value = await cache.db_count_cache(redis, "key1", query, 100) - assert value == query.count() + # We have no cached value yet. + assert cache._redis.get("key1") is None - assert redis.expires["key1"] == 100 + # Add to cache + await cache.db_query_cache("key1", query) + + # It's cached now. + assert cache._redis.get("key1") is not None + + # Modify our user and make sure we got a cached value + user.Username = "changed" + cached = await cache.db_query_cache("key1", query) + assert cached[0].Username != query.all()[0].Username + + # It does not expire + assert cache._redis.ttl("key1") == -1 + + # Cache a query with an expire. + value = await cache.db_query_cache("key2", query, 100) + assert len(value) == query.count() + assert value[0].Username == query.all()[0].Username + + assert cache._redis.ttl("key2") == 100 + + # Test "max_search_entries" options + def mock_max_search_entries(section: str, key: str, fallback: int) -> str: + if section == "cache" and key == "max_search_entries": + return 1 + return config.getint(section, key) + + with mock.patch("aurweb.config.getint", side_effect=mock_max_search_entries): + # Try to add another entry (we already have 2) + await cache.db_query_cache("key3", query) + + # Make sure it was not added because it exceeds our max. + assert cache._redis.get("key3") is None diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index 93dc404a..fb12e65e 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -5,7 +5,7 @@ from unittest import mock import pytest from fastapi.testclient import TestClient -from aurweb import asgi, config, db, time +from aurweb import asgi, cache, config, db, time from aurweb.filters import datetime_display from aurweb.models import License, PackageLicense from aurweb.models.account_type import USER_ID, AccountType @@ -63,6 +63,11 @@ def setup(db_test): return +@pytest.fixture(autouse=True) +def clear_fakeredis_cache(): + cache._redis.flushall() + + @pytest.fixture def client() -> TestClient: """Yield a FastAPI TestClient.""" @@ -815,6 +820,8 @@ def test_packages_search_by_keywords(client: TestClient, packages: list[Package] # And request packages with that keyword, we should get 1 result. with client as request: + # clear fakeredis cache + cache._redis.flushall() response = request.get("/packages", params={"SeB": "k", "K": "testKeyword"}) assert response.status_code == int(HTTPStatus.OK) @@ -870,6 +877,8 @@ def test_packages_search_by_maintainer( # This time, we should get `package` returned, since it's now an orphan. with client as request: + # clear fakeredis cache + cache._redis.flushall() response = request.get("/packages", params={"SeB": "m"}) assert response.status_code == int(HTTPStatus.OK) root = parse_root(response.text) @@ -902,6 +911,8 @@ def test_packages_search_by_comaintainer( # Then test that it's returned by our search. with client as request: + # clear fakeredis cache + cache._redis.flushall() response = request.get( "/packages", params={"SeB": "c", "K": maintainer.Username} ) diff --git a/test/test_util.py b/test/test_util.py index a138d912..042b9ad9 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -5,7 +5,8 @@ import fastapi import pytest from fastapi.responses import JSONResponse -from aurweb import filters, util +from aurweb import db, filters, util +from aurweb.models.user import User from aurweb.testing.requests import Request @@ -146,3 +147,26 @@ def assert_multiple_keys(pks): assert key1 == k1[1] assert pfx2 == k2[0] assert key2 == k2[1] + + +def test_hash_query(): + # No conditions + query = db.query(User) + assert util.hash_query(query) == "75e76026b7d576536e745ec22892cf8f5d7b5d62" + + # With where clause + query = db.query(User).filter(User.Username == "bla") + assert util.hash_query(query) == "4dca710f33b1344c27ec6a3c266970f4fa6a8a00" + + # With where clause and sorting + query = db.query(User).filter(User.Username == "bla").order_by(User.Username) + assert util.hash_query(query) == "ee2c7846fede430776e140f8dfe1d83cd21d2eed" + + # With where clause, sorting and specific columns + query = ( + db.query(User) + .filter(User.Username == "bla") + .order_by(User.Username) + .with_entities(User.Username) + ) + assert util.hash_query(query) == "c1db751be61443d266cf643005eee7a884dac103" From 814ccf6b04e97659c30ecc18dd63607a3ba485e6 Mon Sep 17 00:00:00 2001 From: moson-mo Date: Tue, 4 Jul 2023 09:40:39 +0200 Subject: [PATCH 013/128] feat: add Prometheus metrics for Redis cache Adding a Prometheus counter to be able to monitor cache hits/misses for search queries Signed-off-by: moson-mo --- aurweb/cache.py | 13 +++++++++++-- test/test_metrics.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 test/test_metrics.py diff --git a/aurweb/cache.py b/aurweb/cache.py index 56bb45b7..fe1e5f1d 100644 --- a/aurweb/cache.py +++ b/aurweb/cache.py @@ -1,5 +1,6 @@ import pickle +from prometheus_client import Counter from sqlalchemy import orm from aurweb import config @@ -7,6 +8,11 @@ from aurweb.aur_redis import redis_connection _redis = redis_connection() +# Prometheus metrics +SEARCH_REQUESTS = Counter( + "search_requests", "Number of search requests by cache hit/miss", ["cache"] +) + async def db_count_cache(key: str, query: orm.Query, expire: int = None) -> int: """Store and retrieve a query.count() via redis cache. @@ -24,7 +30,7 @@ async def db_count_cache(key: str, query: orm.Query, expire: int = None) -> int: return int(result) -async def db_query_cache(key: str, query: orm.Query, expire: int = None): +async def db_query_cache(key: str, query: orm.Query, expire: int = None) -> list: """Store and retrieve query results via redis cache. :param key: Redis key @@ -34,10 +40,13 @@ async def db_query_cache(key: str, query: orm.Query, expire: int = None): """ result = _redis.get(key) if result is None: + SEARCH_REQUESTS.labels(cache="miss").inc() if _redis.dbsize() > config.getint("cache", "max_search_entries", 50000): return query.all() - _redis.set(key, (result := pickle.dumps(query.all())), ex=expire) + _redis.set(key, (result := pickle.dumps(query.all()))) if expire: _redis.expire(key, expire) + else: + SEARCH_REQUESTS.labels(cache="hit").inc() return pickle.loads(result) diff --git a/test/test_metrics.py b/test/test_metrics.py new file mode 100644 index 00000000..1859d8cb --- /dev/null +++ b/test/test_metrics.py @@ -0,0 +1,40 @@ +import pytest +from prometheus_client import REGISTRY, generate_latest + +from aurweb import db +from aurweb.cache import db_query_cache +from aurweb.models.account_type import USER_ID +from aurweb.models.user import User + + +@pytest.fixture(autouse=True) +def setup(db_test): + return + + +@pytest.fixture +def user() -> User: + with db.begin(): + user = db.create( + User, + Username="test", + Email="test@example.org", + RealName="Test User", + Passwd="testPassword", + AccountTypeID=USER_ID, + ) + yield user + + +@pytest.mark.asyncio +async def test_search_cache_metrics(user: User): + # Fire off 3 identical queries for caching + for _ in range(3): + await db_query_cache("key", db.query(User)) + + # Get metrics + metrics = str(generate_latest(REGISTRY)) + + # We should have 1 miss and 2 hits + assert 'search_requests_total{cache="miss"} 1.0' in metrics + assert 'search_requests_total{cache="hit"} 2.0' in metrics From 9fe8d524ffabcbd171cbadbbe9b42edc1f5fa91d Mon Sep 17 00:00:00 2001 From: moson Date: Sat, 8 Jul 2023 10:32:26 +0200 Subject: [PATCH 014/128] fix(test): MariaDB 11 upgrade, query result order Fix order of recipients for "FlagNotification" test. Apply sorting to the recipients query. (only relevant for tests, but who knows when they change things again) MariaDB 11 includes some changes related to the query optimizer. Turns out that this might have effects on how records are ordered for certain queries. (in case no ORDER BY clause was specified) https://mariadb.com/kb/en/mariadb-11-0-0-release-notes/ Signed-off-by: moson --- aurweb/scripts/notify.py | 1 + test/test_notify.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/aurweb/scripts/notify.py b/aurweb/scripts/notify.py index ac9022c3..f55254d7 100755 --- a/aurweb/scripts/notify.py +++ b/aurweb/scripts/notify.py @@ -334,6 +334,7 @@ class FlagNotification(Notification): .filter(and_(PackageBase.ID == pkgbase_id, User.Suspended == 0)) .with_entities(User.Email, User.LangPreference) .distinct() + .order_by(User.Email) ) self._recipients = [(u.Email, u.LangPreference) for u in query] diff --git a/test/test_notify.py b/test/test_notify.py index 9e61d9ee..1fd7cd83 100644 --- a/test/test_notify.py +++ b/test/test_notify.py @@ -127,20 +127,20 @@ def test_out_of_date(user: User, user1: User, user2: User, pkgbases: list[Packag # Should've gotten three emails: maintainer + the two comaintainers. assert Email.count() == 3 - # Comaintainer 1. + # Maintainer. first = Email(1).parse() - assert first.headers.get("To") == user1.Email + assert first.headers.get("To") == user.Email expected = f"AUR Out-of-date Notification for {pkgbase.Name}" assert first.headers.get("Subject") == expected - # Comaintainer 2. + # Comaintainer 1. second = Email(2).parse() - assert second.headers.get("To") == user2.Email + assert second.headers.get("To") == user1.Email - # Maintainer. + # Comaintainer 2. third = Email(3).parse() - assert third.headers.get("To") == user.Email + assert third.headers.get("To") == user2.Email def test_reset(user: User): From f3f8c0a8710838ba176f4486eb886ce37565b78a Mon Sep 17 00:00:00 2001 From: moson-mo Date: Sat, 1 Jul 2023 12:55:14 +0200 Subject: [PATCH 015/128] fix: add recipients to BCC when email is hidden Package requests are sent to the ML as well as users (CC). For those who chose to hide their mail address, we should add them to the BCC list instead. Signed-off-by: moson-mo --- aurweb/scripts/notify.py | 21 +++++++++++---- po/aurweb.pot | 8 ++++++ templates/partials/account_form.html | 7 ++++- test/test_notify.py | 38 ++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/aurweb/scripts/notify.py b/aurweb/scripts/notify.py index f55254d7..a85339ce 100755 --- a/aurweb/scripts/notify.py +++ b/aurweb/scripts/notify.py @@ -45,6 +45,9 @@ class Notification: def get_cc(self): return [] + def get_bcc(self): + return [] + def get_body_fmt(self, lang): body = "" for line in self.get_body(lang).splitlines(): @@ -114,7 +117,7 @@ class Notification: server.login(user, passwd) server.set_debuglevel(0) - deliver_to = [to] + self.get_cc() + deliver_to = [to] + self.get_cc() + self.get_bcc() server.sendmail(sender, deliver_to, msg.as_bytes()) server.quit() @@ -578,10 +581,11 @@ class RequestOpenNotification(Notification): ), ) .filter(and_(PackageRequest.ID == reqid, User.Suspended == 0)) - .with_entities(User.Email) + .with_entities(User.Email, User.HideEmail) .distinct() ) - self._cc = [u.Email for u in query] + self._cc = [u.Email for u in query if u.HideEmail == 0] + self._bcc = [u.Email for u in query if u.HideEmail == 1] pkgreq = ( db.query(PackageRequest.Comments).filter(PackageRequest.ID == reqid).first() @@ -598,6 +602,9 @@ class RequestOpenNotification(Notification): def get_cc(self): return self._cc + def get_bcc(self): + return self._bcc + def get_subject(self, lang): return "[PRQ#%d] %s Request for %s" % ( self._reqid, @@ -665,10 +672,11 @@ class RequestCloseNotification(Notification): ), ) .filter(and_(PackageRequest.ID == reqid, User.Suspended == 0)) - .with_entities(User.Email) + .with_entities(User.Email, User.HideEmail) .distinct() ) - self._cc = [u.Email for u in query] + self._cc = [u.Email for u in query if u.HideEmail == 0] + self._bcc = [u.Email for u in query if u.HideEmail == 1] pkgreq = ( db.query(PackageRequest) @@ -695,6 +703,9 @@ class RequestCloseNotification(Notification): def get_cc(self): return self._cc + def get_bcc(self): + return self._bcc + def get_subject(self, lang): return "[PRQ#%d] %s Request for %s %s" % ( self._reqid, diff --git a/po/aurweb.pot b/po/aurweb.pot index b975ab91..77bca3b0 100644 --- a/po/aurweb.pot +++ b/po/aurweb.pot @@ -2366,3 +2366,11 @@ msgstr "" #: templates/requests.html msgid "Package name" msgstr "" + +#: templates/partials/account_form.html +msgid "Note that if you hide your email address, it'll " +"end up on the BCC list for any request notifications. " +"In case someone replies to these notifications, you won't " +"receive an email. However, replies are typically sent to the " +"mailing-list and would then be visible in the archive." +msgstr "" diff --git a/templates/partials/account_form.html b/templates/partials/account_form.html index 28dc0cd5..7595dcaf 100644 --- a/templates/partials/account_form.html +++ b/templates/partials/account_form.html @@ -115,7 +115,12 @@ {{ "If you do not hide your email address, it is " "visible to all registered AUR users. If you hide your " "email address, it is visible to members of the Arch " - "Linux staff only." | tr }} + "Linux staff only." | tr }}
+ {{ "Note that if you hide your email address, it'll " + "end up on the BCC list for any request notifications. " + "In case someone replies to these notifications, you won't " + "receive an email. However, replies are typically sent to the " + "mailing-list and would then be visible in the archive." | tr }}

diff --git a/test/test_notify.py b/test/test_notify.py index 1fd7cd83..fbcf350b 100644 --- a/test/test_notify.py +++ b/test/test_notify.py @@ -479,6 +479,44 @@ def test_close_request_comaintainer_cc( assert email.headers.get("Cc") == ", ".join([user.Email, user2.Email]) +def test_open_close_request_hidden_email( + user2: User, pkgreq: PackageRequest, pkgbases: list[PackageBase] +): + pkgbase = pkgbases[0] + + # Enable the "HideEmail" option for our requester + with db.begin(): + user2.HideEmail = 1 + + # Send an open request notification. + notif = notify.RequestOpenNotification( + user2.ID, pkgreq.ID, pkgreq.RequestType.Name, pkgbase.ID + ) + + # Make sure our address got added to the bcc list + assert user2.Email in notif.get_bcc() + + notif.send() + assert Email.count() == 1 + + email = Email(1).parse() + # Make sure we don't have our address in the Cc header + assert user2.Email not in email.headers.get("Cc") + + # Create a closure notification on the pkgbase we just opened. + notif = notify.RequestCloseNotification(user2.ID, pkgreq.ID, "rejected") + + # Make sure our address got added to the bcc list + assert user2.Email in notif.get_bcc() + + notif.send() + assert Email.count() == 2 + + email = Email(2).parse() + # Make sure we don't have our address in the Cc header + assert user2.Email not in email.headers.get("Cc") + + def test_close_request_closure_comment( user: User, user2: User, pkgreq: PackageRequest, pkgbases: list[PackageBase] ): From 7cde1ca56041afb9aa00d2d0c46bfd10c2291080 Mon Sep 17 00:00:00 2001 From: renovate Date: Sat, 8 Jul 2023 09:25:09 +0000 Subject: [PATCH 016/128] fix(deps): update all non-major dependencies --- poetry.lock | 622 +++++++++++++++++++++++++++------------------------- 1 file changed, 321 insertions(+), 301 deletions(-) diff --git a/poetry.lock b/poetry.lock index 16b0f15a..dcdcf819 100644 --- a/poetry.lock +++ b/poetry.lock @@ -55,14 +55,14 @@ trio = ["trio (>=0.16,<0.22)"] [[package]] name = "asgiref" -version = "3.7.1" +version = "3.7.2" description = "ASGI specs, helper code, and adapters" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "asgiref-3.7.1-py3-none-any.whl", hash = "sha256:33958cb2e4b3cd8b1b06ef295bd8605cde65b11df51d3beab39e2e149a610ab3"}, - {file = "asgiref-3.7.1.tar.gz", hash = "sha256:8de379fcc383bcfe4507e229fc31209ea23d4831c850f74063b2c11639474dd2"}, + {file = "asgiref-3.7.2-py3-none-any.whl", hash = "sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e"}, + {file = "asgiref-3.7.2.tar.gz", hash = "sha256:9e0ce3aa93a819ba5b45120216b23878cf6e8525eb3848653452b4192b92afed"}, ] [package.dependencies] @@ -85,14 +85,14 @@ files = [ [[package]] name = "authlib" -version = "1.2.0" +version = "1.2.1" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." category = "main" optional = false python-versions = "*" files = [ - {file = "Authlib-1.2.0-py2.py3-none-any.whl", hash = "sha256:4ddf4fd6cfa75c9a460b361d4bd9dac71ffda0be879dbe4292a02e92349ad55a"}, - {file = "Authlib-1.2.0.tar.gz", hash = "sha256:4fa3e80883a5915ef9f5bc28630564bc4ed5b5af39812a3ff130ec76bd631e9d"}, + {file = "Authlib-1.2.1-py2.py3-none-any.whl", hash = "sha256:c88984ea00149a90e3537c964327da930779afa4564e354edfd98410bea01911"}, + {file = "Authlib-1.2.1.tar.gz", hash = "sha256:421f7c6b468d907ca2d9afede256f068f87e34d23dd221c07d13d4c234726afb"}, ] [package.dependencies] @@ -355,63 +355,72 @@ files = [ [[package]] name = "coverage" -version = "7.2.6" +version = "7.2.7" description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "coverage-7.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:496b86f1fc9c81a1cd53d8842ef712e950a4611bba0c42d33366a7b91ba969ec"}, - {file = "coverage-7.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fbe6e8c0a9a7193ba10ee52977d4d5e7652957c1f56ccefed0701db8801a2a3b"}, - {file = "coverage-7.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d06b721c2550c01a60e5d3093f417168658fb454e5dfd9a23570e9bffe39a1"}, - {file = "coverage-7.2.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:77a04b84d01f0e12c66f16e69e92616442dc675bbe51b90bfb074b1e5d1c7fbd"}, - {file = "coverage-7.2.6-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35db06450272473eab4449e9c2ad9bc6a0a68dab8e81a0eae6b50d9c2838767e"}, - {file = "coverage-7.2.6-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6727a0d929ff0028b1ed8b3e7f8701670b1d7032f219110b55476bb60c390bfb"}, - {file = "coverage-7.2.6-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aac1d5fdc5378f6bac2c0c7ebe7635a6809f5b4376f6cf5d43243c1917a67087"}, - {file = "coverage-7.2.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1c9e4a5eb1bbc3675ee57bc31f8eea4cd7fb0cbcbe4912cf1cb2bf3b754f4a80"}, - {file = "coverage-7.2.6-cp310-cp310-win32.whl", hash = "sha256:71f739f97f5f80627f1fee2331e63261355fd1e9a9cce0016394b6707ac3f4ec"}, - {file = "coverage-7.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:fde5c7a9d9864d3e07992f66767a9817f24324f354caa3d8129735a3dc74f126"}, - {file = "coverage-7.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bc7b667f8654376e9353dd93e55e12ce2a59fb6d8e29fce40de682273425e044"}, - {file = "coverage-7.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:697f4742aa3f26c107ddcb2b1784a74fe40180014edbd9adaa574eac0529914c"}, - {file = "coverage-7.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:541280dde49ce74a4262c5e395b48ea1207e78454788887118c421cb4ffbfcac"}, - {file = "coverage-7.2.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7f1a8328eeec34c54f1d5968a708b50fc38d31e62ca8b0560e84a968fbf9a9"}, - {file = "coverage-7.2.6-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4bbd58eb5a2371bf160590f4262109f66b6043b0b991930693134cb617bc0169"}, - {file = "coverage-7.2.6-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ae82c5f168d2a39a5d69a12a69d4dc23837a43cf2ca99be60dfe59996ea6b113"}, - {file = "coverage-7.2.6-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f5440cdaf3099e7ab17a5a7065aed59aff8c8b079597b61c1f8be6f32fe60636"}, - {file = "coverage-7.2.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a6f03f87fea579d55e0b690d28f5042ec1368650466520fbc400e7aeaf09e995"}, - {file = "coverage-7.2.6-cp311-cp311-win32.whl", hash = "sha256:dc4d5187ef4d53e0d4c8eaf530233685667844c5fb0b855fea71ae659017854b"}, - {file = "coverage-7.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:c93d52c3dc7b9c65e39473704988602300e3cc1bad08b5ab5b03ca98bbbc68c1"}, - {file = "coverage-7.2.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:42c692b55a647a832025a4c048007034fe77b162b566ad537ce65ad824b12a84"}, - {file = "coverage-7.2.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7786b2fa7809bf835f830779ad285215a04da76293164bb6745796873f0942d"}, - {file = "coverage-7.2.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25bad4196104761bc26b1dae9b57383826542ec689ff0042f7f4f4dd7a815cba"}, - {file = "coverage-7.2.6-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2692306d3d4cb32d2cceed1e47cebd6b1d2565c993d6d2eda8e6e6adf53301e6"}, - {file = "coverage-7.2.6-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:392154d09bd4473b9d11351ab5d63391f3d5d24d752f27b3be7498b0ee2b5226"}, - {file = "coverage-7.2.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:fa079995432037b5e2ef5ddbb270bcd2ded9f52b8e191a5de11fe59a00ea30d8"}, - {file = "coverage-7.2.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d712cefff15c712329113b01088ba71bbcef0f7ea58478ca0bbec63a824844cb"}, - {file = "coverage-7.2.6-cp37-cp37m-win32.whl", hash = "sha256:004948e296149644d208964300cb3d98affc5211e9e490e9979af4030b0d6473"}, - {file = "coverage-7.2.6-cp37-cp37m-win_amd64.whl", hash = "sha256:c1d7a31603c3483ac49c1726723b0934f88f2c011c660e6471e7bd735c2fa110"}, - {file = "coverage-7.2.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3436927d1794fa6763b89b60c896f9e3bd53212001026ebc9080d23f0c2733c1"}, - {file = "coverage-7.2.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44c9b9f1a245f3d0d202b1a8fa666a80b5ecbe4ad5d0859c0fb16a52d9763224"}, - {file = "coverage-7.2.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e3783a286d5a93a2921396d50ce45a909aa8f13eee964465012f110f0cbb611"}, - {file = "coverage-7.2.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cff6980fe7100242170092bb40d2b1cdad79502cd532fd26b12a2b8a5f9aee0"}, - {file = "coverage-7.2.6-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c534431153caffc7c495c3eddf7e6a6033e7f81d78385b4e41611b51e8870446"}, - {file = "coverage-7.2.6-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3062fd5c62df988cea9f2972c593f77fed1182bfddc5a3b12b1e606cb7aba99e"}, - {file = "coverage-7.2.6-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6284a2005e4f8061c58c814b1600ad0074ccb0289fe61ea709655c5969877b70"}, - {file = "coverage-7.2.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:97729e6828643f168a2a3f07848e1b1b94a366b13a9f5aba5484c2215724edc8"}, - {file = "coverage-7.2.6-cp38-cp38-win32.whl", hash = "sha256:dc11b42fa61ff1e788dd095726a0aed6aad9c03d5c5984b54cb9e1e67b276aa5"}, - {file = "coverage-7.2.6-cp38-cp38-win_amd64.whl", hash = "sha256:cbcc874f454ee51f158afd604a315f30c0e31dff1d5d5bf499fc529229d964dd"}, - {file = "coverage-7.2.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d3cacc6a665221108ecdf90517a8028d07a2783df3417d12dcfef1c517e67478"}, - {file = "coverage-7.2.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:272ab31228a9df857ab5df5d67936d8861464dc89c5d3fab35132626e9369379"}, - {file = "coverage-7.2.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a8723ccec4e564d4b9a79923246f7b9a8de4ec55fa03ec4ec804459dade3c4f"}, - {file = "coverage-7.2.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5906f6a84b47f995cd1bf0aca1c72d591c55ee955f98074e93660d64dfc66eb9"}, - {file = "coverage-7.2.6-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52c139b7ab3f0b15f9aad0a3fedef5a1f8c0b2bdc291d88639ca2c97d3682416"}, - {file = "coverage-7.2.6-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a5ffd45c6b93c23a8507e2f436983015c6457aa832496b6a095505ca2f63e8f1"}, - {file = "coverage-7.2.6-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4f3c7c19581d471af0e9cb49d928172cd8492cd78a2b7a4e82345d33662929bb"}, - {file = "coverage-7.2.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e8c0e79820cdd67978e1120983786422d279e07a381dbf89d03bbb23ec670a6"}, - {file = "coverage-7.2.6-cp39-cp39-win32.whl", hash = "sha256:13cde6bb0e58fb67d09e2f373de3899d1d1e866c5a9ff05d93615f2f54fbd2bb"}, - {file = "coverage-7.2.6-cp39-cp39-win_amd64.whl", hash = "sha256:6b9f64526286255735847aed0221b189486e0b9ed943446936e41b7e44b08783"}, - {file = "coverage-7.2.6-pp37.pp38.pp39-none-any.whl", hash = "sha256:6babcbf1e66e46052442f10833cfc4a0d3554d8276aa37af8531a83ed3c1a01d"}, - {file = "coverage-7.2.6.tar.gz", hash = "sha256:2025f913f2edb0272ef15d00b1f335ff8908c921c8eb2013536fcaf61f5a683d"}, + {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, + {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495"}, + {file = "coverage-7.2.7-cp310-cp310-win32.whl", hash = "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818"}, + {file = "coverage-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a"}, + {file = "coverage-7.2.7-cp311-cp311-win32.whl", hash = "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a"}, + {file = "coverage-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562"}, + {file = "coverage-7.2.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d"}, + {file = "coverage-7.2.7-cp312-cp312-win32.whl", hash = "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511"}, + {file = "coverage-7.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3"}, + {file = "coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02"}, + {file = "coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f"}, + {file = "coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f"}, + {file = "coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e"}, + {file = "coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2"}, + {file = "coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb"}, + {file = "coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27"}, + {file = "coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d"}, + {file = "coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59"}, ] [package.dependencies] @@ -531,19 +540,19 @@ testing = ["pre-commit"] [[package]] name = "fakeredis" -version = "2.13.0" -description = "Fake implementation of redis API for testing purposes." +version = "2.16.0" +description = "Python implementation of redis API, can be used for testing purposes." category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ - {file = "fakeredis-2.13.0-py3-none-any.whl", hash = "sha256:df7bb44fb9e593970c626325230e1c321f954ce7b204d4c4452eae5233d554ed"}, - {file = "fakeredis-2.13.0.tar.gz", hash = "sha256:53f00f44f771d2b794f1ea036fa07a33476ab7368f1b0e908daab3eff80336f6"}, + {file = "fakeredis-2.16.0-py3-none-any.whl", hash = "sha256:188514cbd7120ff28c88f2a31e2fddd18fb1b28504478dfa3669c683134c4d82"}, + {file = "fakeredis-2.16.0.tar.gz", hash = "sha256:5abdd734de4ead9d6c7acbd3add1c4aa9b3ab35219339530472d9dd2bdf13057"}, ] [package.dependencies] redis = ">=4" -sortedcontainers = ">=2.4,<3.0" +sortedcontainers = ">=2,<3" [package.extras] json = ["jsonpath-ng (>=1.5,<2.0)"] @@ -588,19 +597,19 @@ python-dateutil = "*" [[package]] name = "filelock" -version = "3.12.0" +version = "3.12.2" description = "A platform independent file lock." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "filelock-3.12.0-py3-none-any.whl", hash = "sha256:ad98852315c2ab702aeb628412cbf7e95b7ce8c3bf9565670b4eaecf1db370a9"}, - {file = "filelock-3.12.0.tar.gz", hash = "sha256:fc03ae43288c013d2ea83c8597001b1129db351aad9c57fe2409327916b8e718"}, + {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, + {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, ] [package.extras] -docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] [[package]] name = "greenlet" @@ -896,96 +905,109 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "lxml" -version = "4.9.2" +version = "4.9.3" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" files = [ - {file = "lxml-4.9.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:76cf573e5a365e790396a5cc2b909812633409306c6531a6877c59061e42c4f2"}, - {file = "lxml-4.9.2-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b1f42b6921d0e81b1bcb5e395bc091a70f41c4d4e55ba99c6da2b31626c44892"}, - {file = "lxml-4.9.2-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9f102706d0ca011de571de32c3247c6476b55bb6bc65a20f682f000b07a4852a"}, - {file = "lxml-4.9.2-cp27-cp27m-win32.whl", hash = "sha256:8d0b4612b66ff5d62d03bcaa043bb018f74dfea51184e53f067e6fdcba4bd8de"}, - {file = "lxml-4.9.2-cp27-cp27m-win_amd64.whl", hash = "sha256:4c8f293f14abc8fd3e8e01c5bd86e6ed0b6ef71936ded5bf10fe7a5efefbaca3"}, - {file = "lxml-4.9.2-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2899456259589aa38bfb018c364d6ae7b53c5c22d8e27d0ec7609c2a1ff78b50"}, - {file = "lxml-4.9.2-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6749649eecd6a9871cae297bffa4ee76f90b4504a2a2ab528d9ebe912b101975"}, - {file = "lxml-4.9.2-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a08cff61517ee26cb56f1e949cca38caabe9ea9fbb4b1e10a805dc39844b7d5c"}, - {file = "lxml-4.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:85cabf64adec449132e55616e7ca3e1000ab449d1d0f9d7f83146ed5bdcb6d8a"}, - {file = "lxml-4.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8340225bd5e7a701c0fa98284c849c9b9fc9238abf53a0ebd90900f25d39a4e4"}, - {file = "lxml-4.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:1ab8f1f932e8f82355e75dda5413a57612c6ea448069d4fb2e217e9a4bed13d4"}, - {file = "lxml-4.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:699a9af7dffaf67deeae27b2112aa06b41c370d5e7633e0ee0aea2e0b6c211f7"}, - {file = "lxml-4.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9cc34af337a97d470040f99ba4282f6e6bac88407d021688a5d585e44a23184"}, - {file = "lxml-4.9.2-cp310-cp310-win32.whl", hash = "sha256:d02a5399126a53492415d4906ab0ad0375a5456cc05c3fc0fc4ca11771745cda"}, - {file = "lxml-4.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:a38486985ca49cfa574a507e7a2215c0c780fd1778bb6290c21193b7211702ab"}, - {file = "lxml-4.9.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c83203addf554215463b59f6399835201999b5e48019dc17f182ed5ad87205c9"}, - {file = "lxml-4.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2a87fa548561d2f4643c99cd13131acb607ddabb70682dcf1dff5f71f781a4bf"}, - {file = "lxml-4.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:d6b430a9938a5a5d85fc107d852262ddcd48602c120e3dbb02137c83d212b380"}, - {file = "lxml-4.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3efea981d956a6f7173b4659849f55081867cf897e719f57383698af6f618a92"}, - {file = "lxml-4.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:df0623dcf9668ad0445e0558a21211d4e9a149ea8f5666917c8eeec515f0a6d1"}, - {file = "lxml-4.9.2-cp311-cp311-win32.whl", hash = "sha256:da248f93f0418a9e9d94b0080d7ebc407a9a5e6d0b57bb30db9b5cc28de1ad33"}, - {file = "lxml-4.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:3818b8e2c4b5148567e1b09ce739006acfaa44ce3156f8cbbc11062994b8e8dd"}, - {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca989b91cf3a3ba28930a9fc1e9aeafc2a395448641df1f387a2d394638943b0"}, - {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:822068f85e12a6e292803e112ab876bc03ed1f03dddb80154c395f891ca6b31e"}, - {file = "lxml-4.9.2-cp35-cp35m-win32.whl", hash = "sha256:be7292c55101e22f2a3d4d8913944cbea71eea90792bf914add27454a13905df"}, - {file = "lxml-4.9.2-cp35-cp35m-win_amd64.whl", hash = "sha256:998c7c41910666d2976928c38ea96a70d1aa43be6fe502f21a651e17483a43c5"}, - {file = "lxml-4.9.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:b26a29f0b7fc6f0897f043ca366142d2b609dc60756ee6e4e90b5f762c6adc53"}, - {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:ab323679b8b3030000f2be63e22cdeea5b47ee0abd2d6a1dc0c8103ddaa56cd7"}, - {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:689bb688a1db722485e4610a503e3e9210dcc20c520b45ac8f7533c837be76fe"}, - {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:f49e52d174375a7def9915c9f06ec4e569d235ad428f70751765f48d5926678c"}, - {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:36c3c175d34652a35475a73762b545f4527aec044910a651d2bf50de9c3352b1"}, - {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a35f8b7fa99f90dd2f5dc5a9fa12332642f087a7641289ca6c40d6e1a2637d8e"}, - {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:58bfa3aa19ca4c0f28c5dde0ff56c520fbac6f0daf4fac66ed4c8d2fb7f22e74"}, - {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc718cd47b765e790eecb74d044cc8d37d58562f6c314ee9484df26276d36a38"}, - {file = "lxml-4.9.2-cp36-cp36m-win32.whl", hash = "sha256:d5bf6545cd27aaa8a13033ce56354ed9e25ab0e4ac3b5392b763d8d04b08e0c5"}, - {file = "lxml-4.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:3ab9fa9d6dc2a7f29d7affdf3edebf6ece6fb28a6d80b14c3b2fb9d39b9322c3"}, - {file = "lxml-4.9.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:05ca3f6abf5cf78fe053da9b1166e062ade3fa5d4f92b4ed688127ea7d7b1d03"}, - {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:a5da296eb617d18e497bcf0a5c528f5d3b18dadb3619fbdadf4ed2356ef8d941"}, - {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:04876580c050a8c5341d706dd464ff04fd597095cc8c023252566a8826505726"}, - {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c9ec3eaf616d67db0764b3bb983962b4f385a1f08304fd30c7283954e6a7869b"}, - {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2a29ba94d065945944016b6b74e538bdb1751a1db6ffb80c9d3c2e40d6fa9894"}, - {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a82d05da00a58b8e4c0008edbc8a4b6ec5a4bc1e2ee0fb6ed157cf634ed7fa45"}, - {file = "lxml-4.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:223f4232855ade399bd409331e6ca70fb5578efef22cf4069a6090acc0f53c0e"}, - {file = "lxml-4.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d17bc7c2ccf49c478c5bdd447594e82692c74222698cfc9b5daae7ae7e90743b"}, - {file = "lxml-4.9.2-cp37-cp37m-win32.whl", hash = "sha256:b64d891da92e232c36976c80ed7ebb383e3f148489796d8d31a5b6a677825efe"}, - {file = "lxml-4.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a0a336d6d3e8b234a3aae3c674873d8f0e720b76bc1d9416866c41cd9500ffb9"}, - {file = "lxml-4.9.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:da4dd7c9c50c059aba52b3524f84d7de956f7fef88f0bafcf4ad7dde94a064e8"}, - {file = "lxml-4.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:821b7f59b99551c69c85a6039c65b75f5683bdc63270fec660f75da67469ca24"}, - {file = "lxml-4.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:e5168986b90a8d1f2f9dc1b841467c74221bd752537b99761a93d2d981e04889"}, - {file = "lxml-4.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8e20cb5a47247e383cf4ff523205060991021233ebd6f924bca927fcf25cf86f"}, - {file = "lxml-4.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:13598ecfbd2e86ea7ae45ec28a2a54fb87ee9b9fdb0f6d343297d8e548392c03"}, - {file = "lxml-4.9.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:880bbbcbe2fca64e2f4d8e04db47bcdf504936fa2b33933efd945e1b429bea8c"}, - {file = "lxml-4.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7d2278d59425777cfcb19735018d897ca8303abe67cc735f9f97177ceff8027f"}, - {file = "lxml-4.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5344a43228767f53a9df6e5b253f8cdca7dfc7b7aeae52551958192f56d98457"}, - {file = "lxml-4.9.2-cp38-cp38-win32.whl", hash = "sha256:925073b2fe14ab9b87e73f9a5fde6ce6392da430f3004d8b72cc86f746f5163b"}, - {file = "lxml-4.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:9b22c5c66f67ae00c0199f6055705bc3eb3fcb08d03d2ec4059a2b1b25ed48d7"}, - {file = "lxml-4.9.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5f50a1c177e2fa3ee0667a5ab79fdc6b23086bc8b589d90b93b4bd17eb0e64d1"}, - {file = "lxml-4.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:090c6543d3696cbe15b4ac6e175e576bcc3f1ccfbba970061b7300b0c15a2140"}, - {file = "lxml-4.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:63da2ccc0857c311d764e7d3d90f429c252e83b52d1f8f1d1fe55be26827d1f4"}, - {file = "lxml-4.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:5b4545b8a40478183ac06c073e81a5ce4cf01bf1734962577cf2bb569a5b3bbf"}, - {file = "lxml-4.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2e430cd2824f05f2d4f687701144556646bae8f249fd60aa1e4c768ba7018947"}, - {file = "lxml-4.9.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6804daeb7ef69e7b36f76caddb85cccd63d0c56dedb47555d2fc969e2af6a1a5"}, - {file = "lxml-4.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a6e441a86553c310258aca15d1c05903aaf4965b23f3bc2d55f200804e005ee5"}, - {file = "lxml-4.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ca34efc80a29351897e18888c71c6aca4a359247c87e0b1c7ada14f0ab0c0fb2"}, - {file = "lxml-4.9.2-cp39-cp39-win32.whl", hash = "sha256:6b418afe5df18233fc6b6093deb82a32895b6bb0b1155c2cdb05203f583053f1"}, - {file = "lxml-4.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1496ea22ca2c830cbcbd473de8f114a320da308438ae65abad6bab7867fe38f"}, - {file = "lxml-4.9.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b264171e3143d842ded311b7dccd46ff9ef34247129ff5bf5066123c55c2431c"}, - {file = "lxml-4.9.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0dc313ef231edf866912e9d8f5a042ddab56c752619e92dfd3a2c277e6a7299a"}, - {file = "lxml-4.9.2-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:16efd54337136e8cd72fb9485c368d91d77a47ee2d42b057564aae201257d419"}, - {file = "lxml-4.9.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0f2b1e0d79180f344ff9f321327b005ca043a50ece8713de61d1cb383fb8ac05"}, - {file = "lxml-4.9.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:7b770ed79542ed52c519119473898198761d78beb24b107acf3ad65deae61f1f"}, - {file = "lxml-4.9.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efa29c2fe6b4fdd32e8ef81c1528506895eca86e1d8c4657fda04c9b3786ddf9"}, - {file = "lxml-4.9.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7e91ee82f4199af8c43d8158024cbdff3d931df350252288f0d4ce656df7f3b5"}, - {file = "lxml-4.9.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b23e19989c355ca854276178a0463951a653309fb8e57ce674497f2d9f208746"}, - {file = "lxml-4.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:01d36c05f4afb8f7c20fd9ed5badca32a2029b93b1750f571ccc0b142531caf7"}, - {file = "lxml-4.9.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7b515674acfdcadb0eb5d00d8a709868173acece5cb0be3dd165950cbfdf5409"}, - {file = "lxml-4.9.2.tar.gz", hash = "sha256:2455cfaeb7ac70338b3257f41e21f0724f4b5b0c0e7702da67ee6c3640835b67"}, + {file = "lxml-4.9.3-cp27-cp27m-macosx_11_0_x86_64.whl", hash = "sha256:b0a545b46b526d418eb91754565ba5b63b1c0b12f9bd2f808c852d9b4b2f9b5c"}, + {file = "lxml-4.9.3-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:075b731ddd9e7f68ad24c635374211376aa05a281673ede86cbe1d1b3455279d"}, + {file = "lxml-4.9.3-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1e224d5755dba2f4a9498e150c43792392ac9b5380aa1b845f98a1618c94eeef"}, + {file = "lxml-4.9.3-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0781a98ff5e6586926293e59480b64ddd46282953203c76ae15dbbbf302e8bb"}, + {file = "lxml-4.9.3-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:cef2502e7e8a96fe5ad686d60b49e1ab03e438bd9123987994528febd569868e"}, + {file = "lxml-4.9.3-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:b86164d2cff4d3aaa1f04a14685cbc072efd0b4f99ca5708b2ad1b9b5988a991"}, + {file = "lxml-4.9.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:42871176e7896d5d45138f6d28751053c711ed4d48d8e30b498da155af39aebd"}, + {file = "lxml-4.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:ae8b9c6deb1e634ba4f1930eb67ef6e6bf6a44b6eb5ad605642b2d6d5ed9ce3c"}, + {file = "lxml-4.9.3-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:411007c0d88188d9f621b11d252cce90c4a2d1a49db6c068e3c16422f306eab8"}, + {file = "lxml-4.9.3-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:cd47b4a0d41d2afa3e58e5bf1f62069255aa2fd6ff5ee41604418ca925911d76"}, + {file = "lxml-4.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0e2cb47860da1f7e9a5256254b74ae331687b9672dfa780eed355c4c9c3dbd23"}, + {file = "lxml-4.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1247694b26342a7bf47c02e513d32225ededd18045264d40758abeb3c838a51f"}, + {file = "lxml-4.9.3-cp310-cp310-win32.whl", hash = "sha256:cdb650fc86227eba20de1a29d4b2c1bfe139dc75a0669270033cb2ea3d391b85"}, + {file = "lxml-4.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:97047f0d25cd4bcae81f9ec9dc290ca3e15927c192df17331b53bebe0e3ff96d"}, + {file = "lxml-4.9.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:1f447ea5429b54f9582d4b955f5f1985f278ce5cf169f72eea8afd9502973dd5"}, + {file = "lxml-4.9.3-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:57d6ba0ca2b0c462f339640d22882acc711de224d769edf29962b09f77129cbf"}, + {file = "lxml-4.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:9767e79108424fb6c3edf8f81e6730666a50feb01a328f4a016464a5893f835a"}, + {file = "lxml-4.9.3-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:71c52db65e4b56b8ddc5bb89fb2e66c558ed9d1a74a45ceb7dcb20c191c3df2f"}, + {file = "lxml-4.9.3-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d73d8ecf8ecf10a3bd007f2192725a34bd62898e8da27eb9d32a58084f93962b"}, + {file = "lxml-4.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0a3d3487f07c1d7f150894c238299934a2a074ef590b583103a45002035be120"}, + {file = "lxml-4.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e28c51fa0ce5674be9f560c6761c1b441631901993f76700b1b30ca6c8378d6"}, + {file = "lxml-4.9.3-cp311-cp311-win32.whl", hash = "sha256:0bfd0767c5c1de2551a120673b72e5d4b628737cb05414f03c3277bf9bed3305"}, + {file = "lxml-4.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:25f32acefac14ef7bd53e4218fe93b804ef6f6b92ffdb4322bb6d49d94cad2bc"}, + {file = "lxml-4.9.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:d3ff32724f98fbbbfa9f49d82852b159e9784d6094983d9a8b7f2ddaebb063d4"}, + {file = "lxml-4.9.3-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:48d6ed886b343d11493129e019da91d4039826794a3e3027321c56d9e71505be"}, + {file = "lxml-4.9.3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9a92d3faef50658dd2c5470af249985782bf754c4e18e15afb67d3ab06233f13"}, + {file = "lxml-4.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b4e4bc18382088514ebde9328da057775055940a1f2e18f6ad2d78aa0f3ec5b9"}, + {file = "lxml-4.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fc9b106a1bf918db68619fdcd6d5ad4f972fdd19c01d19bdb6bf63f3589a9ec5"}, + {file = "lxml-4.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:d37017287a7adb6ab77e1c5bee9bcf9660f90ff445042b790402a654d2ad81d8"}, + {file = "lxml-4.9.3-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:56dc1f1ebccc656d1b3ed288f11e27172a01503fc016bcabdcbc0978b19352b7"}, + {file = "lxml-4.9.3-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:578695735c5a3f51569810dfebd05dd6f888147a34f0f98d4bb27e92b76e05c2"}, + {file = "lxml-4.9.3-cp35-cp35m-win32.whl", hash = "sha256:704f61ba8c1283c71b16135caf697557f5ecf3e74d9e453233e4771d68a1f42d"}, + {file = "lxml-4.9.3-cp35-cp35m-win_amd64.whl", hash = "sha256:c41bfca0bd3532d53d16fd34d20806d5c2b1ace22a2f2e4c0008570bf2c58833"}, + {file = "lxml-4.9.3-cp36-cp36m-macosx_11_0_x86_64.whl", hash = "sha256:64f479d719dc9f4c813ad9bb6b28f8390360660b73b2e4beb4cb0ae7104f1c12"}, + {file = "lxml-4.9.3-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:dd708cf4ee4408cf46a48b108fb9427bfa00b9b85812a9262b5c668af2533ea5"}, + {file = "lxml-4.9.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c31c7462abdf8f2ac0577d9f05279727e698f97ecbb02f17939ea99ae8daa98"}, + {file = "lxml-4.9.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:e3cd95e10c2610c360154afdc2f1480aea394f4a4f1ea0a5eacce49640c9b190"}, + {file = "lxml-4.9.3-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:4930be26af26ac545c3dffb662521d4e6268352866956672231887d18f0eaab2"}, + {file = "lxml-4.9.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4aec80cde9197340bc353d2768e2a75f5f60bacda2bab72ab1dc499589b3878c"}, + {file = "lxml-4.9.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:14e019fd83b831b2e61baed40cab76222139926b1fb5ed0e79225bc0cae14584"}, + {file = "lxml-4.9.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0c0850c8b02c298d3c7006b23e98249515ac57430e16a166873fc47a5d549287"}, + {file = "lxml-4.9.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:aca086dc5f9ef98c512bac8efea4483eb84abbf926eaeedf7b91479feb092458"}, + {file = "lxml-4.9.3-cp36-cp36m-win32.whl", hash = "sha256:50baa9c1c47efcaef189f31e3d00d697c6d4afda5c3cde0302d063492ff9b477"}, + {file = "lxml-4.9.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bef4e656f7d98aaa3486d2627e7d2df1157d7e88e7efd43a65aa5dd4714916cf"}, + {file = "lxml-4.9.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:46f409a2d60f634fe550f7133ed30ad5321ae2e6630f13657fb9479506b00601"}, + {file = "lxml-4.9.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:4c28a9144688aef80d6ea666c809b4b0e50010a2aca784c97f5e6bf143d9f129"}, + {file = "lxml-4.9.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:141f1d1a9b663c679dc524af3ea1773e618907e96075262726c7612c02b149a4"}, + {file = "lxml-4.9.3-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:53ace1c1fd5a74ef662f844a0413446c0629d151055340e9893da958a374f70d"}, + {file = "lxml-4.9.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:17a753023436a18e27dd7769e798ce302963c236bc4114ceee5b25c18c52c693"}, + {file = "lxml-4.9.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7d298a1bd60c067ea75d9f684f5f3992c9d6766fadbc0bcedd39750bf344c2f4"}, + {file = "lxml-4.9.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:081d32421db5df44c41b7f08a334a090a545c54ba977e47fd7cc2deece78809a"}, + {file = "lxml-4.9.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:23eed6d7b1a3336ad92d8e39d4bfe09073c31bfe502f20ca5116b2a334f8ec02"}, + {file = "lxml-4.9.3-cp37-cp37m-win32.whl", hash = "sha256:1509dd12b773c02acd154582088820893109f6ca27ef7291b003d0e81666109f"}, + {file = "lxml-4.9.3-cp37-cp37m-win_amd64.whl", hash = "sha256:120fa9349a24c7043854c53cae8cec227e1f79195a7493e09e0c12e29f918e52"}, + {file = "lxml-4.9.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:4d2d1edbca80b510443f51afd8496be95529db04a509bc8faee49c7b0fb6d2cc"}, + {file = "lxml-4.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8d7e43bd40f65f7d97ad8ef5c9b1778943d02f04febef12def25f7583d19baac"}, + {file = "lxml-4.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:71d66ee82e7417828af6ecd7db817913cb0cf9d4e61aa0ac1fde0583d84358db"}, + {file = "lxml-4.9.3-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:6fc3c450eaa0b56f815c7b62f2b7fba7266c4779adcf1cece9e6deb1de7305ce"}, + {file = "lxml-4.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:65299ea57d82fb91c7f019300d24050c4ddeb7c5a190e076b5f48a2b43d19c42"}, + {file = "lxml-4.9.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:eadfbbbfb41b44034a4c757fd5d70baccd43296fb894dba0295606a7cf3124aa"}, + {file = "lxml-4.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3e9bdd30efde2b9ccfa9cb5768ba04fe71b018a25ea093379c857c9dad262c40"}, + {file = "lxml-4.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fcdd00edfd0a3001e0181eab3e63bd5c74ad3e67152c84f93f13769a40e073a7"}, + {file = "lxml-4.9.3-cp38-cp38-win32.whl", hash = "sha256:57aba1bbdf450b726d58b2aea5fe47c7875f5afb2c4a23784ed78f19a0462574"}, + {file = "lxml-4.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:92af161ecbdb2883c4593d5ed4815ea71b31fafd7fd05789b23100d081ecac96"}, + {file = "lxml-4.9.3-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:9bb6ad405121241e99a86efff22d3ef469024ce22875a7ae045896ad23ba2340"}, + {file = "lxml-4.9.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:8ed74706b26ad100433da4b9d807eae371efaa266ffc3e9191ea436087a9d6a7"}, + {file = "lxml-4.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fbf521479bcac1e25a663df882c46a641a9bff6b56dc8b0fafaebd2f66fb231b"}, + {file = "lxml-4.9.3-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:303bf1edce6ced16bf67a18a1cf8339d0db79577eec5d9a6d4a80f0fb10aa2da"}, + {file = "lxml-4.9.3-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:5515edd2a6d1a5a70bfcdee23b42ec33425e405c5b351478ab7dc9347228f96e"}, + {file = "lxml-4.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:690dafd0b187ed38583a648076865d8c229661ed20e48f2335d68e2cf7dc829d"}, + {file = "lxml-4.9.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b6420a005548ad52154c8ceab4a1290ff78d757f9e5cbc68f8c77089acd3c432"}, + {file = "lxml-4.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bb3bb49c7a6ad9d981d734ef7c7193bc349ac338776a0360cc671eaee89bcf69"}, + {file = "lxml-4.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d27be7405547d1f958b60837dc4c1007da90b8b23f54ba1f8b728c78fdb19d50"}, + {file = "lxml-4.9.3-cp39-cp39-win32.whl", hash = "sha256:8df133a2ea5e74eef5e8fc6f19b9e085f758768a16e9877a60aec455ed2609b2"}, + {file = "lxml-4.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:4dd9a263e845a72eacb60d12401e37c616438ea2e5442885f65082c276dfb2b2"}, + {file = "lxml-4.9.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6689a3d7fd13dc687e9102a27e98ef33730ac4fe37795d5036d18b4d527abd35"}, + {file = "lxml-4.9.3-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:f6bdac493b949141b733c5345b6ba8f87a226029cbabc7e9e121a413e49441e0"}, + {file = "lxml-4.9.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:05186a0f1346ae12553d66df1cfce6f251589fea3ad3da4f3ef4e34b2d58c6a3"}, + {file = "lxml-4.9.3-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c2006f5c8d28dee289f7020f721354362fa304acbaaf9745751ac4006650254b"}, + {file = "lxml-4.9.3-pp38-pypy38_pp73-macosx_11_0_x86_64.whl", hash = "sha256:5c245b783db29c4e4fbbbfc9c5a78be496c9fea25517f90606aa1f6b2b3d5f7b"}, + {file = "lxml-4.9.3-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:4fb960a632a49f2f089d522f70496640fdf1218f1243889da3822e0a9f5f3ba7"}, + {file = "lxml-4.9.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:50670615eaf97227d5dc60de2dc99fb134a7130d310d783314e7724bf163f75d"}, + {file = "lxml-4.9.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:9719fe17307a9e814580af1f5c6e05ca593b12fb7e44fe62450a5384dbf61b4b"}, + {file = "lxml-4.9.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:3331bece23c9ee066e0fb3f96c61322b9e0f54d775fccefff4c38ca488de283a"}, + {file = "lxml-4.9.3-pp39-pypy39_pp73-macosx_11_0_x86_64.whl", hash = "sha256:ed667f49b11360951e201453fc3967344d0d0263aa415e1619e85ae7fd17b4e0"}, + {file = "lxml-4.9.3-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:8b77946fd508cbf0fccd8e400a7f71d4ac0e1595812e66025bac475a8e811694"}, + {file = "lxml-4.9.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:e4da8ca0c0c0aea88fd46be8e44bd49716772358d648cce45fe387f7b92374a7"}, + {file = "lxml-4.9.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fe4bda6bd4340caa6e5cf95e73f8fea5c4bfc55763dd42f1b50a94c1b4a2fbd4"}, + {file = "lxml-4.9.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:f3df3db1d336b9356dd3112eae5f5c2b8b377f3bc826848567f10bfddfee77e9"}, + {file = "lxml-4.9.3.tar.gz", hash = "sha256:48628bd53a426c9eb9bc066a923acaa0878d1e86129fd5359aee99285f4eed9c"}, ] [package.extras] cssselect = ["cssselect (>=0.7)"] html5 = ["html5lib"] htmlsoup = ["BeautifulSoup4"] -source = ["Cython (>=0.29.7)"] +source = ["Cython (>=0.29.35)"] [[package]] name = "mako" @@ -1087,75 +1109,75 @@ files = [ [[package]] name = "mysqlclient" -version = "2.1.1" +version = "2.2.0" description = "Python interface to MySQL" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.8" files = [ - {file = "mysqlclient-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:c1ed71bd6244993b526113cca3df66428609f90e4652f37eb51c33496d478b37"}, - {file = "mysqlclient-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:c812b67e90082a840efb82a8978369e6e69fc62ce1bda4ca8f3084a9d862308b"}, - {file = "mysqlclient-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:0d1cd3a5a4d28c222fa199002810e8146cffd821410b67851af4cc80aeccd97c"}, - {file = "mysqlclient-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:b355c8b5a7d58f2e909acdbb050858390ee1b0e13672ae759e5e784110022994"}, - {file = "mysqlclient-2.1.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:996924f3483fd36a34a5812210c69e71dea5a3d5978d01199b78b7f6d485c855"}, - {file = "mysqlclient-2.1.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:dea88c8d3f5a5d9293dfe7f087c16dd350ceb175f2f6631c9cf4caf3e19b7a96"}, - {file = "mysqlclient-2.1.1.tar.gz", hash = "sha256:828757e419fb11dd6c5ed2576ec92c3efaa93a0f7c39e263586d1ee779c3d782"}, + {file = "mysqlclient-2.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:68837b6bb23170acffb43ae411e47533a560b6360c06dac39aa55700972c93b2"}, + {file = "mysqlclient-2.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:5670679ff1be1cc3fef0fa81bf39f0cd70605ba121141050f02743eb878ac114"}, + {file = "mysqlclient-2.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:004fe1d30d2c2ff8072f8ea513bcec235fd9b896f70dad369461d0ad7e570e98"}, + {file = "mysqlclient-2.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:9c6b142836c7dba4f723bf9c93cc46b6e5081d65b2af807f400dda9eb85a16d0"}, + {file = "mysqlclient-2.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:955dba905a7443ce4788c63fdb9f8d688316260cf60b20ff51ac3b1c77616ede"}, + {file = "mysqlclient-2.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:530ece9995a36cadb6211b9787f0c9e05cdab6702549bdb4236af5e9b535ed6a"}, + {file = "mysqlclient-2.2.0.tar.gz", hash = "sha256:04368445f9c487d8abb7a878e3d23e923e6072c04a6c320f9e0dc8a82efba14e"}, ] [[package]] name = "orjson" -version = "3.8.14" +version = "3.9.2" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "orjson-3.8.14-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:7a7b0fead2d0115ef927fa46ad005d7a3988a77187500bf895af67b365c10d1f"}, - {file = "orjson-3.8.14-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca90db8f551b8960da95b0d4cad6c0489df52ea03585b6979595be7b31a3f946"}, - {file = "orjson-3.8.14-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4ac01a3db4e6a98a8ad1bb1a3e8bfc777928939e87c04e93e0d5006df574a4b"}, - {file = "orjson-3.8.14-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bf6825e160e4eb0ef65ce37d8c221edcab96ff2ffba65e5da2437a60a12b3ad1"}, - {file = "orjson-3.8.14-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f80e62afe49e6bfc706e041faa351d7520b5f86572b8e31455802251ea989613"}, - {file = "orjson-3.8.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6112194c11e611596eed72f46efb0e6b4812682eff3c7b48473d1146c3fa0efb"}, - {file = "orjson-3.8.14-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:739f9f633e1544f2a477fa3bef380f488c8dca6e2521c8dc36424b12554ee31e"}, - {file = "orjson-3.8.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7d3d8faded5a514b80b56d0429eb38b429d7a810f8749d25dc10a0cc15b8a3c8"}, - {file = "orjson-3.8.14-cp310-none-win_amd64.whl", hash = "sha256:0bf00c42333412a9338297bf888d7428c99e281e20322070bde8c2314775508b"}, - {file = "orjson-3.8.14-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:d66966fd94719beb84e8ed84833bc59c3c005d3d2d0c42f11d7552d3267c6de7"}, - {file = "orjson-3.8.14-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:087c0dc93379e8ba2d59e9f586fab8de8c137d164fccf8afd5523a2137570917"}, - {file = "orjson-3.8.14-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04c70dc8ca79b0072a16d82f94b9d9dd6598a43dd753ab20039e9f7d2b14f017"}, - {file = "orjson-3.8.14-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aedba48264fe87e5060c0e9c2b28909f1e60626e46dc2f77e0c8c16939e2e1f7"}, - {file = "orjson-3.8.14-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:01640ab79111dd97515cba9fab7c66cb3b0967b0892cc74756a801ff681a01b6"}, - {file = "orjson-3.8.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b206cca6836a4c6683bcaa523ab467627b5f03902e5e1082dc59cd010e6925f"}, - {file = "orjson-3.8.14-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ee0299b2dda9afce351a5e8c148ea7a886de213f955aa0288fb874fb44829c36"}, - {file = "orjson-3.8.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:31a2a29be559e92dcc5c278787b4166da6f0d45675b59a11c4867f5d1455ebf4"}, - {file = "orjson-3.8.14-cp311-none-win_amd64.whl", hash = "sha256:20b7ffc7736000ea205f9143df322b03961f287b4057606291c62c842ff3c5b5"}, - {file = "orjson-3.8.14-cp37-cp37m-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:de1ee13d6b6727ee1db38722695250984bae81b8fc9d05f1176c74d14b1322d9"}, - {file = "orjson-3.8.14-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ee09bfbf1d54c127d3061f6721a1a11d2ce502b50597c3d0d2e1bd2d235b764"}, - {file = "orjson-3.8.14-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:97ebb7fab5f1ae212a6501f17cb7750a6838ffc2f1cebbaa5dec1a90038ca3c6"}, - {file = "orjson-3.8.14-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38ca39bae7fbc050332a374062d4cdec28095540fa8bb245eada467897a3a0bb"}, - {file = "orjson-3.8.14-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:92374bc35b6da344a927d5a850f7db80a91c7b837de2f0ea90fc870314b1ff44"}, - {file = "orjson-3.8.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9393a63cb0424515ec5e434078b3198de6ec9e057f1d33bad268683935f0a5d5"}, - {file = "orjson-3.8.14-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5fb66f0ac23e861b817c858515ac1f74d1cd9e72e3f82a5b2c9bae9f92286adc"}, - {file = "orjson-3.8.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19415aaf30525a5baff0d72a089fcdd68f19a3674998263c885c3908228c1086"}, - {file = "orjson-3.8.14-cp37-none-win_amd64.whl", hash = "sha256:87ba7882e146e24a7d8b4a7971c20212c2af75ead8096fc3d55330babb1015fb"}, - {file = "orjson-3.8.14-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9f5cf61b6db68f213c805c55bf0aab9b4cb75a4e9c7f5bfbd4deb3a0aef0ec53"}, - {file = "orjson-3.8.14-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33bc310da4ad2ffe8f7f1c9e89692146d9ec5aec2d1c9ef6b67f8dc5e2d63241"}, - {file = "orjson-3.8.14-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:67a7e883b6f782b106683979ccc43d89b98c28a1f4a33fe3a22e253577499bb1"}, - {file = "orjson-3.8.14-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9df820e6c8c84c52ec39ea2cc9c79f7999c839c7d1481a056908dce3b90ce9f9"}, - {file = "orjson-3.8.14-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ebca14ae80814219ea3327e3dfa7ff618621ff335e45781fac26f5cd0b48f2b4"}, - {file = "orjson-3.8.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27967be4c16bd09f4aeff8896d9be9cbd00fd72f5815d5980e4776f821e2f77c"}, - {file = "orjson-3.8.14-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:062829b5e20cd8648bf4c11c3a5ee7cf196fa138e573407b5312c849b0cf354d"}, - {file = "orjson-3.8.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e53bc5beb612df8ddddb065f079d3fd30b5b4e73053518524423549d61177f3f"}, - {file = "orjson-3.8.14-cp38-none-win_amd64.whl", hash = "sha256:d03f29b0369bb1ab55c8a67103eb3a9675daaf92f04388568034fe16be48fa5d"}, - {file = "orjson-3.8.14-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:716a3994e039203f0a59056efa28185d4cac51b922cc5bf27ab9182cfa20e12e"}, - {file = "orjson-3.8.14-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cb35dd3ba062c1d984d57e6477768ed7b62ed9260f31362b2d69106f9c60ebd"}, - {file = "orjson-3.8.14-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0bc6b7abf27f1dc192dadad249df9b513912506dd420ce50fd18864a33789b71"}, - {file = "orjson-3.8.14-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7e2f75b7d9285e35c3d4dff9811185535ff2ea637f06b2b242cb84385f8ffe63"}, - {file = "orjson-3.8.14-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:017de5ba22e58dfa6f41914f5edb8cd052d23f171000684c26b2d2ab219db31e"}, - {file = "orjson-3.8.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09a3bf3154f40299b8bc95e9fb8da47436a59a2106fc22cae15f76d649e062da"}, - {file = "orjson-3.8.14-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:64b4fca0531030040e611c6037aaf05359e296877ab0a8e744c26ef9c32738b9"}, - {file = "orjson-3.8.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8a896a12b38fe201a72593810abc1f4f1597e65b8c869d5fc83bbcf75d93398f"}, - {file = "orjson-3.8.14-cp39-none-win_amd64.whl", hash = "sha256:9725226478d1dafe46d26f758eadecc6cf98dcbb985445e14a9c74aaed6ccfea"}, - {file = "orjson-3.8.14.tar.gz", hash = "sha256:5ea93fd3ef7be7386f2516d728c877156de1559cda09453fc7dd7b696d0439b3"}, + {file = "orjson-3.9.2-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:7323e4ca8322b1ecb87562f1ec2491831c086d9faa9a6c6503f489dadbed37d7"}, + {file = "orjson-3.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1272688ea1865f711b01ba479dea2d53e037ea00892fd04196b5875f7021d9d3"}, + {file = "orjson-3.9.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0b9a26f1d1427a9101a1e8910f2e2df1f44d3d18ad5480ba031b15d5c1cb282e"}, + {file = "orjson-3.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6a5ca55b0d8f25f18b471e34abaee4b175924b6cd62f59992945b25963443141"}, + {file = "orjson-3.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:877872db2c0f41fbe21f852ff642ca842a43bc34895b70f71c9d575df31fffb4"}, + {file = "orjson-3.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a39c2529d75373b7167bf84c814ef9b8f3737a339c225ed6c0df40736df8748"}, + {file = "orjson-3.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:84ebd6fdf138eb0eb4280045442331ee71c0aab5e16397ba6645f32f911bfb37"}, + {file = "orjson-3.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5a60a1cfcfe310547a1946506dd4f1ed0a7d5bd5b02c8697d9d5dcd8d2e9245e"}, + {file = "orjson-3.9.2-cp310-none-win_amd64.whl", hash = "sha256:c290c4f81e8fd0c1683638802c11610b2f722b540f8e5e858b6914b495cf90c8"}, + {file = "orjson-3.9.2-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:02ef014f9a605e84b675060785e37ec9c0d2347a04f1307a9d6840ab8ecd6f55"}, + {file = "orjson-3.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:992af54265ada1c1579500d6594ed73fe333e726de70d64919cf37f93defdd06"}, + {file = "orjson-3.9.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a40958f7af7c6d992ee67b2da4098dca8b770fc3b4b3834d540477788bfa76d3"}, + {file = "orjson-3.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93864dec3e3dd058a2dbe488d11ac0345214a6a12697f53a63e34de7d28d4257"}, + {file = "orjson-3.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16fdf5a82df80c544c3c91516ab3882cd1ac4f1f84eefeafa642e05cef5f6699"}, + {file = "orjson-3.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:275b5a18fd9ed60b2720543d3ddac170051c43d680e47d04ff5203d2c6d8ebf1"}, + {file = "orjson-3.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b9aea6dcb99fcbc9f6d1dd84fca92322fda261da7fb014514bb4689c7c2097a8"}, + {file = "orjson-3.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7d74ae0e101d17c22ef67b741ba356ab896fc0fa64b301c2bf2bb0a4d874b190"}, + {file = "orjson-3.9.2-cp311-none-win_amd64.whl", hash = "sha256:6320b28e7bdb58c3a3a5efffe04b9edad3318d82409e84670a9b24e8035a249d"}, + {file = "orjson-3.9.2-cp37-cp37m-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:368e9cc91ecb7ac21f2aa475e1901204110cf3e714e98649c2502227d248f947"}, + {file = "orjson-3.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58e9e70f0dcd6a802c35887f306b555ff7a214840aad7de24901fc8bd9cf5dde"}, + {file = "orjson-3.9.2-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:00c983896c2e01c94c0ef72fd7373b2aa06d0c0eed0342c4884559f812a6835b"}, + {file = "orjson-3.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ee743e8890b16c87a2f89733f983370672272b61ee77429c0a5899b2c98c1a7"}, + {file = "orjson-3.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7b065942d362aad4818ff599d2f104c35a565c2cbcbab8c09ec49edba91da75"}, + {file = "orjson-3.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e46e9c5b404bb9e41d5555762fd410d5466b7eb1ec170ad1b1609cbebe71df21"}, + {file = "orjson-3.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8170157288714678ffd64f5de33039e1164a73fd8b6be40a8a273f80093f5c4f"}, + {file = "orjson-3.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e3e2f087161947dafe8319ea2cfcb9cea4bb9d2172ecc60ac3c9738f72ef2909"}, + {file = "orjson-3.9.2-cp37-none-win_amd64.whl", hash = "sha256:d7de3dbbe74109ae598692113cec327fd30c5a30ebca819b21dfa4052f7b08ef"}, + {file = "orjson-3.9.2-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:8cd4385c59bbc1433cad4a80aca65d2d9039646a9c57f8084897549b55913b17"}, + {file = "orjson-3.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a74036aab1a80c361039290cdbc51aa7adc7ea13f56e5ef94e9be536abd227bd"}, + {file = "orjson-3.9.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1aaa46d7d4ae55335f635eadc9be0bd9bcf742e6757209fc6dc697e390010adc"}, + {file = "orjson-3.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e52c67ed6bb368083aa2078ea3ccbd9721920b93d4b06c43eb4e20c4c860046"}, + {file = "orjson-3.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a6cdfcf9c7dd4026b2b01fdff56986251dc0cc1e980c690c79eec3ae07b36e7"}, + {file = "orjson-3.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1882a70bb69595b9ec5aac0040a819e94d2833fe54901e2b32f5e734bc259a8b"}, + {file = "orjson-3.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:fc05e060d452145ab3c0b5420769e7356050ea311fc03cb9d79c481982917cca"}, + {file = "orjson-3.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f8bc2c40d9bb26efefb10949d261a47ca196772c308babc538dd9f4b73e8d386"}, + {file = "orjson-3.9.2-cp38-none-win_amd64.whl", hash = "sha256:3164fc20a585ec30a9aff33ad5de3b20ce85702b2b2a456852c413e3f0d7ab09"}, + {file = "orjson-3.9.2-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:7a6ccadf788531595ed4728aa746bc271955448d2460ff0ef8e21eb3f2a281ba"}, + {file = "orjson-3.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3245d230370f571c945f69aab823c279a868dc877352817e22e551de155cb06c"}, + {file = "orjson-3.9.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:205925b179550a4ee39b8418dd4c94ad6b777d165d7d22614771c771d44f57bd"}, + {file = "orjson-3.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0325fe2d69512187761f7368c8cda1959bcb75fc56b8e7a884e9569112320e57"}, + {file = "orjson-3.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:806704cd58708acc66a064a9a58e3be25cf1c3f9f159e8757bd3f515bfabdfa1"}, + {file = "orjson-3.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03fb36f187a0c19ff38f6289418863df8b9b7880cdbe279e920bef3a09d8dab1"}, + {file = "orjson-3.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:20925d07a97c49c6305bff1635318d9fc1804aa4ccacb5fb0deb8a910e57d97a"}, + {file = "orjson-3.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:eebfed53bec5674e981ebe8ed2cf00b3f7bcda62d634733ff779c264307ea505"}, + {file = "orjson-3.9.2-cp39-none-win_amd64.whl", hash = "sha256:869b961df5fcedf6c79f4096119b35679b63272362e9b745e668f0391a892d39"}, + {file = "orjson-3.9.2.tar.gz", hash = "sha256:24257c8f641979bf25ecd3e27251b5cc194cdd3a6e96004aac8446f5e63d9664"}, ] [[package]] @@ -1189,6 +1211,7 @@ category = "main" optional = false python-versions = "*" files = [ + {file = "parse-1.19.0-py2.py3-none-any.whl", hash = "sha256:6ce007645384a91150cb7cd7c8a9db2559e273c2e2542b508cd1e342508c2601"}, {file = "parse-1.19.0.tar.gz", hash = "sha256:9ff82852bcb65d139813e2a5197627a94966245c897796760a3a2a8eb66f020b"}, ] @@ -1269,25 +1292,25 @@ prometheus-client = ">=0.8.0,<1.0.0" [[package]] name = "protobuf" -version = "4.23.2" +version = "4.23.4" description = "" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "protobuf-4.23.2-cp310-abi3-win32.whl", hash = "sha256:384dd44cb4c43f2ccddd3645389a23ae61aeb8cfa15ca3a0f60e7c3ea09b28b3"}, - {file = "protobuf-4.23.2-cp310-abi3-win_amd64.whl", hash = "sha256:09310bce43353b46d73ba7e3bca78273b9bc50349509b9698e64d288c6372c2a"}, - {file = "protobuf-4.23.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:b2cfab63a230b39ae603834718db74ac11e52bccaaf19bf20f5cce1a84cf76df"}, - {file = "protobuf-4.23.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:c52cfcbfba8eb791255edd675c1fe6056f723bf832fa67f0442218f8817c076e"}, - {file = "protobuf-4.23.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:86df87016d290143c7ce3be3ad52d055714ebaebb57cc659c387e76cfacd81aa"}, - {file = "protobuf-4.23.2-cp37-cp37m-win32.whl", hash = "sha256:281342ea5eb631c86697e1e048cb7e73b8a4e85f3299a128c116f05f5c668f8f"}, - {file = "protobuf-4.23.2-cp37-cp37m-win_amd64.whl", hash = "sha256:ce744938406de1e64b91410f473736e815f28c3b71201302612a68bf01517fea"}, - {file = "protobuf-4.23.2-cp38-cp38-win32.whl", hash = "sha256:6c081863c379bb1741be8f8193e893511312b1d7329b4a75445d1ea9955be69e"}, - {file = "protobuf-4.23.2-cp38-cp38-win_amd64.whl", hash = "sha256:25e3370eda26469b58b602e29dff069cfaae8eaa0ef4550039cc5ef8dc004511"}, - {file = "protobuf-4.23.2-cp39-cp39-win32.whl", hash = "sha256:efabbbbac1ab519a514579ba9ec52f006c28ae19d97915951f69fa70da2c9e91"}, - {file = "protobuf-4.23.2-cp39-cp39-win_amd64.whl", hash = "sha256:54a533b971288af3b9926e53850c7eb186886c0c84e61daa8444385a4720297f"}, - {file = "protobuf-4.23.2-py3-none-any.whl", hash = "sha256:8da6070310d634c99c0db7df48f10da495cc283fd9e9234877f0cd182d43ab7f"}, - {file = "protobuf-4.23.2.tar.gz", hash = "sha256:20874e7ca4436f683b64ebdbee2129a5a2c301579a67d1a7dda2cdf62fb7f5f7"}, + {file = "protobuf-4.23.4-cp310-abi3-win32.whl", hash = "sha256:5fea3c64d41ea5ecf5697b83e41d09b9589e6f20b677ab3c48e5f242d9b7897b"}, + {file = "protobuf-4.23.4-cp310-abi3-win_amd64.whl", hash = "sha256:7b19b6266d92ca6a2a87effa88ecc4af73ebc5cfde194dc737cf8ef23a9a3b12"}, + {file = "protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8547bf44fe8cec3c69e3042f5c4fb3e36eb2a7a013bb0a44c018fc1e427aafbd"}, + {file = "protobuf-4.23.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fee88269a090ada09ca63551bf2f573eb2424035bcf2cb1b121895b01a46594a"}, + {file = "protobuf-4.23.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:effeac51ab79332d44fba74660d40ae79985901ac21bca408f8dc335a81aa597"}, + {file = "protobuf-4.23.4-cp37-cp37m-win32.whl", hash = "sha256:c3e0939433c40796ca4cfc0fac08af50b00eb66a40bbbc5dee711998fb0bbc1e"}, + {file = "protobuf-4.23.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9053df6df8e5a76c84339ee4a9f5a2661ceee4a0dab019e8663c50ba324208b0"}, + {file = "protobuf-4.23.4-cp38-cp38-win32.whl", hash = "sha256:e1c915778d8ced71e26fcf43c0866d7499891bca14c4368448a82edc61fdbc70"}, + {file = "protobuf-4.23.4-cp38-cp38-win_amd64.whl", hash = "sha256:351cc90f7d10839c480aeb9b870a211e322bf05f6ab3f55fcb2f51331f80a7d2"}, + {file = "protobuf-4.23.4-cp39-cp39-win32.whl", hash = "sha256:6dd9b9940e3f17077e820b75851126615ee38643c2c5332aa7a359988820c720"}, + {file = "protobuf-4.23.4-cp39-cp39-win_amd64.whl", hash = "sha256:0a5759f5696895de8cc913f084e27fd4125e8fb0914bb729a17816a33819f474"}, + {file = "protobuf-4.23.4-py3-none-any.whl", hash = "sha256:e9d0be5bf34b275b9f87ba7407796556abeeba635455d036c7351f7c183ef8ff"}, + {file = "protobuf-4.23.4.tar.gz", hash = "sha256:ccd9430c0719dce806b93f89c91de7977304729e55377f872a92465d548329a9"}, ] [[package]] @@ -1368,43 +1391,43 @@ email = ["email-validator (>=1.0.3)"] [[package]] name = "pygit2" -version = "1.12.1" +version = "1.12.2" description = "Python bindings for libgit2." category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "pygit2-1.12.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:50a155528aa611e4a217be31a9d2d8da283cfd978dbba07494cd04ea3d7c8768"}, - {file = "pygit2-1.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:248e22ccb1ea31f569373a3da3fa73d110ba2585c6326ff74b03c9579fb7b913"}, - {file = "pygit2-1.12.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e575e672c5a6cb39234b0076423a560e016d6b88cd50947c2df3bf59c5ccdf3d"}, - {file = "pygit2-1.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad9b46b52997d131b31ff46f699b074e9745c8fea8d0efb6b72ace43ab25828c"}, - {file = "pygit2-1.12.1-cp310-cp310-win32.whl", hash = "sha256:a8f495df877da04c572ecec4d532ae195680b4781dbf229bab4e801fa9ef20e9"}, - {file = "pygit2-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:9f1e1355c7fe2938a2bca0d6204a00c02950d13008722879e54a335b3e874006"}, - {file = "pygit2-1.12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8a5c56b0b5dc8a317561070ef7557e180d4937d8b115c5a762d85e0109a216f3"}, - {file = "pygit2-1.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b7c9ca8bc8a722863fc873234748fef3422007d5a6ea90ba3ae338d2907d3d6e"}, - {file = "pygit2-1.12.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71c02a11f10bc4e329ab941f0c70874d39053c8f78544aefeb506f04cedb621a"}, - {file = "pygit2-1.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b3af334adf325b7c973417efa220fd5a9ce946b936262eceabc8ad8d46e0310"}, - {file = "pygit2-1.12.1-cp311-cp311-win32.whl", hash = "sha256:86c393962d1341893bbfa91829b3b8545e8ac7622f8b53b9a0b835b9cc1b5198"}, - {file = "pygit2-1.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:86c7e75ddc76f4e5593b47f9c2074fff242322ed9f4126116749f7c86021520a"}, - {file = "pygit2-1.12.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:939d11677f434024ea25a9137d8a525ef9f9ac474fb8b86399bc9526e6a7bff5"}, - {file = "pygit2-1.12.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:946f9215c0442995042ea512f764f7a6638d3a09f9d0484d3aeedbf8833f89e6"}, - {file = "pygit2-1.12.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd574620d3cc80df0b23bf2b7b08d8726e75a338d0fa1b67e4d6738d3ee56635"}, - {file = "pygit2-1.12.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24d0adeff5c43229913f3bdae71c36e77ed19f36bd8dd6b5c141820964b1f5b3"}, - {file = "pygit2-1.12.1-cp38-cp38-win32.whl", hash = "sha256:ed8e2ef97171e994bf4d46c6c6534a3c12dd2dbbc47741e5995eaf8c2c92f71c"}, - {file = "pygit2-1.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:5318817055a3ca3906bf88344b9a6dc70c640f9b6bc236ac9e767d12bad54361"}, - {file = "pygit2-1.12.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cb9c803151ffeb0b8de52a93381108a2c6a9a446c55d659a135f52645e1650eb"}, - {file = "pygit2-1.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:47bf1e196dc23fe38018ad49b021d425edc319328169c597df45d73cf46b62ef"}, - {file = "pygit2-1.12.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:948479df72223bbcd16b2a88904dc2a3886c15a0107a7cf3b5373c8e34f52f31"}, - {file = "pygit2-1.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4bebe8b310edc2662cbffb94ef1a758252fe2e4c92bc83fac0eaf2bedf8b871"}, - {file = "pygit2-1.12.1-cp39-cp39-win32.whl", hash = "sha256:77bc0ab778ab6fe631f5f9eb831b426376a7b71426c5a913aaa9088382ef1dc9"}, - {file = "pygit2-1.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:e87b2306a266f6abca94ab37dda807033a6f40faad05c4d1e089f9e8354130a8"}, - {file = "pygit2-1.12.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5d5e8a3b67f5d4ba8e3838c492254688997747989b184b5f1a3af4fef7f9f53e"}, - {file = "pygit2-1.12.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2500b749759f2efdfa5096c0aafeb2d92152766708f5700284427bd658e5c407"}, - {file = "pygit2-1.12.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c21759ca9cc755faa2d17180cd49af004486ca84f3166cac089a2083dcb09114"}, - {file = "pygit2-1.12.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d73074ab64b383e3a1ab03e8070f6b195ef89b9d379ca5682c38dd9c289cc6e2"}, - {file = "pygit2-1.12.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:865c0d1925c52426455317f29c1db718187ec69ed5474faaf3e1c68ff2135767"}, - {file = "pygit2-1.12.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ebebbe9125b068337b5415565ec94c9e092c708e430851b2d02e51217bdce4a"}, - {file = "pygit2-1.12.1.tar.gz", hash = "sha256:8218922abedc88a65d5092308d533ca4c4ed634aec86a3493d3bdf1a25aeeff3"}, + {file = "pygit2-1.12.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:79fbd99d3e08ca7478150eeba28ca4d4103f564148eab8d00aba8f1e6fc60654"}, + {file = "pygit2-1.12.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:be3bb0139f464947523022a5af343a2e862c4ff250a57ec9f631449e7c0ba7c0"}, + {file = "pygit2-1.12.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4df3e5745fdf3111a6ccc905eae99f22f1a180728f714795138ca540cc2a50a"}, + {file = "pygit2-1.12.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:214bd214784fcbef7a8494d1d59e0cd3a731c0d24ce0f230dcc843322ee33b08"}, + {file = "pygit2-1.12.2-cp310-cp310-win32.whl", hash = "sha256:336c864ac961e7be8ba06e9ed8c999e4f624a8ccd90121cc4e40956d8b57acac"}, + {file = "pygit2-1.12.2-cp310-cp310-win_amd64.whl", hash = "sha256:fb9eb57b75ce586928053692a25aae2a50fef3ad36661c57c07d4902899b1df3"}, + {file = "pygit2-1.12.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f8f813d35d836c5b0d1962c387754786bcc7f1c3c8e11207b9eeb30238ac4cc7"}, + {file = "pygit2-1.12.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:25a6548930328c5247bfb7c67d29104e63b036cb5390f032d9f91f63efb70434"}, + {file = "pygit2-1.12.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a365ffca23d910381749fdbcc367db52fe808f9aa4852914dd9ef8b711384a32"}, + {file = "pygit2-1.12.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec04c27be5d5af1ceecdcc0464e07081222f91f285f156dc53b23751d146569a"}, + {file = "pygit2-1.12.2-cp311-cp311-win32.whl", hash = "sha256:546091316c9a8c37b9867ddcc6c9f7402ca4d0b9db3f349212a7b5e71988e359"}, + {file = "pygit2-1.12.2-cp311-cp311-win_amd64.whl", hash = "sha256:8bf14196cbfffbcd286f459a1d4fc660c5d5dfa8fb422e21216961df575410d6"}, + {file = "pygit2-1.12.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7bb30ab1fdaa4c30821fed33892958b6d92d50dbd03c76f7775b4e5d62f53a2e"}, + {file = "pygit2-1.12.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e7e705aaecad85b883022e81e054fbd27d26023fc031618ee61c51516580517e"}, + {file = "pygit2-1.12.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac2b5f408eb882e79645ebb43039ac37739c3edd25d857cc97d7482a684b613f"}, + {file = "pygit2-1.12.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7f3ad2b7b0c80be991bb47d8a2f2535cc9bf090746eb8679231ee565fde81"}, + {file = "pygit2-1.12.2-cp38-cp38-win32.whl", hash = "sha256:5b3ab4d6302990f7adb2b015bcbda1f0715277008d0c66440497e6f8313bf9cb"}, + {file = "pygit2-1.12.2-cp38-cp38-win_amd64.whl", hash = "sha256:c74e7601cb8b8dc3d02fd32274e200a7761cffd20ee531442bf1fa115c8f99a5"}, + {file = "pygit2-1.12.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:6a4083ba093c69142e0400114a4ef75e87834637d2bbfd77b964614bf70f624f"}, + {file = "pygit2-1.12.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:926f2e48c4eaa179249d417b8382290b86b0f01dbf41d289f763576209276b9f"}, + {file = "pygit2-1.12.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14ae27491347a0ac4bbe8347b09d752cfe7fea1121c14525415e0cca6db4a836"}, + {file = "pygit2-1.12.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f65483ab5e3563c58f60debe2acc0979fdf6fd633432fcfbddf727a9a265ba4"}, + {file = "pygit2-1.12.2-cp39-cp39-win32.whl", hash = "sha256:8da8517809635ea3da950d9cf99c6d1851352d92b6db309382db88a01c3b0bfd"}, + {file = "pygit2-1.12.2-cp39-cp39-win_amd64.whl", hash = "sha256:b9c2359b99eed8e7fac30c06e6b4ae277a6a0537d6b4b88a190828c3d7eb9ef2"}, + {file = "pygit2-1.12.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:685378852ef8eb081333bc80dbdfc4f1333cf4a8f3baf614c4135e02ad1ee38a"}, + {file = "pygit2-1.12.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdf655e5f801990f5cad721b6ccbe7610962f0a4f1c20373dbf9c0be39374a81"}, + {file = "pygit2-1.12.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:857c5cde635d470f58803d67bfb281dc4f6336065a0253bfbed001f18e2d0767"}, + {file = "pygit2-1.12.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fe35a72af61961dbb7fb4abcdaa36d5f1c85b2cd3daae94137eeb9c07215cdd3"}, + {file = "pygit2-1.12.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f443d3641762b2bb9c76400bb18beb4ba27dd35bc098a8bfae82e6a190c52ab"}, + {file = "pygit2-1.12.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c1e26649e1540b6a774f812e2fc9890320ff4d33f16db1bb02626318b5ceae2"}, + {file = "pygit2-1.12.2.tar.gz", hash = "sha256:56e85d0e66de957d599d1efb2409d39afeefd8f01009bfda0796b42a4b678358"}, ] [package.dependencies] @@ -1412,14 +1435,14 @@ cffi = ">=1.9.1" [[package]] name = "pytest" -version = "7.3.1" +version = "7.4.0" description = "pytest: simple powerful testing with Python" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, - {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, + {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, + {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, ] [package.dependencies] @@ -1431,7 +1454,7 @@ pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-asyncio" @@ -1540,14 +1563,14 @@ dev = ["atomicwrites (==1.2.1)", "attrs (==19.2.0)", "coverage (==6.5.0)", "hatc [[package]] name = "redis" -version = "4.5.5" +version = "4.6.0" description = "Python client for Redis database and key-value store" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "redis-4.5.5-py3-none-any.whl", hash = "sha256:77929bc7f5dab9adf3acba2d3bb7d7658f1e0c2f1cafe7eb36434e751c471119"}, - {file = "redis-4.5.5.tar.gz", hash = "sha256:dc87a0bdef6c8bfe1ef1e1c40be7034390c2ae02d92dcd0c7ca1729443899880"}, + {file = "redis-4.6.0-py3-none-any.whl", hash = "sha256:e2b03db868160ee4591de3cb90d40ebb50a90dd302138775937f6a42b7ed183c"}, + {file = "redis-4.6.0.tar.gz", hash = "sha256:585dc516b9eb042a619ef0a39c3d7d55fe81bdb4df09a52c9cdde0d07bf1aa7d"}, ] [package.dependencies] @@ -1634,53 +1657,50 @@ files = [ [[package]] name = "sqlalchemy" -version = "1.4.48" +version = "1.4.49" description = "Database Abstraction Library" category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "SQLAlchemy-1.4.48-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:4bac3aa3c3d8bc7408097e6fe8bf983caa6e9491c5d2e2488cfcfd8106f13b6a"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:dbcae0e528d755f4522cad5842f0942e54b578d79f21a692c44d91352ea6d64e"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27m-win32.whl", hash = "sha256:cbbe8b8bffb199b225d2fe3804421b7b43a0d49983f81dc654d0431d2f855543"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27m-win_amd64.whl", hash = "sha256:627e04a5d54bd50628fc8734d5fc6df2a1aa5962f219c44aad50b00a6cdcf965"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9af1db7a287ef86e0f5cd990b38da6bd9328de739d17e8864f1817710da2d217"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:ce7915eecc9c14a93b73f4e1c9d779ca43e955b43ddf1e21df154184f39748e5"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5381ddd09a99638f429f4cbe1b71b025bed318f6a7b23e11d65f3eed5e181c33"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:87609f6d4e81a941a17e61a4c19fee57f795e96f834c4f0a30cee725fc3f81d9"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb0808ad34167f394fea21bd4587fc62f3bd81bba232a1e7fbdfa17e6cfa7cd7"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-win32.whl", hash = "sha256:d53cd8bc582da5c1c8c86b6acc4ef42e20985c57d0ebc906445989df566c5603"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-win_amd64.whl", hash = "sha256:4355e5915844afdc5cf22ec29fba1010166e35dd94a21305f49020022167556b"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:066c2b0413e8cb980e6d46bf9d35ca83be81c20af688fedaef01450b06e4aa5e"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c99bf13e07140601d111a7c6f1fc1519914dd4e5228315bbda255e08412f61a4"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ee26276f12614d47cc07bc85490a70f559cba965fb178b1c45d46ffa8d73fda"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-win32.whl", hash = "sha256:49c312bcff4728bffc6fb5e5318b8020ed5c8b958a06800f91859fe9633ca20e"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-win_amd64.whl", hash = "sha256:cef2e2abc06eab187a533ec3e1067a71d7bbec69e582401afdf6d8cad4ba3515"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:3509159e050bd6d24189ec7af373359f07aed690db91909c131e5068176c5a5d"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc2ab4d9f6d9218a5caa4121bdcf1125303482a1cdcfcdbd8567be8518969c0"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e1ddbbcef9bcedaa370c03771ebec7e39e3944782bef49e69430383c376a250b"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f82d8efea1ca92b24f51d3aea1a82897ed2409868a0af04247c8c1e4fef5890"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-win32.whl", hash = "sha256:e3e98d4907805b07743b583a99ecc58bf8807ecb6985576d82d5e8ae103b5272"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-win_amd64.whl", hash = "sha256:25887b4f716e085a1c5162f130b852f84e18d2633942c8ca40dfb8519367c14f"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:0817c181271b0ce5df1aa20949f0a9e2426830fed5ecdcc8db449618f12c2730"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe1dd2562313dd9fe1778ed56739ad5d9aae10f9f43d9f4cf81d65b0c85168bb"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:68413aead943883b341b2b77acd7a7fe2377c34d82e64d1840860247cec7ff7c"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbde5642104ac6e95f96e8ad6d18d9382aa20672008cf26068fe36f3004491df"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-win32.whl", hash = "sha256:11c6b1de720f816c22d6ad3bbfa2f026f89c7b78a5c4ffafb220e0183956a92a"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-win_amd64.whl", hash = "sha256:eb5464ee8d4bb6549d368b578e9529d3c43265007193597ddca71c1bae6174e6"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:92e6133cf337c42bfee03ca08c62ba0f2d9695618c8abc14a564f47503157be9"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44d29a3fc6d9c45962476b470a81983dd8add6ad26fdbfae6d463b509d5adcda"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:005e942b451cad5285015481ae4e557ff4154dde327840ba91b9ac379be3b6ce"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c8cfe951ed074ba5e708ed29c45397a95c4143255b0d022c7c8331a75ae61f3"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-win32.whl", hash = "sha256:2b9af65cc58726129d8414fc1a1a650dcdd594ba12e9c97909f1f57d48e393d3"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-win_amd64.whl", hash = "sha256:2b562e9d1e59be7833edf28b0968f156683d57cabd2137d8121806f38a9d58f4"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:a1fc046756cf2a37d7277c93278566ddf8be135c6a58397b4c940abf837011f4"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d9b55252d2ca42a09bcd10a697fa041e696def9dfab0b78c0aaea1485551a08"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6dab89874e72a9ab5462997846d4c760cdb957958be27b03b49cf0de5e5c327c"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd8b5ee5a3acc4371f820934b36f8109ce604ee73cc668c724abb054cebcb6e"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-win32.whl", hash = "sha256:eee09350fd538e29cfe3a496ec6f148504d2da40dbf52adefb0d2f8e4d38ccc4"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-win_amd64.whl", hash = "sha256:7ad2b0f6520ed5038e795cc2852eb5c1f20fa6831d73301ced4aafbe3a10e1f6"}, - {file = "SQLAlchemy-1.4.48.tar.gz", hash = "sha256:b47bc287096d989a0838ce96f7d8e966914a24da877ed41a7531d44b55cdb8df"}, + {file = "SQLAlchemy-1.4.49-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2e126cf98b7fd38f1e33c64484406b78e937b1a280e078ef558b95bf5b6895f6"}, + {file = "SQLAlchemy-1.4.49-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:03db81b89fe7ef3857b4a00b63dedd632d6183d4ea5a31c5d8a92e000a41fc71"}, + {file = "SQLAlchemy-1.4.49-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:95b9df9afd680b7a3b13b38adf6e3a38995da5e162cc7524ef08e3be4e5ed3e1"}, + {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a63e43bf3f668c11bb0444ce6e809c1227b8f067ca1068898f3008a273f52b09"}, + {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f835c050ebaa4e48b18403bed2c0fda986525896efd76c245bdd4db995e51a4c"}, + {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c21b172dfb22e0db303ff6419451f0cac891d2e911bb9fbf8003d717f1bcf91"}, + {file = "SQLAlchemy-1.4.49-cp310-cp310-win32.whl", hash = "sha256:5fb1ebdfc8373b5a291485757bd6431de8d7ed42c27439f543c81f6c8febd729"}, + {file = "SQLAlchemy-1.4.49-cp310-cp310-win_amd64.whl", hash = "sha256:f8a65990c9c490f4651b5c02abccc9f113a7f56fa482031ac8cb88b70bc8ccaa"}, + {file = "SQLAlchemy-1.4.49-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8923dfdf24d5aa8a3adb59723f54118dd4fe62cf59ed0d0d65d940579c1170a4"}, + {file = "SQLAlchemy-1.4.49-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9ab2c507a7a439f13ca4499db6d3f50423d1d65dc9b5ed897e70941d9e135b0"}, + {file = "SQLAlchemy-1.4.49-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5debe7d49b8acf1f3035317e63d9ec8d5e4d904c6e75a2a9246a119f5f2fdf3d"}, + {file = "SQLAlchemy-1.4.49-cp311-cp311-win32.whl", hash = "sha256:82b08e82da3756765c2e75f327b9bf6b0f043c9c3925fb95fb51e1567fa4ee87"}, + {file = "SQLAlchemy-1.4.49-cp311-cp311-win_amd64.whl", hash = "sha256:171e04eeb5d1c0d96a544caf982621a1711d078dbc5c96f11d6469169bd003f1"}, + {file = "SQLAlchemy-1.4.49-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:36e58f8c4fe43984384e3fbe6341ac99b6b4e083de2fe838f0fdb91cebe9e9cb"}, + {file = "SQLAlchemy-1.4.49-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b31e67ff419013f99ad6f8fc73ee19ea31585e1e9fe773744c0f3ce58c039c30"}, + {file = "SQLAlchemy-1.4.49-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c14b29d9e1529f99efd550cd04dbb6db6ba5d690abb96d52de2bff4ed518bc95"}, + {file = "SQLAlchemy-1.4.49-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c40f3470e084d31247aea228aa1c39bbc0904c2b9ccbf5d3cfa2ea2dac06f26d"}, + {file = "SQLAlchemy-1.4.49-cp36-cp36m-win32.whl", hash = "sha256:706bfa02157b97c136547c406f263e4c6274a7b061b3eb9742915dd774bbc264"}, + {file = "SQLAlchemy-1.4.49-cp36-cp36m-win_amd64.whl", hash = "sha256:a7f7b5c07ae5c0cfd24c2db86071fb2a3d947da7bd487e359cc91e67ac1c6d2e"}, + {file = "SQLAlchemy-1.4.49-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:4afbbf5ef41ac18e02c8dc1f86c04b22b7a2125f2a030e25bbb4aff31abb224b"}, + {file = "SQLAlchemy-1.4.49-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24e300c0c2147484a002b175f4e1361f102e82c345bf263242f0449672a4bccf"}, + {file = "SQLAlchemy-1.4.49-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:201de072b818f8ad55c80d18d1a788729cccf9be6d9dc3b9d8613b053cd4836d"}, + {file = "SQLAlchemy-1.4.49-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7653ed6817c710d0c95558232aba799307d14ae084cc9b1f4c389157ec50df5c"}, + {file = "SQLAlchemy-1.4.49-cp37-cp37m-win32.whl", hash = "sha256:647e0b309cb4512b1f1b78471fdaf72921b6fa6e750b9f891e09c6e2f0e5326f"}, + {file = "SQLAlchemy-1.4.49-cp37-cp37m-win_amd64.whl", hash = "sha256:ab73ed1a05ff539afc4a7f8cf371764cdf79768ecb7d2ec691e3ff89abbc541e"}, + {file = "SQLAlchemy-1.4.49-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:37ce517c011560d68f1ffb28af65d7e06f873f191eb3a73af5671e9c3fada08a"}, + {file = "SQLAlchemy-1.4.49-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1878ce508edea4a879015ab5215546c444233881301e97ca16fe251e89f1c55"}, + {file = "SQLAlchemy-1.4.49-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0e8e608983e6f85d0852ca61f97e521b62e67969e6e640fe6c6b575d4db68557"}, + {file = "SQLAlchemy-1.4.49-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccf956da45290df6e809ea12c54c02ace7f8ff4d765d6d3dfb3655ee876ce58d"}, + {file = "SQLAlchemy-1.4.49-cp38-cp38-win32.whl", hash = "sha256:f167c8175ab908ce48bd6550679cc6ea20ae169379e73c7720a28f89e53aa532"}, + {file = "SQLAlchemy-1.4.49-cp38-cp38-win_amd64.whl", hash = "sha256:45806315aae81a0c202752558f0df52b42d11dd7ba0097bf71e253b4215f34f4"}, + {file = "SQLAlchemy-1.4.49-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:b6d0c4b15d65087738a6e22e0ff461b407533ff65a73b818089efc8eb2b3e1de"}, + {file = "SQLAlchemy-1.4.49-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a843e34abfd4c797018fd8d00ffffa99fd5184c421f190b6ca99def4087689bd"}, + {file = "SQLAlchemy-1.4.49-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1c890421651b45a681181301b3497e4d57c0d01dc001e10438a40e9a9c25ee77"}, + {file = "SQLAlchemy-1.4.49-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d26f280b8f0a8f497bc10573849ad6dc62e671d2468826e5c748d04ed9e670d5"}, + {file = "SQLAlchemy-1.4.49-cp39-cp39-win32.whl", hash = "sha256:ec2268de67f73b43320383947e74700e95c6770d0c68c4e615e9897e46296294"}, + {file = "SQLAlchemy-1.4.49-cp39-cp39-win_amd64.whl", hash = "sha256:bbdf16372859b8ed3f4d05f925a984771cd2abd18bd187042f24be4886c2a15f"}, + {file = "SQLAlchemy-1.4.49.tar.gz", hash = "sha256:06ff25cbae30c396c4b7737464f2a7fc37a67b7da409993b182b024cec80aed9"}, ] [package.dependencies] @@ -1890,14 +1910,14 @@ files = [ [[package]] name = "werkzeug" -version = "2.3.4" +version = "2.3.6" description = "The comprehensive WSGI web application library." category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "Werkzeug-2.3.4-py3-none-any.whl", hash = "sha256:48e5e61472fee0ddee27ebad085614ebedb7af41e88f687aaf881afb723a162f"}, - {file = "Werkzeug-2.3.4.tar.gz", hash = "sha256:1d5a58e0377d1fe39d061a5de4469e414e78ccb1e1e59c0f5ad6fa1c36c52b76"}, + {file = "Werkzeug-2.3.6-py3-none-any.whl", hash = "sha256:935539fa1413afbb9195b24880778422ed620c0fc09670945185cce4d91a8890"}, + {file = "Werkzeug-2.3.6.tar.gz", hash = "sha256:98c774df2f91b05550078891dee5f0eb0cb797a522c757a2452b9cee5b202330"}, ] [package.dependencies] From 81d29b4c66b284459a020b92c8db8de0f2c71bfc Mon Sep 17 00:00:00 2001 From: renovate Date: Sat, 8 Jul 2023 11:24:29 +0000 Subject: [PATCH 017/128] fix(deps): update dependency fastapi to ^0.100.0 --- poetry.lock | 16 +++++++--------- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/poetry.lock b/poetry.lock index dcdcf819..f3f19d11 100644 --- a/poetry.lock +++ b/poetry.lock @@ -560,25 +560,23 @@ lua = ["lupa (>=1.14,<2.0)"] [[package]] name = "fastapi" -version = "0.95.2" +version = "0.100.0" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "fastapi-0.95.2-py3-none-any.whl", hash = "sha256:d374dbc4ef2ad9b803899bd3360d34c534adc574546e25314ab72c0c4411749f"}, - {file = "fastapi-0.95.2.tar.gz", hash = "sha256:4d9d3e8c71c73f11874bcf5e33626258d143252e329a01002f767306c64fb982"}, + {file = "fastapi-0.100.0-py3-none-any.whl", hash = "sha256:271662daf986da8fa98dc2b7c7f61c4abdfdccfb4786d79ed8b2878f172c6d5f"}, + {file = "fastapi-0.100.0.tar.gz", hash = "sha256:acb5f941ea8215663283c10018323ba7ea737c571b67fc7e88e9469c7eb1d12e"}, ] [package.dependencies] -pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0" +pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<3.0.0" starlette = ">=0.27.0,<0.28.0" +typing-extensions = ">=4.5.0" [package.extras] -all = ["email-validator (>=1.1.1)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "python-multipart (>=0.0.5)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] -dev = ["pre-commit (>=2.17.0,<3.0.0)", "ruff (==0.0.138)", "uvicorn[standard] (>=0.12.0,<0.21.0)"] -doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pyyaml (>=5.3.1,<7.0.0)", "typer-cli (>=0.0.13,<0.0.14)", "typer[all] (>=0.6.1,<0.8.0)"] -test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==23.1.0)", "coverage[toml] (>=6.5.0,<8.0)", "databases[sqlite] (>=0.3.2,<0.7.0)", "email-validator (>=1.1.1,<2.0.0)", "flask (>=1.1.2,<3.0.0)", "httpx (>=0.23.0,<0.24.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.982)", "orjson (>=3.2.1,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "peewee (>=3.13.3,<4.0.0)", "pytest (>=7.1.3,<8.0.0)", "python-jose[cryptography] (>=3.3.0,<4.0.0)", "python-multipart (>=0.0.5,<0.0.7)", "pyyaml (>=5.3.1,<7.0.0)", "ruff (==0.0.138)", "sqlalchemy (>=1.3.18,<1.4.43)", "types-orjson (==3.6.2)", "types-ujson (==5.7.0.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)"] +all = ["email-validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.5)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] [[package]] name = "feedgen" @@ -1960,4 +1958,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "caf2a21e3bff699216e53a37697a7a544103fdea9f84a5d54ee94ded3e810973" +content-hash = "bbab458ee508b073ea3693caaa5d8704ff1a800ddecd816bf39a6561729777c0" diff --git a/pyproject.toml b/pyproject.toml index 69f04fab..34c5a135 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,7 +93,7 @@ pytest-xdist = "^3.2.1" filelock = "^3.12.0" posix-ipc = "^1.1.1" pyalpm = "^0.10.6" -fastapi = "^0.95.1" +fastapi = "^0.100.0" srcinfo = "^0.1.2" tomlkit = "^0.11.8" From 1f40f6c5a0a6c22a071e0729d5bdff60018b303c Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Sat, 8 Jul 2023 12:38:19 +0100 Subject: [PATCH 018/128] housekeep: set current maintainers Signed-off-by: Leonidas Spyropoulos --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 34c5a135..012658a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,8 @@ authors = [ "Kevin Morris " ] maintainers = [ - "Eli Schwartz " + "Leonidas Spyropoulos ", + "Mario Oenning " ] packages = [ { include = "aurweb" } From 4821fc131286bcea51ca0ea257d90b9ae20b85b0 Mon Sep 17 00:00:00 2001 From: moson Date: Sat, 8 Jul 2023 13:23:32 +0200 Subject: [PATCH 019/128] fix: show placeholder for deleted user in comments show "" in comment headers in case a user deleted their account. Signed-off-by: moson --- templates/partials/packages/comment.html | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/templates/partials/packages/comment.html b/templates/partials/packages/comment.html index faac0753..3573a914 100644 --- a/templates/partials/packages/comment.html +++ b/templates/partials/packages/comment.html @@ -1,5 +1,9 @@ {% set header_cls = "comment-header" %} {% set article_cls = "article-content" %} +{% set comment_by = comment.User.Username %} +{% if not comment.User %} + {% set comment_by = "<deleted-account>" %} +{% endif %} {% if comment.Deleter %} {% set header_cls = "%s %s" | format(header_cls, "comment-deleted") %} {% set article_cls = "%s %s" | format(article_cls, "comment-deleted") %} @@ -9,15 +13,15 @@ {% if not (request.user.HideDeletedComments and comment.DelTS) %}

{% set commented_at = comment.CommentTS | dt | as_timezone(timezone) %} - {% set view_account_info = 'View account information for %s' | tr | format(comment.User.Username) %} + {% set view_account_info = 'View account information for %s' | tr | format(comment_by) %} {{ "%s commented on %s" | tr | format( ('%s' | format( - comment.User.Username, + comment_by, view_account_info, - comment.User.Username - )) if request.user.is_authenticated() else - (comment.User.Username), + comment_by + )) if request.user.is_authenticated() and comment.User else + (comment_by), '%s' | format( comment.ID, datetime_display(comment.CommentTS) From 225ce23761938dcfbb02809ac2371697038c037b Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Sat, 8 Jul 2023 12:54:43 +0100 Subject: [PATCH 020/128] chore(release): prepare for 6.2.6 Signed-off-by: Leonidas Spyropoulos --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 012658a0..ddd4f638 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ combine_as_imports = true # [tool.poetry] name = "aurweb" -version = "v6.2.5" +version = "v6.2.6" license = "GPL-2.0-only" description = "Source code for the Arch User Repository's website" homepage = "https://aur.archlinux.org" From 5ccfa7c0fdc491df8556550092fac40fb0027284 Mon Sep 17 00:00:00 2001 From: moson Date: Sun, 9 Jul 2023 14:52:15 +0200 Subject: [PATCH 021/128] fix: same ssh key entered multiple times Users might accidentally past their ssh key multiple times when they try to register or edit their account. Convert our of list of keys to a set, removing any double keys. Signed-off-by: moson --- aurweb/util.py | 4 ++-- test/test_accounts_routes.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/aurweb/util.py b/aurweb/util.py index 7050b482..3410e4d8 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -192,9 +192,9 @@ def parse_ssh_key(string: str) -> Tuple[str, str]: return prefix, key -def parse_ssh_keys(string: str) -> list[Tuple[str, str]]: +def parse_ssh_keys(string: str) -> set[Tuple[str, str]]: """Parse a list of SSH public keys.""" - return [parse_ssh_key(e) for e in string.strip().splitlines(True) if e.strip()] + return set([parse_ssh_key(e) for e in string.strip().splitlines(True) if e.strip()]) def shell_exec(cmdline: str, cwd: str) -> Tuple[int, str, str]: diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index d3ddb174..c9d77c1f 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -644,6 +644,18 @@ def test_post_register_with_ssh_pubkey(client: TestClient): assert response.status_code == int(HTTPStatus.OK) + # Let's create another user accidentally pasting their key twice + with db.begin(): + db.query(SSHPubKey).delete() + + pk_double = pk + "\n" + pk + with client as request: + response = post_register( + request, U="doubleKey", E="doubleKey@email.org", PK=pk_double + ) + + assert response.status_code == int(HTTPStatus.OK) + def test_get_account_edit_tu_as_tu(client: TestClient, tu_user: User): """Test edit get route of another TU as a TU.""" @@ -1082,6 +1094,19 @@ def test_post_account_edit_ssh_pub_key(client: TestClient, user: User): assert response.status_code == int(HTTPStatus.OK) + # Accidentally enter the same key twice + pk = make_ssh_pubkey() + post_data["PK"] = pk + "\n" + pk + + with client as request: + request.cookies = {"AURSID": sid} + response = request.post( + "/account/test/edit", + data=post_data, + ) + + assert response.status_code == int(HTTPStatus.OK) + def test_post_account_edit_missing_ssh_pubkey(client: TestClient, user: User): request = Request() From c0bbe21d8183ed2d07eadcb5e0fd27526c70b78f Mon Sep 17 00:00:00 2001 From: moson Date: Sun, 9 Jul 2023 16:13:02 +0200 Subject: [PATCH 022/128] fix(test): correct test for ssh-key parsing Our set of keys returned by "util.parse_ssh_keys" is unordered so we have to adapt our test to not rely on a specific order for multiple keys. Fixes: 5ccfa7c0fdc4 ("fix: same ssh key entered multiple times") Signed-off-by: moson --- test/test_util.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 042b9ad9..1c3b51af 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -142,11 +142,8 @@ def assert_multiple_keys(pks): keys = util.parse_ssh_keys(pks) assert len(keys) == 2 pfx1, key1, pfx2, key2 = pks.split() - k1, k2 = keys - assert pfx1 == k1[0] - assert key1 == k1[1] - assert pfx2 == k2[0] - assert key2 == k2[1] + assert (pfx1, key1) in keys + assert (pfx2, key2) in keys def test_hash_query(): From fa1212f2dee216bd02b321e3747c22fc854b31a5 Mon Sep 17 00:00:00 2001 From: moson Date: Mon, 10 Jul 2023 18:02:20 +0200 Subject: [PATCH 023/128] fix: translations not containing string formatting In some translations we might be missing replacement placeholders (%). This turns out to be problematic when calling the format function. Wrap the jinja2 format function and just return the string unformatted when % is missing. Fixes: #341 Signed-off-by: moson --- aurweb/filters.py | 15 +++++++++++++++ test/test_filters.py | 17 +++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/aurweb/filters.py b/aurweb/filters.py index 893f21af..38322cdf 100644 --- a/aurweb/filters.py +++ b/aurweb/filters.py @@ -8,6 +8,7 @@ from zoneinfo import ZoneInfo import fastapi import paginate from jinja2 import pass_context +from jinja2.filters import do_format import aurweb.models from aurweb import config, l10n @@ -164,3 +165,17 @@ def date_display(context: dict[str, Any], dt: Union[int, datetime]) -> str: @pass_context def datetime_display(context: dict[str, Any], dt: Union[int, datetime]) -> str: return date_strftime(context, dt, "%Y-%m-%d %H:%M (%Z)") + + +@register_filter("format") +def safe_format(value: str, *args: Any, **kwargs: Any) -> str: + """Wrapper for jinja2 format function to perform additional checks.""" + + # If we don't have anything to be formatted, just return the value. + # We have some translations that do not contain placeholders for replacement. + # In these cases the jinja2 function is throwing an error: + # "TypeError: not all arguments converted during string formatting" + if "%" not in value: + return value + + return do_format(value, *args, **kwargs) diff --git a/test/test_filters.py b/test/test_filters.py index e74ddb87..b56b80ab 100644 --- a/test/test_filters.py +++ b/test/test_filters.py @@ -1,6 +1,8 @@ from datetime import datetime from zoneinfo import ZoneInfo +import pytest + from aurweb import filters, time @@ -34,3 +36,18 @@ def test_to_qs(): query = {"a": "b", "c": [1, 2, 3]} qs = filters.to_qs(query) assert qs == "a=b&c=1&c=2&c=3" + + +@pytest.mark.parametrize( + "value, args, expected", + [ + ("", (), ""), + ("a", (), "a"), + ("a", (1,), "a"), + ("%s", ("a",), "a"), + ("%s", ("ab",), "ab"), + ("%s%d", ("a", 1), "a1"), + ], +) +def test_safe_format(value: str, args: tuple, expected: str): + assert filters.safe_format(value, *args) == expected From 27819b4465cd6cde7ef86caca812b1dd6f285880 Mon Sep 17 00:00:00 2001 From: moson Date: Thu, 13 Jul 2023 18:27:02 +0200 Subject: [PATCH 024/128] fix: /rss lazy load issue & perf improvements Some fixes for the /rss endpoints * Load all data in one go: Previously data was lazy loaded thus it made several sub-queries per package (> 200 queries for composing the rss data for a single request). Now we are performing a single SQL query. (request time improvement: 550ms -> 130ms) This also fixes aurweb-errors#510 and alike * Remove some "dead code": The fields "source, author, link" were never included in the rss output (wrong or insufficient data passed to the different entry.xyz functions) Nobody seems to be missing them anyways, so let's remove em. * Remove "Last-Modified" header: Obsolete since nginx can/will only handle "If-Modified-Since" requests in it's current configuration. All requests are passed to fastapi anyways. Signed-off-by: moson --- aurweb/routers/rss.py | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/aurweb/routers/rss.py b/aurweb/routers/rss.py index ee85b738..727d2b6a 100644 --- a/aurweb/routers/rss.py +++ b/aurweb/routers/rss.py @@ -1,5 +1,3 @@ -from datetime import datetime - from fastapi import APIRouter, Request from fastapi.responses import Response from feedgen.feed import FeedGenerator @@ -10,12 +8,11 @@ from aurweb.models import Package, PackageBase router = APIRouter() -def make_rss_feed(request: Request, packages: list, date_attr: str): +def make_rss_feed(request: Request, packages: list): """Create an RSS Feed string for some packages. :param request: A FastAPI request :param packages: A list of packages to add to the RSS feed - :param date_attr: The date attribute (DB column) to use :return: RSS Feed string """ @@ -36,18 +33,11 @@ def make_rss_feed(request: Request, packages: list, date_attr: str): entry = feed.add_entry(order="append") entry.title(pkg.Name) entry.link(href=f"{base}/packages/{pkg.Name}", rel="alternate") - entry.link(href=f"{base}/rss", rel="self", type="application/rss+xml") entry.description(pkg.Description or str()) - - attr = getattr(pkg.PackageBase, date_attr) - dt = filters.timestamp_to_datetime(attr) + dt = filters.timestamp_to_datetime(pkg.Timestamp) dt = filters.as_timezone(dt, request.user.Timezone) entry.pubDate(dt.strftime("%Y-%m-%d %H:%M:%S%z")) - - entry.source(f"{base}") - if pkg.PackageBase.Maintainer: - entry.author(author={"name": pkg.PackageBase.Maintainer.Username}) - entry.guid(f"{pkg.Name} - {attr}") + entry.guid(f"{pkg.Name}-{pkg.Timestamp}") return feed.rss_str() @@ -59,15 +49,15 @@ async def rss(request: Request): .join(PackageBase) .order_by(PackageBase.SubmittedTS.desc()) .limit(100) + .with_entities( + Package.Name, + Package.Description, + PackageBase.SubmittedTS.label("Timestamp"), + ) ) - feed = make_rss_feed(request, packages, "SubmittedTS") + feed = make_rss_feed(request, packages) response = Response(feed, media_type="application/rss+xml") - package = packages.first() - if package: - dt = datetime.utcfromtimestamp(package.PackageBase.SubmittedTS) - modified = dt.strftime("%a, %d %m %Y %H:%M:%S GMT") - response.headers["Last-Modified"] = modified return response @@ -79,14 +69,14 @@ async def rss_modified(request: Request): .join(PackageBase) .order_by(PackageBase.ModifiedTS.desc()) .limit(100) + .with_entities( + Package.Name, + Package.Description, + PackageBase.ModifiedTS.label("Timestamp"), + ) ) - feed = make_rss_feed(request, packages, "ModifiedTS") + feed = make_rss_feed(request, packages) response = Response(feed, media_type="application/rss+xml") - package = packages.first() - if package: - dt = datetime.utcfromtimestamp(package.PackageBase.ModifiedTS) - modified = dt.strftime("%a, %d %m %Y %H:%M:%S GMT") - response.headers["Last-Modified"] = modified return response From 862221f5ce244323208f9034b5c14bb0852d2cf8 Mon Sep 17 00:00:00 2001 From: renovate Date: Sat, 15 Jul 2023 20:27:12 +0000 Subject: [PATCH 025/128] fix(deps): update all non-major dependencies --- poetry.lock | 52 ++++++++++++++++++++------------------------------ pyproject.toml | 2 +- 2 files changed, 22 insertions(+), 32 deletions(-) diff --git a/poetry.lock b/poetry.lock index f3f19d11..368371db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -792,27 +792,27 @@ socks = ["socksio (>=1.0.0,<2.0.0)"] [[package]] name = "hypercorn" -version = "0.14.3" +version = "0.14.4" description = "A ASGI Server based on Hyper libraries and inspired by Gunicorn" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "Hypercorn-0.14.3-py3-none-any.whl", hash = "sha256:7c491d5184f28ee960dcdc14ab45d14633ca79d72ddd13cf4fcb4cb854d679ab"}, - {file = "Hypercorn-0.14.3.tar.gz", hash = "sha256:4a87a0b7bbe9dc75fab06dbe4b301b9b90416e9866c23a377df21a969d6ab8dd"}, + {file = "hypercorn-0.14.4-py3-none-any.whl", hash = "sha256:f956200dbf8677684e6e976219ffa6691d6cf795281184b41dbb0b135ab37b8d"}, + {file = "hypercorn-0.14.4.tar.gz", hash = "sha256:3fa504efc46a271640023c9b88c3184fd64993f47a282e8ae1a13ccb285c2f67"}, ] [package.dependencies] h11 = "*" h2 = ">=3.1.0" priority = "*" -toml = "*" +tomli = {version = "*", markers = "python_version < \"3.11\""} wsproto = ">=0.14.0" [package.extras] docs = ["pydata_sphinx_theme"] h3 = ["aioquic (>=0.9.0,<1.0)"] -trio = ["trio (>=0.11.0)"] +trio = ["exceptiongroup (>=1.1.0)", "trio (>=0.22.0)"] uvloop = ["uvloop"] [[package]] @@ -912,6 +912,8 @@ files = [ {file = "lxml-4.9.3-cp27-cp27m-macosx_11_0_x86_64.whl", hash = "sha256:b0a545b46b526d418eb91754565ba5b63b1c0b12f9bd2f808c852d9b4b2f9b5c"}, {file = "lxml-4.9.3-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:075b731ddd9e7f68ad24c635374211376aa05a281673ede86cbe1d1b3455279d"}, {file = "lxml-4.9.3-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1e224d5755dba2f4a9498e150c43792392ac9b5380aa1b845f98a1618c94eeef"}, + {file = "lxml-4.9.3-cp27-cp27m-win32.whl", hash = "sha256:2c74524e179f2ad6d2a4f7caf70e2d96639c0954c943ad601a9e146c76408ed7"}, + {file = "lxml-4.9.3-cp27-cp27m-win_amd64.whl", hash = "sha256:4f1026bc732b6a7f96369f7bfe1a4f2290fb34dce00d8644bc3036fb351a4ca1"}, {file = "lxml-4.9.3-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0781a98ff5e6586926293e59480b64ddd46282953203c76ae15dbbbf302e8bb"}, {file = "lxml-4.9.3-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:cef2502e7e8a96fe5ad686d60b49e1ab03e438bd9123987994528febd569868e"}, {file = "lxml-4.9.3-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:b86164d2cff4d3aaa1f04a14685cbc072efd0b4f99ca5708b2ad1b9b5988a991"}, @@ -1274,14 +1276,14 @@ twisted = ["twisted"] [[package]] name = "prometheus-fastapi-instrumentator" -version = "6.0.0" +version = "6.1.0" description = "Instrument your FastAPI with Prometheus metrics." category = "main" optional = false python-versions = ">=3.7.0,<4.0.0" files = [ - {file = "prometheus_fastapi_instrumentator-6.0.0-py3-none-any.whl", hash = "sha256:6f66a951a4801667f7311d161f3aebfe0cd202391d0f067fbbe169792e2d987b"}, - {file = "prometheus_fastapi_instrumentator-6.0.0.tar.gz", hash = "sha256:f1ddd0b8ead75e71d055bdf4cb7e995ec6a6ca63543245e7bbc5ca9b14c45191"}, + {file = "prometheus_fastapi_instrumentator-6.1.0-py3-none-any.whl", hash = "sha256:2279ac1cf5b9566a4c3a07f78c9c5ee19648ed90976ab87d73d672abc1bfa017"}, + {file = "prometheus_fastapi_instrumentator-6.1.0.tar.gz", hash = "sha256:1820d7a90389ce100f7d1285495ead388818ae0882e761c1f3e6e62a410bdf13"}, ] [package.dependencies] @@ -1456,14 +1458,14 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no [[package]] name = "pytest-asyncio" -version = "0.21.0" +version = "0.21.1" description = "Pytest support for asyncio" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-asyncio-0.21.0.tar.gz", hash = "sha256:2b38a496aef56f56b0e87557ec313e11e1ab9276fc3863f6a7be0f1d0e415e1b"}, - {file = "pytest_asyncio-0.21.0-py3-none-any.whl", hash = "sha256:f2b3366b7cd501a4056858bd39349d5af19742aed2d81660b7998b6341c7eb9c"}, + {file = "pytest-asyncio-0.21.1.tar.gz", hash = "sha256:40a7eae6dded22c7b604986855ea48400ab15b069ae38116e8c01238e9eeb64d"}, + {file = "pytest_asyncio-0.21.1-py3-none-any.whl", hash = "sha256:8666c1c8ac02631d7c51ba282e0c69a8a452b211ffedf2599099845da5c5c37b"}, ] [package.dependencies] @@ -1494,14 +1496,14 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale [[package]] name = "pytest-tap" -version = "3.3" +version = "3.4" description = "Test Anything Protocol (TAP) reporting plugin for pytest" category = "dev" optional = false python-versions = "*" files = [ - {file = "pytest-tap-3.3.tar.gz", hash = "sha256:5f0919a147cf0396b2f10d64d365a0bf8062e06543e93c675c9d37f5605e983c"}, - {file = "pytest_tap-3.3-py3-none-any.whl", hash = "sha256:4fbbc0e090c2e94f6199bee4e4f68ab3c5e176b37a72a589ad84e0f72a2fce55"}, + {file = "pytest-tap-3.4.tar.gz", hash = "sha256:a7c2a4a3e8b4bf18522e46d74208f8579a191dd972c59182104ad9a4967318fb"}, + {file = "pytest_tap-3.4-py3-none-any.whl", hash = "sha256:d97a2115c94415086f6faec395d243b3c18ea846ce1c1653a4b2588082be35d8"}, ] [package.dependencies] @@ -1774,18 +1776,6 @@ files = [ [package.extras] yaml = ["PyYAML (>=5.1)", "more-itertools"] -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] - [[package]] name = "tomli" version = "2.0.1" @@ -1842,14 +1832,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "uvicorn" -version = "0.22.0" +version = "0.23.0" description = "The lightning-fast ASGI server." category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "uvicorn-0.22.0-py3-none-any.whl", hash = "sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996"}, - {file = "uvicorn-0.22.0.tar.gz", hash = "sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8"}, + {file = "uvicorn-0.23.0-py3-none-any.whl", hash = "sha256:479599b2c0bb1b9b394c6d43901a1eb0c1ec72c7d237b5bafea23c5b2d4cdf10"}, + {file = "uvicorn-0.23.0.tar.gz", hash = "sha256:d38ab90c0e2c6fe3a054cddeb962cfd5d0e0e6608eaaff4a01d5c36a67f3168c"}, ] [package.dependencies] @@ -1958,4 +1948,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "bbab458ee508b073ea3693caaa5d8704ff1a800ddecd816bf39a6561729777c0" +content-hash = "b67f1b1599794a6890b0a31b2b127880d75c84beeeae3df4ecb3ae92296948da" diff --git a/pyproject.toml b/pyproject.toml index ddd4f638..e98e887f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,7 @@ Werkzeug = "^2.3.3" SQLAlchemy = "^1.4.48" # ASGI -uvicorn = "^0.22.0" +uvicorn = "^0.23.0" gunicorn = "^20.1.0" Hypercorn = "^0.14.3" prometheus-fastapi-instrumentator = "^6.0.0" From 5729d6787f43574e70f6b87543065838d475f880 Mon Sep 17 00:00:00 2001 From: moson Date: Sun, 16 Jul 2023 13:27:02 +0200 Subject: [PATCH 026/128] fix: git links in comments for multiple OIDs The chance of finding multiple object IDs when performing lookups with a shortened SHA1 hash (7 digits) seems to be quite high. In those cases pygit2 will throw an error. Let's catch those exceptions and gracefully handle them. Fixes: aurweb-errors#496 (and alike) Signed-off-by: moson --- aurweb/scripts/rendercomment.py | 9 ++++++-- test/test_rendercomment.py | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index e640c1d4..31f3fdd4 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -72,8 +72,13 @@ class GitCommitsInlineProcessor(markdown.inlinepatterns.InlineProcessor): def handleMatch(self, m, data): oid = m.group(1) - if oid not in self._repo: - # Unknown OID; preserve the orginal text. + # Lookup might raise ValueError in case multiple object ID's were found + try: + if oid not in self._repo: + # Unknown OID; preserve the orginal text. + return None, None, None + except ValueError: + # Multiple OID's found; preserve the orginal text. return None, None, None el = Element("a") diff --git a/test/test_rendercomment.py b/test/test_rendercomment.py index 59eb7191..f9edb45b 100644 --- a/test/test_rendercomment.py +++ b/test/test_rendercomment.py @@ -1,5 +1,7 @@ +import os from unittest import mock +import pygit2 import pytest from aurweb import aur_logging, config, db, time @@ -166,6 +168,43 @@ http://example.com/{commit_hash}\ assert comment.RenderedComment == expected +def test_git_commit_link_multiple_oids( + git: GitRepository, user: User, package: Package +): + # Make sure we get reproducible hashes by hardcoding the dates + date = "Sun, 16 Jul 2023 06:06:06 +0200" + os.environ["GIT_COMMITTER_DATE"] = date + os.environ["GIT_AUTHOR_DATE"] = date + + # Package names that cause two object IDs starting with "09a3468" + pkgnames = [ + "bfa3e330-23c5-11ee-9b6c-9c2dcdf2810d", + "54c6a420-23c6-11ee-9b6c-9c2dcdf2810d", + ] + + # Create our commits + for pkgname in pkgnames: + with db.begin(): + package = db.create( + Package, PackageBase=package.PackageBase, Name=pkgname, Version="1.0" + ) + git.commit(package, pkgname) + + repo_path = config.get("serve", "repo-path") + repo = pygit2.Repository(repo_path) + + # Make sure we get an error when we search the git repo for "09a3468" + with pytest.raises(ValueError) as oid_error: + assert "09a3468" in repo + assert "ambiguous OID prefix" in oid_error + + # Create a comment, referencing "09a3468" + comment = create_comment(user, package.PackageBase, "Commit 09a3468 is nasty!") + + # Make sure our comment does not contain a link. + assert comment.RenderedComment == "

Commit 09a3468 is nasty!

" + + def test_flyspray_issue_link(user: User, pkgbase: PackageBase): text = """\ FS#1234567. From bc03d8b8f20ac0a1e6a2b03069632c8a064332f0 Mon Sep 17 00:00:00 2001 From: moson Date: Thu, 20 Jul 2023 18:21:05 +0200 Subject: [PATCH 027/128] fix: Fix middleware checking for accepted terms The current query is a bit mixed up. The intention was to return the number of unaccepted records. Now it does also count all records that were accepted by some other user though. Let's check the total number of terms vs. the number of accepted records (by our user) instead. Signed-off-by: moson --- aurweb/asgi.py | 19 ++++++++----------- test/test_accounts_routes.py | 6 ++++++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/aurweb/asgi.py b/aurweb/asgi.py index eb02413b..1be77ff9 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -14,7 +14,7 @@ from fastapi.responses import RedirectResponse from fastapi.staticfiles import StaticFiles from jinja2 import TemplateNotFound from prometheus_client import multiprocess -from sqlalchemy import and_, or_ +from sqlalchemy import and_ from starlette.exceptions import HTTPException as StarletteHTTPException from starlette.middleware.authentication import AuthenticationMiddleware from starlette.middleware.sessions import SessionMiddleware @@ -277,21 +277,18 @@ async def check_terms_of_service(request: Request, call_next: typing.Callable): """This middleware function redirects authenticated users if they have any outstanding Terms to agree to.""" if request.user.is_authenticated() and request.url.path != "/tos": - unaccepted = ( + accepted = ( query(Term) .join(AcceptedTerm) .filter( - or_( - AcceptedTerm.UsersID != request.user.ID, - and_( - AcceptedTerm.UsersID == request.user.ID, - AcceptedTerm.TermsID == Term.ID, - AcceptedTerm.Revision < Term.Revision, - ), - ) + and_( + AcceptedTerm.UsersID == request.user.ID, + AcceptedTerm.TermsID == Term.ID, + AcceptedTerm.Revision >= Term.Revision, + ), ) ) - if query(Term).count() > unaccepted.count(): + if query(Term).count() - accepted.count() > 0: return RedirectResponse("/tos", status_code=int(http.HTTPStatus.SEE_OTHER)) return await util.error_or_result(call_next, request) diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index c9d77c1f..3c481d0a 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -1915,6 +1915,12 @@ def test_get_terms_of_service(client: TestClient, user: User): # We accepted the term, there's nothing left to accept. assert response.status_code == int(HTTPStatus.SEE_OTHER) + # Make sure we don't get redirected to /tos when browsing "Home" + with client as request: + request.cookies = cookies + response = request.get("/") + assert response.status_code == int(HTTPStatus.OK) + # Bump the term's revision. with db.begin(): term.Revision = 2 From 347c2ce721b5782ff0324eb80fdc5613b4ebe478 Mon Sep 17 00:00:00 2001 From: moson Date: Sat, 22 Jul 2023 10:43:19 +0200 Subject: [PATCH 028/128] change: Change order of commit validation routine We currently validate all commits going from latest -> oldest. It would be nicer to go oldest -> latest so that, in case of errors, we would indicate which commit "introduced" the problem. Signed-off-by: moson --- aurweb/git/update.py | 2 +- test/t1300-git-update.t | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/aurweb/git/update.py b/aurweb/git/update.py index cd7813e0..4c4fff0f 100755 --- a/aurweb/git/update.py +++ b/aurweb/git/update.py @@ -356,7 +356,7 @@ def main(): # noqa: C901 die("denying non-fast-forward (you should pull first)") # Prepare the walker that validates new commits. - walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL) + walker = repo.walk(sha1_new, pygit2.GIT_SORT_REVERSE) if sha1_old != "0" * 40: walker.hide(sha1_old) diff --git a/test/t1300-git-update.t b/test/t1300-git-update.t index 4fdb487b..0fb2da17 100755 --- a/test/t1300-git-update.t +++ b/test/t1300-git-update.t @@ -312,11 +312,16 @@ test_expect_success 'Pushing a tree with a large blob.' ' printf "%256001s" x >aur.git/file && git -C aur.git add file && git -C aur.git commit -q -m "Add large blob" && + first_error=$(git -C aur.git rev-parse HEAD) && + touch aur.git/another.file && + git -C aur.git add another.file && + git -C aur.git commit -q -m "Add another commit" && new=$(git -C aur.git rev-parse HEAD) && test_must_fail \ env AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \ cover "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 && - grep -q "^error: maximum blob size (250.00KiB) exceeded$" actual + grep -q "^error: maximum blob size (250.00KiB) exceeded$" actual && + grep -q "^error: $first_error:$" actual ' test_expect_success 'Pushing .SRCINFO with a non-matching package base.' ' From 44c158b8c2667ace8e44d2cac3b7aed4efcc1464 Mon Sep 17 00:00:00 2001 From: moson Date: Sat, 22 Jul 2023 16:31:50 +0200 Subject: [PATCH 029/128] feat: Implement statistics class & additional metrics The new module/class helps us constructing queries and count records to expose various statistics on the homepage. We also utilize for some new prometheus metrics (package and user gauges). Record counts are being cached with Redis. Signed-off-by: moson --- aurweb/cache.py | 11 ++-- aurweb/prometheus.py | 18 ++++++- aurweb/routers/html.py | 72 ++++---------------------- aurweb/routers/packages.py | 6 +-- aurweb/statistics.py | 102 +++++++++++++++++++++++++++++++++++++ test/test_cache.py | 18 +++---- test/test_metrics.py | 5 +- 7 files changed, 143 insertions(+), 89 deletions(-) create mode 100644 aurweb/statistics.py diff --git a/aurweb/cache.py b/aurweb/cache.py index fe1e5f1d..bb374e57 100644 --- a/aurweb/cache.py +++ b/aurweb/cache.py @@ -1,20 +1,15 @@ import pickle -from prometheus_client import Counter from sqlalchemy import orm from aurweb import config from aurweb.aur_redis import redis_connection +from aurweb.prometheus import SEARCH_REQUESTS _redis = redis_connection() -# Prometheus metrics -SEARCH_REQUESTS = Counter( - "search_requests", "Number of search requests by cache hit/miss", ["cache"] -) - -async def db_count_cache(key: str, query: orm.Query, expire: int = None) -> int: +def db_count_cache(key: str, query: orm.Query, expire: int = None) -> int: """Store and retrieve a query.count() via redis cache. :param key: Redis key @@ -30,7 +25,7 @@ async def db_count_cache(key: str, query: orm.Query, expire: int = None) -> int: return int(result) -async def db_query_cache(key: str, query: orm.Query, expire: int = None) -> list: +def db_query_cache(key: str, query: orm.Query, expire: int = None) -> list: """Store and retrieve query results via redis cache. :param key: Redis key diff --git a/aurweb/prometheus.py b/aurweb/prometheus.py index b8b7984f..d3455551 100644 --- a/aurweb/prometheus.py +++ b/aurweb/prometheus.py @@ -1,6 +1,6 @@ from typing import Any, Callable, Optional -from prometheus_client import Counter +from prometheus_client import Counter, Gauge from prometheus_fastapi_instrumentator import Instrumentator from prometheus_fastapi_instrumentator.metrics import Info from starlette.routing import Match, Route @@ -11,10 +11,26 @@ logger = aur_logging.get_logger(__name__) _instrumentator = Instrumentator() +# Custom metrics +SEARCH_REQUESTS = Counter( + "aur_search_requests", "Number of search requests by cache hit/miss", ["cache"] +) +USERS = Gauge( + "aur_users", "Number of AUR users by type", ["type"], multiprocess_mode="livemax" +) +PACKAGES = Gauge( + "aur_packages", + "Number of AUR packages by state", + ["state"], + multiprocess_mode="livemax", +) + + def instrumentator(): return _instrumentator +# FastAPI metrics # Taken from https://github.com/stephenhillier/starlette_exporter # Their license is included in LICENSES/starlette_exporter. # The code has been modified to remove child route checks diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index fc9f3519..c3bcee49 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -17,11 +17,10 @@ from sqlalchemy import case, or_ import aurweb.config import aurweb.models.package_request from aurweb import aur_logging, cookies, db, models, time, util -from aurweb.cache import db_count_cache from aurweb.exceptions import handle_form_exceptions -from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID from aurweb.models.package_request import PENDING_ID from aurweb.packages.util import query_notified, query_voted, updated_packages +from aurweb.statistics import Statistics, update_prometheus_metrics from aurweb.templates import make_context, render_template logger = aur_logging.get_logger(__name__) @@ -87,68 +86,12 @@ async def index(request: Request): context = make_context(request, "Home") context["ssh_fingerprints"] = util.get_ssh_fingerprints() - bases = db.query(models.PackageBase) - cache_expire = aurweb.config.getint("cache", "expiry_time") + # Package statistics. - context["package_count"] = await db_count_cache( - "package_count", bases, expire=cache_expire - ) - - query = bases.filter(models.PackageBase.MaintainerUID.is_(None)) - context["orphan_count"] = await db_count_cache( - "orphan_count", query, expire=cache_expire - ) - - query = db.query(models.User) - context["user_count"] = await db_count_cache( - "user_count", query, expire=cache_expire - ) - - query = query.filter( - or_( - models.User.AccountTypeID == TRUSTED_USER_ID, - models.User.AccountTypeID == TRUSTED_USER_AND_DEV_ID, - ) - ) - context["trusted_user_count"] = await db_count_cache( - "trusted_user_count", query, expire=cache_expire - ) - - # Current timestamp. - now = time.utcnow() - - seven_days = 86400 * 7 # Seven days worth of seconds. - seven_days_ago = now - seven_days - - one_hour = 3600 - updated = bases.filter( - models.PackageBase.ModifiedTS - models.PackageBase.SubmittedTS >= one_hour - ) - - query = bases.filter(models.PackageBase.SubmittedTS >= seven_days_ago) - context["seven_days_old_added"] = await db_count_cache( - "seven_days_old_added", query, expire=cache_expire - ) - - query = updated.filter(models.PackageBase.ModifiedTS >= seven_days_ago) - context["seven_days_old_updated"] = await db_count_cache( - "seven_days_old_updated", query, expire=cache_expire - ) - - year = seven_days * 52 # Fifty two weeks worth: one year. - year_ago = now - year - query = updated.filter(models.PackageBase.ModifiedTS >= year_ago) - context["year_old_updated"] = await db_count_cache( - "year_old_updated", query, expire=cache_expire - ) - - query = bases.filter( - models.PackageBase.ModifiedTS - models.PackageBase.SubmittedTS < 3600 - ) - context["never_updated"] = await db_count_cache( - "never_updated", query, expire=cache_expire - ) + stats = Statistics(cache_expire) + for counter in stats.HOMEPAGE_COUNTERS: + context[counter] = stats.get_count(counter) # Get the 15 most recently updated packages. context["package_updates"] = updated_packages(15, cache_expire) @@ -193,7 +136,7 @@ async def index(request: Request): ) archive_time = aurweb.config.getint("options", "request_archive_time") - start = now - archive_time + start = time.utcnow() - archive_time # Package requests created by request.user. context["package_requests"] = ( @@ -269,6 +212,9 @@ async def metrics(request: Request): status_code=HTTPStatus.SERVICE_UNAVAILABLE, ) + # update prometheus gauges for packages and users + update_prometheus_metrics() + registry = CollectorRegistry() multiprocess.MultiProcessCollector(registry) data = generate_latest(registry) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 779efb4b..f1b2a138 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -91,9 +91,7 @@ async def packages_get( # increase the amount of time required to collect a count. # we use redis for caching the results of the query cache_expire = config.getint("cache", "expiry_time") - num_packages = await db_count_cache( - hash_query(search.query), search.query, cache_expire - ) + num_packages = db_count_cache(hash_query(search.query), search.query, cache_expire) # Apply user-specified sort column and ordering. search.sort_by(sort_by, sort_order) @@ -118,7 +116,7 @@ async def packages_get( results = results.limit(per_page).offset(offset) # we use redis for caching the results of the query - packages = await db_query_cache(hash_query(results), results, cache_expire) + packages = db_query_cache(hash_query(results), results, cache_expire) context["packages"] = packages context["packages_count"] = num_packages diff --git a/aurweb/statistics.py b/aurweb/statistics.py new file mode 100644 index 00000000..934caa37 --- /dev/null +++ b/aurweb/statistics.py @@ -0,0 +1,102 @@ +from aurweb import config, db, time +from aurweb.cache import db_count_cache +from aurweb.models import PackageBase, User +from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID, USER_ID +from aurweb.prometheus import PACKAGES, USERS + + +class Statistics: + HOMEPAGE_COUNTERS = [ + "package_count", + "orphan_count", + "seven_days_old_added", + "seven_days_old_updated", + "year_old_updated", + "never_updated", + "user_count", + "trusted_user_count", + ] + PROMETHEUS_USER_COUNTERS = [ + ("trusted_user_count", "tu"), + ("regular_user_count", "user"), + ] + PROMETHEUS_PACKAGE_COUNTERS = [ + ("orphan_count", "orphan"), + ("never_updated", "not_updated"), + ("updated_packages", "updated"), + ] + + seven_days = 86400 * 7 + one_hour = 3600 + year = seven_days * 52 + + def __init__(self, cache_expire: int = None) -> "Statistics": + self.expiry_time = cache_expire + self.now = time.utcnow() + self.seven_days_ago = self.now - self.seven_days + self.year_ago = self.now - self.year + self.user_query = db.query(User) + self.bases_query = db.query(PackageBase) + self.updated_query = db.query(PackageBase).filter( + PackageBase.ModifiedTS - PackageBase.SubmittedTS >= self.one_hour + ) + + def get_count(self, counter: str) -> int: + query = None + match counter: + case "package_count": + query = self.bases_query + case "orphan_count": + query = self.bases_query.filter(PackageBase.MaintainerUID.is_(None)) + case "seven_days_old_added": + query = self.bases_query.filter( + PackageBase.SubmittedTS >= self.seven_days_ago + ) + case "seven_days_old_updated": + query = self.updated_query.filter( + PackageBase.ModifiedTS >= self.seven_days_ago + ) + case "year_old_updated": + query = self.updated_query.filter( + PackageBase.ModifiedTS >= self.year_ago + ) + case "never_updated": + query = self.bases_query.filter( + PackageBase.ModifiedTS - PackageBase.SubmittedTS < self.one_hour + ) + case "updated_packages": + query = self.bases_query.filter( + PackageBase.ModifiedTS - PackageBase.SubmittedTS > self.one_hour, + ~PackageBase.MaintainerUID.is_(None), + ) + case "user_count": + query = self.user_query + case "trusted_user_count": + query = self.user_query.filter( + User.AccountTypeID.in_( + ( + TRUSTED_USER_ID, + TRUSTED_USER_AND_DEV_ID, + ) + ) + ) + case "regular_user_count": + query = self.user_query.filter(User.AccountTypeID == USER_ID) + case _: + return -1 + + return db_count_cache(counter, query, expire=self.expiry_time) + + +def update_prometheus_metrics(): + cache_expire = config.getint("cache", "expiry_time") + stats = Statistics(cache_expire) + # Users gauge + for counter, utype in stats.PROMETHEUS_USER_COUNTERS: + count = stats.get_count(counter) + USERS.labels(utype).set(count) + + # Packages gauge + for counter, state in stats.PROMETHEUS_PACKAGE_COUNTERS: + count = stats.get_count(counter) + PACKAGES.labels(state).set(count) diff --git a/test/test_cache.py b/test/test_cache.py index e19fa6a2..a599ab32 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -31,15 +31,14 @@ def clear_fakeredis_cache(): cache._redis.flushall() -@pytest.mark.asyncio -async def test_db_count_cache(user): +def test_db_count_cache(user): query = db.query(User) # We have no cached value yet. assert cache._redis.get("key1") is None # Add to cache - assert await cache.db_count_cache("key1", query) == query.count() + assert cache.db_count_cache("key1", query) == query.count() # It's cached now. assert cache._redis.get("key1") is not None @@ -48,35 +47,34 @@ async def test_db_count_cache(user): assert cache._redis.ttl("key1") == -1 # Cache a query with an expire. - value = await cache.db_count_cache("key2", query, 100) + value = cache.db_count_cache("key2", query, 100) assert value == query.count() assert cache._redis.ttl("key2") == 100 -@pytest.mark.asyncio -async def test_db_query_cache(user): +def test_db_query_cache(user): query = db.query(User) # We have no cached value yet. assert cache._redis.get("key1") is None # Add to cache - await cache.db_query_cache("key1", query) + cache.db_query_cache("key1", query) # It's cached now. assert cache._redis.get("key1") is not None # Modify our user and make sure we got a cached value user.Username = "changed" - cached = await cache.db_query_cache("key1", query) + cached = cache.db_query_cache("key1", query) assert cached[0].Username != query.all()[0].Username # It does not expire assert cache._redis.ttl("key1") == -1 # Cache a query with an expire. - value = await cache.db_query_cache("key2", query, 100) + value = cache.db_query_cache("key2", query, 100) assert len(value) == query.count() assert value[0].Username == query.all()[0].Username @@ -90,7 +88,7 @@ async def test_db_query_cache(user): with mock.patch("aurweb.config.getint", side_effect=mock_max_search_entries): # Try to add another entry (we already have 2) - await cache.db_query_cache("key3", query) + cache.db_query_cache("key3", query) # Make sure it was not added because it exceeds our max. assert cache._redis.get("key3") is None diff --git a/test/test_metrics.py b/test/test_metrics.py index 1859d8cb..6f67d926 100644 --- a/test/test_metrics.py +++ b/test/test_metrics.py @@ -26,11 +26,10 @@ def user() -> User: yield user -@pytest.mark.asyncio -async def test_search_cache_metrics(user: User): +def test_search_cache_metrics(user: User): # Fire off 3 identical queries for caching for _ in range(3): - await db_query_cache("key", db.query(User)) + db_query_cache("key", db.query(User)) # Get metrics metrics = str(generate_latest(REGISTRY)) From 8699457917a05caae41a7cd2b7ecb6d94a7955b7 Mon Sep 17 00:00:00 2001 From: moson Date: Sat, 22 Jul 2023 21:23:16 +0200 Subject: [PATCH 030/128] feat: Separate cache expiry for stats and search Allows us to set different cache eviction timespans for search queries and statistics. Stats and especially "last package updates" should probably be refreshed more often, whereas we might want to cache search results for a bit longer. So this gives us a bit more flexibility playing around with different settings and tweak things. Signed-off-by: moson --- aurweb/routers/html.py | 2 +- aurweb/routers/packages.py | 2 +- aurweb/statistics.py | 2 +- conf/config.defaults | 6 ++++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index c3bcee49..2ec497bd 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -86,7 +86,7 @@ async def index(request: Request): context = make_context(request, "Home") context["ssh_fingerprints"] = util.get_ssh_fingerprints() - cache_expire = aurweb.config.getint("cache", "expiry_time") + cache_expire = aurweb.config.getint("cache", "expiry_time_statistics", 300) # Package statistics. stats = Statistics(cache_expire) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index f1b2a138..3f96d71c 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -90,7 +90,7 @@ async def packages_get( # Including more query operations below, like ordering, will # increase the amount of time required to collect a count. # we use redis for caching the results of the query - cache_expire = config.getint("cache", "expiry_time") + cache_expire = config.getint("cache", "expiry_time_search", 600) num_packages = db_count_cache(hash_query(search.query), search.query, cache_expire) # Apply user-specified sort column and ordering. diff --git a/aurweb/statistics.py b/aurweb/statistics.py index 934caa37..6e9dbe1f 100644 --- a/aurweb/statistics.py +++ b/aurweb/statistics.py @@ -89,7 +89,7 @@ class Statistics: def update_prometheus_metrics(): - cache_expire = config.getint("cache", "expiry_time") + cache_expire = config.getint("cache", "expiry_time_statistics", 300) stats = Statistics(cache_expire) # Users gauge for counter, utype in stats.PROMETHEUS_USER_COUNTERS: diff --git a/conf/config.defaults b/conf/config.defaults index 4e2415ed..ab0a9b67 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -169,5 +169,7 @@ range_end = 172800 [cache] ; maximum number of keys/entries (for search results) in our redis cache, default is 50000 max_search_entries = 50000 -; number of seconds after a cache entry expires, default is 3 minutes -expiry_time = 180 +; number of seconds after a cache entry for search queries expires, default is 10 minutes +expiry_time_search = 600 +; number of seconds after a cache entry for statistics queries expires, default is 5 minutes +expiry_time_statistics = 300 From 6cd70a5c9fb57c42af7c2254045eba9fe6aa17e0 Mon Sep 17 00:00:00 2001 From: moson Date: Sun, 23 Jul 2023 11:34:50 +0200 Subject: [PATCH 031/128] test: Add tests for user/package statistics Signed-off-by: moson --- test/test_statistics.py | 125 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 test/test_statistics.py diff --git a/test/test_statistics.py b/test/test_statistics.py new file mode 100644 index 00000000..dda7b357 --- /dev/null +++ b/test/test_statistics.py @@ -0,0 +1,125 @@ +import pytest +from prometheus_client import REGISTRY, generate_latest + +from aurweb import cache, db, time +from aurweb.models.account_type import TRUSTED_USER_ID, USER_ID +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.user import User +from aurweb.statistics import Statistics, update_prometheus_metrics + + +@pytest.fixture(autouse=True) +def setup(db_test): + return + + +@pytest.fixture(autouse=True) +def clear_fakeredis_cache(): + cache._redis.flushall() + + +@pytest.fixture +def test_data(): + # Create some test data (users and packages) + with db.begin(): + for i in range(10): + user = db.create( + User, + Username=f"test{i}", + Email=f"test{i}@example.org", + RealName=f"Test User {i}", + Passwd="testPassword", + AccountTypeID=USER_ID, + ) + + now = time.utcnow() + old = now - 60 * 60 * 24 * 8 # 8 days + older = now - 60 * 60 * 24 * 400 # 400 days + + pkgbase = db.create( + PackageBase, + Name=f"test-package{i}", + Maintainer=user, + SubmittedTS=old, + ModifiedTS=now, + ) + db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + + # Modify some data to get some variances for our counters + if i == 1: + user.AccountTypeID = TRUSTED_USER_ID + pkgbase.Maintainer = None + pkgbase.SubmittedTS = now + + if i == 2: + pkgbase.SubmittedTS = older + + if i == 3: + pkgbase.SubmittedTS = older + pkgbase.ModifiedTS = old + yield + + +@pytest.fixture +def stats() -> Statistics: + yield Statistics() + + +@pytest.mark.parametrize( + "counter, expected", + [ + ("package_count", 10), + ("orphan_count", 1), + ("seven_days_old_added", 1), + ("seven_days_old_updated", 8), + ("year_old_updated", 9), + ("never_updated", 1), + ("user_count", 10), + ("trusted_user_count", 1), + ("regular_user_count", 9), + ("updated_packages", 9), + ("nonsense", -1), + ], +) +def test_get_count(stats: Statistics, test_data, counter: str, expected: int): + assert stats.get_count(counter) == expected + + +def test_get_count_change(stats: Statistics, test_data): + pkgs_before = stats.get_count("package_count") + tus_before = stats.get_count("trusted_user_count") + + assert pkgs_before == 10 + assert tus_before == 1 + + # Let's delete a package and promote a user to TU + with db.begin(): + pkgbase = db.query(PackageBase).first() + db.delete(pkgbase) + + user = db.query(User).filter(User.AccountTypeID == USER_ID).first() + user.AccountTypeID = TRUSTED_USER_ID + + # Values should end up in (fake) redis cache so they should be the same + assert stats.get_count("package_count") == pkgs_before + assert stats.get_count("trusted_user_count") == tus_before + + # Let's clear the cache and check again + cache._redis.flushall() + assert stats.get_count("package_count") != pkgs_before + assert stats.get_count("trusted_user_count") != tus_before + + +def test_update_prometheus_metrics(test_data): + metrics = str(generate_latest(REGISTRY)) + + assert "aur_users{" not in metrics + assert "aur_packages{" not in metrics + + # Let's update our metrics. We should find our gauges now + update_prometheus_metrics() + metrics = str(generate_latest(REGISTRY)) + + assert 'aur_users{type="user"} 9.0' in metrics + assert 'aur_packages{state="updated"} 9.0' in metrics From e45878a058071e59f0f08eb3dca597f560448298 Mon Sep 17 00:00:00 2001 From: moson Date: Sun, 23 Jul 2023 18:53:58 +0200 Subject: [PATCH 032/128] fix: Fix issue with requests totals Problem is that we join with PackageBase, thus we are missing requests for packages that were deleted. Fixes: #483 Signed-off-by: moson --- aurweb/routers/html.py | 11 ++--- aurweb/routers/requests.py | 16 ++----- aurweb/statistics.py | 95 ++++++++++++++++++++++++++++---------- test/test_statistics.py | 32 ++++++++++++- 4 files changed, 111 insertions(+), 43 deletions(-) diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index 2ec497bd..63cc3bb8 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -16,11 +16,10 @@ from sqlalchemy import case, or_ import aurweb.config import aurweb.models.package_request -from aurweb import aur_logging, cookies, db, models, time, util +from aurweb import aur_logging, cookies, db, models, statistics, time, util from aurweb.exceptions import handle_form_exceptions from aurweb.models.package_request import PENDING_ID from aurweb.packages.util import query_notified, query_voted, updated_packages -from aurweb.statistics import Statistics, update_prometheus_metrics from aurweb.templates import make_context, render_template logger = aur_logging.get_logger(__name__) @@ -89,9 +88,9 @@ async def index(request: Request): cache_expire = aurweb.config.getint("cache", "expiry_time_statistics", 300) # Package statistics. - stats = Statistics(cache_expire) - for counter in stats.HOMEPAGE_COUNTERS: - context[counter] = stats.get_count(counter) + counts = statistics.get_homepage_counts() + for k in counts: + context[k] = counts[k] # Get the 15 most recently updated packages. context["package_updates"] = updated_packages(15, cache_expire) @@ -213,7 +212,7 @@ async def metrics(request: Request): ) # update prometheus gauges for packages and users - update_prometheus_metrics() + statistics.update_prometheus_metrics() registry = CollectorRegistry() multiprocess.MultiProcessCollector(registry) diff --git a/aurweb/routers/requests.py b/aurweb/routers/requests.py index 4cfda269..a67419fe 100644 --- a/aurweb/routers/requests.py +++ b/aurweb/routers/requests.py @@ -16,6 +16,7 @@ from aurweb.models.package_request import ( ) from aurweb.requests.util import get_pkgreq_by_id from aurweb.scripts import notify +from aurweb.statistics import get_request_counts from aurweb.templates import make_context, render_template FILTER_PARAMS = { @@ -31,7 +32,7 @@ router = APIRouter() @router.get("/requests") @requires_auth -async def requests( +async def requests( # noqa: C901 request: Request, O: int = Query(default=defaults.O), PP: int = Query(default=defaults.PP), @@ -74,18 +75,11 @@ async def requests( .join(User, PackageRequest.UsersID == User.ID, isouter=True) .join(Maintainer, PackageBase.MaintainerUID == Maintainer.ID, isouter=True) ) - # query = db.query(PackageRequest).join(User) # Requests statistics - context["total_requests"] = query.count() - pending_count = 0 + query.filter(PackageRequest.Status == PENDING_ID).count() - context["pending_requests"] = pending_count - closed_count = 0 + query.filter(PackageRequest.Status == CLOSED_ID).count() - context["closed_requests"] = closed_count - accepted_count = 0 + query.filter(PackageRequest.Status == ACCEPTED_ID).count() - context["accepted_requests"] = accepted_count - rejected_count = 0 + query.filter(PackageRequest.Status == REJECTED_ID).count() - context["rejected_requests"] = rejected_count + counts = get_request_counts() + for k in counts: + context[k] = counts[k] # Apply status filters in_filters = [] diff --git a/aurweb/statistics.py b/aurweb/statistics.py index 6e9dbe1f..3c1298b7 100644 --- a/aurweb/statistics.py +++ b/aurweb/statistics.py @@ -1,31 +1,46 @@ from aurweb import config, db, time from aurweb.cache import db_count_cache -from aurweb.models import PackageBase, User +from aurweb.models import PackageBase, PackageRequest, User from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID, USER_ID +from aurweb.models.package_request import ( + ACCEPTED_ID, + CLOSED_ID, + PENDING_ID, + REJECTED_ID, +) from aurweb.prometheus import PACKAGES, USERS +cache_expire = config.getint("cache", "expiry_time_statistics", 300) + +HOMEPAGE_COUNTERS = [ + "package_count", + "orphan_count", + "seven_days_old_added", + "seven_days_old_updated", + "year_old_updated", + "never_updated", + "user_count", + "trusted_user_count", +] +REQUEST_COUNTERS = [ + "total_requests", + "pending_requests", + "closed_requests", + "accepted_requests", + "rejected_requests", +] +PROMETHEUS_USER_COUNTERS = [ + ("trusted_user_count", "tu"), + ("regular_user_count", "user"), +] +PROMETHEUS_PACKAGE_COUNTERS = [ + ("orphan_count", "orphan"), + ("never_updated", "not_updated"), + ("updated_packages", "updated"), +] + class Statistics: - HOMEPAGE_COUNTERS = [ - "package_count", - "orphan_count", - "seven_days_old_added", - "seven_days_old_updated", - "year_old_updated", - "never_updated", - "user_count", - "trusted_user_count", - ] - PROMETHEUS_USER_COUNTERS = [ - ("trusted_user_count", "tu"), - ("regular_user_count", "user"), - ] - PROMETHEUS_PACKAGE_COUNTERS = [ - ("orphan_count", "orphan"), - ("never_updated", "not_updated"), - ("updated_packages", "updated"), - ] - seven_days = 86400 * 7 one_hour = 3600 year = seven_days * 52 @@ -35,15 +50,18 @@ class Statistics: self.now = time.utcnow() self.seven_days_ago = self.now - self.seven_days self.year_ago = self.now - self.year + self.user_query = db.query(User) self.bases_query = db.query(PackageBase) self.updated_query = db.query(PackageBase).filter( PackageBase.ModifiedTS - PackageBase.SubmittedTS >= self.one_hour ) + self.request_query = db.query(PackageRequest) def get_count(self, counter: str) -> int: query = None match counter: + # Packages case "package_count": query = self.bases_query case "orphan_count": @@ -69,6 +87,7 @@ class Statistics: PackageBase.ModifiedTS - PackageBase.SubmittedTS > self.one_hour, ~PackageBase.MaintainerUID.is_(None), ) + # Users case "user_count": query = self.user_query case "trusted_user_count": @@ -82,6 +101,18 @@ class Statistics: ) case "regular_user_count": query = self.user_query.filter(User.AccountTypeID == USER_ID) + + # Requests + case "total_requests": + query = self.request_query + case "pending_requests": + query = self.request_query.filter(PackageRequest.Status == PENDING_ID) + case "closed_requests": + query = self.request_query.filter(PackageRequest.Status == CLOSED_ID) + case "accepted_requests": + query = self.request_query.filter(PackageRequest.Status == ACCEPTED_ID) + case "rejected_requests": + query = self.request_query.filter(PackageRequest.Status == REJECTED_ID) case _: return -1 @@ -89,14 +120,30 @@ class Statistics: def update_prometheus_metrics(): - cache_expire = config.getint("cache", "expiry_time_statistics", 300) stats = Statistics(cache_expire) # Users gauge - for counter, utype in stats.PROMETHEUS_USER_COUNTERS: + for counter, utype in PROMETHEUS_USER_COUNTERS: count = stats.get_count(counter) USERS.labels(utype).set(count) # Packages gauge - for counter, state in stats.PROMETHEUS_PACKAGE_COUNTERS: + for counter, state in PROMETHEUS_PACKAGE_COUNTERS: count = stats.get_count(counter) PACKAGES.labels(state).set(count) + + +def _get_counts(counters: list[str]) -> dict[str, int]: + stats = Statistics(cache_expire) + result = dict() + for counter in counters: + result[counter] = stats.get_count(counter) + + return result + + +def get_homepage_counts() -> dict[str, int]: + return _get_counts(HOMEPAGE_COUNTERS) + + +def get_request_counts() -> dict[str, int]: + return _get_counts(REQUEST_COUNTERS) diff --git a/test/test_statistics.py b/test/test_statistics.py index dda7b357..a6a814c5 100644 --- a/test/test_statistics.py +++ b/test/test_statistics.py @@ -2,9 +2,15 @@ import pytest from prometheus_client import REGISTRY, generate_latest from aurweb import cache, db, time +from aurweb.models import Package, PackageBase, PackageRequest from aurweb.models.account_type import TRUSTED_USER_ID, USER_ID -from aurweb.models.package import Package -from aurweb.models.package_base import PackageBase +from aurweb.models.package_request import ( + ACCEPTED_ID, + CLOSED_ID, + PENDING_ID, + REJECTED_ID, +) +from aurweb.models.request_type import DELETION_ID, ORPHAN_ID from aurweb.models.user import User from aurweb.statistics import Statistics, update_prometheus_metrics @@ -45,19 +51,36 @@ def test_data(): ModifiedTS=now, ) db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + pkgreq = db.create( + PackageRequest, + ReqTypeID=ORPHAN_ID, + User=user, + PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + RequestTS=now, + Comments=str(), + ClosureComment=str(), + ) # Modify some data to get some variances for our counters if i == 1: user.AccountTypeID = TRUSTED_USER_ID pkgbase.Maintainer = None pkgbase.SubmittedTS = now + pkgreq.Status = PENDING_ID + pkgreq.ReqTypeID = DELETION_ID if i == 2: pkgbase.SubmittedTS = older + pkgreq.Status = ACCEPTED_ID if i == 3: pkgbase.SubmittedTS = older pkgbase.ModifiedTS = old + pkgreq.Status = CLOSED_ID + + if i == 4: + pkgreq.Status = REJECTED_ID yield @@ -79,6 +102,11 @@ def stats() -> Statistics: ("trusted_user_count", 1), ("regular_user_count", 9), ("updated_packages", 9), + ("total_requests", 10), + ("pending_requests", 7), + ("closed_requests", 1), + ("accepted_requests", 1), + ("rejected_requests", 1), ("nonsense", -1), ], ) From 375895f08011c2c91b52b79a2d41fe1504524acf Mon Sep 17 00:00:00 2001 From: moson Date: Sun, 23 Jul 2023 22:46:44 +0200 Subject: [PATCH 033/128] feat: Add Prometheus metrics for requests Adds gauge for requests by type and status Signed-off-by: moson --- aurweb/prometheus.py | 6 ++++++ aurweb/statistics.py | 22 +++++++++++++++++++--- test/test_statistics.py | 6 ++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/aurweb/prometheus.py b/aurweb/prometheus.py index d3455551..40b99a90 100644 --- a/aurweb/prometheus.py +++ b/aurweb/prometheus.py @@ -24,6 +24,12 @@ PACKAGES = Gauge( ["state"], multiprocess_mode="livemax", ) +REQUESTS = Gauge( + "aur_requests", + "Number of AUR requests by type and status", + ["type", "status"], + multiprocess_mode="livemax", +) def instrumentator(): diff --git a/aurweb/statistics.py b/aurweb/statistics.py index 3c1298b7..f301b59c 100644 --- a/aurweb/statistics.py +++ b/aurweb/statistics.py @@ -1,6 +1,8 @@ +from sqlalchemy import func + from aurweb import config, db, time -from aurweb.cache import db_count_cache -from aurweb.models import PackageBase, PackageRequest, User +from aurweb.cache import db_count_cache, db_query_cache +from aurweb.models import PackageBase, PackageRequest, RequestType, User from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID, USER_ID from aurweb.models.package_request import ( ACCEPTED_ID, @@ -8,7 +10,7 @@ from aurweb.models.package_request import ( PENDING_ID, REJECTED_ID, ) -from aurweb.prometheus import PACKAGES, USERS +from aurweb.prometheus import PACKAGES, REQUESTS, USERS cache_expire = config.getint("cache", "expiry_time_statistics", 300) @@ -131,6 +133,20 @@ def update_prometheus_metrics(): count = stats.get_count(counter) PACKAGES.labels(state).set(count) + # Requests gauge + query = ( + db.get_session() + .query(PackageRequest, func.count(PackageRequest.ID), RequestType.Name) + .join(RequestType) + .group_by(RequestType.Name, PackageRequest.Status) + ) + results = db_query_cache("request_metrics", query, cache_expire) + for record in results: + status = record[0].status_display() + count = record[1] + rtype = record[2] + REQUESTS.labels(type=rtype, status=status).set(count) + def _get_counts(counters: list[str]) -> dict[str, int]: stats = Statistics(cache_expire) diff --git a/test/test_statistics.py b/test/test_statistics.py index a6a814c5..db262fa3 100644 --- a/test/test_statistics.py +++ b/test/test_statistics.py @@ -144,6 +144,7 @@ def test_update_prometheus_metrics(test_data): assert "aur_users{" not in metrics assert "aur_packages{" not in metrics + assert "aur_requests{" not in metrics # Let's update our metrics. We should find our gauges now update_prometheus_metrics() @@ -151,3 +152,8 @@ def test_update_prometheus_metrics(test_data): assert 'aur_users{type="user"} 9.0' in metrics assert 'aur_packages{state="updated"} 9.0' in metrics + assert 'aur_requests{status="Pending",type="orphan"} 6.0' in metrics + assert 'aur_requests{status="Closed",type="orphan"} 1.0' in metrics + assert 'aur_requests{status="Accepted",type="orphan"} 1.0' in metrics + assert 'aur_requests{status="Rejected",type="orphan"} 1.0' in metrics + assert 'aur_requests{status="Pending",type="deletion"} 1.0' in metrics From f74f94b50170d82c1d3f899037dc0debd2222725 Mon Sep 17 00:00:00 2001 From: renovate Date: Mon, 24 Jul 2023 11:24:26 +0000 Subject: [PATCH 034/128] fix(deps): update dependency gunicorn to v21 --- poetry.lock | 27 +++++---------------------- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/poetry.lock b/poetry.lock index 368371db..42010de5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -685,18 +685,18 @@ test = ["objgraph", "psutil"] [[package]] name = "gunicorn" -version = "20.1.0" +version = "21.2.0" description = "WSGI HTTP Server for UNIX" category = "main" optional = false python-versions = ">=3.5" files = [ - {file = "gunicorn-20.1.0-py3-none-any.whl", hash = "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e"}, - {file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"}, + {file = "gunicorn-21.2.0-py3-none-any.whl", hash = "sha256:3213aa5e8c24949e792bcacfc176fef362e7aac80b76c56f6b5122bf350722f0"}, + {file = "gunicorn-21.2.0.tar.gz", hash = "sha256:88ec8bff1d634f98e61b9f65bc4bf3cd918a90806c6f5c48bc5603849ec81033"}, ] [package.dependencies] -setuptools = ">=3.0" +packaging = "*" [package.extras] eventlet = ["eventlet (>=0.24.1)"] @@ -1602,23 +1602,6 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] -[[package]] -name = "setuptools" -version = "67.7.2" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" -optional = false -python-versions = ">=3.7" -files = [ - {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"}, - {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"}, -] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - [[package]] name = "six" version = "1.16.0" @@ -1948,4 +1931,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "b67f1b1599794a6890b0a31b2b127880d75c84beeeae3df4ecb3ae92296948da" +content-hash = "48d66bc7145b8cdac8da9977d6d2b0d554b382193a28f275743697d0a17d2f58" diff --git a/pyproject.toml b/pyproject.toml index e98e887f..e743e675 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,7 +87,7 @@ SQLAlchemy = "^1.4.48" # ASGI uvicorn = "^0.23.0" -gunicorn = "^20.1.0" +gunicorn = "^21.0.0" Hypercorn = "^0.14.3" prometheus-fastapi-instrumentator = "^6.0.0" pytest-xdist = "^3.2.1" From 969b84afe4f2d74a005bd35594d9a76de5351e95 Mon Sep 17 00:00:00 2001 From: renovate Date: Tue, 25 Jul 2023 11:24:30 +0000 Subject: [PATCH 035/128] fix(deps): update all non-major dependencies --- poetry.lock | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 42010de5..9897fafb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -540,14 +540,14 @@ testing = ["pre-commit"] [[package]] name = "fakeredis" -version = "2.16.0" +version = "2.17.0" description = "Python implementation of redis API, can be used for testing purposes." category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ - {file = "fakeredis-2.16.0-py3-none-any.whl", hash = "sha256:188514cbd7120ff28c88f2a31e2fddd18fb1b28504478dfa3669c683134c4d82"}, - {file = "fakeredis-2.16.0.tar.gz", hash = "sha256:5abdd734de4ead9d6c7acbd3add1c4aa9b3ab35219339530472d9dd2bdf13057"}, + {file = "fakeredis-2.17.0-py3-none-any.whl", hash = "sha256:a99ef6e5642c31e91d36be78809fec3743e2bf7aaa682685b0d65a849fecd148"}, + {file = "fakeredis-2.17.0.tar.gz", hash = "sha256:e304bc7addb2f862c3550cb7db58548418a0fadd4cd78a4de66464c84fbc2195"}, ] [package.dependencies] @@ -1815,19 +1815,20 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "uvicorn" -version = "0.23.0" +version = "0.23.1" description = "The lightning-fast ASGI server." category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "uvicorn-0.23.0-py3-none-any.whl", hash = "sha256:479599b2c0bb1b9b394c6d43901a1eb0c1ec72c7d237b5bafea23c5b2d4cdf10"}, - {file = "uvicorn-0.23.0.tar.gz", hash = "sha256:d38ab90c0e2c6fe3a054cddeb962cfd5d0e0e6608eaaff4a01d5c36a67f3168c"}, + {file = "uvicorn-0.23.1-py3-none-any.whl", hash = "sha256:1d55d46b83ee4ce82b4e82f621f2050adb3eb7b5481c13f9af1744951cae2f1f"}, + {file = "uvicorn-0.23.1.tar.gz", hash = "sha256:da9b0c8443b2d7ee9db00a345f1eee6db7317432c9d4400f5049cc8d358383be"}, ] [package.dependencies] click = ">=7.0" h11 = ">=0.8" +typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} [package.extras] standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] From 7a44f379687cb10817b8b2b3f6a636e289832b36 Mon Sep 17 00:00:00 2001 From: renovate Date: Thu, 27 Jul 2023 19:24:28 +0000 Subject: [PATCH 036/128] fix(deps): update dependency fastapi to v0.100.1 --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9897fafb..c4d3adb6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -560,14 +560,14 @@ lua = ["lupa (>=1.14,<2.0)"] [[package]] name = "fastapi" -version = "0.100.0" +version = "0.100.1" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "fastapi-0.100.0-py3-none-any.whl", hash = "sha256:271662daf986da8fa98dc2b7c7f61c4abdfdccfb4786d79ed8b2878f172c6d5f"}, - {file = "fastapi-0.100.0.tar.gz", hash = "sha256:acb5f941ea8215663283c10018323ba7ea737c571b67fc7e88e9469c7eb1d12e"}, + {file = "fastapi-0.100.1-py3-none-any.whl", hash = "sha256:ec6dd52bfc4eff3063cfcd0713b43c87640fefb2687bbbe3d8a08d94049cdf32"}, + {file = "fastapi-0.100.1.tar.gz", hash = "sha256:522700d7a469e4a973d92321ab93312448fbe20fca9c8da97effc7e7bc56df23"}, ] [package.dependencies] From 94b62d2949c0b627bbeac4107c681ab7eccfff7d Mon Sep 17 00:00:00 2001 From: moson Date: Fri, 4 Aug 2023 14:12:50 +0200 Subject: [PATCH 037/128] fix: Check if user exists when editing account We should check if a user (target) exists before validating permissions. Otherwise things crash when a TU is trying to edit an account that does not exist. Fixes: aurweb-errors#529 Signed-off-by: moson --- aurweb/routers/accounts.py | 3 +++ test/test_accounts_routes.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 010aae58..1c81ec1d 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -374,6 +374,9 @@ def cannot_edit( :param user: Target user to be edited :return: RedirectResponse if approval != granted else None """ + # raise 404 if user does not exist + if not user: + raise HTTPException(status_code=HTTPStatus.NOT_FOUND) approved = request.user.can_edit_user(user) if not approved and (to := "/"): if user: diff --git a/test/test_accounts_routes.py b/test/test_accounts_routes.py index 3c481d0a..3ff6291a 100644 --- a/test/test_accounts_routes.py +++ b/test/test_accounts_routes.py @@ -764,6 +764,17 @@ def test_get_account_edit_unauthorized(client: TestClient, user: User): assert response.headers.get("location") == expected +def test_get_account_edit_not_exists(client: TestClient, tu_user: User): + """Test that users do not have an Account Type field.""" + cookies = {"AURSID": tu_user.login(Request(), "testPassword")} + endpoint = "/account/doesnotexist/edit" + + with client as request: + request.cookies = cookies + response = request.get(endpoint) + assert response.status_code == int(HTTPStatus.NOT_FOUND) + + def test_post_account_edit(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") @@ -872,6 +883,19 @@ def test_post_account_edit_dev(client: TestClient, tu_user: User): assert expected in response.content.decode() +def test_post_account_edit_not_exists(client: TestClient, tu_user: User): + request = Request() + sid = tu_user.login(request, "testPassword") + + post_data = {"U": "test", "E": "test666@example.org", "passwd": "testPassword"} + + endpoint = "/account/doesnotexist/edit" + with client as request: + request.cookies = {"AURSID": sid} + response = request.post(endpoint, data=post_data) + assert response.status_code == int(HTTPStatus.NOT_FOUND) + + def test_post_account_edit_language(client: TestClient, user: User): request = Request() sid = user.login(request, "testPassword") From 8ad03522de34a40992165649fc0605390db93a98 Mon Sep 17 00:00:00 2001 From: renovate Date: Fri, 4 Aug 2023 14:25:22 +0000 Subject: [PATCH 038/128] fix(deps): update all non-major dependencies --- poetry.lock | 27 ++++++++++++++------------- pyproject.toml | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/poetry.lock b/poetry.lock index c4d3adb6..623884a0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -14,14 +14,14 @@ files = [ [[package]] name = "alembic" -version = "1.11.1" +version = "1.11.2" description = "A database migration tool for SQLAlchemy." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "alembic-1.11.1-py3-none-any.whl", hash = "sha256:dc871798a601fab38332e38d6ddb38d5e734f60034baeb8e2db5b642fccd8ab8"}, - {file = "alembic-1.11.1.tar.gz", hash = "sha256:6a810a6b012c88b33458fceb869aef09ac75d6ace5291915ba7fae44de372c01"}, + {file = "alembic-1.11.2-py3-none-any.whl", hash = "sha256:7981ab0c4fad4fe1be0cf183aae17689fe394ff874fd2464adb774396faf0796"}, + {file = "alembic-1.11.2.tar.gz", hash = "sha256:678f662130dc540dac12de0ea73de9f89caea9dbea138f60ef6263149bf84657"}, ] [package.dependencies] @@ -1031,20 +1031,21 @@ testing = ["pytest"] [[package]] name = "markdown" -version = "3.4.3" +version = "3.4.4" description = "Python implementation of John Gruber's Markdown." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "Markdown-3.4.3-py3-none-any.whl", hash = "sha256:065fd4df22da73a625f14890dd77eb8040edcbd68794bcd35943be14490608b2"}, - {file = "Markdown-3.4.3.tar.gz", hash = "sha256:8bf101198e004dc93e84a12a7395e31aac6a9c9942848ae1d99b9d72cf9b3520"}, + {file = "Markdown-3.4.4-py3-none-any.whl", hash = "sha256:a4c1b65c0957b4bd9e7d86ddc7b3c9868fb9670660f6f99f6d1bca8954d5a941"}, + {file = "Markdown-3.4.4.tar.gz", hash = "sha256:225c6123522495d4119a90b3a3ba31a1e87a70369e03f14799ea9c0d7183a3d6"}, ] [package.dependencies] importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} [package.extras] +docs = ["mdx-gh-links (>=0.2)", "mkdocs (>=1.0)", "mkdocs-nature (>=0.4)"] testing = ["coverage", "pyyaml"] [[package]] @@ -1773,14 +1774,14 @@ files = [ [[package]] name = "tomlkit" -version = "0.11.8" +version = "0.12.1" description = "Style preserving TOML library" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "tomlkit-0.11.8-py3-none-any.whl", hash = "sha256:8c726c4c202bdb148667835f68d68780b9a003a9ec34167b6c673b38eff2a171"}, - {file = "tomlkit-0.11.8.tar.gz", hash = "sha256:9330fc7faa1db67b541b28e62018c17d20be733177d290a13b24c62d1614e0c3"}, + {file = "tomlkit-0.12.1-py3-none-any.whl", hash = "sha256:712cbd236609acc6a3e2e97253dfc52d4c2082982a88f61b640ecf0817eab899"}, + {file = "tomlkit-0.12.1.tar.gz", hash = "sha256:38e1ff8edb991273ec9f6181244a6a391ac30e9f5098e7535640ea6be97a7c86"}, ] [[package]] @@ -1815,14 +1816,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "uvicorn" -version = "0.23.1" +version = "0.23.2" description = "The lightning-fast ASGI server." category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "uvicorn-0.23.1-py3-none-any.whl", hash = "sha256:1d55d46b83ee4ce82b4e82f621f2050adb3eb7b5481c13f9af1744951cae2f1f"}, - {file = "uvicorn-0.23.1.tar.gz", hash = "sha256:da9b0c8443b2d7ee9db00a345f1eee6db7317432c9d4400f5049cc8d358383be"}, + {file = "uvicorn-0.23.2-py3-none-any.whl", hash = "sha256:1f9be6558f01239d4fdf22ef8126c39cb1ad0addf76c40e760549d2c2f43ab53"}, + {file = "uvicorn-0.23.2.tar.gz", hash = "sha256:4d3cc12d7727ba72b64d12d3cc7743124074c0a69f7b201512fc50c3e3f1569a"}, ] [package.dependencies] @@ -1932,4 +1933,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "48d66bc7145b8cdac8da9977d6d2b0d554b382193a28f275743697d0a17d2f58" +content-hash = "ac9dbb5b28292c4a3dd2318a2f5c9120dfaa117ee834fac05995c5d0cbdc460d" diff --git a/pyproject.toml b/pyproject.toml index e743e675..f4682bad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,7 @@ posix-ipc = "^1.1.1" pyalpm = "^0.10.6" fastapi = "^0.100.0" srcinfo = "^0.1.2" -tomlkit = "^0.11.8" +tomlkit = "^0.12.0" [tool.poetry.dev-dependencies] coverage = "^7.2.5" From f05f1dbac798c5fd41e0f19c9b0419fa302c9bf4 Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Fri, 4 Aug 2023 19:18:38 +0300 Subject: [PATCH 039/128] chore(release): prepare for 6.2.7 Signed-off-by: Leonidas Spyropoulos --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f4682bad..359923a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ combine_as_imports = true # [tool.poetry] name = "aurweb" -version = "v6.2.6" +version = "v6.2.7" license = "GPL-2.0-only" description = "Source code for the Arch User Repository's website" homepage = "https://aur.archlinux.org" From 3005e82f607e7b20ac5e32d50f0ffb124a8737e0 Mon Sep 17 00:00:00 2001 From: moson Date: Fri, 18 Aug 2023 22:04:55 +0200 Subject: [PATCH 040/128] fix: Cleanup prometheus metrics for dead workers The current "cleanup" function that is removing orphan prometheus files is actually never invoked. We move this to a default gunicorn config file to register our hook(s). https://docs.gunicorn.org/en/stable/configure.html https://docs.gunicorn.org/en/stable/settings.html#child-exit Signed-off-by: moson --- aurweb/asgi.py | 7 ------- gunicorn.conf.py | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 gunicorn.conf.py diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 1be77ff9..9b6ffcb3 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -13,7 +13,6 @@ from fastapi import FastAPI, HTTPException, Request, Response from fastapi.responses import RedirectResponse from fastapi.staticfiles import StaticFiles from jinja2 import TemplateNotFound -from prometheus_client import multiprocess from sqlalchemy import and_ from starlette.exceptions import HTTPException as StarletteHTTPException from starlette.middleware.authentication import AuthenticationMiddleware @@ -91,12 +90,6 @@ async def app_startup(): get_engine() -def child_exit(server, worker): # pragma: no cover - """This function is required for gunicorn customization - of prometheus multiprocessing.""" - multiprocess.mark_process_dead(worker.pid) - - async def internal_server_error(request: Request, exc: Exception) -> Response: """ Catch all uncaught Exceptions thrown in a route. diff --git a/gunicorn.conf.py b/gunicorn.conf.py new file mode 100644 index 00000000..4f1c3a8c --- /dev/null +++ b/gunicorn.conf.py @@ -0,0 +1,7 @@ +from prometheus_client import multiprocess + + +def child_exit(server, worker): # pragma: no cover + """This function is required for gunicorn customization + of prometheus multiprocessing.""" + multiprocess.mark_process_dead(worker.pid) From 6c610b26a39a56db562e4b1ed3c95420a73ee766 Mon Sep 17 00:00:00 2001 From: Kristian Klausen Date: Fri, 28 Jul 2023 22:42:44 +0200 Subject: [PATCH 041/128] feat: Add terraform config for review-app[1] Also removed the logic for deploying to the long gone aur-dev box. Ansible will be added in a upcoming commit for configurating and deploying aurweb on the VM. [1] https://docs.gitlab.com/ee/ci/review_apps/ --- .gitignore | 4 +++ .gitlab-ci.yml | 71 +++++++++++++++++++++++---------------- ci/tf/.terraform.lock.hcl | 61 +++++++++++++++++++++++++++++++++ ci/tf/main.tf | 67 ++++++++++++++++++++++++++++++++++++ ci/tf/terraform.tfvars | 4 +++ ci/tf/variables.tf | 36 ++++++++++++++++++++ ci/tf/versions.tf | 13 +++++++ 7 files changed, 227 insertions(+), 29 deletions(-) create mode 100644 ci/tf/.terraform.lock.hcl create mode 100644 ci/tf/main.tf create mode 100644 ci/tf/terraform.tfvars create mode 100644 ci/tf/variables.tf create mode 100644 ci/tf/versions.tf diff --git a/.gitignore b/.gitignore index 68de7cd5..97157118 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,7 @@ test-emails/ env/ venv/ .venv/ + +# Ignore some terraform files +/ci/tf/.terraform +/ci/tf/terraform.tfstate* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 10dd1787..4bd71920 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -61,34 +61,47 @@ test: coverage_format: cobertura path: coverage.xml -deploy: - stage: deploy - tags: - - secure - rules: - - if: $CI_COMMIT_BRANCH == "pu" - when: manual - variables: - FASTAPI_BACKEND: gunicorn - FASTAPI_WORKERS: 5 - AURWEB_FASTAPI_PREFIX: https://aur-dev.archlinux.org - AURWEB_SSHD_PREFIX: ssh://aur@aur-dev.archlinux.org:2222 - COMMIT_HASH: $CI_COMMIT_SHA - GIT_DATA_DIR: git_data - script: - - pacman -Syu --noconfirm docker docker-compose socat openssh - - chmod 600 ${SSH_KEY} - - socat "UNIX-LISTEN:/tmp/docker.sock,reuseaddr,fork" EXEC:"ssh -o UserKnownHostsFile=${SSH_KNOWN_HOSTS} -Ti ${SSH_KEY} ${SSH_USER}@${SSH_HOST}" & - - export DOCKER_HOST="unix:///tmp/docker.sock" - # Set secure login config for aurweb. - - sed -ri "s/^(disable_http_login).*$/\1 = 1/" conf/config.dev - - docker-compose build - - docker-compose -f docker-compose.yml -f docker-compose.aur-dev.yml down --remove-orphans - - docker-compose -f docker-compose.yml -f docker-compose.aur-dev.yml up -d - - docker image prune -f - - docker container prune -f - - docker volume prune -f +.init_tf: &init_tf + - pacman -Syu --needed --noconfirm --cachedir .pkg-cache terraform + - export TF_VAR_name="aurweb-${CI_COMMIT_REF_SLUG}" + - TF_ADDRESS="${CI_API_V4_URL}/projects/${TF_STATE_PROJECT}/terraform/state/${CI_COMMIT_REF_SLUG}" + - cd ci/tf + - > + terraform init \ + -backend-config="address=${TF_ADDRESS}" \ + -backend-config="lock_address=${TF_ADDRESS}/lock" \ + -backend-config="unlock_address=${TF_ADDRESS}/lock" \ + -backend-config="username=x-access-token" \ + -backend-config="password=${TF_STATE_GITLAB_ACCESS_TOKEN}" \ + -backend-config="lock_method=POST" \ + -backend-config="unlock_method=DELETE" \ + -backend-config="retry_wait_min=5" +deploy_review: + stage: deploy + script: + - *init_tf + - terraform apply -auto-approve environment: - name: development - url: https://aur-dev.archlinux.org + name: review/$CI_COMMIT_REF_NAME + url: https://aurweb-$CI_ENVIRONMENT_SLUG.sandbox.archlinux.page + on_stop: stop_review + auto_stop_in: 1 week + rules: + - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" + when: manual + +stop_review: + stage: deploy + needs: + - deploy_review + script: + - *init_tf + - terraform destroy -auto-approve + - 'curl --silent --show-error --fail --header "Private-Token: ${TF_STATE_GITLAB_ACCESS_TOKEN}" --request DELETE "${CI_API_V4_URL}/projects/${TF_STATE_PROJECT}/terraform/state/${CI_COMMIT_REF_SLUG}"' + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + rules: + - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" + when: manual diff --git a/ci/tf/.terraform.lock.hcl b/ci/tf/.terraform.lock.hcl new file mode 100644 index 00000000..aa5501c4 --- /dev/null +++ b/ci/tf/.terraform.lock.hcl @@ -0,0 +1,61 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/dns" { + version = "3.3.2" + hashes = [ + "h1:HjskPLRqmCw8Q/kiSuzti3iJBSpcAvcBFdlwFFQuoDE=", + "zh:05d2d50e301318362a4a82e6b7a9734ace07bc01abaaa649c566baf98814755f", + "zh:1e9fd1c3bfdda777e83e42831dd45b7b9e794250a0f351e5fd39762e8a0fe15b", + "zh:40e715fc7a2ede21f919567249b613844692c2f8a64f93ee64e5b68bae7ac2a2", + "zh:454d7aa83000a6e2ba7a7bfde4bcf5d7ed36298b22d760995ca5738ab02ee468", + "zh:46124ded51b4153ad90f12b0305fdbe0c23261b9669aa58a94a31c9cca2f4b19", + "zh:55a4f13d20f73534515a6b05701abdbfc54f4e375ba25b2dffa12afdad20e49d", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7903b1ceb8211e2b8c79290e2e70906a4b88f4fba71c900eb3a425ce12f1716a", + "zh:b79fc4f444ef7a2fd7111a80428c070ad824f43a681699e99ab7f83074dfedbd", + "zh:ca9f45e0c4cb94e7d62536c226024afef3018b1de84f1ea4608b51bcd497a2a0", + "zh:ddc8bd894559d7d176e0ceb0bb1ae266519b01b315362ebfee8327bb7e7e5fa8", + "zh:e77334c0794ef8f9354b10e606040f6b0b67b373f5ff1db65bddcdd4569b428b", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.4" + hashes = [ + "h1:pe9vq86dZZKCm+8k1RhzARwENslF3SXb9ErHbQfgjXU=", + "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55", + "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848", + "zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be", + "zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5", + "zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe", + "zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e", + "zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48", + "zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8", + "zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60", + "zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e", + "zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hetznercloud/hcloud" { + version = "1.42.0" + hashes = [ + "h1:cr9lh26H3YbWSHb7OUnCoYw169cYO3Cjpt3yPnRhXS0=", + "zh:153b5f39d780e9a18bc1ea377d872647d328d943813cbd25d3d20863f8a37782", + "zh:35b9e95760c58cca756e34ad5f4138ac6126aa3e8c41b4a0f1d5dc9ee5666c73", + "zh:47a3cdbce982f2b4e17f73d4934bdb3e905a849b36fb59b80f87d852496ed049", + "zh:6a718c244c2ba300fbd43791661a061ad1ab16225ef3e8aeaa3db8c9eff12c85", + "zh:a2cbfc95c5e2c9422ed0a7b6292192c38241220d5b7813c678f937ab3ef962ae", + "zh:b837e118e08fd36aa8be48af7e9d0d3d112d2680c79cfc71cfe2501fb40dbefa", + "zh:bf66db8c680e18b77e16dc1f20ed1cdcc7876bfb7848c320ccb86f0fb80661ed", + "zh:c1ad80bbe48dc8a272a02dcdb4b12f019606f445606651c01e561b9d72d816b1", + "zh:d4e616701128ad14a6b5a427b0e9145ece4cad02aa3b5f9945c6d0b9ada8ab70", + "zh:d9d01f727037d028720100a5bc9fd213cb01e63e4b439a16f2f482c147976530", + "zh:dea047ee4d679370d4376fb746c4b959bf51dd06047c1c2656b32789c2433643", + "zh:e5ad7a3c556894bd40b28a874e7d2f6924876fa75fa443136a7d6ab9a00abbaa", + "zh:edf6e7e129157bd45e3da4a330d1ace17a336d417c3b77c620f302d440c368e8", + "zh:f610bc729866d58da9cffa4deae34dbfdba96655e855a87c6bb2cb7b35a8961c", + ] +} diff --git a/ci/tf/main.tf b/ci/tf/main.tf new file mode 100644 index 00000000..b149a621 --- /dev/null +++ b/ci/tf/main.tf @@ -0,0 +1,67 @@ +terraform { + backend "http" { + } +} + +provider "hcloud" { + token = var.hcloud_token +} + +provider "dns" { + update { + server = var.dns_server + key_name = var.dns_tsig_key + key_algorithm = var.dns_tsig_algorithm + key_secret = var.dns_tsig_secret + } +} + +resource "tls_private_key" "this" { + algorithm = "ED25519" +} + +resource "hcloud_ssh_key" "this" { + name = var.name + public_key = tls_private_key.this.public_key_openssh +} + +data "hcloud_image" "this" { + with_selector = "custom_image=archlinux" + most_recent = true + with_status = ["available"] +} + +resource "hcloud_server" "this" { + name = var.name + image = data.hcloud_image.this.id + server_type = var.server_type + datacenter = var.datacenter + ssh_keys = [hcloud_ssh_key.this.name] + + public_net { + ipv4_enabled = true + ipv6_enabled = true + } +} + +resource "hcloud_rdns" "this" { + for_each = { ipv4 : hcloud_server.this.ipv4_address, ipv6 : hcloud_server.this.ipv6_address } + + server_id = hcloud_server.this.id + ip_address = each.value + dns_ptr = "${var.name}.${var.dns_zone}" +} + +resource "dns_a_record_set" "this" { + zone = "${var.dns_zone}." + name = var.name + addresses = [hcloud_server.this.ipv4_address] + ttl = 300 +} + +resource "dns_aaaa_record_set" "this" { + zone = "${var.dns_zone}." + name = var.name + addresses = [hcloud_server.this.ipv6_address] + ttl = 300 +} diff --git a/ci/tf/terraform.tfvars b/ci/tf/terraform.tfvars new file mode 100644 index 00000000..14818592 --- /dev/null +++ b/ci/tf/terraform.tfvars @@ -0,0 +1,4 @@ +server_type = "cpx11" +datacenter = "fsn1-dc14" +dns_server = "redirect.archlinux.org" +dns_zone = "sandbox.archlinux.page" diff --git a/ci/tf/variables.tf b/ci/tf/variables.tf new file mode 100644 index 00000000..a4e710ee --- /dev/null +++ b/ci/tf/variables.tf @@ -0,0 +1,36 @@ +variable "hcloud_token" { + type = string + sensitive = true +} + +variable "dns_server" { + type = string +} + +variable "dns_tsig_key" { + type = string +} + +variable "dns_tsig_algorithm" { + type = string +} + +variable "dns_tsig_secret" { + type = string +} + +variable "dns_zone" { + type = string +} + +variable "name" { + type = string +} + +variable "server_type" { + type = string +} + +variable "datacenter" { + type = string +} diff --git a/ci/tf/versions.tf b/ci/tf/versions.tf new file mode 100644 index 00000000..2c72215a --- /dev/null +++ b/ci/tf/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + tls = { + source = "hashicorp/tls" + } + hcloud = { + source = "hetznercloud/hcloud" + } + dns = { + source = "hashicorp/dns" + } + } +} From 9eda6a42c69581dfdc14dc1b0d51f744985c7202 Mon Sep 17 00:00:00 2001 From: moson Date: Sun, 27 Aug 2023 13:54:39 +0200 Subject: [PATCH 042/128] feat: Add ansible provisioning step for review-app Clone infrastructure repository and run playbook to provision our VM with aurweb. Signed-off-by: moson --- .gitlab-ci.yml | 54 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4bd71920..cf80ab24 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,8 @@ variables: TEST_RECURSION_LIMIT: 10000 CURRENT_DIR: "$(pwd)" LOG_CONFIG: logging.test.conf + DEV_FQDN: aurweb-$CI_COMMIT_REF_SLUG.sandbox.archlinux.page + INFRASTRUCTURE_REPO: https://gitlab.archlinux.org/archlinux/infrastructure.git lint: stage: .pre @@ -84,13 +86,63 @@ deploy_review: - terraform apply -auto-approve environment: name: review/$CI_COMMIT_REF_NAME - url: https://aurweb-$CI_ENVIRONMENT_SLUG.sandbox.archlinux.page + url: https://$DEV_FQDN on_stop: stop_review auto_stop_in: 1 week rules: - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" when: manual +provision_review: + stage: deploy + needs: + - deploy_review + script: + - *init_tf + - pacman -Syu --noconfirm --needed --cachedir .pkg-cache ansible git openssh jq + # Get ssh key from terraform state file + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - terraform show -json | + jq -r '.values.root_module.resources[] | + select(.address == "tls_private_key.this") | + .values.private_key_openssh' > ~/.ssh/id_ed25519 + - chmod 400 ~/.ssh/id_ed25519 + # Clone infra repo + - git clone $INFRASTRUCTURE_REPO + - cd infrastructure + # Remove vault files + - rm $(git grep -l 'ANSIBLE_VAULT;1.1;AES256$') + # Remove vault config + - sed -i '/^vault/d' ansible.cfg + # Add host config + - mkdir -p host_vars/$DEV_FQDN + - 'echo "filesystem: btrfs" > host_vars/$DEV_FQDN/misc' + # Add host + - echo "$DEV_FQDN" > hosts + # Add our pubkey and hostkeys + - ssh-keyscan $DEV_FQDN >> ~/.ssh/known_hosts + - ssh-keygen -f ~/.ssh/id_ed25519 -y > pubkeys/aurweb-dev.pub + # Run our ansible playbook + - > + ansible-playbook playbooks/aur-dev.archlinux.org.yml \ + -e "aurdev_fqdn=$DEV_FQDN" \ + -e "aurweb_repository=$CI_REPOSITORY_URL" \ + -e "aurweb_version=$CI_COMMIT_SHA" \ + -e "{\"vault_mariadb_users\":{\"root\":\"aur\"}}" \ + -e "vault_aurweb_db_password=aur" \ + -e "vault_aurweb_gitlab_instance=https://does.not.exist" \ + -e "vault_aurweb_error_project=aur" \ + -e "vault_aurweb_error_token=aur" \ + -e "vault_aurweb_secret=aur" \ + -e "vault_goaurrpc_metrics_token=aur" \ + -e '{"root_additional_keys": ["moson.pub", "aurweb-dev.pub"]}' + environment: + name: review/$CI_COMMIT_REF_NAME + action: access + rules: + - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" + stop_review: stage: deploy needs: From 5699e9bb41638fc1d6040f3e70a90fab38257458 Mon Sep 17 00:00:00 2001 From: moson Date: Sat, 26 Aug 2023 14:47:21 +0200 Subject: [PATCH 043/128] fix(test): Remove file locking and semaphore All tests within a file run in the same worker and out test DB names are unique per file as well. We don't really need a locking mechanism here. Same is valid for the test-emails. The only potential issue is that it might try to create the same directory multiple times and thus run into an error. However, that can be covered by specifying "exist_ok=True" with os.makedirs such that those errors are ignored. Signed-off-by: moson --- test/conftest.py | 40 +++++++++++----------------------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 15a982aa..c36f78dd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -43,7 +43,6 @@ from multiprocessing import Lock import py import pytest -from posix_ipc import O_CREAT, Semaphore from sqlalchemy import create_engine from sqlalchemy.engine import URL from sqlalchemy.engine.base import Engine @@ -54,7 +53,6 @@ import aurweb.config import aurweb.db from aurweb import aur_logging, initdb, testing from aurweb.testing.email import Email -from aurweb.testing.filelock import FileLock from aurweb.testing.git import GitRepository logger = aur_logging.get_logger(__name__) @@ -133,20 +131,16 @@ def _drop_database(engine: Engine, dbname: str) -> None: def setup_email(): - # TODO: Fix this data race! This try/catch is ugly; why is it even - # racing here? Perhaps we need to multiproc + multithread lock - # inside of setup_database to block the check? - with Semaphore("/test-emails", flags=O_CREAT, initial_value=1): - if not os.path.exists(Email.TEST_DIR): - # Create the directory. - os.makedirs(Email.TEST_DIR) + if not os.path.exists(Email.TEST_DIR): + # Create the directory. + os.makedirs(Email.TEST_DIR, exist_ok=True) - # Cleanup all email files for this test suite. - prefix = Email.email_prefix(suite=True) - files = os.listdir(Email.TEST_DIR) - for file in files: - if file.startswith(prefix): - os.remove(os.path.join(Email.TEST_DIR, file)) + # Cleanup all email files for this test suite. + prefix = Email.email_prefix(suite=True) + files = os.listdir(Email.TEST_DIR) + for file in files: + if file.startswith(prefix): + os.remove(os.path.join(Email.TEST_DIR, file)) @pytest.fixture(scope="module") @@ -155,20 +149,8 @@ def setup_database(tmp_path_factory: pathlib.Path, worker_id: str) -> None: engine = test_engine() dbname = aurweb.db.name() - if worker_id == "master": # pragma: no cover - # If we're not running tests through multiproc pytest-xdist. - setup_email() - yield _create_database(engine, dbname) - _drop_database(engine, dbname) - return - - def setup(path): - setup_email() - _create_database(engine, dbname) - - tmpdir = tmp_path_factory.getbasetemp().parent - file_lock = FileLock(tmpdir, dbname) - file_lock.lock(on_create=setup) + setup_email() + _create_database(engine, dbname) yield # Run the test function depending on this fixture. _drop_database(engine, dbname) # Cleanup the database. From 1433553c05993b097e812e43496bf140df49144c Mon Sep 17 00:00:00 2001 From: moson Date: Sat, 26 Aug 2023 17:08:36 +0200 Subject: [PATCH 044/128] fix(test): Clear previous prometheus data for test It could happen that test data is already generated by a previous test. (running in the same worker) Make sure we clear everything before performing our checks. Signed-off-by: moson --- test/test_statistics.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_statistics.py b/test/test_statistics.py index db262fa3..80223cbd 100644 --- a/test/test_statistics.py +++ b/test/test_statistics.py @@ -1,7 +1,7 @@ import pytest from prometheus_client import REGISTRY, generate_latest -from aurweb import cache, db, time +from aurweb import cache, db, prometheus, time from aurweb.models import Package, PackageBase, PackageRequest from aurweb.models.account_type import TRUSTED_USER_ID, USER_ID from aurweb.models.package_request import ( @@ -140,6 +140,11 @@ def test_get_count_change(stats: Statistics, test_data): def test_update_prometheus_metrics(test_data): + # Make sure any previous data is cleared + prometheus.USERS.clear() + prometheus.PACKAGES.clear() + prometheus.REQUESTS.clear() + metrics = str(generate_latest(REGISTRY)) assert "aur_users{" not in metrics From 0a7b02956feeaaeea4813650b12ead15cfc822af Mon Sep 17 00:00:00 2001 From: moson Date: Sun, 3 Sep 2023 14:17:11 +0200 Subject: [PATCH 045/128] feat: Indicate dependency source Dependencies might reside in the AUR or official repositories. Add "AUR" as superscript letters to indicate if a package/provider is present in the AUR. Signed-off-by: moson --- aurweb/models/package_dependency.py | 7 +++- aurweb/packages/util.py | 8 ++-- .../partials/packages/package_metadata.html | 7 +++- test/test_package_dependency.py | 17 ++++++++ test/test_packages_util.py | 40 +++++++++++++++++++ 5 files changed, 72 insertions(+), 7 deletions(-) diff --git a/aurweb/models/package_dependency.py b/aurweb/models/package_dependency.py index 587ba68d..9cf1eda0 100644 --- a/aurweb/models/package_dependency.py +++ b/aurweb/models/package_dependency.py @@ -57,14 +57,17 @@ class PackageDependency(Base): params=("NULL"), ) - def is_package(self) -> bool: + def is_aur_package(self) -> bool: pkg = db.query(_Package).filter(_Package.Name == self.DepName).exists() + return db.query(pkg).scalar() + + def is_package(self) -> bool: official = ( db.query(_OfficialProvider) .filter(_OfficialProvider.Name == self.DepName) .exists() ) - return db.query(pkg).scalar() or db.query(official).scalar() + return self.is_aur_package() or db.query(official).scalar() def provides(self) -> list[PackageRelation]: from aurweb.models.relation_type import PROVIDES_ID diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index 78d79508..cfd1e9e9 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -83,9 +83,11 @@ def package_link(package: Union[Package, OfficialProvider]) -> str: @register_filter("provides_markup") def provides_markup(provides: Providers) -> str: - return ", ".join( - [f'{pkg.Name}' for pkg in provides] - ) + links = [] + for pkg in provides: + aur = "ᴬᵁᴿ" if not pkg.is_official else "" + links.append(f'{pkg.Name}{aur}') + return ", ".join(links) def get_pkg_or_base( diff --git a/templates/partials/packages/package_metadata.html b/templates/partials/packages/package_metadata.html index 50d38b48..c8d583a1 100644 --- a/templates/partials/packages/package_metadata.html +++ b/templates/partials/packages/package_metadata.html @@ -14,12 +14,15 @@ {% endif %} {{ dep.DepName }} - {% if broken %} + {%- if broken %} {% if not provides %} {% endif %} - {% else %} + {% else -%} + {%- if dep.is_aur_package() -%} + ᴬᵁᴿ + {% endif %} {% endif %} {% if provides %} diff --git a/test/test_package_dependency.py b/test/test_package_dependency.py index 9366bb55..1cd2d305 100644 --- a/test/test_package_dependency.py +++ b/test/test_package_dependency.py @@ -4,6 +4,7 @@ from sqlalchemy.exc import IntegrityError from aurweb import db from aurweb.models.account_type import USER_ID from aurweb.models.dependency_type import DEPENDS_ID +from aurweb.models.official_provider import OfficialProvider from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.package_dependency import PackageDependency @@ -58,6 +59,22 @@ def test_package_dependencies(user: User, package: Package): db.create(Package, PackageBase=base, Name=pkgdep.DepName) assert pkgdep.is_package() + assert pkgdep.is_aur_package() + + # Test with OfficialProvider + with db.begin(): + pkgdep = db.create( + PackageDependency, + Package=package, + DepTypeID=DEPENDS_ID, + DepName="test-repo-pkg", + ) + db.create( + OfficialProvider, Name=pkgdep.DepName, Repo="extra", Provides=pkgdep.DepName + ) + + assert pkgdep.is_package() + assert not pkgdep.is_aur_package() def test_package_dependencies_null_package_raises(): diff --git a/test/test_packages_util.py b/test/test_packages_util.py index bae84614..b429181b 100644 --- a/test/test_packages_util.py +++ b/test/test_packages_util.py @@ -10,8 +10,10 @@ from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_notification import PackageNotification +from aurweb.models.package_relation import PackageRelation from aurweb.models.package_source import PackageSource from aurweb.models.package_vote import PackageVote +from aurweb.models.relation_type import PROVIDES_ID from aurweb.models.user import User from aurweb.packages import util @@ -155,3 +157,41 @@ def test_pkg_required(package: Package): # We should have 1 record assert qry.count() == 1 + + +def test_provides_markup(package: Package): + # Create dependency and provider for AUR pkg + with db.begin(): + dep = db.create( + PackageDependency, + Package=package, + DepName="test", + DepTypeID=DEPENDS_ID, + ) + rel_pkg = db.create(Package, PackageBase=package.PackageBase, Name=dep.DepName) + db.create( + PackageRelation, + Package=rel_pkg, + RelName=dep.DepName, + RelTypeID=PROVIDES_ID, + ) + + # AUR provider links should end with ᴬᵁᴿ + link = util.provides_markup(dep.provides()) + assert link.endswith("ᴬᵁᴿ") + assert OFFICIAL_BASE not in link + + # Remove AUR provider and add official one + with db.begin(): + db.delete(rel_pkg) + db.create( + OfficialProvider, + Name="official-pkg", + Repo="extra", + Provides=dep.DepName, + ) + + # Repo provider links should not have any suffix + link = util.provides_markup(dep.provides()) + assert link.endswith("") + assert OFFICIAL_BASE in link From 7466e964498cd9d19b93e1f38394ae358a5e6a5f Mon Sep 17 00:00:00 2001 From: moson Date: Tue, 26 Sep 2023 13:47:03 +0200 Subject: [PATCH 046/128] fix(ci): Exclude review-app jobs for renovate MR's Signed-off-by: moson --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf80ab24..fb40d414 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,6 +90,8 @@ deploy_review: on_stop: stop_review auto_stop_in: 1 week rules: + - if: $CI_COMMIT_REF_NAME =~ /^renovate\// + when: never - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" when: manual @@ -141,6 +143,8 @@ provision_review: name: review/$CI_COMMIT_REF_NAME action: access rules: + - if: $CI_COMMIT_REF_NAME =~ /^renovate\// + when: never - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" stop_review: @@ -155,5 +159,7 @@ stop_review: name: review/$CI_COMMIT_REF_NAME action: stop rules: + - if: $CI_COMMIT_REF_NAME =~ /^renovate\// + when: never - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" when: manual From 1702075875514de8170b4393d5326cb61e7c5e6e Mon Sep 17 00:00:00 2001 From: moson Date: Fri, 1 Sep 2023 13:25:21 +0200 Subject: [PATCH 047/128] housekeep: TU rename - code changes Renaming of symbols. Functions, variables, values, DB values, etc. Basically everything that is not user-facing. This only covers "Trusted User" things: tests, comments, etc. will covered in a following commit. --- aurweb/auth/__init__.py | 4 +- aurweb/auth/creds.py | 76 ++++++++++--------- aurweb/initdb.py | 4 +- aurweb/models/account_type.py | 12 +-- aurweb/models/user.py | 12 +-- aurweb/pkgbase/actions.py | 2 +- aurweb/routers/__init__.py | 4 +- aurweb/routers/accounts.py | 16 ++-- ...{trusted_user.py => package_maintainer.py} | 59 +++++++------- aurweb/statistics.py | 16 ++-- aurweb/users/validate.py | 2 +- ...d126029_rename_tu_to_package_maintainer.py | 37 +++++++++ schema/gendummydata.py | 28 +++---- templates/addvote.html | 12 +-- templates/partials/archdev-navbar.html | 2 +- .../partials/packages/search_actions.html | 2 +- templates/partials/packages/statistics.html | 2 +- templates/partials/tu/proposals.html | 2 +- templates/tu/index.html | 6 +- test/test_accounts_routes.py | 48 ++++++------ test/test_adduser.py | 4 +- test/test_auth.py | 4 +- test/test_homepage.py | 2 +- test/test_html.py | 10 +-- test/test_notify.py | 4 +- test/test_packages_routes.py | 4 +- test/test_pkgbase_routes.py | 4 +- test/test_requests.py | 4 +- test/test_statistics.py | 14 ++-- test/test_trusted_user_routes.py | 14 ++-- test/test_tu_vote.py | 4 +- test/test_tu_voteinfo.py | 4 +- test/test_tuvotereminder.py | 8 +- test/test_user.py | 42 +++++----- 34 files changed, 265 insertions(+), 203 deletions(-) rename aurweb/routers/{trusted_user.py => package_maintainer.py} (88%) create mode 100644 migrations/versions/6a64dd126029_rename_tu_to_package_maintainer.py diff --git a/aurweb/auth/__init__.py b/aurweb/auth/__init__.py index 83dd424c..e895dcdb 100644 --- a/aurweb/auth/__init__.py +++ b/aurweb/auth/__init__.py @@ -71,7 +71,7 @@ class AnonymousUser: return False @staticmethod - def is_trusted_user(): + def is_package_maintainer(): return False @staticmethod @@ -205,7 +205,7 @@ def account_type_required(one_of: set): @router.get('/some_route') @auth_required(True) - @account_type_required({"Trusted User", "Trusted User & Developer"}) + @account_type_required({"Package Maintainer", "Package Maintainer & Developer"}) async def some_route(request: fastapi.Request): return Response() diff --git a/aurweb/auth/creds.py b/aurweb/auth/creds.py index 17d02a5b..594188ca 100644 --- a/aurweb/auth/creds.py +++ b/aurweb/auth/creds.py @@ -1,7 +1,7 @@ from aurweb.models.account_type import ( DEVELOPER_ID, - TRUSTED_USER_AND_DEV_ID, - TRUSTED_USER_ID, + PACKAGE_MAINTAINER_AND_DEV_ID, + PACKAGE_MAINTAINER_ID, USER_ID, ) from aurweb.models.user import User @@ -30,47 +30,49 @@ PKGBASE_VOTE = 16 PKGREQ_FILE = 23 PKGREQ_CLOSE = 17 PKGREQ_LIST = 18 -TU_ADD_VOTE = 19 -TU_LIST_VOTES = 20 -TU_VOTE = 21 +PM_ADD_VOTE = 19 +PM_LIST_VOTES = 20 +PM_VOTE = 21 PKGBASE_MERGE = 29 -user_developer_or_trusted_user = set( - [USER_ID, TRUSTED_USER_ID, DEVELOPER_ID, TRUSTED_USER_AND_DEV_ID] +user_developer_or_package_maintainer = set( + [USER_ID, PACKAGE_MAINTAINER_ID, DEVELOPER_ID, PACKAGE_MAINTAINER_AND_DEV_ID] ) -trusted_user_or_dev = set([TRUSTED_USER_ID, DEVELOPER_ID, TRUSTED_USER_AND_DEV_ID]) -developer = set([DEVELOPER_ID, TRUSTED_USER_AND_DEV_ID]) -trusted_user = set([TRUSTED_USER_ID, TRUSTED_USER_AND_DEV_ID]) +package_maintainer_or_dev = set( + [PACKAGE_MAINTAINER_ID, DEVELOPER_ID, PACKAGE_MAINTAINER_AND_DEV_ID] +) +developer = set([DEVELOPER_ID, PACKAGE_MAINTAINER_AND_DEV_ID]) +package_maintainer = set([PACKAGE_MAINTAINER_ID, PACKAGE_MAINTAINER_AND_DEV_ID]) cred_filters = { - PKGBASE_FLAG: user_developer_or_trusted_user, - PKGBASE_NOTIFY: user_developer_or_trusted_user, - PKGBASE_VOTE: user_developer_or_trusted_user, - PKGREQ_FILE: user_developer_or_trusted_user, - ACCOUNT_CHANGE_TYPE: trusted_user_or_dev, - ACCOUNT_EDIT: trusted_user_or_dev, - ACCOUNT_LAST_LOGIN: trusted_user_or_dev, - ACCOUNT_LIST_COMMENTS: trusted_user_or_dev, - ACCOUNT_SEARCH: trusted_user_or_dev, - COMMENT_DELETE: trusted_user_or_dev, - COMMENT_UNDELETE: trusted_user_or_dev, - COMMENT_VIEW_DELETED: trusted_user_or_dev, - COMMENT_EDIT: trusted_user_or_dev, - COMMENT_PIN: trusted_user_or_dev, - PKGBASE_ADOPT: trusted_user_or_dev, - PKGBASE_SET_KEYWORDS: trusted_user_or_dev, - PKGBASE_DELETE: trusted_user_or_dev, - PKGBASE_EDIT_COMAINTAINERS: trusted_user_or_dev, - PKGBASE_DISOWN: trusted_user_or_dev, - PKGBASE_LIST_VOTERS: trusted_user_or_dev, - PKGBASE_UNFLAG: trusted_user_or_dev, - PKGREQ_CLOSE: trusted_user_or_dev, - PKGREQ_LIST: trusted_user_or_dev, - TU_ADD_VOTE: trusted_user, - TU_LIST_VOTES: trusted_user_or_dev, - TU_VOTE: trusted_user, + PKGBASE_FLAG: user_developer_or_package_maintainer, + PKGBASE_NOTIFY: user_developer_or_package_maintainer, + PKGBASE_VOTE: user_developer_or_package_maintainer, + PKGREQ_FILE: user_developer_or_package_maintainer, + ACCOUNT_CHANGE_TYPE: package_maintainer_or_dev, + ACCOUNT_EDIT: package_maintainer_or_dev, + ACCOUNT_LAST_LOGIN: package_maintainer_or_dev, + ACCOUNT_LIST_COMMENTS: package_maintainer_or_dev, + ACCOUNT_SEARCH: package_maintainer_or_dev, + COMMENT_DELETE: package_maintainer_or_dev, + COMMENT_UNDELETE: package_maintainer_or_dev, + COMMENT_VIEW_DELETED: package_maintainer_or_dev, + COMMENT_EDIT: package_maintainer_or_dev, + COMMENT_PIN: package_maintainer_or_dev, + PKGBASE_ADOPT: package_maintainer_or_dev, + PKGBASE_SET_KEYWORDS: package_maintainer_or_dev, + PKGBASE_DELETE: package_maintainer_or_dev, + PKGBASE_EDIT_COMAINTAINERS: package_maintainer_or_dev, + PKGBASE_DISOWN: package_maintainer_or_dev, + PKGBASE_LIST_VOTERS: package_maintainer_or_dev, + PKGBASE_UNFLAG: package_maintainer_or_dev, + PKGREQ_CLOSE: package_maintainer_or_dev, + PKGREQ_LIST: package_maintainer_or_dev, + PM_ADD_VOTE: package_maintainer, + PM_LIST_VOTES: package_maintainer_or_dev, + PM_VOTE: package_maintainer, ACCOUNT_EDIT_DEV: developer, - PKGBASE_MERGE: trusted_user_or_dev, + PKGBASE_MERGE: package_maintainer_or_dev, } diff --git a/aurweb/initdb.py b/aurweb/initdb.py index ee59212c..7181ea3e 100644 --- a/aurweb/initdb.py +++ b/aurweb/initdb.py @@ -13,9 +13,9 @@ def feed_initial_data(conn): aurweb.schema.AccountTypes.insert(), [ {"ID": 1, "AccountType": "User"}, - {"ID": 2, "AccountType": "Trusted User"}, + {"ID": 2, "AccountType": "Package Maintainer"}, {"ID": 3, "AccountType": "Developer"}, - {"ID": 4, "AccountType": "Trusted User & Developer"}, + {"ID": 4, "AccountType": "Package Maintainer & Developer"}, ], ) conn.execute( diff --git a/aurweb/models/account_type.py b/aurweb/models/account_type.py index 315800a7..70bfc2c5 100644 --- a/aurweb/models/account_type.py +++ b/aurweb/models/account_type.py @@ -2,21 +2,21 @@ from aurweb import schema from aurweb.models.declarative import Base USER = "User" -TRUSTED_USER = "Trusted User" +PACKAGE_MAINTAINER = "Package Maintainer" DEVELOPER = "Developer" -TRUSTED_USER_AND_DEV = "Trusted User & Developer" +PACKAGE_MAINTAINER_AND_DEV = "Package Maintainer & Developer" USER_ID = 1 -TRUSTED_USER_ID = 2 +PACKAGE_MAINTAINER_ID = 2 DEVELOPER_ID = 3 -TRUSTED_USER_AND_DEV_ID = 4 +PACKAGE_MAINTAINER_AND_DEV_ID = 4 # Map string constants to integer constants. ACCOUNT_TYPE_ID = { USER: USER_ID, - TRUSTED_USER: TRUSTED_USER_ID, + PACKAGE_MAINTAINER: PACKAGE_MAINTAINER_ID, DEVELOPER: DEVELOPER_ID, - TRUSTED_USER_AND_DEV: TRUSTED_USER_AND_DEV_ID, + PACKAGE_MAINTAINER_AND_DEV: PACKAGE_MAINTAINER_AND_DEV_ID, } # Reversed ACCOUNT_TYPE_ID mapping. diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 8612c259..f90d19eb 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -157,25 +157,25 @@ class User(Base): with db.begin(): db.delete(self.session) - def is_trusted_user(self): + def is_package_maintainer(self): return self.AccountType.ID in { - aurweb.models.account_type.TRUSTED_USER_ID, - aurweb.models.account_type.TRUSTED_USER_AND_DEV_ID, + aurweb.models.account_type.PACKAGE_MAINTAINER_ID, + aurweb.models.account_type.PACKAGE_MAINTAINER_AND_DEV_ID, } def is_developer(self): return self.AccountType.ID in { aurweb.models.account_type.DEVELOPER_ID, - aurweb.models.account_type.TRUSTED_USER_AND_DEV_ID, + aurweb.models.account_type.PACKAGE_MAINTAINER_AND_DEV_ID, } def is_elevated(self): """A User is 'elevated' when they have either a Trusted User or Developer AccountType.""" return self.AccountType.ID in { - aurweb.models.account_type.TRUSTED_USER_ID, + aurweb.models.account_type.PACKAGE_MAINTAINER_ID, aurweb.models.account_type.DEVELOPER_ID, - aurweb.models.account_type.TRUSTED_USER_AND_DEV_ID, + aurweb.models.account_type.PACKAGE_MAINTAINER_AND_DEV_ID, } def can_edit_user(self, target: "User") -> bool: diff --git a/aurweb/pkgbase/actions.py b/aurweb/pkgbase/actions.py index 00efc1ff..f3688f54 100644 --- a/aurweb/pkgbase/actions.py +++ b/aurweb/pkgbase/actions.py @@ -187,7 +187,7 @@ def pkgbase_merge_instance( # Log this out for accountability purposes. logger.info( - f"Trusted User '{request.user.Username}' merged " + f"Package Maintainer '{request.user.Username}' merged " f"'{pkgbasename}' into '{target.Name}'." ) diff --git a/aurweb/routers/__init__.py b/aurweb/routers/__init__.py index f77bce4f..ccd70662 100644 --- a/aurweb/routers/__init__.py +++ b/aurweb/routers/__init__.py @@ -7,13 +7,13 @@ from . import ( accounts, auth, html, + package_maintainer, packages, pkgbase, requests, rpc, rss, sso, - trusted_user, ) """ @@ -28,7 +28,7 @@ APP_ROUTES = [ packages, pkgbase, requests, - trusted_user, + package_maintainer, rss, rpc, sso, diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 1c81ec1d..a2d167bc 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -184,9 +184,9 @@ def make_account_form_context( lambda e: request.user.AccountTypeID >= e[0], [ (at.USER_ID, f"Normal {at.USER}"), - (at.TRUSTED_USER_ID, at.TRUSTED_USER), + (at.PACKAGE_MAINTAINER_ID, at.PACKAGE_MAINTAINER), (at.DEVELOPER_ID, at.DEVELOPER), - (at.TRUSTED_USER_AND_DEV_ID, at.TRUSTED_USER_AND_DEV), + (at.PACKAGE_MAINTAINER_AND_DEV_ID, at.PACKAGE_MAINTAINER_AND_DEV), ], ) ) @@ -520,7 +520,9 @@ async def account_comments(request: Request, username: str): @router.get("/accounts") @requires_auth -@account_type_required({at.TRUSTED_USER, at.DEVELOPER, at.TRUSTED_USER_AND_DEV}) +@account_type_required( + {at.PACKAGE_MAINTAINER, at.DEVELOPER, at.PACKAGE_MAINTAINER_AND_DEV} +) async def accounts(request: Request): context = make_context(request, "Accounts") return render_template(request, "account/search.html", context) @@ -529,7 +531,9 @@ async def accounts(request: Request): @router.post("/accounts") @handle_form_exceptions @requires_auth -@account_type_required({at.TRUSTED_USER, at.DEVELOPER, at.TRUSTED_USER_AND_DEV}) +@account_type_required( + {at.PACKAGE_MAINTAINER, at.DEVELOPER, at.PACKAGE_MAINTAINER_AND_DEV} +) async def accounts_post( request: Request, O: int = Form(default=0), # Offset @@ -564,9 +568,9 @@ async def accounts_post( # Convert parameter T to an AccountType ID. account_types = { "u": at.USER_ID, - "t": at.TRUSTED_USER_ID, + "t": at.PACKAGE_MAINTAINER_ID, "d": at.DEVELOPER_ID, - "td": at.TRUSTED_USER_AND_DEV_ID, + "td": at.PACKAGE_MAINTAINER_AND_DEV_ID, } account_type_id = account_types.get(T, None) diff --git a/aurweb/routers/trusted_user.py b/aurweb/routers/package_maintainer.py similarity index 88% rename from aurweb/routers/trusted_user.py rename to aurweb/routers/package_maintainer.py index 4248347d..c5e70dcf 100644 --- a/aurweb/routers/trusted_user.py +++ b/aurweb/routers/package_maintainer.py @@ -11,7 +11,10 @@ from aurweb import aur_logging, db, l10n, models, time from aurweb.auth import creds, requires_auth from aurweb.exceptions import handle_form_exceptions from aurweb.models import User -from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID +from aurweb.models.account_type import ( + PACKAGE_MAINTAINER_AND_DEV_ID, + PACKAGE_MAINTAINER_ID, +) from aurweb.templates import make_context, make_variable_context, render_template router = APIRouter() @@ -26,32 +29,32 @@ ADDVOTE_SPECIFICS = { # When a proposal is added, duration is added to the current # timestamp. # "addvote_type": (duration, quorum) - "add_tu": (7 * 24 * 60 * 60, 0.66), - "remove_tu": (7 * 24 * 60 * 60, 0.75), - "remove_inactive_tu": (5 * 24 * 60 * 60, 0.66), + "add_pm": (7 * 24 * 60 * 60, 0.66), + "remove_pm": (7 * 24 * 60 * 60, 0.75), + "remove_inactive_pm": (5 * 24 * 60 * 60, 0.66), "bylaws": (7 * 24 * 60 * 60, 0.75), } -def populate_trusted_user_counts(context: dict[str, Any]) -> None: - tu_query = db.query(User).filter( +def populate_package_maintainer_counts(context: dict[str, Any]) -> None: + pm_query = db.query(User).filter( or_( - User.AccountTypeID == TRUSTED_USER_ID, - User.AccountTypeID == TRUSTED_USER_AND_DEV_ID, + User.AccountTypeID == PACKAGE_MAINTAINER_ID, + User.AccountTypeID == PACKAGE_MAINTAINER_AND_DEV_ID, ) ) - context["trusted_user_count"] = tu_query.count() + context["package_maintainer_count"] = pm_query.count() # In case any records have a None InactivityTS. - active_tu_query = tu_query.filter( + active_pm_query = pm_query.filter( or_(User.InactivityTS.is_(None), User.InactivityTS == 0) ) - context["active_trusted_user_count"] = active_tu_query.count() + context["active_package_maintainer_count"] = active_pm_query.count() @router.get("/tu") @requires_auth -async def trusted_user( +async def package_maintainer( request: Request, coff: int = 0, # current offset cby: str = "desc", # current by @@ -63,7 +66,7 @@ async def trusted_user( if not request.user.has_credential(creds.TU_LIST_VOTES): return RedirectResponse("/", status_code=HTTPStatus.SEE_OTHER) - context = make_context(request, "Trusted User") + context = make_context(request, "Package Maintainer") current_by, past_by = cby, pby current_off, past_off = coff, poff @@ -108,7 +111,7 @@ async def trusted_user( context["past_off"] = past_off last_vote = func.max(models.TUVote.VoteID).label("LastVote") - last_votes_by_tu = ( + last_votes_by_pm = ( db.query(models.TUVote) .join(models.User) .join(models.TUVoteInfo, models.TUVoteInfo.ID == models.TUVote.VoteID) @@ -124,12 +127,12 @@ async def trusted_user( .group_by(models.TUVote.UserID) .order_by(last_vote.desc(), models.User.Username.asc()) ) - context["last_votes_by_tu"] = last_votes_by_tu.all() + context["last_votes_by_pm"] = last_votes_by_pm.all() context["current_by_next"] = "asc" if current_by == "desc" else "desc" context["past_by_next"] = "asc" if past_by == "desc" else "desc" - populate_trusted_user_counts(context) + populate_package_maintainer_counts(context) context["q"] = { "coff": current_off, @@ -178,11 +181,11 @@ def render_proposal( @router.get("/tu/{proposal}") @requires_auth -async def trusted_user_proposal(request: Request, proposal: int): +async def package_maintainer_proposal(request: Request, proposal: int): if not request.user.has_credential(creds.TU_LIST_VOTES): return RedirectResponse("/tu", status_code=HTTPStatus.SEE_OTHER) - context = await make_variable_context(request, "Trusted User") + context = await make_variable_context(request, "Package Maintainer") proposal = int(proposal) voteinfo = ( @@ -221,13 +224,13 @@ async def trusted_user_proposal(request: Request, proposal: int): @router.post("/tu/{proposal}") @handle_form_exceptions @requires_auth -async def trusted_user_proposal_post( +async def package_maintainer_proposal_post( request: Request, proposal: int, decision: str = Form(...) ): if not request.user.has_credential(creds.TU_LIST_VOTES): return RedirectResponse("/tu", status_code=HTTPStatus.SEE_OTHER) - context = await make_variable_context(request, "Trusted User") + context = await make_variable_context(request, "Package Maintainer") proposal = int(proposal) # Make sure it's an int. voteinfo = ( @@ -285,8 +288,8 @@ async def trusted_user_proposal_post( @router.get("/addvote") @requires_auth -async def trusted_user_addvote( - request: Request, user: str = str(), type: str = "add_tu", agenda: str = str() +async def package_maintainer_addvote( + request: Request, user: str = str(), type: str = "add_pm", agenda: str = str() ): if not request.user.has_credential(creds.TU_ADD_VOTE): return RedirectResponse("/tu", status_code=HTTPStatus.SEE_OTHER) @@ -295,7 +298,7 @@ async def trusted_user_addvote( if type not in ADDVOTE_SPECIFICS: context["error"] = "Invalid type." - type = "add_tu" # Default it. + type = "add_pm" # Default it. context["user"] = user context["type"] = type @@ -308,7 +311,7 @@ async def trusted_user_addvote( @router.post("/addvote") @handle_form_exceptions @requires_auth -async def trusted_user_addvote_post( +async def package_maintainer_addvote_post( request: Request, user: str = Form(default=str()), type: str = Form(default=str()), @@ -352,7 +355,7 @@ async def trusted_user_addvote_post( if type not in ADDVOTE_SPECIFICS: context["error"] = "Invalid type." - context["type"] = type = "add_tu" # Default for rendering. + context["type"] = type = "add_pm" # Default for rendering. return render_addvote(context, HTTPStatus.BAD_REQUEST) if not agenda: @@ -364,11 +367,11 @@ async def trusted_user_addvote_post( timestamp = time.utcnow() # Active TU types we filter for. - types = {TRUSTED_USER_ID, TRUSTED_USER_AND_DEV_ID} + types = {PACKAGE_MAINTAINER_ID, PACKAGE_MAINTAINER_AND_DEV_ID} # Create a new TUVoteInfo (proposal)! with db.begin(): - active_tus = ( + active_pms = ( db.query(User) .filter( and_( @@ -386,7 +389,7 @@ async def trusted_user_addvote_post( Submitted=timestamp, End=(timestamp + duration), Quorum=quorum, - ActiveTUs=active_tus, + ActiveTUs=active_pms, Submitter=request.user, ) diff --git a/aurweb/statistics.py b/aurweb/statistics.py index f301b59c..00a5c151 100644 --- a/aurweb/statistics.py +++ b/aurweb/statistics.py @@ -3,7 +3,11 @@ from sqlalchemy import func from aurweb import config, db, time from aurweb.cache import db_count_cache, db_query_cache from aurweb.models import PackageBase, PackageRequest, RequestType, User -from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID, USER_ID +from aurweb.models.account_type import ( + PACKAGE_MAINTAINER_AND_DEV_ID, + PACKAGE_MAINTAINER_ID, + USER_ID, +) from aurweb.models.package_request import ( ACCEPTED_ID, CLOSED_ID, @@ -22,7 +26,7 @@ HOMEPAGE_COUNTERS = [ "year_old_updated", "never_updated", "user_count", - "trusted_user_count", + "package_maintainer_count", ] REQUEST_COUNTERS = [ "total_requests", @@ -32,7 +36,7 @@ REQUEST_COUNTERS = [ "rejected_requests", ] PROMETHEUS_USER_COUNTERS = [ - ("trusted_user_count", "tu"), + ("package_maintainer_count", "package_maintainer"), ("regular_user_count", "user"), ] PROMETHEUS_PACKAGE_COUNTERS = [ @@ -92,12 +96,12 @@ class Statistics: # Users case "user_count": query = self.user_query - case "trusted_user_count": + case "package_maintainer_count": query = self.user_query.filter( User.AccountTypeID.in_( ( - TRUSTED_USER_ID, - TRUSTED_USER_AND_DEV_ID, + PACKAGE_MAINTAINER_ID, + PACKAGE_MAINTAINER_AND_DEV_ID, ) ) ) diff --git a/aurweb/users/validate.py b/aurweb/users/validate.py index 8fc68864..5f1fcd43 100644 --- a/aurweb/users/validate.py +++ b/aurweb/users/validate.py @@ -220,7 +220,7 @@ def invalid_account_type( raise ValidationError([error]) logger.debug( - f"Trusted User '{request.user.Username}' has " + f"Package Maintainer '{request.user.Username}' has " f"modified '{user.Username}' account's type to" f" {name}." ) diff --git a/migrations/versions/6a64dd126029_rename_tu_to_package_maintainer.py b/migrations/versions/6a64dd126029_rename_tu_to_package_maintainer.py new file mode 100644 index 00000000..005549b0 --- /dev/null +++ b/migrations/versions/6a64dd126029_rename_tu_to_package_maintainer.py @@ -0,0 +1,37 @@ +"""Rename TU to Package Maintainer + +Revision ID: 6a64dd126029 +Revises: c5a6a9b661a0 +Create Date: 2023-09-01 13:48:15.315244 + +""" +from aurweb import db +from aurweb.models import AccountType + +# revision identifiers, used by Alembic. +revision = "6a64dd126029" +down_revision = "c5a6a9b661a0" +branch_labels = None +depends_on = None + +# AccountTypes +# ID 2 -> Trusted User / Package Maintainer +# ID 4 -> Trusted User & Developer / Package Maintainer & Developer + + +def upgrade(): + with db.begin(): + tu = db.query(AccountType).filter(AccountType.ID == 2).first() + tudev = db.query(AccountType).filter(AccountType.ID == 4).first() + + tu.AccountType = "Package Maintainer" + tudev.AccountType = "Package Maintainer & Developer" + + +def downgrade(): + with db.begin(): + pm = db.query(AccountType).filter(AccountType.ID == 2).first() + pmdev = db.query(AccountType).filter(AccountType.ID == 4).first() + + pm.AccountType = "Trusted User" + pmdev.AccountType = "Trusted User & Developer" diff --git a/schema/gendummydata.py b/schema/gendummydata.py index dfc8eee5..25f85c74 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -156,9 +156,9 @@ contents = None # developer/tu IDs # developers = [] -trustedusers = [] +packagemaintainers = [] has_devs = 0 -has_tus = 0 +has_pms = 0 # Just let python throw the errors if any happen # @@ -170,7 +170,7 @@ out.write("BEGIN;\n") log.debug("Creating SQL statements for users.") for u in user_keys: account_type = 1 # default to normal user - if not has_devs or not has_tus: + if not has_devs or not has_pms: account_type = random.randrange(1, 4) if account_type == 3 and not has_devs: # this will be a dev account @@ -178,12 +178,12 @@ for u in user_keys: developers.append(seen_users[u]) if len(developers) >= MAX_DEVS * MAX_USERS: has_devs = 1 - elif account_type == 2 and not has_tus: + elif account_type == 2 and not has_pms: # this will be a trusted user account # - trustedusers.append(seen_users[u]) - if len(trustedusers) >= MAX_TUS * MAX_USERS: - has_tus = 1 + packagemaintainers.append(seen_users[u]) + if len(packagemaintainers) >= MAX_TUS * MAX_USERS: + has_pms = 1 else: # a normal user account # @@ -205,8 +205,10 @@ for u in user_keys: out.write(s) log.debug("Number of developers: %d" % len(developers)) -log.debug("Number of trusted users: %d" % len(trustedusers)) -log.debug("Number of users: %d" % (MAX_USERS - len(developers) - len(trustedusers))) +log.debug("Number of package maintainers: %d" % len(packagemaintainers)) +log.debug( + "Number of users: %d" % (MAX_USERS - len(developers) - len(packagemaintainers)) +) log.debug("Number of packages: %d" % MAX_PKGS) log.debug("Gathering text from fortune file...") @@ -224,8 +226,8 @@ for p in list(seen_pkgs.keys()): muid = developers[random.randrange(0, len(developers))] puid = developers[random.randrange(0, len(developers))] else: - muid = trustedusers[random.randrange(0, len(trustedusers))] - puid = trustedusers[random.randrange(0, len(trustedusers))] + muid = packagemaintainers[random.randrange(0, len(packagemaintainers))] + puid = packagemaintainers[random.randrange(0, len(packagemaintainers))] if count % 20 == 0: # every so often, there are orphans... muid = "NULL" @@ -339,7 +341,7 @@ for p in seen_pkgs_keys: # Create trusted user proposals # -log.debug("Creating SQL statements for trusted user proposals.") +log.debug("Creating SQL statements for package maintainer proposals.") count = 0 for t in range(0, OPEN_PROPOSALS + CLOSE_PROPOSALS): now = int(time.time()) @@ -353,7 +355,7 @@ for t in range(0, OPEN_PROPOSALS + CLOSE_PROPOSALS): user = "" else: user = user_keys[random.randrange(0, len(user_keys))] - suid = trustedusers[random.randrange(0, len(trustedusers))] + suid = packagemaintainers[random.randrange(0, len(packagemaintainers))] s = ( "INSERT INTO TU_VoteInfo (Agenda, User, Submitted, End," " Quorum, SubmitterID) VALUES ('%s', '%s', %d, %d, 0.0, %d);\n" diff --git a/templates/addvote.html b/templates/addvote.html index 8777cbf3..30b65c0e 100644 --- a/templates/addvote.html +++ b/templates/addvote.html @@ -19,22 +19,22 @@

- + - +

diff --git a/templates/addvote.html b/templates/addvote.html index 30b65c0e..cc12f42b 100644 --- a/templates/addvote.html +++ b/templates/addvote.html @@ -24,21 +24,21 @@ selected {% endif %} > - {{ "Addition of a TU" | tr }} + {{ "Addition of a Package Maintainer" | tr }}

  • - {% trans %}Trusted User{% endtrans %} + {% trans %}Package Maintainer{% endtrans %}
    • {% endif %} diff --git a/templates/partials/packages/statistics.html b/templates/partials/packages/statistics.html index 7c3c3ef6..7ce5fba1 100644 --- a/templates/partials/packages/statistics.html +++ b/templates/partials/packages/statistics.html @@ -42,7 +42,7 @@ - {{ "Trusted Users" | tr }} + {{ "Package Maintainers" | tr }} {{ package_maintainer_count }} diff --git a/templates/partials/support.html b/templates/partials/support.html index a2890cc5..b175a040 100644 --- a/templates/partials/support.html +++ b/templates/partials/support.html @@ -10,7 +10,7 @@

    • {% trans %}Orphan Request{% endtrans %}: {% trans %}Request a package to be disowned, e.g. when the maintainer is inactive and the package has been flagged out-of-date for a long time.{% endtrans %}
      • -
    • {% trans %}Deletion Request{% endtrans %}: {%trans %}Request a package to be removed from the Arch User Repository. Please do not use this if a package is broken and can be fixed easily. Instead, contact the package maintainer and file orphan request if necessary.{% endtrans %}
      • +
    • {% trans %}Deletion Request{% endtrans %}: {%trans %}Request a package to be removed from the Arch User Repository. Please do not use this if a package is broken and can be fixed easily. Instead, contact the maintainer and file orphan request if necessary.{% endtrans %}
    • {% trans %}Merge Request{% endtrans %}: {% trans %}Request a package to be merged into another one. Can be used when a package needs to be renamed or replaced by a split package.{% endtrans %}

    @@ -44,7 +44,7 @@

    {% trans %}Discussion{% endtrans %}

    - {{ "General discussion regarding the Arch User Repository (AUR) and Trusted User structure takes place on %saur-general%s. For discussion relating to the development of the AUR web interface, use the %saur-dev%s mailing list." + {{ "General discussion regarding the Arch User Repository (AUR) and Package Maintainer structure takes place on %saur-general%s. For discussion relating to the development of the AUR web interface, use the %saur-dev%s mailing list." | tr | format('', "", '', "") @@ -55,7 +55,7 @@

    {% trans %}Bug Reporting{% endtrans %}

    - {{ "If you find a bug in the AUR web interface, please fill out a bug report on our %sbug tracker%s. Use the tracker to report bugs in the AUR web interface %sonly%s. To report packaging bugs contact the package maintainer or leave a comment on the appropriate package page." + {{ "If you find a bug in the AUR web interface, please fill out a bug report on our %sbug tracker%s. Use the tracker to report bugs in the AUR web interface %sonly%s. To report packaging bugs contact the maintainer or leave a comment on the appropriate package page." | tr | format('', "", "", "") diff --git a/templates/partials/tu/proposal/details.html b/templates/partials/tu/proposal/details.html index 4cbee9ad..c74a5c5e 100644 --- a/templates/partials/tu/proposal/details.html +++ b/templates/partials/tu/proposal/details.html @@ -22,7 +22,7 @@

    - {{ "Active" | tr }} {{ "Trusted Users" | tr }} {{ "assigned" | tr }}: + {{ "Active" | tr }} {{ "Package Maintainers" | tr }} {{ "assigned" | tr }}: {{ voteinfo.ActiveTUs }}
    diff --git a/templates/pkgbase/request.html b/templates/pkgbase/request.html index 61654a49..3ffa2d2d 100644 --- a/templates/pkgbase/request.html +++ b/templates/pkgbase/request.html @@ -69,8 +69,8 @@

    {{ - "By submitting a deletion request, you ask a Trusted " - "User to delete the package base. This type of " + "By submitting a deletion request, you ask a Package " + "Maintainer to delete the package base. This type of " "request should be used for duplicates, software " "abandoned by upstream, as well as illegal and " "irreparably broken packages." | tr @@ -79,8 +79,8 @@