From 29989b7fdbb6f8a5bacfd6edef48cb66b483b722 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 17 Nov 2021 08:04:33 -0800 Subject: [PATCH] change(aurblup): converted to use aurweb.db ORM Introduces: - aurweb.testing.alpm.AlpmDatabase - Used to mock up and manage a remote repository. - templates/testing/alpm_package.j2 - Used to generate a single ALPM package desc. - Removed aurblup sharness test Signed-off-by: Kevin Morris --- aurweb/scripts/aurblup.py | 50 ++++++++++------- aurweb/templates.py | 5 ++ aurweb/testing/alpm.py | 87 ++++++++++++++++++++++++++++++ aurweb/util.py | 17 ++++++ templates/testing/alpm_package.j2 | 16 ++++++ test/t2400-aurblup.t | 53 ------------------ test/test_aurblup.py | 90 +++++++++++++++++++++++++++++++ 7 files changed, 246 insertions(+), 72 deletions(-) create mode 100644 aurweb/testing/alpm.py create mode 100644 templates/testing/alpm_package.j2 delete mode 100755 test/t2400-aurblup.t create mode 100644 test/test_aurblup.py diff --git a/aurweb/scripts/aurblup.py b/aurweb/scripts/aurblup.py index e32937ce..9c9059ec 100755 --- a/aurweb/scripts/aurblup.py +++ b/aurweb/scripts/aurblup.py @@ -4,30 +4,34 @@ import re import pyalpm +from sqlalchemy import and_ + import aurweb.config -import aurweb.db -db_path = aurweb.config.get('aurblup', 'db-path') -sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ') -server = aurweb.config.get('aurblup', 'server') +from aurweb import db, util +from aurweb.models import OfficialProvider -def main(): +def _main(force: bool = False): blacklist = set() providers = set() repomap = dict() + db_path = aurweb.config.get("aurblup", "db-path") + sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ') + server = aurweb.config.get('aurblup', 'server') + h = pyalpm.Handle("/", db_path) for sync_db in sync_dbs: repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) repo.servers = [server.replace("%s", sync_db)] t = h.init_transaction() - repo.update(False) + repo.update(force) t.release() for pkg in repo.pkgcache: blacklist.add(pkg.name) - [blacklist.add(x) for x in pkg.replaces] + util.apply_all(pkg.replaces, blacklist.add) providers.add((pkg.name, pkg.name)) repomap[(pkg.name, pkg.name)] = repo.name for provision in pkg.provides: @@ -35,21 +39,29 @@ def main(): providers.add((pkg.name, provisionname)) repomap[(pkg.name, provisionname)] = repo.name - conn = aurweb.db.Connection() + with db.begin(): + old_providers = set( + db.query(OfficialProvider).with_entities( + OfficialProvider.Name.label("Name"), + OfficialProvider.Provides.label("Provides") + ).distinct().order_by("Name").all() + ) - cur = conn.execute("SELECT Name, Provides FROM OfficialProviders") - oldproviders = set(cur.fetchall()) + for name, provides in old_providers.difference(providers): + db.delete_all(db.query(OfficialProvider).filter( + and_(OfficialProvider.Name == name, + OfficialProvider.Provides == provides) + )) - for pkg, provides in oldproviders.difference(providers): - conn.execute("DELETE FROM OfficialProviders " - "WHERE Name = ? AND Provides = ?", [pkg, provides]) - for pkg, provides in providers.difference(oldproviders): - repo = repomap[(pkg, provides)] - conn.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " - "VALUES (?, ?, ?)", [pkg, repo, provides]) + for name, provides in providers.difference(old_providers): + repo = repomap.get((name, provides)) + db.create(OfficialProvider, Name=name, + Repo=repo, Provides=provides) - conn.commit() - conn.close() + +def main(force: bool = False): + db.get_engine() + _main(force) if __name__ == '__main__': diff --git a/aurweb/templates.py b/aurweb/templates.py index 0039535d..a7102ae1 100644 --- a/aurweb/templates.py +++ b/aurweb/templates.py @@ -125,6 +125,11 @@ async def make_variable_context(request: Request, title: str, next: str = None): return context +def base_template(path: str): + templates = copy.copy(_env) + return templates.get_template(path) + + def render_raw_template(request: Request, path: str, context: dict): """ Render a Jinja2 multi-lingual template with some context. """ # Create a deep copy of our jinja2 _environment. The _environment in diff --git a/aurweb/testing/alpm.py b/aurweb/testing/alpm.py new file mode 100644 index 00000000..6015d859 --- /dev/null +++ b/aurweb/testing/alpm.py @@ -0,0 +1,87 @@ +import hashlib +import os +import re +import shutil +import subprocess + +from typing import List + +from aurweb import logging, util +from aurweb.templates import base_template + +logger = logging.get_logger(__name__) + + +class AlpmDatabase: + """ + Fake libalpm database management class. + + This class can be used to add or remove packages from a + test repository. + """ + repo = "test" + + def __init__(self, database_root: str): + self.root = database_root + self.local = os.path.join(self.root, "local") + self.remote = os.path.join(self.root, "remote") + self.repopath = os.path.join(self.remote, self.repo) + + # Make directories. + os.makedirs(self.local) + os.makedirs(self.remote) + + def _get_pkgdir(self, pkgname: str, pkgver: str, repo: str) -> str: + pkgfile = f"{pkgname}-{pkgver}-1" + pkgdir = os.path.join(self.remote, repo, pkgfile) + os.makedirs(pkgdir) + return pkgdir + + def add(self, pkgname: str, pkgver: str, arch: str, + provides: List[str] = []) -> None: + context = { + "pkgname": pkgname, + "pkgver": pkgver, + "arch": arch, + "provides": provides + } + template = base_template("testing/alpm_package.j2") + pkgdir = self._get_pkgdir(pkgname, pkgver, self.repo) + desc = os.path.join(pkgdir, "desc") + with open(desc, "w") as f: + f.write(template.render(context)) + + self.compile() + + def remove(self, pkgname: str): + files = os.listdir(self.repopath) + logger.info(f"Files: {files}") + expr = "^" + pkgname + r"-[0-9.]+-1$" + logger.info(f"Expression: {expr}") + to_delete = filter(lambda e: re.match(expr, e), files) + + for target in to_delete: + logger.info(f"Deleting {target}") + path = os.path.join(self.repopath, target) + shutil.rmtree(path) + + self.compile() + + def clean(self) -> None: + db_file = os.path.join(self.remote, "test.db") + try: + os.remove(db_file) + except Exception: + pass + + def compile(self) -> None: + self.clean() + cmdline = ["bash", "-c", "bsdtar -czvf ../test.db *"] + proc = subprocess.run(cmdline, cwd=self.repopath) + assert proc.returncode == 0, \ + f"Bad return code while creating alpm database: {proc.returncode}" + + # Print out the md5 hash value of the new test.db. + test_db = os.path.join(self.remote, "test.db") + db_hash = util.file_hash(test_db, hashlib.md5) + logger.debug(f"{test_db}: {db_hash}") diff --git a/aurweb/util.py b/aurweb/util.py index 62575c71..bf2d6e4b 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -176,3 +176,20 @@ def strtobool(value: str) -> bool: if isinstance(value, str): return _strtobool(value) return value + + +def file_hash(filepath: str, hash_function: Callable) -> str: + """ + Return a hash of filepath contents using `hash_function`. + + `hash_function` can be any one of the hashlib module's hash + functions which implement the `hexdigest()` method -- e.g. + hashlib.sha1, hashlib.md5, etc. + + :param filepath: Path to file you want to hash + :param hash_function: hashlib hash function + :return: hash_function(filepath_content).hexdigest() + """ + with open(filepath, "rb") as f: + hash_ = hash_function(f.read()) + return hash_.hexdigest() diff --git a/templates/testing/alpm_package.j2 b/templates/testing/alpm_package.j2 new file mode 100644 index 00000000..0e741729 --- /dev/null +++ b/templates/testing/alpm_package.j2 @@ -0,0 +1,16 @@ +%FILENAME% +{{ pkgname }}-{{ pkgver }}-{{ arch }}.pkg.tar.xz + +%NAME% +{{ pkgname }} + +%VERSION% +{{ pkgver }}-1 + +%ARCH% +{{ arch }} + +{% if provides %} +%PROVIDES% +{{ provides | join("\n") }} +{% endif %} diff --git a/test/t2400-aurblup.t b/test/t2400-aurblup.t deleted file mode 100755 index 42da6791..00000000 --- a/test/t2400-aurblup.t +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh - -test_description='aurblup tests' - -. "$(dirname "$0")/setup.sh" - -test_expect_success 'Test official provider update script.' ' - mkdir -p remote/test/foobar-1.0-1 && - cat <<-EOD >remote/test/foobar-1.0-1/desc && - %FILENAME% - foobar-1.0-any.pkg.tar.xz - - %NAME% - foobar - - %VERSION% - 1.0-1 - - %ARCH% - any - EOD - mkdir -p remote/test/foobar2-1.0-1 && - cat <<-EOD >remote/test/foobar2-1.0-1/desc && - %FILENAME% - foobar2-1.0-any.pkg.tar.xz - - %NAME% - foobar2 - - %VERSION% - 1.0-1 - - %ARCH% - any - - %PROVIDES% - foobar3 - foobar4 - EOD - ( cd remote/test && bsdtar -czf ../test.db * ) && - mkdir sync && - cover "$AURBLUP" && - cat <<-EOD >expected && - foobar|test|foobar - foobar2|test|foobar2 - foobar2|test|foobar3 - foobar2|test|foobar4 - EOD - echo "SELECT Name, Repo, Provides FROM OfficialProviders ORDER BY Provides;" | sqlite3 aur.db >actual && - test_cmp actual expected -' - -test_done diff --git a/test/test_aurblup.py b/test/test_aurblup.py new file mode 100644 index 00000000..7eaae556 --- /dev/null +++ b/test/test_aurblup.py @@ -0,0 +1,90 @@ +import tempfile + +from unittest import mock + +import pytest + +from aurweb import config, db +from aurweb.models import OfficialProvider +from aurweb.scripts import aurblup +from aurweb.testing.alpm import AlpmDatabase + + +@pytest.fixture +def tempdir() -> str: + with tempfile.TemporaryDirectory() as name: + yield name + + +@pytest.fixture +def alpm_db(tempdir: str) -> AlpmDatabase: + yield AlpmDatabase(tempdir) + + +@pytest.fixture(autouse=True) +def setup(db_test, alpm_db: AlpmDatabase, tempdir: str) -> None: + config_get = config.get + + def mock_config_get(section: str, key: str) -> str: + value = config_get(section, key) + if section == "aurblup": + if key == "db-path": + return alpm_db.local + elif key == "server": + return f'file://{alpm_db.remote}' + elif key == "sync-dbs": + return alpm_db.repo + return value + + with mock.patch("aurweb.config.get", side_effect=mock_config_get): + config.rehash() + yield + config.rehash() + + +def test_aurblup(alpm_db: AlpmDatabase): + # Test that we can add a package. + alpm_db.add("pkg", "1.0", "x86_64", provides=["pkg2", "pkg3"]) + alpm_db.add("pkg2", "2.0", "x86_64") + aurblup.main() + + # Test that the package got added to the database. + for name in ("pkg", "pkg2"): + pkg = db.query(OfficialProvider).filter( + OfficialProvider.Name == name).first() + assert pkg is not None + + # Test that we can remove the package. + alpm_db.remove("pkg") + + # Run aurblup again with forced repository update. + aurblup.main(True) + + # Expect that the database got updated accordingly. + pkg = db.query(OfficialProvider).filter( + OfficialProvider.Name == "pkg").first() + assert pkg is None + pkg2 = db.query(OfficialProvider).filter( + OfficialProvider.Name == "pkg2").first() + assert pkg2 is not None + + +def test_aurblup_cleanup(alpm_db: AlpmDatabase): + # Add a package and sync up the database. + alpm_db.add("pkg", "1.0", "x86_64", provides=["pkg2", "pkg3"]) + aurblup.main() + + # Now, let's insert an OfficialPackage that doesn't exist, + # then exercise the old provider deletion path. + with db.begin(): + db.create(OfficialProvider, Name="fake package", + Repo="test", Provides="package") + + # Run aurblup again. + aurblup.main() + + # Expect that the fake package got deleted because it's + # not in alpm_db anymore. + providers = db.query(OfficialProvider).filter( + OfficialProvider.Name == "fake package").all() + assert len(providers) == 0