change(aurblup): converted to use aurweb.db ORM

Introduces:
- aurweb.testing.alpm.AlpmDatabase
    - Used to mock up and manage a remote repository.
- templates/testing/alpm_package.j2
    - Used to generate a single ALPM package desc.
- Removed aurblup sharness test

Signed-off-by: Kevin Morris <kevr@0cost.org>

aurweb/scripts/aurblup.py

@@ -4,30 +4,34 @@ import re
 
 import pyalpm
 
+from sqlalchemy import and_
+
 import aurweb.config
-import aurweb.db
 
-db_path = aurweb.config.get('aurblup', 'db-path')
+from aurweb import db, util
-sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ')
+from aurweb.models import OfficialProvider
-server = aurweb.config.get('aurblup', 'server')
 
 
-def main():
+def _main(force: bool = False):
     blacklist = set()
     providers = set()
     repomap = dict()
 
+    db_path = aurweb.config.get("aurblup", "db-path")
+    sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ')
+    server = aurweb.config.get('aurblup', 'server')
+
     h = pyalpm.Handle("/", db_path)
     for sync_db in sync_dbs:
         repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL)
         repo.servers = [server.replace("%s", sync_db)]
         t = h.init_transaction()
-        repo.update(False)
+        repo.update(force)
         t.release()
 
         for pkg in repo.pkgcache:
             blacklist.add(pkg.name)
-            [blacklist.add(x) for x in pkg.replaces]
+            util.apply_all(pkg.replaces, blacklist.add)
             providers.add((pkg.name, pkg.name))
             repomap[(pkg.name, pkg.name)] = repo.name
             for provision in pkg.provides:
@@ -35,21 +39,29 @@ def main():
                 providers.add((pkg.name, provisionname))
                 repomap[(pkg.name, provisionname)] = repo.name
 
-    conn = aurweb.db.Connection()
+    with db.begin():
+        old_providers = set(
+            db.query(OfficialProvider).with_entities(
+                OfficialProvider.Name.label("Name"),
+                OfficialProvider.Provides.label("Provides")
+            ).distinct().order_by("Name").all()
+        )
 
-    cur = conn.execute("SELECT Name, Provides FROM OfficialProviders")
+        for name, provides in old_providers.difference(providers):
-    oldproviders = set(cur.fetchall())
+            db.delete_all(db.query(OfficialProvider).filter(
+                and_(OfficialProvider.Name == name,
+                     OfficialProvider.Provides == provides)
+            ))
 
-    for pkg, provides in oldproviders.difference(providers):
+        for name, provides in providers.difference(old_providers):
-        conn.execute("DELETE FROM OfficialProviders "
+            repo = repomap.get((name, provides))
-                     "WHERE Name = ? AND Provides = ?", [pkg, provides])
+            db.create(OfficialProvider, Name=name,
-    for pkg, provides in providers.difference(oldproviders):
+                      Repo=repo, Provides=provides)
-        repo = repomap[(pkg, provides)]
-        conn.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) "
-                     "VALUES (?, ?, ?)", [pkg, repo, provides])
 
-    conn.commit()
+
-    conn.close()
+def main(force: bool = False):
+    db.get_engine()
+    _main(force)
 
 
 if __name__ == '__main__':

aurweb/templates.py

@@ -125,6 +125,11 @@ async def make_variable_context(request: Request, title: str, next: str = None):
     return context
 
 
+def base_template(path: str):
+    templates = copy.copy(_env)
+    return templates.get_template(path)
+
+
 def render_raw_template(request: Request, path: str, context: dict):
     """ Render a Jinja2 multi-lingual template with some context. """
     # Create a deep copy of our jinja2 _environment. The _environment in

aurweb/testing/alpm.py

@@ -0,0 +1,87 @@
+import hashlib
+import os
+import re
+import shutil
+import subprocess
+
+from typing import List
+
+from aurweb import logging, util
+from aurweb.templates import base_template
+
+logger = logging.get_logger(__name__)
+
+
+class AlpmDatabase:
+    """
+    Fake libalpm database management class.
+
+    This class can be used to add or remove packages from a
+    test repository.
+    """
+    repo = "test"
+
+    def __init__(self, database_root: str):
+        self.root = database_root
+        self.local = os.path.join(self.root, "local")
+        self.remote = os.path.join(self.root, "remote")
+        self.repopath = os.path.join(self.remote, self.repo)
+
+        # Make directories.
+        os.makedirs(self.local)
+        os.makedirs(self.remote)
+
+    def _get_pkgdir(self, pkgname: str, pkgver: str, repo: str) -> str:
+        pkgfile = f"{pkgname}-{pkgver}-1"
+        pkgdir = os.path.join(self.remote, repo, pkgfile)
+        os.makedirs(pkgdir)
+        return pkgdir
+
+    def add(self, pkgname: str, pkgver: str, arch: str,
+            provides: List[str] = []) -> None:
+        context = {
+            "pkgname": pkgname,
+            "pkgver": pkgver,
+            "arch": arch,
+            "provides": provides
+        }
+        template = base_template("testing/alpm_package.j2")
+        pkgdir = self._get_pkgdir(pkgname, pkgver, self.repo)
+        desc = os.path.join(pkgdir, "desc")
+        with open(desc, "w") as f:
+            f.write(template.render(context))
+
+        self.compile()
+
+    def remove(self, pkgname: str):
+        files = os.listdir(self.repopath)
+        logger.info(f"Files: {files}")
+        expr = "^" + pkgname + r"-[0-9.]+-1$"
+        logger.info(f"Expression: {expr}")
+        to_delete = filter(lambda e: re.match(expr, e), files)
+
+        for target in to_delete:
+            logger.info(f"Deleting {target}")
+            path = os.path.join(self.repopath, target)
+            shutil.rmtree(path)
+
+        self.compile()
+
+    def clean(self) -> None:
+        db_file = os.path.join(self.remote, "test.db")
+        try:
+            os.remove(db_file)
+        except Exception:
+            pass
+
+    def compile(self) -> None:
+        self.clean()
+        cmdline = ["bash", "-c", "bsdtar -czvf ../test.db *"]
+        proc = subprocess.run(cmdline, cwd=self.repopath)
+        assert proc.returncode == 0, \
+            f"Bad return code while creating alpm database: {proc.returncode}"
+
+        # Print out the md5 hash value of the new test.db.
+        test_db = os.path.join(self.remote, "test.db")
+        db_hash = util.file_hash(test_db, hashlib.md5)
+        logger.debug(f"{test_db}: {db_hash}")

aurweb/util.py

@@ -176,3 +176,20 @@ def strtobool(value: str) -> bool:
     if isinstance(value, str):
         return _strtobool(value)
     return value
+
+
+def file_hash(filepath: str, hash_function: Callable) -> str:
+    """
+    Return a hash of filepath contents using `hash_function`.
+
+    `hash_function` can be any one of the hashlib module's hash
+    functions which implement the `hexdigest()` method -- e.g.
+    hashlib.sha1, hashlib.md5, etc.
+
+    :param filepath: Path to file you want to hash
+    :param hash_function: hashlib hash function
+    :return: hash_function(filepath_content).hexdigest()
+    """
+    with open(filepath, "rb") as f:
+        hash_ = hash_function(f.read())
+    return hash_.hexdigest()

templates/testing/alpm_package.j2

@@ -0,0 +1,16 @@
+%FILENAME%
+{{ pkgname }}-{{ pkgver }}-{{ arch }}.pkg.tar.xz
+
+%NAME%
+{{ pkgname }}
+
+%VERSION%
+{{ pkgver }}-1
+
+%ARCH%
+{{ arch }}
+
+{% if provides %}
+%PROVIDES%
+{{ provides | join("\n") }}
+{% endif %}

test/t2400-aurblup.t

@@ -1,53 +0,0 @@
-#!/bin/sh
-
-test_description='aurblup tests'
-
-. "$(dirname "$0")/setup.sh"
-
-test_expect_success 'Test official provider update script.' '
-	mkdir -p remote/test/foobar-1.0-1 &&
-	cat <<-EOD >remote/test/foobar-1.0-1/desc &&
-	%FILENAME%
-	foobar-1.0-any.pkg.tar.xz
-
-	%NAME%
-	foobar
-
-	%VERSION%
-	1.0-1
-
-	%ARCH%
-	any
-	EOD
-	mkdir -p remote/test/foobar2-1.0-1 &&
-	cat <<-EOD >remote/test/foobar2-1.0-1/desc &&
-	%FILENAME%
-	foobar2-1.0-any.pkg.tar.xz
-
-	%NAME%
-	foobar2
-
-	%VERSION%
-	1.0-1
-
-	%ARCH%
-	any
-
-	%PROVIDES%
-	foobar3
-	foobar4
-	EOD
-	( cd remote/test && bsdtar -czf ../test.db * ) &&
-	mkdir sync &&
-	cover "$AURBLUP" &&
-	cat <<-EOD >expected &&
-	foobar|test|foobar
-	foobar2|test|foobar2
-	foobar2|test|foobar3
-	foobar2|test|foobar4
-	EOD
-	echo "SELECT Name, Repo, Provides FROM OfficialProviders ORDER BY Provides;" | sqlite3 aur.db >actual &&
-	test_cmp actual expected
-'
-
-test_done

test/test_aurblup.py

@@ -0,0 +1,90 @@
+import tempfile
+
+from unittest import mock
+
+import pytest
+
+from aurweb import config, db
+from aurweb.models import OfficialProvider
+from aurweb.scripts import aurblup
+from aurweb.testing.alpm import AlpmDatabase
+
+
+@pytest.fixture
+def tempdir() -> str:
+    with tempfile.TemporaryDirectory() as name:
+        yield name
+
+
+@pytest.fixture
+def alpm_db(tempdir: str) -> AlpmDatabase:
+    yield AlpmDatabase(tempdir)
+
+
+@pytest.fixture(autouse=True)
+def setup(db_test, alpm_db: AlpmDatabase, tempdir: str) -> None:
+    config_get = config.get
+
+    def mock_config_get(section: str, key: str) -> str:
+        value = config_get(section, key)
+        if section == "aurblup":
+            if key == "db-path":
+                return alpm_db.local
+            elif key == "server":
+                return f'file://{alpm_db.remote}'
+            elif key == "sync-dbs":
+                return alpm_db.repo
+        return value
+
+    with mock.patch("aurweb.config.get", side_effect=mock_config_get):
+        config.rehash()
+        yield
+    config.rehash()
+
+
+def test_aurblup(alpm_db: AlpmDatabase):
+    # Test that we can add a package.
+    alpm_db.add("pkg", "1.0", "x86_64", provides=["pkg2", "pkg3"])
+    alpm_db.add("pkg2", "2.0", "x86_64")
+    aurblup.main()
+
+    # Test that the package got added to the database.
+    for name in ("pkg", "pkg2"):
+        pkg = db.query(OfficialProvider).filter(
+            OfficialProvider.Name == name).first()
+        assert pkg is not None
+
+    # Test that we can remove the package.
+    alpm_db.remove("pkg")
+
+    # Run aurblup again with forced repository update.
+    aurblup.main(True)
+
+    # Expect that the database got updated accordingly.
+    pkg = db.query(OfficialProvider).filter(
+        OfficialProvider.Name == "pkg").first()
+    assert pkg is None
+    pkg2 = db.query(OfficialProvider).filter(
+        OfficialProvider.Name == "pkg2").first()
+    assert pkg2 is not None
+
+
+def test_aurblup_cleanup(alpm_db: AlpmDatabase):
+    # Add a package and sync up the database.
+    alpm_db.add("pkg", "1.0", "x86_64", provides=["pkg2", "pkg3"])
+    aurblup.main()
+
+    # Now, let's insert an OfficialPackage that doesn't exist,
+    # then exercise the old provider deletion path.
+    with db.begin():
+        db.create(OfficialProvider, Name="fake package",
+                  Repo="test", Provides="package")
+
+    # Run aurblup again.
+    aurblup.main()
+
+    # Expect that the fake package got deleted because it's
+    # not in alpm_db anymore.
+    providers = db.query(OfficialProvider).filter(
+        OfficialProvider.Name == "fake package").all()
+    assert len(providers) == 0