#!/usr/bin/env python3 import os import re import subprocess import sys import time import pygit2 import srcinfo.parse import srcinfo.utils import aurweb.config import aurweb.db notify_cmd = aurweb.config.get("notifications", "notify-cmd") repo_path = aurweb.config.get("serve", "repo-path") repo_regex = aurweb.config.get("serve", "repo-regex") max_blob_size = aurweb.config.getint("update", "max-blob-size") def size_humanize(num): for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB"]: if abs(num) < 2048.0: if isinstance(num, int): return "{}{}".format(num, unit) else: return "{:.2f}{}".format(num, unit) num /= 1024.0 return "{:.2f}{}".format(num, "YiB") def extract_arch_fields(pkginfo, field): values = [] if field in pkginfo: for val in pkginfo[field]: values.append({"value": val, "arch": None}) for arch in pkginfo["arch"]: if field + "_" + arch in pkginfo: for val in pkginfo[field + "_" + arch]: values.append({"value": val, "arch": arch}) return values def parse_dep(depstring): dep, _, desc = depstring.partition(": ") depname = re.sub(r"(<|=|>).*", "", dep) depcond = dep[len(depname) :] return depname, desc, depcond def create_pkgbase(conn, pkgbase, user): cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) userid = cur.fetchone()[0] now = int(time.time()) cur = conn.execute( "INSERT INTO PackageBases (Name, SubmittedTS, " + "ModifiedTS, SubmitterUID, MaintainerUID, " + "FlaggerComment) VALUES (?, ?, ?, ?, ?, '')", [pkgbase, now, now, userid, userid], ) pkgbase_id = cur.lastrowid cur = conn.execute( "INSERT INTO PackageNotifications " + "(PackageBaseID, UserID) VALUES (?, ?)", [pkgbase_id, userid], ) conn.commit() return pkgbase_id def save_metadata(metadata, conn, user): # noqa: C901 # Obtain package base ID and previous maintainer. pkgbase = metadata["pkgbase"] cur = conn.execute( "SELECT ID, MaintainerUID FROM PackageBases " "WHERE Name = ?", [pkgbase] ) (pkgbase_id, maintainer_uid) = cur.fetchone() was_orphan = not maintainer_uid # Obtain the user ID of the new maintainer. cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) user_id = int(cur.fetchone()[0]) # Update package base details and delete current packages. now = int(time.time()) conn.execute( "UPDATE PackageBases SET ModifiedTS = ?, " + "PackagerUID = ?, OutOfDateTS = NULL WHERE ID = ?", [now, user_id, pkgbase_id], ) conn.execute( "UPDATE PackageBases SET MaintainerUID = ? " + "WHERE ID = ? AND MaintainerUID IS NULL", [user_id, pkgbase_id], ) for table in ("Sources", "Depends", "Relations", "Licenses", "Groups"): conn.execute( "DELETE FROM Package" + table + " WHERE EXISTS (" + "SELECT * FROM Packages " + "WHERE Packages.PackageBaseID = ? AND " + "Package" + table + ".PackageID = Packages.ID)", [pkgbase_id], ) conn.execute("DELETE FROM Packages WHERE PackageBaseID = ?", [pkgbase_id]) for pkgname in srcinfo.utils.get_package_names(metadata): pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) if "epoch" in pkginfo and int(pkginfo["epoch"]) > 0: ver = "{:d}:{:s}-{:s}".format( int(pkginfo["epoch"]), pkginfo["pkgver"], pkginfo["pkgrel"] ) else: ver = "{:s}-{:s}".format(pkginfo["pkgver"], pkginfo["pkgrel"]) for field in ("pkgdesc", "url"): if field not in pkginfo: pkginfo[field] = None # Create a new package. cur = conn.execute( "INSERT INTO Packages (PackageBaseID, Name, " + "Version, Description, URL) " + "VALUES (?, ?, ?, ?, ?)", [pkgbase_id, pkginfo["pkgname"], ver, pkginfo["pkgdesc"], pkginfo["url"]], ) conn.commit() pkgid = cur.lastrowid # Add package sources. for source_info in extract_arch_fields(pkginfo, "source"): conn.execute( "INSERT INTO PackageSources (PackageID, Source, " + "SourceArch) VALUES (?, ?, ?)", [pkgid, source_info["value"], source_info["arch"]], ) # Add package dependencies. for deptype in ("depends", "makedepends", "checkdepends", "optdepends"): cur = conn.execute( "SELECT ID FROM DependencyTypes WHERE Name = ?", [deptype] ) deptypeid = cur.fetchone()[0] for dep_info in extract_arch_fields(pkginfo, deptype): depname, depdesc, depcond = parse_dep(dep_info["value"]) deparch = dep_info["arch"] conn.execute( "INSERT INTO PackageDepends (PackageID, " + "DepTypeID, DepName, DepDesc, DepCondition, " + "DepArch) VALUES (?, ?, ?, ?, ?, ?)", [pkgid, deptypeid, depname, depdesc, depcond, deparch], ) # Add package relations (conflicts, provides, replaces). for reltype in ("conflicts", "provides", "replaces"): cur = conn.execute("SELECT ID FROM RelationTypes WHERE Name = ?", [reltype]) reltypeid = cur.fetchone()[0] for rel_info in extract_arch_fields(pkginfo, reltype): relname, _, relcond = parse_dep(rel_info["value"]) relarch = rel_info["arch"] conn.execute( "INSERT INTO PackageRelations (PackageID, " + "RelTypeID, RelName, RelCondition, RelArch) " + "VALUES (?, ?, ?, ?, ?)", [pkgid, reltypeid, relname, relcond, relarch], ) # Add package licenses. if "license" in pkginfo: for license in pkginfo["license"]: cur = conn.execute("SELECT ID FROM Licenses WHERE Name = ?", [license]) row = cur.fetchone() if row: licenseid = row[0] else: cur = conn.execute( "INSERT INTO Licenses (Name) " + "VALUES (?)", [license] ) conn.commit() licenseid = cur.lastrowid conn.execute( "INSERT INTO PackageLicenses (PackageID, " + "LicenseID) VALUES (?, ?)", [pkgid, licenseid], ) # Add package groups. if "groups" in pkginfo: for group in pkginfo["groups"]: cur = conn.execute("SELECT ID FROM `Groups` WHERE Name = ?", [group]) row = cur.fetchone() if row: groupid = row[0] else: cur = conn.execute( "INSERT INTO `Groups` (Name) VALUES (?)", [group] ) conn.commit() groupid = cur.lastrowid conn.execute( "INSERT INTO PackageGroups (PackageID, " "GroupID) VALUES (?, ?)", [pkgid, groupid], ) # Add user to notification list on adoption. if was_orphan: cur = conn.execute( "SELECT COUNT(*) FROM PackageNotifications WHERE " + "PackageBaseID = ? AND UserID = ?", [pkgbase_id, user_id], ) if cur.fetchone()[0] == 0: conn.execute( "INSERT INTO PackageNotifications " + "(PackageBaseID, UserID) VALUES (?, ?)", [pkgbase_id, user_id], ) conn.commit() def update_notify(conn, user, pkgbase_id): # Obtain the user ID of the new maintainer. cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) user_id = int(cur.fetchone()[0]) # Execute the notification script. subprocess.Popen((notify_cmd, "update", str(user_id), str(pkgbase_id))) def die(msg): sys.stderr.write("error: {:s}\n".format(msg)) exit(1) def warn(msg): sys.stderr.write("warning: {:s}\n".format(msg)) def die_commit(msg, commit): sys.stderr.write("error: The following error " + "occurred when parsing commit\n") sys.stderr.write("error: {:s}:\n".format(commit)) sys.stderr.write("error: {:s}\n".format(msg)) exit(1) def main(): # noqa: C901 repo = pygit2.Repository(repo_path) user = os.environ.get("AUR_USER") pkgbase = os.environ.get("AUR_PKGBASE") privileged = os.environ.get("AUR_PRIVILEGED", "0") == "1" allow_overwrite = (os.environ.get("AUR_OVERWRITE", "0") == "1") and privileged warn_or_die = warn if privileged else die if len(sys.argv) == 2 and sys.argv[1] == "restore": if "refs/heads/" + pkgbase not in repo.listall_references(): die("{:s}: repository not found: {:s}".format(sys.argv[1], pkgbase)) refname = "refs/heads/master" branchref = "refs/heads/" + pkgbase sha1_old = sha1_new = repo.lookup_reference(branchref).target elif len(sys.argv) == 4: refname, sha1_old, sha1_new = sys.argv[1:4] else: die("invalid arguments") if refname != "refs/heads/master": die("pushing to a branch other than master is restricted") conn = aurweb.db.Connection() # Detect and deny non-fast-forwards. if sha1_old != "0" * 40 and not allow_overwrite: walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL) walker.hide(sha1_new) if next(walker, None) is not None: die("denying non-fast-forward (you should pull first)") # Prepare the walker that validates new commits. walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL) if sha1_old != "0" * 40: walker.hide(sha1_old) # Validate all new commits. for commit in walker: for fname in (".SRCINFO", "PKGBUILD"): if fname not in commit.tree: die_commit("missing {:s}".format(fname), str(commit.id)) for treeobj in commit.tree: blob = repo[treeobj.id] if isinstance(blob, pygit2.Tree): die_commit( "the repository must not contain subdirectories", str(commit.id) ) if not isinstance(blob, pygit2.Blob): die_commit("not a blob object: {:s}".format(treeobj), str(commit.id)) if blob.size > max_blob_size: die_commit( "maximum blob size ({:s}) exceeded".format( size_humanize(max_blob_size) ), str(commit.id), ) metadata_raw = repo[commit.tree[".SRCINFO"].id].data.decode() (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) if errors: sys.stderr.write( "error: The following errors occurred " "when parsing .SRCINFO in commit\n" ) sys.stderr.write("error: {:s}:\n".format(str(commit.id))) for error in errors: for err in error["error"]: sys.stderr.write( "error: line {:d}: {:s}\n".format(error["line"], err) ) exit(1) try: metadata_pkgbase = metadata["pkgbase"] except KeyError: die_commit( "invalid .SRCINFO, does not contain a pkgbase (is the file empty?)", str(commit.id), ) if not re.match(repo_regex, metadata_pkgbase): die_commit("invalid pkgbase: {:s}".format(metadata_pkgbase), str(commit.id)) if not metadata["packages"]: die_commit("missing pkgname entry", str(commit.id)) for pkgname in set(metadata["packages"].keys()): pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) for field in ("pkgver", "pkgrel", "pkgname"): if field not in pkginfo: die_commit( "missing mandatory field: {:s}".format(field), str(commit.id) ) if "epoch" in pkginfo and not pkginfo["epoch"].isdigit(): die_commit( "invalid epoch: {:s}".format(pkginfo["epoch"]), str(commit.id) ) if not re.match(r"[a-z0-9][a-z0-9\.+_-]*$", pkginfo["pkgname"]): die_commit( "invalid package name: {:s}".format(pkginfo["pkgname"]), str(commit.id), ) max_len = {"pkgname": 255, "pkgdesc": 255, "url": 8000} for field in max_len.keys(): if field in pkginfo and len(pkginfo[field]) > max_len[field]: die_commit( "{:s} field too long: {:s}".format(field, pkginfo[field]), str(commit.id), ) for field in ("install", "changelog"): if field in pkginfo and not pkginfo[field] in commit.tree: die_commit( "missing {:s} file: {:s}".format(field, pkginfo[field]), str(commit.id), ) for field in extract_arch_fields(pkginfo, "source"): fname = field["value"] if len(fname) > 8000: die_commit( "source entry too long: {:s}".format(fname), str(commit.id) ) if "://" in fname or "lp:" in fname: continue if fname not in commit.tree: die_commit( "missing source file: {:s}".format(fname), str(commit.id) ) # Display a warning if .SRCINFO is unchanged. if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new): srcinfo_id_old = repo[sha1_old].tree[".SRCINFO"].id srcinfo_id_new = repo[sha1_new].tree[".SRCINFO"].id if srcinfo_id_old == srcinfo_id_new: warn(".SRCINFO unchanged. " "The package database will not be updated!") # Read .SRCINFO from the HEAD commit. metadata_raw = repo[repo[sha1_new].tree[".SRCINFO"].id].data.decode() (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) # Ensure that the package base name matches the repository name. metadata_pkgbase = metadata["pkgbase"] if metadata_pkgbase != pkgbase: die("invalid pkgbase: {:s}, expected {:s}".format(metadata_pkgbase, pkgbase)) # Ensure that packages are neither blacklisted nor overwritten. pkgbase = metadata["pkgbase"] cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) row = cur.fetchone() pkgbase_id = row[0] if row else 0 cur = conn.execute("SELECT Name FROM PackageBlacklist") blacklist = [row[0] for row in cur.fetchall()] cur = conn.execute("SELECT Name, Repo FROM OfficialProviders") providers = dict(cur.fetchall()) for pkgname in srcinfo.utils.get_package_names(metadata): pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) pkgname = pkginfo["pkgname"] if pkgname in blacklist: warn_or_die("package is blacklisted: {:s}".format(pkgname)) if pkgname in providers: warn_or_die( "package already provided by [{:s}]: {:s}".format( providers[pkgname], pkgname ) ) cur = conn.execute( "SELECT COUNT(*) FROM Packages WHERE Name = ? " + "AND PackageBaseID <> ?", [pkgname, pkgbase_id], ) if cur.fetchone()[0] > 0: die("cannot overwrite package: {:s}".format(pkgname)) # Create a new package base if it does not exist yet. if pkgbase_id == 0: pkgbase_id = create_pkgbase(conn, pkgbase, user) # Store package base details in the database. save_metadata(metadata, conn, user) # Create (or update) a branch with the name of the package base for better # accessibility. branchref = "refs/heads/" + pkgbase repo.create_reference(branchref, sha1_new, True) # Work around a Git bug: The HEAD ref is not updated when using # gitnamespaces. This can be removed once the bug fix is included in Git # mainline. See # http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref-namespace-td7632149.html # for details. headref = "refs/namespaces/" + pkgbase + "/HEAD" repo.create_reference(headref, sha1_new, True) # Send package update notifications. update_notify(conn, user, pkgbase_id) # Close the database. cur.close() conn.close() if __name__ == "__main__": main()