git-update: Replace aurinfo.py with python-srcinfo

python-srcinfo is a more transparent and simpler library for parsing
SRCINFO files.

Signed-off-by: Johannes Löthberg <johannes@kyriasis.com>
Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org>

INSTALL

@@ -53,7 +53,7 @@ Setup on Arch Linux
 
 7) Install needed Python modules:
 
-    # pacman -S python-mysql-connector python-pygit2
+    # pacman -S python-mysql-connector python-pygit2 python-srcinfo
 
 8) Install the git-auth wrapper script:
 

git-interface/aurinfo.py

@@ -1,208 +0,0 @@
-#!/usr/bin/env python
-
-from copy import copy, deepcopy
-import pprint
-import sys
-
-class Attr(object):
-    def __init__(self, name, is_multivalued=False, allow_arch_extensions=False):
-        self.name = name
-        self.is_multivalued = is_multivalued
-        self.allow_arch_extensions = allow_arch_extensions
-
-PKGBUILD_ATTRIBUTES = {
-    'arch':         Attr('arch',            True),
-    'backup':       Attr('backup',          True),
-    'changelog':    Attr('changelog',       False),
-    'checkdepends': Attr('checkdepends',    True),
-    'conflicts':    Attr('conflicts',       True, True),
-    'depends':      Attr('depends',         True, True),
-    'epoch':        Attr('epoch',           False),
-    'groups':       Attr('groups',          True),
-    'install':      Attr('install',         False),
-    'license':      Attr('license',         True),
-    'makedepends':  Attr('makedepends',     True, True),
-    'md5sums':      Attr('md5sums',         True, True),
-    'noextract':    Attr('noextract',       True),
-    'optdepends':   Attr('optdepends',      True, True),
-    'options':      Attr('options',         True),
-    'pkgname':      Attr('pkgname',         False),
-    'pkgrel':       Attr('pkgrel',          False),
-    'pkgver':       Attr('pkgver',          False),
-    'provides':     Attr('provides',        True, True),
-    'replaces':     Attr('replaces',        True, True),
-    'sha1sums':     Attr('sha1sums',        True, True),
-    'sha224sums':   Attr('sha224sums',      True, True),
-    'sha256sums':   Attr('sha256sums',      True, True),
-    'sha384sums':   Attr('sha384sums',      True, True),
-    'sha512sums':   Attr('sha512sums',      True, True),
-    'source':       Attr('source',          True, True),
-    'url':          Attr('url',             False),
-    'validpgpkeys': Attr('validpgpkeys',    True),
-}
-
-def find_attr(attrname):
-    # exact match
-    attr = PKGBUILD_ATTRIBUTES.get(attrname, None)
-    if attr:
-        return attr
-
-    # prefix match
-    # XXX: this could break in the future if PKGBUILD(5) ever
-    # introduces a key which is a subset of another.
-    for k in PKGBUILD_ATTRIBUTES.keys():
-        if attrname.startswith(k + '_'):
-            return PKGBUILD_ATTRIBUTES[k]
-
-def IsMultiValued(attrname):
-    attr = find_attr(attrname)
-    return attr and attr.is_multivalued
-
-class AurInfo(object):
-    def __init__(self):
-        self._pkgbase = {}
-        self._packages = {}
-
-    def GetPackageNames(self):
-        return self._packages.keys()
-
-    def GetMergedPackage(self, pkgname):
-        package = deepcopy(self._pkgbase)
-        package['pkgname'] = pkgname
-        for k, v in self._packages.get(pkgname).items():
-            package[k] = deepcopy(v)
-        return package
-
-    def AddPackage(self, pkgname):
-        self._packages[pkgname] = {}
-        return self._packages[pkgname]
-
-    def SetPkgbase(self, pkgbasename):
-        self._pkgbase = {'pkgname' : pkgbasename}
-        return self._pkgbase
-
-
-class StderrECatcher(object):
-    def Catch(self, lineno, error):
-        print('ERROR[{:d}]: {:s}'.format(lineno, error), file=sys.stderr)
-
-
-class CollectionECatcher(object):
-    def __init__(self):
-        self._errors = []
-
-    def Catch(self, lineno, error):
-        self._errors.append((lineno, error))
-
-    def HasErrors(self):
-        return len(self._errors) > 0
-
-    def Errors(self):
-        return copy(self._errors)
-
-
-def ParseAurinfoFromIterable(iterable, ecatcher=None):
-    aurinfo = AurInfo()
-
-    if ecatcher is None:
-        ecatcher = StderrECatcher()
-
-    current_package = None
-    lineno = 0
-
-    for line in iterable:
-        lineno += 1
-
-        if line.startswith('#'):
-            continue
-
-        if not line.strip():
-            # end of package
-            current_package = None
-            continue
-
-        if not (line.startswith('\t') or line.startswith(' ')):
-            # start of new package
-            try:
-                key, value = map(str.strip, line.split('=', 1))
-            except ValueError:
-                ecatcher.Catch(lineno, 'unexpected header format in section={:s}'.format(
-                               current_package['pkgname']))
-                continue
-
-            if key == 'pkgbase':
-                current_package = aurinfo.SetPkgbase(value)
-            elif key == 'pkgname':
-                current_package = aurinfo.AddPackage(value)
-            else:
-                ecatcher.Catch(lineno, 'unexpected new section not starting '
-                               'with \'pkgname\' found')
-                continue
-        else:
-            # package attribute
-            if current_package is None:
-                ecatcher.Catch(lineno, 'package attribute found outside of '
-                               'a package section')
-                continue
-
-            try:
-                key, value = map(str.strip, line.split('=', 1))
-            except ValueError:
-                ecatcher.Catch(lineno, 'unexpected attribute format in '
-                                       'section={:s}'.format(current_package['pkgname']))
-
-            if IsMultiValued(key):
-                if not current_package.get(key):
-                    current_package[key] = []
-                if value:
-                    current_package[key].append(value)
-            else:
-                if not current_package.get(key):
-                    current_package[key] = value
-                else:
-                    ecatcher.Catch(lineno, 'overwriting attribute '
-                                           '{:s}: {:s} -> {:s}'.format(key,
-                                           current_package[key], value))
-
-    return aurinfo
-
-
-def ParseAurinfo(filename='.AURINFO', ecatcher=None):
-    with open(filename) as f:
-        return ParseAurinfoFromIterable(f, ecatcher)
-
-
-def ValidateAurinfo(filename='.AURINFO'):
-    ecatcher = CollectionECatcher()
-    ParseAurinfo(filename, ecatcher)
-    errors = ecatcher.Errors()
-    for error in errors:
-        print('error on line {:d}: {:s}'.format(error), file=sys.stderr)
-    return not errors
-
-
-if __name__ == '__main__':
-    pp = pprint.PrettyPrinter(indent=4)
-
-    if len(sys.argv) == 1:
-        print('error: not enough arguments')
-        sys.exit(1)
-    elif len(sys.argv) == 2:
-        action = sys.argv[1]
-        filename = '.AURINFO'
-    else:
-        action, filename = sys.argv[1:3]
-
-    if action == 'parse':
-        aurinfo = ParseAurinfo(filename)
-        for pkgname in aurinfo.GetPackageNames():
-            print(">>> merged package: {:s}".format(pkgname))
-            pp.pprint(aurinfo.GetMergedPackage(pkgname))
-            print()
-    elif action == 'validate':
-        sys.exit(not ValidateAurinfo(filename))
-    else:
-        print('unknown action: {:s}'.format(action))
-        sys.exit(1)
-
-# vim: set et ts=4 sw=4:

git-interface/git-update.py

@@ -7,7 +7,8 @@ import pygit2
 import re
 import sys
 
-import aurinfo
+import srcinfo.parse
+import srcinfo.utils
 
 config = configparser.RawConfigParser()
 config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config")
@@ -48,9 +49,9 @@ def parse_dep(depstring):
         return (depname, depcond)
 
 
-def save_srcinfo(srcinfo, db, cur, user):
+def save_metadata(metadata, db, cur, user):
     # Obtain package base ID and previous maintainer.
-    pkgbase = srcinfo._pkgbase['pkgname']
+    pkgbase = metadata['pkgbase']
     cur.execute("SELECT ID, MaintainerUID FROM PackageBases "
                 "WHERE Name = %s", [pkgbase])
     (pkgbase_id, maintainer_uid) = cur.fetchone()
@@ -70,8 +71,8 @@ def save_srcinfo(srcinfo, db, cur, user):
     cur.execute("DELETE FROM Packages WHERE PackageBaseID = %s",
                 [pkgbase_id])
 
-    for pkgname in srcinfo.GetPackageNames():
+    for pkgname in srcinfo.utils.get_package_names(metadata):
-        pkginfo = srcinfo.GetMergedPackage(pkgname)
+        pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
 
         if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0:
             ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']),
@@ -245,26 +246,24 @@ for commit in walker:
         if blob.size > 250000:
             die_commit("maximum blob size (250kB) exceeded", str(commit.id))
 
-    srcinfo_raw = repo[commit.tree['.SRCINFO'].id].data.decode()
+    metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode()
-    srcinfo_raw = srcinfo_raw.split('\n')
+    (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
-    ecatcher = aurinfo.CollectionECatcher()
-    srcinfo = aurinfo.ParseAurinfoFromIterable(srcinfo_raw, ecatcher)
-    errors = ecatcher.Errors()
     if errors:
         sys.stderr.write("error: The following errors occurred "
                          "when parsing .SRCINFO in commit\n")
         sys.stderr.write("error: {:s}:\n".format(str(commit.id)))
         for error in errors:
-            sys.stderr.write("error: line {:d}: {:s}\n".format(*error))
+            for err in error['error']:
+                sys.stderr.write("error: line {:d}: {:s}\n".format(error['line'], err))
         exit(1)
 
-    srcinfo_pkgbase = srcinfo._pkgbase['pkgname']
+    metadata_pkgbase = metadata['pkgbase']
-    if not re.match(repo_regex, srcinfo_pkgbase):
+    if not re.match(repo_regex, metadata_pkgbase):
-        die_commit('invalid pkgbase: {:s}'.format(srcinfo_pkgbase),
+        die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase),
                    str(commit.id))
 
-    for pkgname in srcinfo.GetPackageNames():
+    for pkgname in set(metadata['packages'].keys()):
-        pkginfo = srcinfo.GetMergedPackage(pkgname)
+        pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
 
         for field in ('pkgver', 'pkgrel', 'pkgname'):
             if field not in pkginfo:
@@ -306,28 +305,28 @@ if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new):
         warn(".SRCINFO unchanged. The package database will not be updated!")
 
 # Read .SRCINFO from the HEAD commit.
-srcinfo_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode()
+metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode()
-srcinfo_raw = srcinfo_raw.split('\n')
+(metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
-srcinfo = aurinfo.ParseAurinfoFromIterable(srcinfo_raw)
 
 # Ensure that the package base name matches the repository name.
-srcinfo_pkgbase = srcinfo._pkgbase['pkgname']
+metadata_pkgbase = metadata['pkgbase']
-if srcinfo_pkgbase != pkgbase:
+if metadata_pkgbase != pkgbase:
-    die('invalid pkgbase: {:s}, expected {:s}'.format(srcinfo_pkgbase, pkgbase))
+    die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, pkgbase))
 
 # Ensure that packages are neither blacklisted nor overwritten.
+pkgbase = metadata['pkgbase']
 cur.execute("SELECT ID FROM PackageBases WHERE Name = %s", [pkgbase])
 pkgbase_id = cur.fetchone()[0] if cur.rowcount == 1 else 0
 
 cur.execute("SELECT Name FROM PackageBlacklist")
 blacklist = [row[0] for row in cur.fetchall()]
 
-for pkgname in srcinfo.GetPackageNames():
+for pkgname in srcinfo.utils.get_package_names(metadata):
-    pkginfo = srcinfo.GetMergedPackage(pkgname)
+    pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
     pkgname = pkginfo['pkgname']
 
     if pkgname in blacklist and not privileged:
-        die('package is blacklisted: {:s}'.format(pkginfo['pkgname']))
+        die('package is blacklisted: {:s}'.format(pkgname))
 
     cur.execute("SELECT COUNT(*) FROM Packages WHERE Name = %s AND " +
                 "PackageBaseID <> %s", [pkgname, pkgbase_id])
@@ -335,7 +334,7 @@ for pkgname in srcinfo.GetPackageNames():
         die('cannot overwrite package: {:s}'.format(pkgname))
 
 # Store package base details in the database.
-save_srcinfo(srcinfo, db, cur, user)
+save_metadata(metadata, db, cur, user)
 
 db.close()