aurweb/tupkg/update/tupkgupdate

#!/usr/bin/python -O

import re
import os
import sys
import pacman
import getopt
import MySQLdb
import MySQLdb.connections
import ConfigParser
from subprocess import Popen, PIPE

###########################################################
# Deal with configuration
###########################################################

conffile = '/etc/tupkgs.conf'

config = ConfigParser.ConfigParser()

############################################################

# Define some classes we need
class Version:
  def __init__(self):
    self.version = None
    self.file = None

class Package:
  def __init__(self):
    self.name = None
    self.category = None
    self.old = None
    self.new = None
    self.desc = None
    self.url = None
    self.depends = None
    self.sources = None

class PackageDatabase:
  def __init__(self, host, user, password, dbname):
    self.host = host
    self.user = user
    self.password = password
    self.dbname = dbname
    self.connection = MySQLdb.connect(host=host, user=user, passwd=password, db=dbname)

  def cursor(self):
    try:
      self.connection.ping()
    except MySQLdb.OperationalError:
      self.connection = MySQLdb.connect(host=self.host, user=self.user, passwd=self.password, db=self.dbname)
    return self.connection.cursor()

  def lookup(self, packagename):
    warning("DB: Looking up package: " + packagename)
    q = self.cursor()
    q.execute("SELECT ID FROM Packages WHERE Name = '" +
               MySQLdb.escape_string(packagename) + "'")
    if (q.rowcount != 0):
      row = q.fetchone()
      return row[0]
    return None

  def getCategoryID(self, package):
    category_id = self.lookupCategory(package.category)
    if (category_id == None):
      category_id = 1
    warning("DB: Got category ID '" + str(category_id) + "' for package '" + package.name + "'")
    return category_id

  def insert(self, package, locationId):
    warning("DB: Inserting package: " + package.name)
    global repo_dir
    q = self.cursor()
    q.execute("INSERT INTO Packages " +
      "(Name, CategoryID, Version, FSPath, LocationID, SubmittedTS, Description, URL) VALUES ('" +
      MySQLdb.escape_string(package.name) + "', " +
      str(self.getCategoryID(package)) + ", '" +
      MySQLdb.escape_string(package.new.version) + "', '" +
      MySQLdb.escape_string(
        os.path.join(repo_dir, os.path.basename(package.new.file))) + "', " +
      str(locationId) + ", " +
      "UNIX_TIMESTAMP(), '" +
      MySQLdb.escape_string(str(package.desc)) + "', '" +
      MySQLdb.escape_string(str(package.url)) + "')")
    id = self.lookup(package.name)
    self.insertNewInfo(package, id, locationId)

  def update(self, id, package, locationId):
    warning("DB: Updating package: " + package.name + " with id " + str(id))
    global repo_dir
    q = self.cursor()
    if (self.isdummy(package.name)):
      q.execute("UPDATE Packages SET " +
        "Version = '" + MySQLdb.escape_string(package.new.version) + "', " +
        "CategoryID = " + str(self.getCategoryID(package)) + ", " +
        "FSPath = '" + MySQLdb.escape_string(
          os.path.join(repo_dir, os.path.basename(package.new.file))) + "', " +
        "Description = '" + MySQLdb.escape_string(str(package.desc)) + "', " +
        "DummyPkg = 0, " +
        "SubmittedTS = UNIX_TIMESTAMP(), " +
        "URL = '" + MySQLdb.escape_string(str(package.url)) + "' " +
        "WHERE ID = " + str(id))
    else:
      q.execute("UPDATE Packages SET " +
        "Version = '" + MySQLdb.escape_string(package.new.version) + "', " +
        "CategoryID = " + str(self.getCategoryID(package)) + ", " +
        "FSPath = '" + MySQLdb.escape_string(
          os.path.join(repo_dir, os.path.basename(package.new.file))) + "', " +
        "Description = '" + MySQLdb.escape_string(str(package.desc)) + "', " +
        "ModifiedTS = UNIX_TIMESTAMP(), " +
        "URL = '" + MySQLdb.escape_string(str(package.url)) + "' " +
        "WHERE ID = " + str(id))
    self.insertNewInfo(package, id, locationId)

    # Check to see if this is a move of a package from unsupported
    # to community, because we have to reset maintainer and location.

    q = self.cursor()
    q.execute("SELECT LocationID FROM Packages WHERE ID = " + str(id))
    if (q.rowcount != 0):
      row = q.fetchone()
      if (row[0] != 3):
        q = self.cursor()
        q.execute("UPDATE Packages SET LocationID = 3, MaintainerUID = null WHERE ID = " + str(id))

  def remove(self, id, locationId):
    warning("DB: Removing package with id: " + str(id))
    q = self.cursor()
    q.execute("DELETE FROM Packages WHERE " +
      "LocationID = " + str(locationId) + " AND ID = " + str(id))

  def clearOldInfo(self, id):
    warning("DB: Clearing old info for package with id : " + str(id))
    q = self.cursor()
    q.execute("DELETE FROM PackageContents WHERE PackageID = " + str(id))
    q.execute("DELETE FROM PackageDepends WHERE PackageID = " + str(id))
    q.execute("DELETE FROM PackageSources WHERE PackageID = " + str(id))

  def lookupOrDummy(self, packagename):
    retval = self.lookup(packagename)
    if (retval != None):
      return retval
    return self.createDummy(packagename)

  def lookupCategory(self, categoryname):
    warning("DB: Looking up category: " + categoryname)
    q = self.cursor()
    q.execute("SELECT ID from PackageCategories WHERE Category = '" + MySQLdb.escape_string(categoryname) + "'")
    if (q.rowcount != 0):
      row = q.fetchone()
      return row[0]
    return None

  def createDummy(self, packagename):
    warning("DB: Creating dummy package for: " + packagename)
    q = self.cursor()
    q.execute("INSERT INTO Packages " +
      "(Name, Description, LocationID, DummyPkg) " +
      "VALUES ('" +
      MySQLdb.escape_string(packagename) + "', '" +
      MySQLdb.escape_string("A dummy package") + "', 1, 1)")
    return self.lookup(packagename)

  def insertNewInfo(self, package, id, locationId):
    q = self.cursor()

    # First delete the old.
    self.clearOldInfo(id)

    warning("DB: Inserting new package info for " + package.name +
            " with id " + str(id))

    # PackageSources
    for source in package.sources:
      q.execute("INSERT INTO PackageSources (PackageID, Source) " +
        "VALUES (" + str(id) + ", '" + MySQLdb.escape_string(source) + "')")

    # PackageDepends
    for dep in package.depends:
      depid = self.lookupOrDummy(dep)
      q.execute("INSERT INTO PackageDepends (PackageID, DepPkgID) " +
        "VALUES (" + str(id) + ", " + str(depid) + ")")

  def isdummy(self, packagename):
    warning("DB: Looking up package: " + packagename)
    q = self.cursor()
    q.execute("SELECT * FROM Packages WHERE Name = '" +
      MySQLdb.escape_string(packagename) + "' AND DummyPkg = 1")
    if (q.rowcount != 0):
      return True
    return False

############################################################
# Functions for walking the file trees
############################################################

def filesForRegexp(topdir, regexp):
  retval = []
  def matchfile(regexp, dirpath, namelist):
    for name in namelist:
      if (regexp.match(name)):
        retval.append(os.path.join(dirpath, name))
  os.path.walk(topdir, matchfile, regexp)
  return retval

def packagesInTree(topdir):
  return filesForRegexp(topdir, re.compile("^.*\.pkg\.tar\.gz$"))

def pkgbuildsInTree(topdir):
  return filesForRegexp(topdir, re.compile("^PKGBUILD$"))

############################################################
# Function for testing if two files are identical
############################################################

def areFilesIdentical(file_a, file_b):
  command = "cmp '" + file_a + "' '" + file_b + "' >/dev/null"
  retval = os.system(command)
  if (retval == 0):
    return True
  return False

############################################################
# Function for fetching info from PKGBUILDs and packages
############################################################

def infoFromPackageFile(filename):
  pkg = os.path.basename(filename)
  m = re.compile("(?P<pkgname>.*)-(?P<pkgver>.*)-(?P<pkgrel>.*).pkg.tar.gz").search(pkg)
  if not m:
    raise Exception("Non-standard filename")
  else:
    return m.group('pkgname'), m.group('pkgver') + "-" + m.group('pkgrel')

def infoFromPkgbuildFile(filename):
  # first grab the category based on the file path
  pkgdirectory = os.path.dirname(filename)
  catdirectory = os.path.dirname(pkgdirectory)
  m = re.match(r".*/([^/]+)$", catdirectory)
  if (m):
    category = m.group(1)
  else:
    category = "none"

  # open and source the file
  pf = Popen("/bin/bash",
      shell=True, bufsize=0, stdin=PIPE, stdout=PIPE, close_fds=True)

  print >>pf.stdin, ". " + filename
  #print "PKGBUILD: " + filename

  # get pkgname
  print >>pf.stdin, 'echo $pkgname'
  pkgname = pf.stdout.readline().strip()

  #print "PKGBUILD: pkgname: " + pkgname

  # get pkgver
  print >>pf.stdin, 'echo $pkgver'
  pkgver = pf.stdout.readline().strip()
  #print "PKGBUILD: pkgver: " + pkgver

  # get pkgrel
  print >>pf.stdin, 'echo $pkgrel'
  pkgrel = pf.stdout.readline().strip()
  #print "PKGBUILD: pkgrel: " + pkgrel

  # get url
  print >>pf.stdin, 'echo $url'
  url = pf.stdout.readline().strip()
  #print "PKGBUILD: url: " + url

  # get desc
  print >>pf.stdin, 'echo $pkgdesc'
  pkgdesc = pf.stdout.readline().strip()
  #print "PKGBUILD: pkgdesc: " + pkgdesc

  # get source array
  print >>pf.stdin, 'echo ${source[*]}'
  source = (pf.stdout.readline().strip()).split(" ")

  # get depends array
  print >>pf.stdin, 'echo ${depends[*]}'
  depends = (pf.stdout.readline().strip()).split(" ")

  # clean up
  pf.stdin.close()
  pf.stdout.close()

  return pkgname, pkgver + "-" + pkgrel, pkgdesc, url, depends, source, category

def infoFromPkgbuildFileWorse(filename):
  # load the file with pacman library
  pkg = pacman.load(filename)
  return (pkg.name, pkg.version + "-" + pkg.release, pkg.desc,
    pkg.url, pkg.depends, pkg.source)

############################################################
# Functions for doing the final steps of execution
############################################################

def execute(command):
  global switches
  print(command)
  if not (switches.get("-n") == True):
    return os.system(command)
  return 0

def copyFileToRepo(filename, repodir):
  destfile = os.path.join(repodir, os.path.basename(filename))
  command = "cp --preserve=timestamps '" + filename + "' '" + destfile + "'"
  return execute(command)

def deleteFile(filename):
  command = "rm '" + filename + "'"
  return execute(command)

def runRepoAdd(repo, package):
  global havefakeroot
  targetDB = os.path.join(repo, "community.db.tar.gz")
  destfile = os.path.join(repo, os.path.basename(package.new.file))
  if havefakeroot:
    command = "fakeroot repo-add '" + targetDB + "' '" + destfile + "'"
  else:
    command = "repo-add '" + targetDB + "' '" + destfile + "'"
  return execute(command)

def runRepoRemove(repo, pkgname):
  global havefakeroot
  targetDB = os.path.join(repo, "community.db.tar.gz")
  if havefakeroot:
    command = "fakeroot repo-remove '" + targetDB + "' '"+ pkgname + "'"
  else:
    command = "repo-remove '" + targetDB + "' '" + pkgname +"'"
  return execute(command)

############################################################
# Functions for error handling
############################################################

def warning(string):
  print >>sys.stderr, string

had_error = 0
def error(string):
  global had_error
  warning(string)
  had_error = 1

def usage(name):
  print "Usage: %s [options] <repo_dir> <pkgbuild_tree> <build_tree>" % name
  print "Options:"
  print "  -c, --config        Specify a path to the config file."
  print "  -n                  Don't actually perform any action on the repo."
  print "  --delete            Delete duplicate and temporary pkgs."
  print "  --paranoid          Warn of duplicate pkgs that aren't identical."
  sys.exit(1)

############################################################
# MAIN
############################################################

# ARGUMENTS
# See usage() for specifying arguments.

try:
  optlist, args = getopt.getopt(sys.argv[1:], 'c:n',
    ['config=', 'delete', 'paranoid'])
except getopt.GetoptError:
  usage(sys.argv[0])

switches = {}
for opt in optlist:
  switches[opt[0]] = 1

# Check for required arguments.
if (len(args) < 3):
  usage(sys.argv[0])

for opt, value in optlist:
  if opt in ('-c', '--config'):
    conffile = value

try:
  repo_dir, pkgbuild_dir, build_dir = args
except ValueError:
  usage(sys.argv[0])

if not os.path.isfile(conffile):
  print "Error: cannot access config file (%s)" % conffile
  sys.exit(1)

config.read(conffile)
config_use_db = config.has_section('mysql')

# Make sure we can use fakeroot, warn if not
havefakeroot = False
if os.access('/usr/bin/fakeroot', os.X_OK):
  havefakeroot = True
else:
  warning("Not using fakeroot for repo db generation")

# Open the database if we need it so we find out now if we can't!
if config_use_db:
  try:
    db = PackageDatabase(config.get('mysql', 'host'),
      config.get('mysql', 'username'),
      config.get('mysql', 'password'),
      config.get('mysql', 'db'))
  except:
    print "Error: Could not connect to the database %s at %s." % (
        config.get('mysql', 'db'), config.get('mysql', 'host'))
    sys.exit(1)

# Set up the lists and tables
packages = dict()
copy = list()
delete = list()

dbremove = list()
dbmodify = list()

# PASS 1: PARSING/LOCATING
#
# A) Go through the PKGBUILD tree
#    For each PKGBUILD, create a Package with new Version containing
#    parsed version and and None for file

a_files = pkgbuildsInTree(pkgbuild_dir)
for a_file in a_files:
  pkgname, ver, desc, url, depends, sources, category = infoFromPkgbuildFile(a_file)

  # Error (and skip) if we encounter any invalid PKGBUILD files
  if (pkgname == None or ver == None):
    error("Pkgbuild '" + a_file + "' is invalid!")
    continue

  # Error (and skip) if we encounter any duplicate package names
  # in the PKGBUILDs
  if (packages.get(pkgname)):
    error("Pkgbuild '" + a_file + "' is a duplicate!")
    continue

  version = Version()
  version.version = ver
  version.file = None

  package = Package()
  package.name = pkgname
  package.category = category
  package.desc = desc
  package.url = url
  package.depends = depends
  package.sources = sources
  package.new = version

#  print "Package: desc " + desc

  packages[pkgname] = package

# B) Go through the old repo dir
#    For each package file we encounter, create a Package with old
#    Version containing parsed version and filepath

b_files = packagesInTree(repo_dir)
for b_file in b_files:
  pkgname, ver = infoFromPackageFile(b_file)

  version = Version()
  version.version = ver
  version.file = b_file

  package = packages.get(pkgname)
  if (package == None):
    package = Package()
    package.name = pkgname
    packages[pkgname] = package
  package.old = version

# C) Go through the build tree
#    For each package file we encounter:
#      1 - look up the package name; if it fails, ignore the file (no error)
#      2 - if package.new == None, ignore the package (no error)
#      3 - if package.new.version doesn't match, then skip (no error)
#      4 - if package.new.file == None, point it to this file
#          otherwise, log an error (and skip)

c_files = packagesInTree(build_dir)
for c_file in c_files:
  pkgname, ver = infoFromPackageFile(c_file)

  # 1
  package = packages.get(pkgname)
  if (package == None):
    continue

  # 2
  if (package.new == None):
    continue

  # 3
  if (package.new.version != ver):
    continue

  # 4
  if (package.new.file == None):
    package.new.file = c_file
    continue
  else:
    error("Duplicate new file '" + c_file + "'")
    continue

# PASS 2: CHECKING
#
# Go through the package collection
#   1 - if package has no new, place its old file on the "delete" list (and package on "dbremove")
#   2 - if package has a new but no new.file, and old file doesn't
#       have the same version, then error (because gensync won't rebuild)
#   3 - if package has no old, add new file to "copy" list into repo dir (and package on "dbmodify")
#   4 - if new == old and paranoid is set, compare the files and error if not the same;
#       otherwise just skip (no update)
#   5 - if we got here, it's a legit nontrivial new version which we allow
#       add entry to "delete" list for old file and "copy" list for
#       new file into repo dir (and package to "dbmodify")

for package in packages.values():
  # 1
  if (package.new == None):
    delete.append(package.old.file)
    dbremove.append(package)
    continue

  # 2
  if (package.new.file == None):
    if (package.old == None or package.old.file == None or
      package.old.version != package.new.version):
      errstr = "No new package supplied for " + package.name + " " + package.new.version + "!"
      error(errstr)
      continue

  # 3
  if (package.old == None):
    copy.append(package.new.file)
    dbmodify.append(package)
    continue

  # 4
  if (package.old.version == package.new.version):
    if (switches.get("--paranoid") == True and package.new.file != None):
      if not (areFilesIdentical(package.old.file, package.new.file)):
        warning("New package file with identical version '" +
          package.new.file + "' is different than the old one:")
        if (switches.get("--delete") == True):
          warning("  Deleting the new file.")
          delete.append(package.new.file)
        else:
          warning("  Ignoring the new file.")
    continue

  # 5
  delete.append(package.old.file)
  copy.append(package.new.file)
  dbmodify.append(package)
  continue

## IF WE HAVE HAD ANY ERRORS AT THIS POINT, ABORT! ##
if (had_error == 1):
  error("Aborting due to errors.")
  sys.exit(-1)

# PASS 3: EXECUTION
#

if config_use_db:
  # First, do all the database updates if asked for
  for package in dbremove:
    id = db.lookup(package.name)
    # Note: this could remove a package from unsupported; probably want to restrict to locationId and/or non-dummy
    if (id != None):
      db.clearOldInfo(id)
      db.remove(id, 3)

  for package in dbmodify:
    warning("DB: Package in dbmodify: " + package.name)
    id = db.lookup(package.name)
    if (id == None):
      db.insert(package, 3)
    else:
      db.update(id, package, 3)

# Copy
for file in copy:
  retval = copyFileToRepo(file, repo_dir)
  if (retval != 0):
    error("Could not copy file to repo: '" + file + "'")
    sys.exit(-1)

# Delete (second, for safety's sake)
for file in delete:
  deleteFile(file)

# Now that we've copied new files and deleted, we should delete the source
# files, if we're supposed to
if (switches.get("--delete") == True):
  for file in copy:
    deleteFile(file)

# Run updatesync where it is needed
for package in dbremove:
  retval = runRepoRemove(repo_dir, package.name)
  if (retval != 0):
    error("repo-remove returned an error!")
    sys.exit(-1)
for package in dbmodify:
  retval = runRepoAdd(repo_dir, package)
  if (retval != 0):
    error("repo-add returned an error!")
    sys.exit(-1)

# vim: ft=python ts=2 sw=2 et