external/aurweb
0
0
Fork 0
mirror of https://gitlab.archlinux.org/archlinux/aurweb.git synced 2025-02-03 10:43:03 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Rewrite aurblup in Python

Browse source 
The AUR backend already uses several Python scripts, rewrite the aurblup
helper as well. This has several advantages:

* We can easily use the main configuration file without using any shell
  script wrappers.

* aurblup does not need to be recompiled on libalpm soname bumps.

Signed-off-by: Lukas Fleischer <archlinux@cryptocrack.de>
This commit is contained in:
Lukas Fleischer 2015-01-09 10:53:44 +01:00
parent 7dd78de3fd
commit 3c171d353f
9 changed files with 53 additions and 383 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
conf/config.proto
View file

@ -37,3 +37,8 @@ repo-base = /srv/http/aur/repos/
repo-regex = [a-z0-9][a-z0-9.+_-]*$ repo-regex = [a-z0-9][a-z0-9.+_-]*$
git-update-hook = /srv/http/aur/scripts/git-integration/git-update.py git-update-hook = /srv/http/aur/scripts/git-integration/git-update.py
git-shell-cmd = /usr/bin/git-shell git-shell-cmd = /usr/bin/git-shell
[aurblup]
db-path = /srv/http/aur/scripts/aurblup/
sync-dbs = core extra community multilib testing community-testing
servers = ftp://mirrors.kernel.org/archlinux/%s/os/x86_64

3
scripts/aurblup/.gitignore vendored
View file

@ -1,3 +0,0 @@
config.h
*.o
aurblup

26
scripts/aurblup/Makefile
View file

@ -1,26 +0,0 @@
include config.mk
SRC = aurblup.c
OBJ = ${SRC:.c=.o}
all: aurblup
config.h:
cp config.h.proto config.h
${OBJ}: config.h
aurblup: ${OBJ}
install: aurblup
install -Dm0755 aurblup "${DESTDIR}${PREFIX}/bin/aurblup"
install -dm0755 "${DESTDIR}/var/lib/aurblup/"
uninstall:
rm -f "${DESTDIR}${PREFIX}/bin/aurblup"
rm -f "${DESTDIR}/var/lib/aurblup/"
clean:
rm -f aurblup ${OBJ}
.PHONY: all install uninstall clean

28
scripts/aurblup/README
View file

@ -1,28 +0,0 @@
aurblup
=======
aurblup is a small and lightweight tool that updates the package blacklist of
an AUR MySQL database using one (or more) package databases. It does the
following things:
- Sync a bunch of local package databases with a remote server.
- Get a list of packages in those databases.
- Update the MySQL blacklist table to match those packages, including those
packages' provides and replaces.
Requirements
------------
You need the libalpm and libmysqlclient header files to build aurblup.
Installation
------------
Edit the "config.h" (copy from "config.h.proto" if doesn't exist) and
"config.mk" configuration files to match your setup and enter the following
command to build and install aurblup:
make install

21
scripts/aurblup/aurblup-wrapper
View file

@ -1,21 +0,0 @@
#!/usr/bin/php
<?php
$dir = $argv[1];
if (empty($dir)) {
echo "Please specify AUR directory.\n";
exit;
}
set_include_path(get_include_path() . PATH_SEPARATOR . "$dir/lib");
include("confparser.inc.php");
$user = config_get('database', 'user');
$password = config_get('database', 'password');
$name = config_get('database', 'name');
exec($dir . "/../scripts/aurblup/aurblup " .
"-S /var/run/mysqld/mysqld.sock " .
"-u " . escapeshellarg($user) . " " .
"-p " . escapeshellarg($password) . " " .
"-D " . escapeshellarg($name));

287
scripts/aurblup/aurblup.c
View file

@ -1,287 +0,0 @@
/* aurblup - AUR blacklist updater
*
* Small utility to update the AUR package blacklist. Can be used in a cronjob.
* Check the "README" file for details.
*/
#include <alpm.h>
#include <getopt.h>
#include <mysql.h>
#include <stdio.h>
#include <string.h>
#include "config.h"
#define alpm_die(...) die(__VA_ARGS__, alpm_strerror(alpm_errno(handle)));
#define mysql_die(...) die(__VA_ARGS__, mysql_error(c));
static void die(const char *, ...);
static alpm_list_t *pkglist_append(alpm_list_t *, const char *);
static alpm_list_t *blacklist_get_pkglist();
static void blacklist_add(const char *);
static void blacklist_remove(const char *);
static void blacklist_sync(alpm_list_t *, alpm_list_t *);
static alpm_list_t *dblist_get_pkglist(alpm_list_t *);
static alpm_list_t *dblist_create(void);
static int parse_options(int, char **);
static void init(void);
static void cleanup(void);
static char *mysql_host = "localhost";
static char *mysql_socket = NULL;
static char *mysql_user = "aur";
static char *mysql_passwd = "aur";
static char *mysql_db = "AUR";
static MYSQL *c;
static alpm_handle_t *handle;
static void
die(const char *format, ...)
{
va_list arg;
va_start(arg, format);
fprintf(stderr, "aurblup: ");
vfprintf(stderr, format, arg);
va_end(arg);
cleanup();
exit(1);
}
static alpm_list_t *
pkglist_append(alpm_list_t *pkglist, const char *pkgname)
{
int len = strcspn(pkgname, "<=>");
if (!len)
len = strlen(pkgname);
char *s = malloc(len + 1);
strncpy(s, pkgname, len);
s[len] = '\0';
if (alpm_list_find_str(pkglist, s))
free(s);
else
pkglist = alpm_list_add(pkglist, s);
return pkglist;
}
static alpm_list_t *
blacklist_get_pkglist()
{
MYSQL_RES *res;
MYSQL_ROW row;
alpm_list_t *pkglist = NULL;
if (mysql_query(c, "SELECT Name FROM PackageBlacklist"))
mysql_die("failed to read blacklist from MySQL database: %s\n");
if (!(res = mysql_store_result(c)))
mysql_die("failed to store MySQL result: %s\n");
while ((row = mysql_fetch_row(res)))
pkglist = pkglist_append(pkglist, row[0]);
mysql_free_result(res);
return pkglist;
}
static void
blacklist_add(const char *name)
{
char *esc = malloc(strlen(name) * 2 + 1);
char query[1024];
mysql_real_escape_string(c, esc, name, strlen(name));
snprintf(query, 1024, "INSERT INTO PackageBlacklist (Name) "
"VALUES ('%s')", esc);
free(esc);
if (mysql_query(c, query))
mysql_die("failed to query MySQL database (\"%s\"): %s\n", query);
}
static void
blacklist_remove(const char *name)
{
char *esc = malloc(strlen(name) * 2 + 1);
char query[1024];
mysql_real_escape_string(c, esc, name, strlen(name));
snprintf(query, 1024, "DELETE FROM PackageBlacklist WHERE Name = '%s'", esc);
free(esc);
if (mysql_query(c, query))
mysql_die("failed to query MySQL database (\"%s\"): %s\n", query);
}
static void
blacklist_sync(alpm_list_t *pkgs_cur, alpm_list_t *pkgs_new)
{
alpm_list_t *pkgs_add, *pkgs_rem, *p;
pkgs_add = alpm_list_diff(pkgs_new, pkgs_cur, (alpm_list_fn_cmp)strcmp);
pkgs_rem = alpm_list_diff(pkgs_cur, pkgs_new, (alpm_list_fn_cmp)strcmp);
if (mysql_query(c, "START TRANSACTION"))
mysql_die("failed to start MySQL transaction: %s\n");
for (p = pkgs_add; p; p = alpm_list_next(p))
blacklist_add(p->data);
for (p = pkgs_rem; p; p = alpm_list_next(p))
blacklist_remove(p->data);
if (mysql_query(c, "COMMIT"))
mysql_die("failed to commit MySQL transaction: %s\n");
alpm_list_free(pkgs_add);
alpm_list_free(pkgs_rem);
}
static alpm_list_t *
dblist_get_pkglist(alpm_list_t *dblist)
{
alpm_list_t *d, *p, *q;
alpm_list_t *pkglist = NULL;
for (d = dblist; d; d = alpm_list_next(d)) {
alpm_db_t *db = d->data;
if (alpm_trans_init(handle, 0))
alpm_die("failed to initialize ALPM transaction: %s\n");
if (alpm_db_update(0, db) < 0)
alpm_die("failed to update ALPM database: %s\n");
if (alpm_trans_release(handle))
alpm_die("failed to release ALPM transaction: %s\n");
for (p = alpm_db_get_pkgcache(db); p; p = alpm_list_next(p)) {
alpm_pkg_t *pkg = p->data;
pkglist = pkglist_append(pkglist, alpm_pkg_get_name(pkg));
for (q = alpm_pkg_get_replaces(pkg); q; q = alpm_list_next(q)) {
alpm_depend_t *replace = q->data;
pkglist = pkglist_append(pkglist, replace->name);
}
}
}
return pkglist;
}
static alpm_list_t *
dblist_create(void)
{
alpm_list_t *d;
alpm_list_t *dblist = NULL;
int i;
for (i = 0; i < sizeof(alpm_repos) / sizeof(char *); i++) {
if (!alpm_register_syncdb(handle, alpm_repos[i], 0))
alpm_die("failed to register sync db \"%s\": %s\n", alpm_repos[i]);
}
if (!(dblist = alpm_get_syncdbs(handle)))
alpm_die("failed to get sync DBs: %s\n");
for (d = dblist; d; d = alpm_list_next(d)) {
alpm_db_t *db = d->data;
char server[1024];
snprintf(server, 1024, ALPM_MIRROR, alpm_db_get_name(db));
if (alpm_db_add_server(db, server))
alpm_die("failed to set server \"%s\": %s\n", server);
}
return dblist;
}
static int parse_options(int argc, char **argv)
{
int opt;
static const struct option opts[] = {
{ "mysql-host", required_argument, 0, 'h' },
{ "mysql-socket", required_argument, 0, 'S' },
{ "mysql-user", required_argument, 0, 'u' },
{ "mysql-passwd", required_argument, 0, 'p' },
{ "mysql-db", required_argument, 0, 'D' },
{ 0, 0, 0, 0 }
};
while((opt = getopt_long(argc, argv, "h:S:u:p:D:", opts, NULL)) != -1) {
switch(opt) {
case 'h':
mysql_host = optarg;
break;;
case 'S':
mysql_socket = optarg;
break;;
case 'u':
mysql_user = optarg;
break;;
case 'p':
mysql_passwd = optarg;
break;;
case 'D':
mysql_db = optarg;
break;;
default:
return 0;
}
}
return 1;
}
static void
init(void)
{
enum _alpm_errno_t alpm_err;
if (mysql_library_init(0, NULL, NULL))
mysql_die("could not initialize MySQL library: %s\n");
if (!(c = mysql_init(NULL)))
mysql_die("failed to setup MySQL client: %s\n");
if (!mysql_real_connect(c, mysql_host, mysql_user, mysql_passwd,
mysql_db, 0, mysql_socket, 0))
mysql_die("failed to initiate MySQL connection to %s: %s\n", mysql_host);
if ((handle = alpm_initialize("/", ALPM_DBPATH, &alpm_err)) == NULL)
die("failed to initialize ALPM: %s\n", alpm_strerror(alpm_err));
}
static void
cleanup(void)
{
alpm_release(handle);
mysql_close(c);
mysql_library_end();
}
int main(int argc, char *argv[])
{
alpm_list_t *pkgs_cur, *pkgs_new;
if (!parse_options(argc, argv))
return 1;
init();
pkgs_cur = blacklist_get_pkglist();
pkgs_new = dblist_get_pkglist(dblist_create());
blacklist_sync(pkgs_cur, pkgs_new);
FREELIST(pkgs_new);
FREELIST(pkgs_cur);
cleanup();
return 0;
}

48
scripts/aurblup/aurblup.py Executable file
View file

@ -0,0 +1,48 @@
#!/usr/bin/python3
import configparser
import mysql.connector
import os
import pyalpm
config = configparser.RawConfigParser()
config.read(os.path.dirname(os.path.realpath(__file__)) + "/../../conf/config")
aur_db_host = config.get('database', 'host')
aur_db_name = config.get('database', 'name')
aur_db_user = config.get('database', 'user')
aur_db_pass = config.get('database', 'password')
aur_db_socket = config.get('database', 'socket')
db_path = config.get('aurblup', 'db-path')
sync_dbs = config.get('aurblup', 'sync-dbs').split(' ')
servers = config.get('aurblup', 'servers').split(' ')
blacklist = set()
h = pyalpm.Handle("/", db_path)
for sync_db in sync_dbs:
repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL)
repo.servers = [server.replace("%s", sync_db) for server in servers]
t = h.init_transaction()
repo.update(False)
t.release()
for pkg in repo.pkgcache:
blacklist.add(pkg.name)
[blacklist.add(x) for x in pkg.replaces]
db = mysql.connector.connect(host=aur_db_host, user=aur_db_user,
passwd=aur_db_pass, db=aur_db_name,
unix_socket=aur_db_socket, buffered=True)
cur = db.cursor()
cur.execute("SELECT Name FROM PackageBlacklist")
oldblacklist = set([row[0] for row in cur.fetchall()])
for pkg in blacklist.difference(oldblacklist):
cur.execute("INSERT INTO PackageBlacklist (Name) VALUES (%s)", [pkg])
for pkg in oldblacklist.difference(blacklist):
cur.execute("DELETE FROM PackageBlacklist WHERE Name = %s", [pkg])
db.commit()
db.close()

12
scripts/aurblup/config.h.proto
View file

@ -1,12 +0,0 @@
/* libalpm options */
#define ALPM_DBPATH "/var/lib/aurblup/"
#define ALPM_MIRROR "ftp://mirrors.kernel.org/archlinux/%s/os/x86_64"
static const char *alpm_repos[] = {
"core",
"community",
"community-testing",
"extra",
"multilib",
"testing"
};

6
scripts/aurblup/config.mk
View file

@ -1,6 +0,0 @@
PREFIX = /usr/local
CFLAGS = -g -O2 -std=c99 -pedantic -Wall -I/usr/include/mysql
LDFLAGS = -g -lalpm -lmysqlclient
CC = cc