diff --git a/.env b/.env index bf6c48c4..24a2dd43 100644 --- a/.env +++ b/.env @@ -1,6 +1,5 @@ FASTAPI_BACKEND="uvicorn" FASTAPI_WORKERS=2 -MARIADB_SOCKET_DIR="/var/run/mysqld/" AURWEB_FASTAPI_PREFIX=https://localhost:8444 AURWEB_SSHD_PREFIX=ssh://aur@localhost:2222 GIT_DATA_DIR="./aur.git/" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 385736ae..8727b673 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ cache: - .pre-commit variables: - AUR_CONFIG: conf/config # Default MySQL config setup in before_script. + AUR_CONFIG: conf/config # Default PostgresSQL config setup in before_script. DB_HOST: localhost TEST_RECURSION_LIMIT: 10000 CURRENT_DIR: "$(pwd)" @@ -40,12 +40,12 @@ test: - source .venv/bin/activate # Enable our virtualenv cache - ./docker/scripts/install-python-deps.sh - useradd -U -d /aurweb -c 'AUR User' aur - - ./docker/mariadb-entrypoint.sh - - (cd '/usr' && /usr/bin/mysqld_safe --datadir='/var/lib/mysql') & - - 'until : > /dev/tcp/127.0.0.1/3306; do sleep 1s; done' + - ./docker/postgres-entrypoint.sh + - su postgres -c '/usr/bin/postgres -D /var/lib/postgres/data' & + - 'until : > /dev/tcp/127.0.0.1/5432; do sleep 1s; done' - cp -v conf/config.dev conf/config - sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config - - ./docker/test-mysql-entrypoint.sh # Create mysql AUR_CONFIG. + - ./docker/test-postgres-entrypoint.sh # Create postgres AUR_CONFIG. - make -C po all install # Compile translations. - make -C doc # Compile asciidoc. - make -C test clean # Cleanup coverage. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1957ae22..2e6338f3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -91,7 +91,7 @@ browser if desired. Accessible services (on the host): - https://localhost:8444 (python via nginx) -- localhost:13306 (mariadb) +- localhost:15432 (postgresql) - localhost:16379 (redis) Docker services, by default, are setup to be hot reloaded when source code diff --git a/INSTALL b/INSTALL index 23fb6c3d..617da0a1 100644 --- a/INSTALL +++ b/INSTALL @@ -14,7 +14,7 @@ read the instructions below. $ cd aurweb $ poetry install -2) Setup a web server with MySQL. The following block can be used with nginx: +2) Setup a web server with PostgreSQL. The following block can be used with nginx: server { # https is preferred and can be done easily with LetsEncrypt @@ -100,7 +100,7 @@ read the instructions below. 6b) Setup Services aurweb utilizes the following systemd services: -- mariadb +- postgresql - redis (optional, requires [options] cache 'redis') - `examples/aurweb.service` diff --git a/TESTING b/TESTING index e9cbf33b..19233490 100644 --- a/TESTING +++ b/TESTING @@ -31,10 +31,10 @@ Containerized environment 6) [Optionally] populate the database with dummy data: - # docker compose exec mariadb /bin/bash + # docker compose exec postgres /bin/bash # pacman -S --noconfirm words fortune-mod # poetry run schema/gendummydata.py dummy_data.sql - # mariadb -uaur -paur aurweb < dummy_data.sql + # su postgres -q -c 'psql aurweb < dummy_data.sql' # exit Inspect `dummy_data.sql` for test credentials. @@ -62,7 +62,7 @@ INSTALL. 2) Install the necessary packages: - # pacman -S --needed python-poetry mariadb words fortune-mod nginx + # pacman -S --needed python-poetry postgresql words fortune-mod nginx 3) Install the package/dependencies via `poetry`: @@ -76,21 +76,24 @@ INSTALL. Note that when the upstream config.dev is updated, you should compare it to your conf/config, or regenerate your configuration with the command above. -5) Set up mariadb: +5) Set up postgres: - # mariadb-install-db --user=mysql --basedir=/usr --datadir=/var/lib/mysql - # systemctl start mariadb - # mariadb -u root - > CREATE USER 'aur'@'localhost' IDENTIFIED BY 'aur'; - > GRANT ALL ON *.* TO 'aur'@'localhost' WITH GRANT OPTION; - > CREATE DATABASE aurweb; + # su postgres + $ pg_ctl initdb -D /var/lib/postgres/data + $ pg_ctl start -D /var/lib/postgres/data + $ psql + > create database aurweb; + > create role aur superuser login password 'aur'; > exit + For the sake of simplicity in this example we just created a superuser account. + You might want to set up more granular permissions... + 6) Prepare a database and insert dummy data: $ AUR_CONFIG=conf/config poetry run python -m aurweb.initdb $ poetry run schema/gendummydata.py dummy_data.sql - $ mariadb -uaur -paur aurweb < dummy_data.sql + $ psql -U aur aurweb < dummy_data.sql 7) Run the test server: @@ -121,7 +124,7 @@ In case you did the bare-metal install, steps 2, 3, 4 and 5 should be skipped. 1) Install the necessary packages: - # pacman -S --needed python-poetry mariadb-libs asciidoc openssh + # pacman -S --needed python-poetry postgresql-libs asciidoc openssh 2) Install the package/dependencies via `poetry`: @@ -135,24 +138,24 @@ In case you did the bare-metal install, steps 2, 3, 4 and 5 should be skipped. Note that when the upstream config.dev is updated, you should compare it to your conf/config, or regenerate your configuration with the command above. -4) Edit the config file conf/config and change the mysql/mariadb portion +4) Edit the config file conf/config and change the postgres portion - We can make use of our mariadb docker container instead of having to install - mariadb. Change the config as follows: + We can make use of our postgres docker container instead of having to install + postgres. Change the config as follows: --------------------------------------------------------------------- - ; MySQL database information. User defaults to root for containerized - ; testing with mysqldb. This should be set to a non-root user. - user = root + ; PostgreSQL database information. User defaults to root for containerized + ; testing with postgres. This should be set to a non-root user. + user = aur password = aur host = 127.0.0.1 - port = 13306 - ;socket = /var/run/mysqld/mysqld.sock + port = 15432 + ;socket = /run/postgresql --------------------------------------------------------------------- -5) Start our mariadb docker container +5) Start our postgres docker container - # docker compose start mariadb + # docker compose start postgres 6) Set environment variables diff --git a/aurweb/auth/__init__.py b/aurweb/auth/__init__.py index e895dcdb..57b01de9 100644 --- a/aurweb/auth/__init__.py +++ b/aurweb/auth/__init__.py @@ -50,7 +50,7 @@ class AnonymousUser: LangPreference = aurweb.config.get("options", "default_lang") Timezone = aurweb.config.get("options", "default_timezone") - Suspended = 0 + Suspended = False InactivityTS = 0 # A stub ssh_pub_key relationship. @@ -120,7 +120,7 @@ class BasicAuthBackend(AuthenticationBackend): # At this point, we cannot have an invalid user if the record # exists, due to ForeignKey constraints in the schema upheld - # by mysqlclient. + # by the database system. user = db.query(User).filter(User.ID == record.UsersID).first() user.nonce = util.make_nonce() user.authenticated = True diff --git a/aurweb/db.py b/aurweb/db.py index 8311f2be..d4af6e3c 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -1,5 +1,7 @@ +from sqlalchemy.orm import Session + # Supported database drivers. -DRIVERS = {"mysql": "mysql+mysqldb"} +DRIVERS = {"postgres": "postgresql+psycopg2"} def make_random_value(table: str, column: str, length: int): @@ -65,7 +67,7 @@ def name() -> str: _sessions = dict() -def get_session(engine=None): +def get_session(engine=None) -> Session: """Return aurweb.db's global session.""" dbname = name() @@ -221,22 +223,21 @@ def get_sqlalchemy_url(): constructor = URL.create aur_db_backend = aurweb.config.get("database", "backend") - if aur_db_backend == "mysql": - param_query = {} + if aur_db_backend == "postgres": port = aurweb.config.get_with_fallback("database", "port", None) + host = aurweb.config.get_with_fallback("database", "host", None) + socket = None if not port: - param_query["unix_socket"] = aurweb.config.get("database", "socket") - + socket = aurweb.config.get("database", "socket") return constructor( DRIVERS.get(aur_db_backend), username=aurweb.config.get("database", "user"), password=aurweb.config.get_with_fallback( "database", "password", fallback=None ), - host=aurweb.config.get("database", "host"), + host=socket if socket else host, database=name(), port=port, - query=param_query, ) elif aur_db_backend == "sqlite": return constructor( @@ -352,7 +353,7 @@ class ConnectionExecutor: backend = backend or aurweb.config.get("database", "backend") self._conn = conn - if backend == "mysql": + if backend == "postgres": self._paramstyle = "format" elif backend == "sqlite": import sqlite3 @@ -393,20 +394,21 @@ class Connection: aur_db_backend = aurweb.config.get("database", "backend") - if aur_db_backend == "mysql": - import MySQLdb + if aur_db_backend == "postgres": + import psycopg2 - aur_db_host = aurweb.config.get("database", "host") + aur_db_host = aurweb.config.get_with_fallback("database", "host", None) aur_db_name = name() aur_db_user = aurweb.config.get("database", "user") aur_db_pass = aurweb.config.get_with_fallback("database", "password", str()) - aur_db_socket = aurweb.config.get("database", "socket") - self._conn = MySQLdb.connect( - host=aur_db_host, + aur_db_socket = aurweb.config.get_with_fallback("database", "socket", None) + aur_db_port = aurweb.config.get_with_fallback("database", "port", None) + self._conn = psycopg2.connect( + host=aur_db_host if not aur_db_socket else aur_db_socket, user=aur_db_user, - passwd=aur_db_pass, - db=aur_db_name, - unix_socket=aur_db_socket, + password=aur_db_pass, + dbname=aur_db_name, + port=aur_db_port if not aur_db_socket else None, ) elif aur_db_backend == "sqlite": # pragma: no cover # TODO: SQLite support has been removed in FastAPI. It remains diff --git a/aurweb/git/auth.py b/aurweb/git/auth.py index 759fce89..e991f1ad 100755 --- a/aurweb/git/auth.py +++ b/aurweb/git/auth.py @@ -39,7 +39,7 @@ def main(): cur = conn.execute( "SELECT Users.Username, Users.AccountTypeID FROM Users " "INNER JOIN SSHPubKeys ON SSHPubKeys.UserID = Users.ID " - "WHERE SSHPubKeys.PubKey = ? AND Users.Suspended = 0 " + "WHERE SSHPubKeys.PubKey = ? AND Users.Suspended = False " "AND NOT Users.Passwd = ''", (keytype + " " + keytext,), ) diff --git a/aurweb/git/update.py b/aurweb/git/update.py index 4c4fff0f..d08f3e7a 100755 --- a/aurweb/git/update.py +++ b/aurweb/git/update.py @@ -63,10 +63,10 @@ def create_pkgbase(conn, pkgbase, user): cur = conn.execute( "INSERT INTO PackageBases (Name, SubmittedTS, " + "ModifiedTS, SubmitterUID, MaintainerUID, " - + "FlaggerComment) VALUES (?, ?, ?, ?, ?, '')", + + "FlaggerComment) VALUES (?, ?, ?, ?, ?, '') RETURNING id", [pkgbase, now, now, userid, userid], ) - pkgbase_id = cur.lastrowid + pkgbase_id = cur.fetchone()[0] cur = conn.execute( "INSERT INTO PackageNotifications " + "(PackageBaseID, UserID) VALUES (?, ?)", @@ -135,11 +135,11 @@ def save_metadata(metadata, conn, user): # noqa: C901 cur = conn.execute( "INSERT INTO Packages (PackageBaseID, Name, " + "Version, Description, URL) " - + "VALUES (?, ?, ?, ?, ?)", + + "VALUES (?, ?, ?, ?, ?) RETURNING id", [pkgbase_id, pkginfo["pkgname"], ver, pkginfo["pkgdesc"], pkginfo["url"]], ) + pkgid = cur.fetchone()[0] conn.commit() - pkgid = cur.lastrowid # Add package sources. for source_info in extract_arch_fields(pkginfo, "source"): @@ -188,10 +188,11 @@ def save_metadata(metadata, conn, user): # noqa: C901 licenseid = row[0] else: cur = conn.execute( - "INSERT INTO Licenses (Name) " + "VALUES (?)", [license] + "INSERT INTO Licenses (Name) " + "VALUES (?) RETURNING id", + [license], ) + licenseid = cur.fetchone()[0] conn.commit() - licenseid = cur.lastrowid conn.execute( "INSERT INTO PackageLicenses (PackageID, " + "LicenseID) VALUES (?, ?)", @@ -201,16 +202,16 @@ def save_metadata(metadata, conn, user): # noqa: C901 # Add package groups. if "groups" in pkginfo: for group in pkginfo["groups"]: - cur = conn.execute("SELECT ID FROM `Groups` WHERE Name = ?", [group]) + cur = conn.execute("SELECT ID FROM Groups WHERE Name = ?", [group]) row = cur.fetchone() if row: groupid = row[0] else: cur = conn.execute( - "INSERT INTO `Groups` (Name) VALUES (?)", [group] + "INSERT INTO Groups (Name) VALUES (?) RETURNING id", [group] ) + groupid = cur.fetchone()[0] conn.commit() - groupid = cur.lastrowid conn.execute( "INSERT INTO PackageGroups (PackageID, " "GroupID) VALUES (?, ?)", [pkgid, groupid], diff --git a/aurweb/initdb.py b/aurweb/initdb.py index 7181ea3e..8dcf73f6 100644 --- a/aurweb/initdb.py +++ b/aurweb/initdb.py @@ -12,35 +12,35 @@ def feed_initial_data(conn): conn.execute( aurweb.schema.AccountTypes.insert(), [ - {"ID": 1, "AccountType": "User"}, - {"ID": 2, "AccountType": "Package Maintainer"}, - {"ID": 3, "AccountType": "Developer"}, - {"ID": 4, "AccountType": "Package Maintainer & Developer"}, + {"AccountType": "User"}, + {"AccountType": "Package Maintainer"}, + {"AccountType": "Developer"}, + {"AccountType": "Package Maintainer & Developer"}, ], ) conn.execute( aurweb.schema.DependencyTypes.insert(), [ - {"ID": 1, "Name": "depends"}, - {"ID": 2, "Name": "makedepends"}, - {"ID": 3, "Name": "checkdepends"}, - {"ID": 4, "Name": "optdepends"}, + {"Name": "depends"}, + {"Name": "makedepends"}, + {"Name": "checkdepends"}, + {"Name": "optdepends"}, ], ) conn.execute( aurweb.schema.RelationTypes.insert(), [ - {"ID": 1, "Name": "conflicts"}, - {"ID": 2, "Name": "provides"}, - {"ID": 3, "Name": "replaces"}, + {"Name": "conflicts"}, + {"Name": "provides"}, + {"Name": "replaces"}, ], ) conn.execute( aurweb.schema.RequestTypes.insert(), [ - {"ID": 1, "Name": "deletion"}, - {"ID": 2, "Name": "orphan"}, - {"ID": 3, "Name": "merge"}, + {"Name": "deletion"}, + {"Name": "orphan"}, + {"Name": "merge"}, ], ) @@ -57,8 +57,9 @@ def run(args): alembic_config.attributes["configure_logger"] = False engine = aurweb.db.get_engine(echo=(args.verbose >= 1)) - aurweb.schema.metadata.create_all(engine) conn = engine.connect() + # conn.execute("CREATE COLLATION ci (provider = icu, locale = 'und-u-ks-level2', deterministic = false)") # noqa: E501 + aurweb.schema.metadata.create_all(engine) feed_initial_data(conn) conn.close() diff --git a/aurweb/models/declarative.py b/aurweb/models/declarative.py index 22df31c7..5ec5c71c 100644 --- a/aurweb/models/declarative.py +++ b/aurweb/models/declarative.py @@ -6,7 +6,8 @@ from aurweb import util def to_dict(model): - return {c.name: getattr(model, c.name) for c in model.__table__.columns} + return {c.origname: getattr(model, c.origname) for c in model.__table__.columns} + # return {c.name: getattr(model, c.name) for c in model.__table__.columns} def to_json(model, indent: int = None): diff --git a/aurweb/models/package_request.py b/aurweb/models/package_request.py index 94ff064b..3e54ccca 100644 --- a/aurweb/models/package_request.py +++ b/aurweb/models/package_request.py @@ -21,6 +21,13 @@ CLOSED_ID = 1 ACCEPTED_ID = 2 REJECTED_ID = 3 +STATUS_DISPLAY = { + PENDING_ID: PENDING, + CLOSED_ID: CLOSED, + ACCEPTED_ID: ACCEPTED, + REJECTED_ID: REJECTED, +} + class PackageRequest(Base): __table__ = schema.PackageRequests @@ -51,13 +58,6 @@ class PackageRequest(Base): foreign_keys=[__table__.c.ClosedUID], ) - STATUS_DISPLAY = { - PENDING_ID: PENDING, - CLOSED_ID: CLOSED, - ACCEPTED_ID: ACCEPTED, - REJECTED_ID: REJECTED, - } - def __init__(self, **kwargs): super().__init__(**kwargs) @@ -105,7 +105,7 @@ class PackageRequest(Base): def status_display(self) -> str: """Return a display string for the Status column.""" - return self.STATUS_DISPLAY[self.Status] + return STATUS_DISPLAY[self.Status] def ml_message_id_hash(self) -> str: """Return the X-Message-ID-Hash that is used in the mailing list archive.""" diff --git a/aurweb/packages/search.py b/aurweb/packages/search.py index 78b27a9a..a4b3d995 100644 --- a/aurweb/packages/search.py +++ b/aurweb/packages/search.py @@ -1,6 +1,6 @@ from typing import Set -from sqlalchemy import and_, case, or_, orm +from sqlalchemy import and_, case, func, or_, orm from aurweb import db, models from aurweb.models import Group, Package, PackageBase, User @@ -106,7 +106,7 @@ class PackageSearch: self.query = self.query.filter( or_( Package.Name.like(f"%{keywords}%"), - Package.Description.like(f"%{keywords}%"), + func.lower(Package.Description).like(f"%{keywords}%"), ) ) return self @@ -136,9 +136,9 @@ class PackageSearch: self._join_user() self._join_keywords() keywords = set(k.lower() for k in keywords) - self.query = self.query.filter(PackageKeyword.Keyword.in_(keywords)).group_by( - models.Package.Name - ) + self.query = self.query.filter( + func.lower(PackageKeyword.Keyword).in_(keywords) + ).distinct() return self @@ -146,7 +146,10 @@ class PackageSearch: self._join_user() if keywords: self.query = self.query.filter( - and_(User.Username == keywords, User.ID == PackageBase.MaintainerUID) + and_( + func.lower(User.Username) == keywords, + User.ID == PackageBase.MaintainerUID, + ) ) else: self.query = self.query.filter(PackageBase.MaintainerUID.is_(None)) @@ -155,7 +158,7 @@ class PackageSearch: def _search_by_comaintainer(self, keywords: str) -> orm.Query: self._join_user() self._join_comaint() - user = db.query(User).filter(User.Username == keywords).first() + user = db.query(User).filter(func.lower(User.Username) == keywords).first() uid = 0 if not user else user.ID self.query = self.query.filter(PackageComaintainer.UsersID == uid) return self @@ -163,7 +166,7 @@ class PackageSearch: def _search_by_co_or_maintainer(self, keywords: str) -> orm.Query: self._join_user() self._join_comaint(True) - user = db.query(User).filter(User.Username == keywords).first() + user = db.query(User).filter(func.lower(User.Username) == keywords).first() uid = 0 if not user else user.ID self.query = self.query.filter( or_(PackageComaintainer.UsersID == uid, User.ID == uid) @@ -174,7 +177,7 @@ class PackageSearch: self._join_user() uid = 0 - user = db.query(User).filter(User.Username == keywords).first() + user = db.query(User).filter(func.lower(User.Username) == keywords).first() if user: uid = user.ID diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index a2c6cbaa..3acd27b7 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -102,7 +102,7 @@ def get_pkg_or_base( :raises HTTPException: With status code 404 if record doesn't exist :return: {Package,PackageBase} instance """ - instance = db.query(cls).filter(cls.Name == name).first() + instance = db.query(cls).filter(cls.Name == name.lower()).first() if not instance: raise HTTPException(status_code=HTTPStatus.NOT_FOUND) return instance diff --git a/aurweb/pkgbase/util.py b/aurweb/pkgbase/util.py index 695a2a38..2c6c9f17 100644 --- a/aurweb/pkgbase/util.py +++ b/aurweb/pkgbase/util.py @@ -4,7 +4,7 @@ from fastapi import Request from sqlalchemy import and_ from aurweb import config, db, defaults, l10n, time, util -from aurweb.models import PackageBase, User +from aurweb.models import PackageBase, PackageKeyword, User from aurweb.models.package_base import popularity from aurweb.models.package_comaintainer import PackageComaintainer from aurweb.models.package_comment import PackageComment @@ -46,7 +46,7 @@ def make_context( context["unflaggers"].extend([pkgbase.Maintainer, pkgbase.Flagger]) context["packages_count"] = pkgbase.packages.count() - context["keywords"] = pkgbase.keywords + context["keywords"] = pkgbase.keywords.order_by(PackageKeyword.Keyword) context["comments_total"] = pkgbase.comments.order_by( PackageComment.CommentTS.desc() ).count() diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index a2d167bc..790ac2ae 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -47,7 +47,7 @@ async def passreset_post( # The user parameter being required, we can match against criteria = or_(models.User.Username == user, models.User.Email == user) - db_user = db.query(models.User, and_(criteria, models.User.Suspended == 0)).first() + db_user = db.query(models.User, and_(criteria, ~models.User.Suspended)).first() if db_user is None: context["errors"] = ["Invalid e-mail."] return render_template( @@ -584,11 +584,11 @@ async def accounts_post( v for k, v in [ (account_type_id is not None, models.AccountType.ID == account_type_id), - (bool(U), models.User.Username.like(f"%{U}%")), + (bool(U), models.User.Username.ilike(f"%{U}%")), (bool(S), models.User.Suspended == S), - (bool(E), models.User.Email.like(f"%{E}%")), - (bool(R), models.User.RealName.like(f"%{R}%")), - (bool(I), models.User.IRCNick.like(f"%{I}%")), + (bool(E), models.User.Email.ilike(f"%{E}%")), + (bool(R), models.User.RealName.ilike(f"%{R}%")), + (bool(I), models.User.IRCNick.ilike(f"%{I}%")), (bool(K), models.User.PGPKey.like(f"%{K}%")), ] if k diff --git a/aurweb/routers/auth.py b/aurweb/routers/auth.py index 88eaa0e6..fb563b91 100644 --- a/aurweb/routers/auth.py +++ b/aurweb/routers/auth.py @@ -2,7 +2,7 @@ from http import HTTPStatus from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse -from sqlalchemy import or_ +from sqlalchemy import func, or_ import aurweb.config from aurweb import cookies, db @@ -57,8 +57,8 @@ async def login_post( db.query(User) .filter( or_( - User.Username == user, - User.Email == user, + func.lower(User.Username) == user.lower(), + func.lower(User.Email) == user.lower(), ) ) .first() diff --git a/aurweb/routers/package_maintainer.py b/aurweb/routers/package_maintainer.py index 9ce38d07..4526e38c 100644 --- a/aurweb/routers/package_maintainer.py +++ b/aurweb/routers/package_maintainer.py @@ -124,7 +124,7 @@ async def package_maintainer( ) ) .with_entities(models.Vote.UserID, last_vote, models.User.Username) - .group_by(models.Vote.UserID) + .group_by(models.Vote.UserID, models.User.Username) .order_by(last_vote.desc(), models.User.Username.asc()) ) context["last_votes_by_pm"] = last_votes_by_pm.all() @@ -371,7 +371,7 @@ async def package_maintainer_addvote_post( db.query(User) .filter( and_( - User.Suspended == 0, + ~User.Suspended, User.InactivityTS.isnot(None), User.AccountTypeID.in_(types), ) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 30d0d896..9847af6b 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -54,6 +54,7 @@ async def packages_get( # This means that for any sentences separated by spaces, # they are used as if they were ANDed. keywords = context["K"] = request.query_params.get("K", str()) + keywords = keywords.lower() keywords = keywords.split(" ") if search_by == "k": diff --git a/aurweb/routers/requests.py b/aurweb/routers/requests.py index a67419fe..b987dec9 100644 --- a/aurweb/routers/requests.py +++ b/aurweb/routers/requests.py @@ -95,7 +95,9 @@ async def requests( # noqa: C901 # Name filter (contains) if filter_pkg_name: - filtered = filtered.filter(PackageBase.Name.like(f"%{filter_pkg_name}%")) + filtered = filtered.filter( + PackageBase.Name.like(f"%{filter_pkg_name.lower()}%") + ) # Additionally filter for requests made from package maintainer if filter_maintainer_requests: diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 5fcbbb78..e2f0961b 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -218,7 +218,7 @@ class RPC: models.User.Username.label("Maintainer"), Submitter.Username.label("Submitter"), ) - .group_by(models.Package.ID) + .distinct() ) return query @@ -465,6 +465,9 @@ class RPC: # Convert by to its aliased value if it has one. by = RPC.BY_ALIASES.get(by, by) + # lowercase all args + args = [arg.lower() for arg in args] + # Process the requested handler. try: results = self._handle_callback(by, args) diff --git a/aurweb/schema.py b/aurweb/schema.py index 683f427d..7a38a568 100644 --- a/aurweb/schema.py +++ b/aurweb/schema.py @@ -7,7 +7,6 @@ usually be automatically generated. See `migrations/README` for details. from sqlalchemy import ( - CHAR, TIMESTAMP, Column, ForeignKey, @@ -16,19 +15,23 @@ from sqlalchemy import ( String, Table, Text, + event, text, ) -from sqlalchemy.dialects.mysql import BIGINT, DECIMAL, INTEGER, TINYINT +from sqlalchemy.dialects.postgresql import BIGINT, BOOLEAN, INTEGER, NUMERIC, SMALLINT from sqlalchemy.ext.compiler import compiles import aurweb.config +# from sqlalchemy import event + + db_backend = aurweb.config.get("database", "backend") -@compiles(TINYINT, "sqlite") -def compile_tinyint_sqlite(type_, compiler, **kw): # pragma: no cover - """TINYINT is not supported on SQLite. Substitute it with INTEGER.""" +@compiles(SMALLINT, "sqlite") +def compile_smallint_sqlite(type_, compiler, **kw): # pragma: no cover + """SMALLINT is not supported on SQLite. Substitute it with INTEGER.""" return "INTEGER" @@ -43,17 +46,26 @@ def compile_bigint_sqlite(type_, compiler, **kw): # pragma: no cover return "INTEGER" +@event.listens_for(Column, "before_parent_attach") +def attach_column(column: Column, parent, **kw): + column.origname = column.name + column.name = column.name.lower() + + +@event.listens_for(Index, "before_parent_attach") +def attach_index(index, parent, **kw): + index.name = index.name.lower() + + metadata = MetaData() # Define the Account Types for the AUR. AccountTypes = Table( "AccountTypes", metadata, - Column("ID", TINYINT(unsigned=True), primary_key=True), + Column("ID", SMALLINT(), primary_key=True), Column("AccountType", String(32), nullable=False, server_default=text("''")), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -61,62 +73,51 @@ AccountTypes = Table( Users = Table( "Users", metadata, - Column("ID", INTEGER(unsigned=True), primary_key=True), + Column("ID", INTEGER(), primary_key=True), Column( "AccountTypeID", ForeignKey("AccountTypes.ID", ondelete="NO ACTION"), nullable=False, server_default=text("1"), ), - Column( - "Suspended", TINYINT(unsigned=True), nullable=False, server_default=text("0") - ), + Column("Suspended", BOOLEAN(), nullable=False, server_default=text("False")), Column("Username", String(32), nullable=False, unique=True), Column("Email", String(254), nullable=False, unique=True), Column("BackupEmail", String(254)), - Column( - "HideEmail", TINYINT(unsigned=True), nullable=False, server_default=text("0") - ), + Column("HideEmail", BOOLEAN(), nullable=False, server_default=text("False")), Column("Passwd", String(255), nullable=False), - Column("Salt", CHAR(32), nullable=False, server_default=text("''")), - Column("ResetKey", CHAR(32), nullable=False, server_default=text("''")), + Column("Salt", String(32), nullable=False, server_default=text("''")), + Column("ResetKey", String(32), nullable=False, server_default=text("''")), Column("RealName", String(64), nullable=False, server_default=text("''")), Column("LangPreference", String(6), nullable=False, server_default=text("'en'")), Column("Timezone", String(32), nullable=False, server_default=text("'UTC'")), Column("Homepage", Text), Column("IRCNick", String(32), nullable=False, server_default=text("''")), Column("PGPKey", String(40)), - Column( - "LastLogin", BIGINT(unsigned=True), nullable=False, server_default=text("0") - ), + Column("LastLogin", BIGINT(), nullable=False, server_default=text("0")), Column("LastLoginIPAddress", String(45)), - Column( - "LastSSHLogin", BIGINT(unsigned=True), nullable=False, server_default=text("0") - ), + Column("LastSSHLogin", BIGINT(), nullable=False, server_default=text("0")), Column("LastSSHLoginIPAddress", String(45)), - Column( - "InactivityTS", BIGINT(unsigned=True), nullable=False, server_default=text("0") - ), + Column("InactivityTS", BIGINT(), nullable=False, server_default=text("0")), Column( "RegistrationTS", TIMESTAMP, - nullable=False, server_default=text("CURRENT_TIMESTAMP"), ), - Column("CommentNotify", TINYINT(1), nullable=False, server_default=text("1")), - Column("UpdateNotify", TINYINT(1), nullable=False, server_default=text("0")), - Column("OwnershipNotify", TINYINT(1), nullable=False, server_default=text("1")), + Column("CommentNotify", BOOLEAN(), nullable=False, server_default=text("True")), + Column("UpdateNotify", BOOLEAN(), nullable=False, server_default=text("False")), + Column("OwnershipNotify", BOOLEAN(), nullable=False, server_default=text("True")), Column("SSOAccountID", String(255), nullable=True, unique=True), Index("UsersAccountTypeID", "AccountTypeID"), Column( "HideDeletedComments", - TINYINT(unsigned=True), + BOOLEAN(), nullable=False, - server_default=text("0"), + server_default=text("False"), ), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + Index("UsernameLowerUnique", text("lower(username)"), unique=True), + Index("EmailLowerUnique", text("lower(email)"), unique=True), + quote=False, ) @@ -127,9 +128,7 @@ SSHPubKeys = Table( Column("UserID", ForeignKey("Users.ID", ondelete="CASCADE"), nullable=False), Column("Fingerprint", String(44), primary_key=True), Column("PubKey", String(4096), nullable=False), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_bin", + quote=False, ) @@ -138,11 +137,9 @@ Sessions = Table( "Sessions", metadata, Column("UsersID", ForeignKey("Users.ID", ondelete="CASCADE"), nullable=False), - Column("SessionID", CHAR(32), nullable=False, unique=True), - Column("LastUpdateTS", BIGINT(unsigned=True), nullable=False), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_bin", + Column("SessionID", String(32), nullable=False, unique=True), + Column("LastUpdateTS", BIGINT(), nullable=False), + quote=False, ) @@ -150,14 +147,12 @@ Sessions = Table( PackageBases = Table( "PackageBases", metadata, - Column("ID", INTEGER(unsigned=True), primary_key=True), + Column("ID", INTEGER(), primary_key=True), Column("Name", String(255), nullable=False, unique=True), - Column( - "NumVotes", INTEGER(unsigned=True), nullable=False, server_default=text("0") - ), + Column("NumVotes", INTEGER(), nullable=False, server_default=text("0")), Column( "Popularity", - DECIMAL(10, 6, unsigned=True) if db_backend == "mysql" else String(17), + NUMERIC(10, 6) if db_backend == "postgres" else String(17), nullable=False, server_default=text("0"), ), @@ -167,10 +162,10 @@ PackageBases = Table( nullable=False, server_default=text("'1970-01-01 00:00:01.000000'"), ), - Column("OutOfDateTS", BIGINT(unsigned=True)), + Column("OutOfDateTS", BIGINT()), Column("FlaggerComment", Text, nullable=False), - Column("SubmittedTS", BIGINT(unsigned=True), nullable=False), - Column("ModifiedTS", BIGINT(unsigned=True), nullable=False), + Column("SubmittedTS", BIGINT(), nullable=False), + Column("ModifiedTS", BIGINT(), nullable=False), Column( "FlaggerUID", ForeignKey("Users.ID", ondelete="SET NULL") ), # who flagged the package out-of-date? @@ -184,9 +179,8 @@ PackageBases = Table( Index("BasesNumVotes", "NumVotes"), Index("BasesPackagerUID", "PackagerUID"), Index("BasesSubmitterUID", "SubmitterUID"), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + Index("BasesNameLowerUnique", text("lower(name)"), unique=True), + quote=False, ) @@ -208,9 +202,7 @@ PackageKeywords = Table( server_default=text("''"), ), Index("KeywordsPackageBaseID", "PackageBaseID"), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -218,7 +210,7 @@ PackageKeywords = Table( Packages = Table( "Packages", metadata, - Column("ID", INTEGER(unsigned=True), primary_key=True), + Column("ID", INTEGER(), primary_key=True), Column( "PackageBaseID", ForeignKey("PackageBases.ID", ondelete="CASCADE"), @@ -228,9 +220,8 @@ Packages = Table( Column("Version", String(255), nullable=False, server_default=text("''")), Column("Description", String(255)), Column("URL", String(8000)), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + Index("PackagesNameLowerUnique", text("lower(name)"), unique=True), + quote=False, ) @@ -238,11 +229,9 @@ Packages = Table( Licenses = Table( "Licenses", metadata, - Column("ID", INTEGER(unsigned=True), primary_key=True), + Column("ID", INTEGER(), primary_key=True), Column("Name", String(255), nullable=False, unique=True), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -262,7 +251,7 @@ PackageLicenses = Table( primary_key=True, nullable=True, ), - mysql_engine="InnoDB", + quote=False, ) @@ -270,11 +259,9 @@ PackageLicenses = Table( Groups = Table( "Groups", metadata, - Column("ID", INTEGER(unsigned=True), primary_key=True), + Column("ID", INTEGER(), primary_key=True), Column("Name", String(255), nullable=False, unique=True), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -294,7 +281,7 @@ PackageGroups = Table( primary_key=True, nullable=True, ), - mysql_engine="InnoDB", + quote=False, ) @@ -302,11 +289,9 @@ PackageGroups = Table( DependencyTypes = Table( "DependencyTypes", metadata, - Column("ID", TINYINT(unsigned=True), primary_key=True), + Column("ID", SMALLINT(), primary_key=True), Column("Name", String(32), nullable=False, server_default=text("''")), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -326,9 +311,7 @@ PackageDepends = Table( Column("DepArch", String(255)), Index("DependsDepName", "DepName"), Index("DependsPackageID", "PackageID"), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -336,11 +319,9 @@ PackageDepends = Table( RelationTypes = Table( "RelationTypes", metadata, - Column("ID", TINYINT(unsigned=True), primary_key=True), + Column("ID", SMALLINT(), primary_key=True), Column("Name", String(32), nullable=False, server_default=text("''")), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -359,9 +340,7 @@ PackageRelations = Table( Column("RelArch", String(255)), Index("RelationsPackageID", "PackageID"), Index("RelationsRelName", "RelName"), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -373,9 +352,7 @@ PackageSources = Table( Column("Source", String(8000), nullable=False, server_default=text("'/dev/null'")), Column("SourceArch", String(255)), Index("SourcesPackageID", "PackageID"), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -389,11 +366,11 @@ PackageVotes = Table( ForeignKey("PackageBases.ID", ondelete="CASCADE"), nullable=False, ), - Column("VoteTS", BIGINT(unsigned=True), nullable=False), + Column("VoteTS", BIGINT(), nullable=False), Index("VoteUsersIDPackageID", "UsersID", "PackageBaseID", unique=True), Index("VotesPackageBaseID", "PackageBaseID"), Index("VotesUsersID", "UsersID"), - mysql_engine="InnoDB", + quote=False, ) @@ -401,7 +378,7 @@ PackageVotes = Table( PackageComments = Table( "PackageComments", metadata, - Column("ID", BIGINT(unsigned=True), primary_key=True), + Column("ID", BIGINT(), primary_key=True), Column( "PackageBaseID", ForeignKey("PackageBases.ID", ondelete="CASCADE"), @@ -410,19 +387,15 @@ PackageComments = Table( Column("UsersID", ForeignKey("Users.ID", ondelete="SET NULL")), Column("Comments", Text, nullable=False), Column("RenderedComment", Text, nullable=False), - Column( - "CommentTS", BIGINT(unsigned=True), nullable=False, server_default=text("0") - ), - Column("EditedTS", BIGINT(unsigned=True)), + Column("CommentTS", BIGINT(), nullable=False, server_default=text("0")), + Column("EditedTS", BIGINT()), Column("EditedUsersID", ForeignKey("Users.ID", ondelete="SET NULL")), - Column("DelTS", BIGINT(unsigned=True)), + Column("DelTS", BIGINT()), Column("DelUsersID", ForeignKey("Users.ID", ondelete="CASCADE")), - Column("PinnedTS", BIGINT(unsigned=True), nullable=False, server_default=text("0")), + Column("PinnedTS", BIGINT(), nullable=False, server_default=text("0")), Index("CommentsPackageBaseID", "PackageBaseID"), Index("CommentsUsersID", "UsersID"), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -436,10 +409,10 @@ PackageComaintainers = Table( ForeignKey("PackageBases.ID", ondelete="CASCADE"), nullable=False, ), - Column("Priority", INTEGER(unsigned=True), nullable=False), + Column("Priority", INTEGER(), nullable=False), Index("ComaintainersPackageBaseID", "PackageBaseID"), Index("ComaintainersUsersID", "UsersID"), - mysql_engine="InnoDB", + quote=False, ) @@ -454,7 +427,7 @@ PackageNotifications = Table( ), Column("UserID", ForeignKey("Users.ID", ondelete="CASCADE"), nullable=False), Index("NotifyUserIDPkgID", "UserID", "PackageBaseID", unique=True), - mysql_engine="InnoDB", + quote=False, ) @@ -462,11 +435,9 @@ PackageNotifications = Table( PackageBlacklist = Table( "PackageBlacklist", metadata, - Column("ID", INTEGER(unsigned=True), primary_key=True), + Column("ID", INTEGER(), primary_key=True), Column("Name", String(64), nullable=False, unique=True), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -474,14 +445,12 @@ PackageBlacklist = Table( OfficialProviders = Table( "OfficialProviders", metadata, - Column("ID", INTEGER(unsigned=True), primary_key=True), + Column("ID", INTEGER(), primary_key=True), Column("Name", String(64), nullable=False), Column("Repo", String(64), nullable=False), Column("Provides", String(64), nullable=False), Index("ProviderNameProvides", "Name", "Provides", unique=True), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_bin", + quote=False, ) @@ -489,11 +458,9 @@ OfficialProviders = Table( RequestTypes = Table( "RequestTypes", metadata, - Column("ID", TINYINT(unsigned=True), primary_key=True), + Column("ID", SMALLINT(), primary_key=True), Column("Name", String(32), nullable=False, server_default=text("''")), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -501,7 +468,7 @@ RequestTypes = Table( PackageRequests = Table( "PackageRequests", metadata, - Column("ID", BIGINT(unsigned=True), primary_key=True), + Column("ID", BIGINT(), primary_key=True), Column( "ReqTypeID", ForeignKey("RequestTypes.ID", ondelete="NO ACTION"), nullable=False ), @@ -511,17 +478,13 @@ PackageRequests = Table( Column("UsersID", ForeignKey("Users.ID", ondelete="SET NULL")), Column("Comments", Text, nullable=False), Column("ClosureComment", Text, nullable=False), - Column( - "RequestTS", BIGINT(unsigned=True), nullable=False, server_default=text("0") - ), - Column("ClosedTS", BIGINT(unsigned=True)), + Column("RequestTS", BIGINT(), nullable=False, server_default=text("0")), + Column("ClosedTS", BIGINT()), Column("ClosedUID", ForeignKey("Users.ID", ondelete="SET NULL")), - Column("Status", TINYINT(unsigned=True), nullable=False, server_default=text("0")), + Column("Status", SMALLINT(), nullable=False, server_default=text("0")), Index("RequestsPackageBaseID", "PackageBaseID"), Index("RequestsUsersID", "UsersID"), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -529,31 +492,27 @@ PackageRequests = Table( VoteInfo = Table( "VoteInfo", metadata, - Column("ID", INTEGER(unsigned=True), primary_key=True), + Column("ID", INTEGER(), primary_key=True), Column("Agenda", Text, nullable=False), Column("User", String(32), nullable=False), - Column("Submitted", BIGINT(unsigned=True), nullable=False), - Column("End", BIGINT(unsigned=True), nullable=False), + Column("Submitted", BIGINT(), nullable=False), + Column("End", BIGINT(), nullable=False), Column( "Quorum", - DECIMAL(2, 2, unsigned=True) if db_backend == "mysql" else String(5), + NUMERIC(2, 2) if db_backend == "postgres" else String(5), nullable=False, ), Column("SubmitterID", ForeignKey("Users.ID", ondelete="CASCADE"), nullable=False), - Column("Yes", INTEGER(unsigned=True), nullable=False, server_default=text("'0'")), - Column("No", INTEGER(unsigned=True), nullable=False, server_default=text("'0'")), - Column( - "Abstain", INTEGER(unsigned=True), nullable=False, server_default=text("'0'") - ), + Column("Yes", INTEGER(), nullable=False, server_default=text("'0'")), + Column("No", INTEGER(), nullable=False, server_default=text("'0'")), + Column("Abstain", INTEGER(), nullable=False, server_default=text("'0'")), Column( "ActiveUsers", - INTEGER(unsigned=True), + INTEGER(), nullable=False, server_default=text("'0'"), ), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -563,7 +522,7 @@ Votes = Table( metadata, Column("VoteID", ForeignKey("VoteInfo.ID", ondelete="CASCADE"), nullable=False), Column("UserID", ForeignKey("Users.ID", ondelete="CASCADE"), nullable=False), - mysql_engine="InnoDB", + quote=False, ) @@ -573,9 +532,7 @@ Bans = Table( metadata, Column("IPAddress", String(45), primary_key=True), Column("BanTS", TIMESTAMP, nullable=False), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) @@ -583,15 +540,11 @@ Bans = Table( Terms = Table( "Terms", metadata, - Column("ID", INTEGER(unsigned=True), primary_key=True), + Column("ID", INTEGER(), primary_key=True), Column("Description", String(255), nullable=False), Column("URL", String(8000), nullable=False), - Column( - "Revision", INTEGER(unsigned=True), nullable=False, server_default=text("1") - ), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + Column("Revision", INTEGER(), nullable=False, server_default=text("1")), + quote=False, ) @@ -601,10 +554,8 @@ AcceptedTerms = Table( metadata, Column("UsersID", ForeignKey("Users.ID", ondelete="CASCADE"), nullable=False), Column("TermsID", ForeignKey("Terms.ID", ondelete="CASCADE"), nullable=False), - Column( - "Revision", INTEGER(unsigned=True), nullable=False, server_default=text("0") - ), - mysql_engine="InnoDB", + Column("Revision", INTEGER(), nullable=False, server_default=text("0")), + quote=False, ) @@ -613,10 +564,8 @@ ApiRateLimit = Table( "ApiRateLimit", metadata, Column("IP", String(45), primary_key=True, unique=True, default=str()), - Column("Requests", INTEGER(11), nullable=False), - Column("WindowStart", BIGINT(20), nullable=False), + Column("Requests", INTEGER(), nullable=False), + Column("WindowStart", BIGINT(), nullable=False), Index("ApiRateLimitWindowStart", "WindowStart"), - mysql_engine="InnoDB", - mysql_charset="utf8mb4", - mysql_collate="utf8mb4_general_ci", + quote=False, ) diff --git a/aurweb/scripts/notify.py b/aurweb/scripts/notify.py index 5bb81e20..9ad6454b 100755 --- a/aurweb/scripts/notify.py +++ b/aurweb/scripts/notify.py @@ -139,7 +139,7 @@ class ResetKeyNotification(Notification): def __init__(self, uid): user = ( db.query(User) - .filter(and_(User.ID == uid, User.Suspended == 0)) + .filter(and_(User.ID == uid, ~User.Suspended)) .with_entities( User.Username, User.Email, @@ -209,10 +209,10 @@ class CommentNotification(Notification): .join(PackageNotification) .filter( and_( - User.CommentNotify == 1, + User.CommentNotify, PackageNotification.UserID != uid, PackageNotification.PackageBaseID == pkgbase_id, - User.Suspended == 0, + ~User.Suspended, ) ) .with_entities(User.Email, User.LangPreference) @@ -274,10 +274,10 @@ class UpdateNotification(Notification): .join(PackageNotification) .filter( and_( - User.UpdateNotify == 1, + User.UpdateNotify, PackageNotification.UserID != uid, PackageNotification.PackageBaseID == pkgbase_id, - User.Suspended == 0, + ~User.Suspended, ) ) .with_entities(User.Email, User.LangPreference) @@ -337,7 +337,7 @@ class FlagNotification(Notification): PackageBase.ID == PackageComaintainer.PackageBaseID, ), ) - .filter(and_(PackageBase.ID == pkgbase_id, User.Suspended == 0)) + .filter(and_(PackageBase.ID == pkgbase_id, ~User.Suspended)) .with_entities(User.Email, User.LangPreference) .distinct() .order_by(User.Email) @@ -388,10 +388,10 @@ class OwnershipEventNotification(Notification): .join(PackageNotification) .filter( and_( - User.OwnershipNotify == 1, + User.OwnershipNotify, PackageNotification.UserID != uid, PackageNotification.PackageBaseID == pkgbase_id, - User.Suspended == 0, + ~User.Suspended, ) ) .with_entities(User.Email, User.LangPreference) @@ -507,7 +507,7 @@ class DeleteNotification(Notification): and_( PackageNotification.UserID != uid, PackageNotification.PackageBaseID == old_pkgbase_id, - User.Suspended == 0, + ~User.Suspended, ) ) .with_entities(User.Email, User.LangPreference) @@ -584,12 +584,12 @@ class RequestOpenNotification(Notification): User.ID == PackageComaintainer.UsersID, ), ) - .filter(and_(PackageRequest.ID == reqid, User.Suspended == 0)) + .filter(and_(PackageRequest.ID == reqid, ~User.Suspended)) .with_entities(User.Email, User.HideEmail) .distinct() ) - self._cc = [u.Email for u in query if u.HideEmail == 0] - self._bcc = [u.Email for u in query if u.HideEmail == 1] + self._cc = [u.Email for u in query if not u.HideEmail] + self._bcc = [u.Email for u in query if u.HideEmail] pkgreq = ( db.query(PackageRequest.Comments).filter(PackageRequest.ID == reqid).first() @@ -679,12 +679,12 @@ class RequestCloseNotification(Notification): User.ID == PackageComaintainer.UsersID, ), ) - .filter(and_(PackageRequest.ID == reqid, User.Suspended == 0)) + .filter(and_(PackageRequest.ID == reqid, ~User.Suspended)) .with_entities(User.Email, User.HideEmail) .distinct() ) - self._cc = [u.Email for u in query if u.HideEmail == 0] - self._bcc = [u.Email for u in query if u.HideEmail == 1] + self._cc = [u.Email for u in query if not u.HideEmail] + self._bcc = [u.Email for u in query if u.HideEmail] pkgreq = ( db.query(PackageRequest) @@ -766,7 +766,7 @@ class VoteReminderNotification(Notification): and_( User.AccountTypeID.in_((2, 4)), ~User.ID.in_(subquery), - User.Suspended == 0, + ~User.Suspended, ) ) .with_entities(User.Email, User.LangPreference) diff --git a/aurweb/statistics.py b/aurweb/statistics.py index 00a5c151..78ae9b2e 100644 --- a/aurweb/statistics.py +++ b/aurweb/statistics.py @@ -13,6 +13,7 @@ from aurweb.models.package_request import ( CLOSED_ID, PENDING_ID, REJECTED_ID, + STATUS_DISPLAY, ) from aurweb.prometheus import PACKAGES, REQUESTS, USERS @@ -143,10 +144,13 @@ def update_prometheus_metrics(): .query(PackageRequest, func.count(PackageRequest.ID), RequestType.Name) .join(RequestType) .group_by(RequestType.Name, PackageRequest.Status) + .with_entities( + PackageRequest.Status, func.count(PackageRequest.ID), RequestType.Name + ) ) results = db_query_cache("request_metrics", query, cache_expire) for record in results: - status = record[0].status_display() + status = STATUS_DISPLAY[record[0]] count = record[1] rtype = record[2] REQUESTS.labels(type=rtype, status=status).set(count) diff --git a/aurweb/testing/__init__.py b/aurweb/testing/__init__.py index b9b1d263..b01277e6 100644 --- a/aurweb/testing/__init__.py +++ b/aurweb/testing/__init__.py @@ -56,8 +56,8 @@ def setup_test_db(*args): models.User.__tablename__, ] - aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 0") + aurweb.db.get_session().execute("SET session_replication_role = 'replica'") for table in tables: aurweb.db.get_session().execute(f"DELETE FROM {table}") - aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 1") + aurweb.db.get_session().execute("SET session_replication_role = 'origin';") aurweb.db.get_session().expunge_all() diff --git a/aurweb/users/util.py b/aurweb/users/util.py index e9635f08..dd510505 100644 --- a/aurweb/users/util.py +++ b/aurweb/users/util.py @@ -1,6 +1,7 @@ from http import HTTPStatus from fastapi import HTTPException +from sqlalchemy import func from aurweb import db from aurweb.models import User @@ -13,7 +14,7 @@ def get_user_by_name(username: str) -> User: :param username: User.Username :return: User instance """ - user = db.query(User).filter(User.Username == username).first() + user = db.query(User).filter(func.lower(User.Username) == username.lower()).first() if not user: raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) return db.refresh(user) diff --git a/aurweb/users/validate.py b/aurweb/users/validate.py index e49b0bc1..c4388f71 100644 --- a/aurweb/users/validate.py +++ b/aurweb/users/validate.py @@ -7,7 +7,7 @@ All functions in this module raise aurweb.exceptions.ValidationError when encountering invalid criteria and return silently otherwise. """ from fastapi import Request -from sqlalchemy import and_ +from sqlalchemy import and_, func from aurweb import aur_logging, config, db, l10n, models, time, util from aurweb.auth import creds @@ -157,7 +157,11 @@ def username_in_use( ) -> None: exists = ( db.query(models.User) - .filter(and_(models.User.ID != user.ID, models.User.Username == U)) + .filter( + and_( + models.User.ID != user.ID, func.lower(models.User.Username) == U.lower() + ) + ) .exists() ) if db.query(exists).scalar(): @@ -175,7 +179,9 @@ def email_in_use( ) -> None: exists = ( db.query(models.User) - .filter(and_(models.User.ID != user.ID, models.User.Email == E)) + .filter( + and_(models.User.ID != user.ID, func.lower(models.User.Email) == E.lower()) + ) .exists() ) if db.query(exists).scalar(): diff --git a/conf/config.defaults b/conf/config.defaults index db885b65..9151a316 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -1,9 +1,9 @@ [database] -backend = mysql -host = localhost -socket = /var/run/mysqld/mysqld.sock -;port = 3306 -name = AUR +backend = postgres +;host = localhost +socket = /run/postgresql +;port = 5432 +name = aurweb user = aur ;password = aur diff --git a/conf/config.dev b/conf/config.dev index f3b0ee21..3a2aa33e 100644 --- a/conf/config.dev +++ b/conf/config.dev @@ -6,19 +6,13 @@ ; development-specific options too. [database] -; FastAPI options: mysql. -backend = mysql - -; If using sqlite, set name to the database file path. +backend = postgres +;host = localhost +socket = /run/postgresql +;port = 5432 name = aurweb - -; MySQL database information. User defaults to root for containerized -; testing with mysqldb. This should be set to a non-root user. -user = root -;password = aur -host = localhost -;port = 3306 -socket = /var/run/mysqld/mysqld.sock +user = aur +password = aur [options] aurwebdir = YOUR_AUR_ROOT diff --git a/doc/docker.md b/doc/docker.md index c54184b8..5cf905b0 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -62,7 +62,7 @@ Services |---------------------|-----------------| | [ca](#ca) | | | [cron](#cron) | | -| [mariadb](#mariadb) | 127.0.0.1:13306 | +| [postgres](#postgres) | 127.0.0.1:15432 | | [git](#git) | 127.0.0.1:2222 | | redis | 127.0.0.1:16379 | | [fastapi](#fastapi) | 127.0.0.1:18000 | @@ -88,13 +88,10 @@ anchors or browsers for SSL verification. The _cron_ service includes all scripts recommended in `doc/maintenance.txt`. -#### mariadb +#### postgres - When used with the [default](#default) profile, a Docker-driven - mariadb service is used. -- When used with the [aur-dev](#aur-dev) profile, `MARIADB_SOCKET_DIR` - (defaulted to `/var/run/mysqld/`) can be defined to bind-mount a - host-driven mariadb socket to the container. + postgresql service is used. #### git diff --git a/docker-compose.aur-dev.yml b/docker-compose.aur-dev.yml index 1763f427..09efbfae 100644 --- a/docker-compose.aur-dev.yml +++ b/docker-compose.aur-dev.yml @@ -9,7 +9,7 @@ services: redis: restart: always - mariadb: + postgres: restart: always git: @@ -37,7 +37,7 @@ services: cron: volumes: # Exclude ./aurweb:/aurweb in production. - - mariadb_run:/var/run/mysqld + - postgres_run:/run/postgresql - archives:/var/lib/aurweb/archives fastapi: @@ -60,8 +60,8 @@ services: - smartgit_run:/var/run/smartgit volumes: - mariadb_run: {} # Share /var/run/mysqld - mariadb_data: {} # Share /var/lib/mysql + postgres_run: {} + postgres_data: {} git_data: {} # Share aurweb/aur.git smartgit_run: {} data: {} diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 6580de30..923bec23 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -6,9 +6,9 @@ services: - ./data:/data - step:/root/.step - mariadb_init: + postgres_init: depends_on: - mariadb: + postgres: condition: service_healthy git: @@ -22,7 +22,7 @@ services: - ./data:/data - smartgit_run:/var/run/smartgit depends_on: - mariadb: + postgres: condition: service_healthy fastapi: diff --git a/docker-compose.yml b/docker-compose.yml index 0973fc0e..66c6564b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,10 +3,10 @@ # # Notable services: # - `sharness` - Run sharness test suites -# - `pytest-mysql` - Run pytest suites with MariaDB +# - `pytest-postgres` - Run pytest suites with PostgreSQL # - `pytest-sqlite` - Run pytest suites with SQLite -# - `test` - Run sharness, pytest-mysql and pytest-sqlite -# - `mariadb` - `port 13306` - MariaDB server for docker +# - `test` - Run sharness, pytest-postgres and pytest-sqlite +# - `postgres` - `port 15432` - PostgreSQL server for docker # - `ca` - Certificate Authority generation # - `git` - `port 2222` - Git over SSH server # - `fastapi` - hypercorn service for aurweb's FastAPI app @@ -45,53 +45,34 @@ services: ports: - "127.0.0.1:16379:6379" - mariadb: + postgres: image: aurweb:latest init: true - entrypoint: /docker/mariadb-entrypoint.sh - command: /usr/bin/mysqld_safe --datadir=/var/lib/mysql + entrypoint: /docker/postgres-entrypoint.sh + command: su postgres -c '/usr/bin/postgres -D /var/lib/postgres/data' ports: - # This will expose mariadbd on 127.0.0.1:13306 in the host. - # Ex: `mysql -uaur -paur -h 127.0.0.1 -P 13306 aurweb` - - "127.0.0.1:13306:3306" + - "127.0.0.1:15432:5432" volumes: - - mariadb_run:/var/run/mysqld # Bind socket in this volume. - - mariadb_data:/var/lib/mysql + - postgres_run:/run/postgresql + - postgres_data:/var/lib/postgres healthcheck: - test: "bash /docker/health/mariadb.sh" + test: "bash /docker/health/postgres.sh" interval: 3s + shm_size: 2gb - mariadb_init: + postgres_init: image: aurweb:latest init: true environment: - AUR_CONFIG_IMMUTABLE=${AUR_CONFIG_IMMUTABLE:-0} - entrypoint: /docker/mariadb-init-entrypoint.sh - command: echo "MariaDB tables initialized." + entrypoint: /docker/postgres-init-entrypoint.sh + command: echo "Postgres tables initialized." volumes: - - mariadb_run:/var/run/mysqld + - postgres_run:/run/postgresql depends_on: - mariadb: + postgres: condition: service_healthy - mariadb_test: - # Test database. - image: aurweb:latest - init: true - environment: - - MARIADB_PRIVILEGED=1 - entrypoint: /docker/mariadb-entrypoint.sh - command: /usr/bin/mysqld_safe --datadir=/var/lib/mysql - ports: - # This will expose mariadbd on 127.0.0.1:13307 in the host. - # Ex: `mysql -uaur -paur -h 127.0.0.1 -P 13306 aurweb` - - "127.0.0.1:13307:3306" - volumes: - - mariadb_test_run:/var/run/mysqld # Bind socket in this volume. - healthcheck: - test: "bash /docker/health/mariadb.sh" - interval: 3s - git: image: aurweb:latest init: true @@ -107,10 +88,10 @@ services: test: "bash /docker/health/sshd.sh" interval: 3s depends_on: - mariadb_init: + postgres_init: condition: service_started volumes: - - mariadb_run:/var/run/mysqld + - postgres_run:/run/postgresql smartgit: image: aurweb:latest @@ -152,11 +133,11 @@ services: entrypoint: /docker/cron-entrypoint.sh command: /docker/scripts/run-cron.sh depends_on: - mariadb_init: + postgres_init: condition: service_started volumes: - ./aurweb:/aurweb/aurweb - - mariadb_run:/var/run/mysqld + - postgres_run:/run/postgresql - archives:/var/lib/aurweb/archives fastapi: @@ -184,7 +165,7 @@ services: condition: service_started volumes: - archives:/var/lib/aurweb/archives - - mariadb_run:/var/run/mysqld + - postgres_run:/run/postgresql ports: - "127.0.0.1:18000:8000" @@ -222,7 +203,7 @@ services: stdin_open: true tty: true depends_on: - mariadb_test: + postgres: condition: service_healthy volumes: - ./data:/data @@ -231,7 +212,7 @@ services: - ./test:/aurweb/test - ./templates:/aurweb/templates - pytest-mysql: + pytest-postgres: image: aurweb:latest profiles: ["dev"] init: true @@ -240,17 +221,17 @@ services: - TEST_RECURSION_LIMIT=${TEST_RECURSION_LIMIT} - PROMETHEUS_MULTIPROC_DIR=/tmp_prometheus - LOG_CONFIG=logging.test.conf - entrypoint: /docker/test-mysql-entrypoint.sh + entrypoint: /docker/test-postgres-entrypoint.sh command: /docker/scripts/run-pytests.sh clean stdin_open: true tty: true depends_on: - mariadb_test: + postgres: condition: service_healthy tmpfs: - /tmp volumes: - - mariadb_test_run:/var/run/mysqld + - postgres_run:/run/postgresql - ./data:/data - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations @@ -266,15 +247,15 @@ services: - TEST_RECURSION_LIMIT=${TEST_RECURSION_LIMIT} - PROMETHEUS_MULTIPROC_DIR=/tmp_prometheus - LOG_CONFIG=logging.test.conf - entrypoint: /docker/test-mysql-entrypoint.sh + entrypoint: /docker/test-postgres-entrypoint.sh command: /docker/scripts/run-tests.sh stdin_open: true tty: true depends_on: - mariadb_test: + postgres: condition: service_healthy volumes: - - mariadb_test_run:/var/run/mysqld + - postgres_run:/run/postgresql - ./data:/data - ./aurweb:/aurweb/aurweb - ./migrations:/aurweb/migrations @@ -282,9 +263,8 @@ services: - ./templates:/aurweb/templates volumes: - mariadb_test_run: {} - mariadb_run: {} # Share /var/run/mysqld/mysqld.sock - mariadb_data: {} # Share /var/lib/mysql + postgres_run: {} + postgres_data: {} git_data: {} # Share aurweb/aur.git smartgit_run: {} archives: {} diff --git a/docker/README.md b/docker/README.md index 51e485f6..1ea6d0ac 100644 --- a/docker/README.md +++ b/docker/README.md @@ -47,7 +47,7 @@ Luckily such data can be generated. docker compose exec fastapi /bin/bash pacman -S words fortune-mod ./schema/gendummydata.py dummy.sql -mysql aurweb < dummy.sql +su postgres -q -c 'psql aurweb < dummy.sql' ``` The generation script may prompt you to install other Arch packages before it diff --git a/docker/ca-entrypoint.sh b/docker/ca-entrypoint.sh index 55c7cd75..b7c4d0d7 100755 --- a/docker/ca-entrypoint.sh +++ b/docker/ca-entrypoint.sh @@ -71,7 +71,7 @@ start_step_ca() { kill_step_ca() { # Stop the step-ca web server. - killall step-ca >/dev/null 2>&1 || /bin/true + killall -w step-ca >/dev/null 2>&1 || /bin/true } install_step_ca() { @@ -105,8 +105,6 @@ if [ ! -d /root/.step/config ]; then echo -n "WARN: Your certificates are being regenerated to resolve " echo -n "an inconsistent step-ca state. You will need to re-import " echo "the root CA certificate into your browser." -else - exec "$@" fi # Set permissions to /data to rwx for everybody. diff --git a/docker/cron-entrypoint.sh b/docker/cron-entrypoint.sh index 5b69ab19..b1eb4758 100755 --- a/docker/cron-entrypoint.sh +++ b/docker/cron-entrypoint.sh @@ -2,7 +2,7 @@ set -eou pipefail # Setup the DB. -NO_INITDB=1 /docker/mariadb-init-entrypoint.sh +/docker/postgres-init-entrypoint.sh # Create aurblup's directory. AURBLUP_DIR="/aurweb/aurblup/" diff --git a/docker/fastapi-entrypoint.sh b/docker/fastapi-entrypoint.sh index c6597313..9f42cd04 100755 --- a/docker/fastapi-entrypoint.sh +++ b/docker/fastapi-entrypoint.sh @@ -2,7 +2,7 @@ set -eou pipefail # Setup database. -NO_INITDB=1 /docker/mariadb-init-entrypoint.sh +/docker/postgres-init-entrypoint.sh # Setup some other options. aurweb-config set options cache 'redis' diff --git a/docker/git-entrypoint.sh b/docker/git-entrypoint.sh index c9f1ec30..ca83cb42 100755 --- a/docker/git-entrypoint.sh +++ b/docker/git-entrypoint.sh @@ -39,7 +39,7 @@ Match User aur EOF # Setup database. -NO_INITDB=1 /docker/mariadb-init-entrypoint.sh +/docker/postgres-init-entrypoint.sh # Setup some other options. aurweb-config set serve repo-path '/aurweb/aur.git/' diff --git a/docker/health/mariadb.sh b/docker/health/mariadb.sh deleted file mode 100755 index cbae37bd..00000000 --- a/docker/health/mariadb.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -exec mysqladmin ping --silent diff --git a/docker/health/postgres.sh b/docker/health/postgres.sh new file mode 100755 index 00000000..64a60334 --- /dev/null +++ b/docker/health/postgres.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec su postgres -c 'pg_isready' diff --git a/docker/mariadb-entrypoint.sh b/docker/mariadb-entrypoint.sh deleted file mode 100755 index a6fb9a76..00000000 --- a/docker/mariadb-entrypoint.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -eou pipefail - -MYSQL_DATA=/var/lib/mysql - -mariadb-install-db --user=mysql --basedir=/usr --datadir=$MYSQL_DATA - -# Start it up. -mysqld_safe --datadir=$MYSQL_DATA --skip-networking & -while ! mysqladmin ping 2>/dev/null; do - sleep 1s -done - -# Configure databases. -DATABASE="aurweb" # Persistent database for fastapi. - -echo "Taking care of primary database '${DATABASE}'..." -mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'localhost' IDENTIFIED BY 'aur';" -mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'%' IDENTIFIED BY 'aur';" -mysql -u root -e "CREATE DATABASE IF NOT EXISTS $DATABASE;" - -mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'%' IDENTIFIED BY 'aur';" -mysql -u root -e "GRANT ALL ON aurweb.* TO 'aur'@'localhost';" -mysql -u root -e "GRANT ALL ON aurweb.* TO 'aur'@'%';" - -mysql -u root -e "CREATE USER IF NOT EXISTS 'root'@'%' IDENTIFIED BY 'aur';" -mysql -u root -e "GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION;" - -mysqladmin -uroot shutdown - -exec "$@" diff --git a/docker/mariadb-init-entrypoint.sh b/docker/mariadb-init-entrypoint.sh deleted file mode 100755 index 74980031..00000000 --- a/docker/mariadb-init-entrypoint.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -eou pipefail - -# Setup a config for our mysql db. -aurweb-config set database name 'aurweb' -aurweb-config set database user 'aur' -aurweb-config set database password 'aur' -aurweb-config set database host 'localhost' -aurweb-config set database socket '/var/run/mysqld/mysqld.sock' -aurweb-config unset database port - -if [ ! -z ${NO_INITDB+x} ]; then - exec "$@" -fi - -python -m aurweb.initdb 2>/dev/null || /bin/true -exec "$@" diff --git a/docker/postgres-entrypoint.sh b/docker/postgres-entrypoint.sh new file mode 100755 index 00000000..896a9b3a --- /dev/null +++ b/docker/postgres-entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -eou pipefail + +PGDATA=/var/lib/postgres/data +DATABASE="aurweb" + +# Initialize and setup postgres +if [ ! -f "$PGDATA/../init" ]; then + echo "Preparing postgres instance..." + touch $PGDATA/../init + + # Init db directory + su postgres -c "pg_ctl initdb -D $PGDATA" + su postgres -c "echo \"listen_addresses='*'\" >> $PGDATA/postgresql.conf" + su postgres -c "echo \"host all all 0.0.0.0/0 scram-sha-256\" >> $PGDATA/pg_hba.conf" + install -d -o postgres -g postgres /run/postgresql + + # Start postgres + su postgres -c "pg_ctl start -D $PGDATA" + + # Configure database & user + echo "Taking care of primary database '$DATABASE'..." + su postgres -c "psql -c \"create database $DATABASE;\"" + su postgres -c "psql -c \"create role aur superuser login password 'aur';\""; + + # Provision database + python -m aurweb.initdb 2>/dev/null || /bin/true + + # Stop postgres + su postgres -c "pg_ctl stop -D $PGDATA" + +fi + +exec "$@" diff --git a/docker/postgres-init-entrypoint.sh b/docker/postgres-init-entrypoint.sh new file mode 100755 index 00000000..3fbbdfdf --- /dev/null +++ b/docker/postgres-init-entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -eou pipefail + +# Setup a config for our postgres db via socket connection. +aurweb-config set database name 'aurweb' +aurweb-config set database user 'aur' +aurweb-config set database socket '/run/postgresql' +aurweb-config unset database host +aurweb-config unset database port +aurweb-config unset database password + +exec "$@" diff --git a/docker/scripts/install-deps.sh b/docker/scripts/install-deps.sh index 7aa225fa..2d300cd1 100755 --- a/docker/scripts/install-deps.sh +++ b/docker/scripts/install-deps.sh @@ -14,7 +14,7 @@ pacman -Sy --noconfirm --noprogressbar archlinux-keyring # Install other OS dependencies. pacman -Syu --noconfirm --noprogressbar \ --cachedir .pkg-cache git gpgme nginx redis openssh \ - mariadb mariadb-libs cgit-aurweb uwsgi uwsgi-plugin-cgi \ + postgresql cgit-aurweb uwsgi uwsgi-plugin-cgi \ python-pip pyalpm python-srcinfo curl libeatmydata cronie \ python-poetry python-poetry-core step-cli step-ca asciidoc \ python-virtualenv python-pre-commit diff --git a/docker/scripts/run-tests.sh b/docker/scripts/run-tests.sh index 75e562b0..2164986f 100755 --- a/docker/scripts/run-tests.sh +++ b/docker/scripts/run-tests.sh @@ -8,7 +8,7 @@ make -C test clean # Run sharness tests. bash $dir/run-sharness.sh -# Run Python tests with MariaDB database. +# Run Python tests with PostgreSQL database. # Pass --silence to avoid reporting coverage. We will do that below. bash $dir/run-pytests.sh --no-coverage diff --git a/docker/test-mysql-entrypoint.sh b/docker/test-mysql-entrypoint.sh deleted file mode 100755 index b3464256..00000000 --- a/docker/test-mysql-entrypoint.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -eou pipefail - -# We use the root user for testing in Docker. -# The test user must be able to create databases and drop them. -aurweb-config set database user 'root' -aurweb-config set database host 'localhost' -aurweb-config set database socket '/var/run/mysqld/mysqld.sock' - -# Remove possibly problematic configuration options. -# We depend on the database socket within Docker and -# being run as the root user. -aurweb-config unset database password -aurweb-config unset database port - -# Setup notifications for testing. -aurweb-config set notifications sendmail "$(pwd)/util/sendmail" - -exec "$@" diff --git a/docker/test-postgres-entrypoint.sh b/docker/test-postgres-entrypoint.sh new file mode 100755 index 00000000..790c02b3 --- /dev/null +++ b/docker/test-postgres-entrypoint.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -eou pipefail + +# Setup a config for our postgres db via socket connection. +aurweb-config set database name 'aurweb' +aurweb-config set database user 'aur' +aurweb-config set database socket '/run/postgresql' +aurweb-config unset database host +aurweb-config unset database port +aurweb-config unset database password + +# Setup notifications for testing. +aurweb-config set notifications sendmail "$(pwd)/util/sendmail" + +exec "$@" diff --git a/docker/tests-entrypoint.sh b/docker/tests-entrypoint.sh index 145bee6e..6253249b 100755 --- a/docker/tests-entrypoint.sh +++ b/docker/tests-entrypoint.sh @@ -2,6 +2,6 @@ set -eou pipefail dir="$(dirname $0)" -bash $dir/test-mysql-entrypoint.sh +bash $dir/test-postgres-entrypoint.sh exec "$@" diff --git a/poetry.lock b/poetry.lock index 7b8e7d07..80100a83 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1089,22 +1089,6 @@ files = [ {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, ] -[[package]] -name = "mysqlclient" -version = "2.2.0" -description = "Python interface to MySQL" -optional = false -python-versions = ">=3.8" -files = [ - {file = "mysqlclient-2.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:68837b6bb23170acffb43ae411e47533a560b6360c06dac39aa55700972c93b2"}, - {file = "mysqlclient-2.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:5670679ff1be1cc3fef0fa81bf39f0cd70605ba121141050f02743eb878ac114"}, - {file = "mysqlclient-2.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:004fe1d30d2c2ff8072f8ea513bcec235fd9b896f70dad369461d0ad7e570e98"}, - {file = "mysqlclient-2.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:9c6b142836c7dba4f723bf9c93cc46b6e5081d65b2af807f400dda9eb85a16d0"}, - {file = "mysqlclient-2.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:955dba905a7443ce4788c63fdb9f8d688316260cf60b20ff51ac3b1c77616ede"}, - {file = "mysqlclient-2.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:530ece9995a36cadb6211b9787f0c9e05cdab6702549bdb4236af5e9b535ed6a"}, - {file = "mysqlclient-2.2.0.tar.gz", hash = "sha256:04368445f9c487d8abb7a878e3d23e923e6072c04a6c320f9e0dc8a82efba14e"}, -] - [[package]] name = "orjson" version = "3.9.10" @@ -1286,6 +1270,28 @@ files = [ {file = "protobuf-4.25.1.tar.gz", hash = "sha256:57d65074b4f5baa4ab5da1605c02be90ac20c8b40fb137d6a8df9f416b0d0ce2"}, ] +[[package]] +name = "psycopg2" +version = "2.9.9" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg2-2.9.9-cp310-cp310-win32.whl", hash = "sha256:38a8dcc6856f569068b47de286b472b7c473ac7977243593a288ebce0dc89516"}, + {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, + {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, + {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, + {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, + {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, + {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, + {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, + {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, + {file = "psycopg2-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:bac58c024c9922c23550af2a581998624d6e02350f4ae9c5f0bc642c633a2d5e"}, + {file = "psycopg2-2.9.9-cp39-cp39-win32.whl", hash = "sha256:c92811b2d4c9b6ea0285942b2e7cac98a59e166d59c588fe5cfe1eda58e72d59"}, + {file = "psycopg2-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913"}, + {file = "psycopg2-2.9.9.tar.gz", hash = "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156"}, +] + [[package]] name = "pyalpm" version = "0.10.6" @@ -2007,4 +2013,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "3c931b9e7957fc045d5e2356688606356f730c7a814958eb64ba9d5079f670e9" +content-hash = "7cc2869b398d51b38a3849b2dfcc0e11fb82333eca0a0658d310ee67da373588" diff --git a/pyproject.toml b/pyproject.toml index 36c43373..482892e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,6 @@ paginate = "^0.5.6" # SQL alembic = "^1.12.1" -mysqlclient = "^2.2.0" Authlib = "^1.2.1" Jinja2 = "^3.1.2" Markdown = "^3.5.1" @@ -97,6 +96,7 @@ pyalpm = "^0.10.6" fastapi = "^0.104.1" srcinfo = "^0.1.2" tomlkit = "^0.12.0" +psycopg2 = {extras = ["c"], version = "^2.9.7"} [tool.poetry.dev-dependencies] coverage = "^7.3.2" diff --git a/schema/gendummydata.py b/schema/gendummydata.py index f83de525..dac8bf04 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -357,7 +357,7 @@ for t in range(0, OPEN_PROPOSALS + CLOSE_PROPOSALS): user = user_keys[random.randrange(0, len(user_keys))] suid = packagemaintainers[random.randrange(0, len(packagemaintainers))] s = ( - "INSERT INTO VoteInfo (Agenda, User, Submitted, End," + 'INSERT INTO VoteInfo (Agenda, "user", Submitted, "end",' " Quorum, SubmitterID) VALUES ('%s', '%s', %d, %d, 0.0, %d);\n" ) s = s % (genFortune(), user, start, end, suid) diff --git a/templates/partials/packages/details.html b/templates/partials/packages/details.html index 5f242414..85126069 100644 --- a/templates/partials/packages/details.html +++ b/templates/partials/packages/details.html @@ -43,7 +43,7 @@
@@ -51,7 +51,7 @@ {% else %} - {% for keyword in pkgbase.keywords.all() %} + {% for keyword in keywords.all() %} diff --git a/test/README.md b/test/README.md index 0d7a85e2..9f7e51eb 100644 --- a/test/README.md +++ b/test/README.md @@ -60,7 +60,7 @@ other user: GRANT ALL ON *.* TO 'user'@'localhost' WITH GRANT OPTION -The aurweb platform is intended to use the `mysql` backend, but +The aurweb platform is intended to use the `postgresql` backend, but the `sqlite` backend is still used for sharness tests. These tests will soon be replaced with pytest suites and `sqlite` removed. diff --git a/test/conftest.py b/test/conftest.py index 575e9b98..ac30c74c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -52,6 +52,7 @@ from sqlalchemy.orm import scoped_session import aurweb.config import aurweb.db +import aurweb.schema from aurweb import aur_logging, initdb, testing from aurweb.testing.email import Email from aurweb.testing.git import GitRepository @@ -68,25 +69,28 @@ values.ValueClass = values.MutexValue def test_engine() -> Engine: """ - Return a privileged SQLAlchemy engine with no database. + Return a privileged SQLAlchemy engine with default database. This method is particularly useful for providing an engine that can be used to create and drop databases from an SQL server. - :return: SQLAlchemy Engine instance (not connected to a database) + :return: SQLAlchemy Engine instance (connected to a default) """ - unix_socket = aurweb.config.get_with_fallback("database", "socket", None) + socket = aurweb.config.get_with_fallback("database", "socket", None) + host = aurweb.config.get_with_fallback("database", "host", None) + port = aurweb.config.get_with_fallback("database", "port", None) + kwargs = { + "database": aurweb.config.get("database", "name"), "username": aurweb.config.get("database", "user"), "password": aurweb.config.get_with_fallback("database", "password", None), - "host": aurweb.config.get("database", "host"), - "port": aurweb.config.get_with_fallback("database", "port", None), - "query": {"unix_socket": unix_socket}, + "host": socket if socket else host, + "port": port if not socket else None, } backend = aurweb.config.get("database", "backend") driver = aurweb.db.DRIVERS.get(backend) - return create_engine(URL.create(driver, **kwargs)) + return create_engine(URL.create(driver, **kwargs), isolation_level="AUTOCOMMIT") class AlembicArgs: @@ -116,7 +120,7 @@ def _create_database(engine: Engine, dbname: str) -> None: # a ProgrammingError. Just drop the database and try # again. If at that point things still fail, any # exception will be propogated up to the caller. - conn.execute(f"DROP DATABASE {dbname}") + conn.execute(f"DROP DATABASE {dbname} WITH (FORCE)") conn.execute(f"CREATE DATABASE {dbname}") conn.close() initdb.run(AlembicArgs) @@ -129,9 +133,8 @@ def _drop_database(engine: Engine, dbname: str) -> None: :param engine: Engine returned by test_engine() :param dbname: Database name to drop """ - aurweb.schema.metadata.drop_all(bind=engine) conn = engine.connect() - conn.execute(f"DROP DATABASE {dbname}") + conn.execute(f"DROP DATABASE {dbname} WITH (FORCE)") conn.close() @@ -178,6 +181,10 @@ def db_session(setup_database: None) -> scoped_session: session.close() aurweb.db.pop_session(dbname) + # Dispose engine and close connections + aurweb.db.get_engine(dbname).dispose() + aurweb.db.pop_engine(dbname) + @pytest.fixture def db_test(db_session: scoped_session) -> None: diff --git a/test/test_auth_routes.py b/test/test_auth_routes.py index 066457c4..4d9702b0 100644 --- a/test/test_auth_routes.py +++ b/test/test_auth_routes.py @@ -14,7 +14,7 @@ from aurweb.models.user import User from aurweb.testing.html import get_errors # Some test global constants. -TEST_USERNAME = "test" +TEST_USERNAME = "Test" TEST_EMAIL = "test@example.org" TEST_REFERER = { "referer": aurweb.config.get("options", "aur_location") + "/login", @@ -54,36 +54,37 @@ def user() -> User: def test_login_logout(client: TestClient, user: User): - post_data = {"user": "test", "passwd": "testPassword", "next": "/"} + for username in ["test", "TEst"]: + post_data = {"user": username, "passwd": "testPassword", "next": "/"} - with client as request: - # First, let's test get /login. - response = request.get("/login") - assert response.status_code == int(HTTPStatus.OK) + with client as request: + # First, let's test get /login. + response = request.get("/login") + assert response.status_code == int(HTTPStatus.OK) - response = request.post("/login", data=post_data) - assert response.status_code == int(HTTPStatus.SEE_OTHER) + response = request.post("/login", data=post_data) + assert response.status_code == int(HTTPStatus.SEE_OTHER) - # Simulate following the redirect location from above's response. - response = request.get(response.headers.get("location")) - assert response.status_code == int(HTTPStatus.OK) + # Simulate following the redirect location from above's response. + response = request.get(response.headers.get("location")) + assert response.status_code == int(HTTPStatus.OK) - response = request.post("/logout", data=post_data) - assert response.status_code == int(HTTPStatus.SEE_OTHER) + response = request.post("/logout", data=post_data) + assert response.status_code == int(HTTPStatus.SEE_OTHER) - request.cookies = {"AURSID": response.cookies.get("AURSID")} - response = request.post( - "/logout", - data=post_data, - ) - assert response.status_code == int(HTTPStatus.SEE_OTHER) + request.cookies = {"AURSID": response.cookies.get("AURSID")} + response = request.post( + "/logout", + data=post_data, + ) + assert response.status_code == int(HTTPStatus.SEE_OTHER) - assert "AURSID" not in response.cookies + assert "AURSID" not in response.cookies def test_login_suspended(client: TestClient, user: User): with db.begin(): - user.Suspended = 1 + user.Suspended = True data = {"user": user.Username, "passwd": "testPassword", "next": "/"} with client as request: @@ -184,23 +185,23 @@ def test_secure_login(getboolean: mock.Mock, client: TestClient, user: User): def test_authenticated_login(client: TestClient, user: User): - post_data = {"user": user.Username, "passwd": "testPassword", "next": "/"} + for username in [user.Username.lower(), user.Username.upper()]: + post_data = {"user": username, "passwd": "testPassword", "next": "/"} - with client as request: - # Try to login. - response = request.post("/login", data=post_data) - assert response.status_code == int(HTTPStatus.SEE_OTHER) - assert response.headers.get("location") == "/" + with client as request: + # Try to login. + request.cookies = {} + response = request.post("/login", data=post_data) + assert response.status_code == int(HTTPStatus.SEE_OTHER) + assert response.headers.get("location") == "/" - # Now, let's verify that we get the logged in rendering - # when requesting GET /login as an authenticated user. - # Now, let's verify that we receive 403 Forbidden when we - # try to get /login as an authenticated user. - request.cookies = response.cookies - response = request.get("/login") + # Now, let's verify that we get the logged in rendering + # when requesting GET /login as an authenticated user. + request.cookies = response.cookies + response = request.get("/login") - assert response.status_code == int(HTTPStatus.OK) - assert "Logged-in as: test" in response.text + assert response.status_code == int(HTTPStatus.OK) + assert f"Logged-in as: {user.Username}" in response.text def test_unauthenticated_logout_unauthorized(client: TestClient): @@ -370,5 +371,4 @@ def test_generate_unique_sid_exhausted( assert re.search(expr, caplog.text) assert "IntegrityError" in caplog.text - expr = r"Duplicate entry .+ for key .+SessionID.+" - assert re.search(expr, response.text) + assert "duplicate key value" in response.text diff --git a/test/test_db.py b/test/test_db.py index 22dbdd36..20c7c618 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -93,9 +93,9 @@ def make_temp_sqlite_config(): ) -def make_temp_mysql_config(): +def make_temp_postgres_config(): return make_temp_config( - (r"backend = .*", "backend = mysql"), (r"name = .*", "name = aurweb_test") + (r"backend = .*", "backend = postgres"), (r"name = .*", "name = aurweb_test") ) @@ -114,8 +114,8 @@ def test_sqlalchemy_sqlite_url(): aurweb.config.rehash() -def test_sqlalchemy_mysql_url(): - tmpctx, tmp = make_temp_mysql_config() +def test_sqlalchemy_postgres_url(): + tmpctx, tmp = make_temp_postgres_config() with tmpctx: with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): aurweb.config.rehash() @@ -123,8 +123,8 @@ def test_sqlalchemy_mysql_url(): aurweb.config.rehash() -def test_sqlalchemy_mysql_port_url(): - tmpctx, tmp = make_temp_config((r";port = 3306", "port = 3306")) +def test_sqlalchemy_postgres_port_url(): + tmpctx, tmp = make_temp_config((r";port = 5432", "port = 5432")) with tmpctx: with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): @@ -133,7 +133,7 @@ def test_sqlalchemy_mysql_port_url(): aurweb.config.rehash() -def test_sqlalchemy_mysql_socket_url(): +def test_sqlalchemy_postgres_socket_url(): tmpctx, tmp = make_temp_config() with tmpctx: @@ -170,16 +170,6 @@ def test_connection_class_unsupported_backend(): aurweb.config.rehash() -@mock.patch("MySQLdb.connect", mock.MagicMock(return_value=True)) -def test_connection_mysql(): - tmpctx, tmp = make_temp_mysql_config() - with tmpctx: - with mock.patch.dict(os.environ, {"AUR_CONFIG": tmp}): - aurweb.config.rehash() - db.Connection() - aurweb.config.rehash() - - def test_create_delete(): with db.begin(): account_type = db.create(AccountType, AccountType="test") @@ -212,8 +202,8 @@ def test_add_commit(): db.delete(account_type) -def test_connection_executor_mysql_paramstyle(): - executor = db.ConnectionExecutor(None, backend="mysql") +def test_connection_executor_postgres_paramstyle(): + executor = db.ConnectionExecutor(None, backend="postgres") assert executor.paramstyle() == "format" diff --git a/test/test_initdb.py b/test/test_initdb.py index db5edf74..0f85484d 100644 --- a/test/test_initdb.py +++ b/test/test_initdb.py @@ -20,7 +20,7 @@ def test_run(): from aurweb.schema import metadata aurweb.db.kill_engine() - metadata.drop_all(aurweb.db.get_engine()) + metadata.drop_all(aurweb.db.get_engine(), checkfirst=False) aurweb.initdb.run(Args()) # Check that constant table rows got added via initdb. diff --git a/test/test_notify.py b/test/test_notify.py index bc6d878f..89e3e144 100644 --- a/test/test_notify.py +++ b/test/test_notify.py @@ -227,7 +227,7 @@ please go to the package page [2] and select "Disable notifications". def test_update(user: User, user2: User, pkgbases: list[PackageBase]): pkgbase = pkgbases[0] with db.begin(): - user.UpdateNotify = 1 + user.UpdateNotify = True notif = notify.UpdateNotification(user2.ID, pkgbase.ID) notif.send() @@ -331,7 +331,7 @@ You were removed from the co-maintainer list of {pkgbase.Name} [1]. def test_suspended_ownership_change(user: User, pkgbases: list[PackageBase]): with db.begin(): - user.Suspended = 1 + user.Suspended = True pkgbase = pkgbases[0] notif = notify.ComaintainerAddNotification(user.ID, pkgbase.ID) @@ -491,7 +491,7 @@ def test_open_close_request_hidden_email( # Enable the "HideEmail" option for our requester with db.begin(): - user2.HideEmail = 1 + user2.HideEmail = True # Send an open request notification. notif = notify.RequestOpenNotification( diff --git a/test/test_package_maintainer_routes.py b/test/test_package_maintainer_routes.py index 1824556b..6dd1ad88 100644 --- a/test/test_package_maintainer_routes.py +++ b/test/test_package_maintainer_routes.py @@ -350,7 +350,7 @@ def test_pm_index_table_paging(client, pm_user): VoteInfo, Agenda=f"Agenda #{i}", User=pm_user.Username, - Submitted=(ts - 5), + Submitted=(ts - 5 - i), End=(ts + 1000), Quorum=0.0, Submitter=pm_user, @@ -362,7 +362,7 @@ def test_pm_index_table_paging(client, pm_user): VoteInfo, Agenda=f"Agenda #{25 + i}", User=pm_user.Username, - Submitted=(ts - 1000), + Submitted=(ts - 1000 - i), End=(ts - 5), Quorum=0.0, Submitter=pm_user, diff --git a/test/test_packages_routes.py b/test/test_packages_routes.py index e1c989da..58b2b1e6 100644 --- a/test/test_packages_routes.py +++ b/test/test_packages_routes.py @@ -742,14 +742,15 @@ def test_packages_empty(client: TestClient): def test_packages_search_by_name(client: TestClient, packages: list[Package]): - with client as request: - response = request.get("/packages", params={"SeB": "n", "K": "pkg_"}) - assert response.status_code == int(HTTPStatus.OK) + for keyword in ["pkg_", "PkG_"]: + with client as request: + response = request.get("/packages", params={"SeB": "n", "K": keyword}) + assert response.status_code == int(HTTPStatus.OK) - root = parse_root(response.text) + root = parse_root(response.text) - rows = root.xpath('//table[@class="results"]/tbody/tr') - assert len(rows) == 50 # Default per-page + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 50 # Default per-page def test_packages_search_by_exact_name(client: TestClient, packages: list[Package]): @@ -763,26 +764,28 @@ def test_packages_search_by_exact_name(client: TestClient, packages: list[Packag # There is no package named exactly 'pkg_', we get 0 results. assert len(rows) == 0 - with client as request: - response = request.get("/packages", params={"SeB": "N", "K": "pkg_1"}) - assert response.status_code == int(HTTPStatus.OK) + for keyword in ["pkg_1", "PkG_1"]: + with client as request: + response = request.get("/packages", params={"SeB": "N", "K": keyword}) + assert response.status_code == int(HTTPStatus.OK) - root = parse_root(response.text) - rows = root.xpath('//table[@class="results"]/tbody/tr') + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') - # There's just one package named 'pkg_1', we get 1 result. - assert len(rows) == 1 + # There's just one package named 'pkg_1', we get 1 result. + assert len(rows) == 1 def test_packages_search_by_pkgbase(client: TestClient, packages: list[Package]): - with client as request: - response = request.get("/packages", params={"SeB": "b", "K": "pkg_"}) - assert response.status_code == int(HTTPStatus.OK) + for keyword in ["pkg_", "PkG_"]: + with client as request: + response = request.get("/packages", params={"SeB": "b", "K": "pkg_"}) + assert response.status_code == int(HTTPStatus.OK) - root = parse_root(response.text) + root = parse_root(response.text) - rows = root.xpath('//table[@class="results"]/tbody/tr') - assert len(rows) == 50 + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 50 def test_packages_search_by_exact_pkgbase(client: TestClient, packages: list[Package]): @@ -794,13 +797,14 @@ def test_packages_search_by_exact_pkgbase(client: TestClient, packages: list[Pac rows = root.xpath('//table[@class="results"]/tbody/tr') assert len(rows) == 0 - with client as request: - response = request.get("/packages", params={"SeB": "B", "K": "pkg_1"}) - assert response.status_code == int(HTTPStatus.OK) + for keyword in ["pkg_1", "PkG_1"]: + with client as request: + response = request.get("/packages", params={"SeB": "B", "K": "pkg_1"}) + assert response.status_code == int(HTTPStatus.OK) - root = parse_root(response.text) - rows = root.xpath('//table[@class="results"]/tbody/tr') - assert len(rows) == 1 + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 def test_packages_search_by_keywords(client: TestClient, packages: list[Package]): @@ -821,15 +825,16 @@ def test_packages_search_by_keywords(client: TestClient, packages: list[Package] ) # And request packages with that keyword, we should get 1 result. - with client as request: - # clear fakeredis cache - cache._redis.flushall() - response = request.get("/packages", params={"SeB": "k", "K": "testKeyword"}) - assert response.status_code == int(HTTPStatus.OK) + for keyword in ["testkeyword", "TestKeyWord"]: + with client as request: + # clear fakeredis cache + cache._redis.flushall() + response = request.get("/packages", params={"SeB": "k", "K": keyword}) + assert response.status_code == int(HTTPStatus.OK) - root = parse_root(response.text) - rows = root.xpath('//table[@class="results"]/tbody/tr') - assert len(rows) == 1 + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 # Now let's add another keyword to the same package with db.begin(): @@ -854,14 +859,13 @@ def test_packages_search_by_maintainer( ): # We should expect that searching by `package`'s maintainer # returns `package` in the results. - with client as request: - response = request.get( - "/packages", params={"SeB": "m", "K": maintainer.Username} - ) - assert response.status_code == int(HTTPStatus.OK) - root = parse_root(response.text) - rows = root.xpath('//table[@class="results"]/tbody/tr') - assert len(rows) == 1 + for keyword in [maintainer.Username, maintainer.Username.upper()]: + with client as request: + response = request.get("/packages", params={"SeB": "m", "K": keyword}) + assert response.status_code == int(HTTPStatus.OK) + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 # Search again by maintainer with no keywords given. # This kind of search returns all orphans instead. @@ -912,17 +916,16 @@ def test_packages_search_by_comaintainer( ) # Then test that it's returned by our search. - with client as request: - # clear fakeredis cache - cache._redis.flushall() - response = request.get( - "/packages", params={"SeB": "c", "K": maintainer.Username} - ) - assert response.status_code == int(HTTPStatus.OK) + for keyword in [maintainer.Username, maintainer.Username.upper()]: + with client as request: + # clear fakeredis cache + cache._redis.flushall() + response = request.get("/packages", params={"SeB": "c", "K": keyword}) + assert response.status_code == int(HTTPStatus.OK) - root = parse_root(response.text) - rows = root.xpath('//table[@class="results"]/tbody/tr') - assert len(rows) == 1 + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 def test_packages_search_by_co_or_maintainer( @@ -954,27 +957,27 @@ def test_packages_search_by_co_or_maintainer( PackageComaintainer, PackageBase=package.PackageBase, User=user, Priority=1 ) - with client as request: - response = request.get("/packages", params={"SeB": "M", "K": user.Username}) - assert response.status_code == int(HTTPStatus.OK) + for keyword in [user.Username, user.Username.upper()]: + with client as request: + response = request.get("/packages", params={"SeB": "M", "K": keyword}) + assert response.status_code == int(HTTPStatus.OK) - root = parse_root(response.text) - rows = root.xpath('//table[@class="results"]/tbody/tr') - assert len(rows) == 1 + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 def test_packages_search_by_submitter( client: TestClient, maintainer: User, package: Package ): - with client as request: - response = request.get( - "/packages", params={"SeB": "s", "K": maintainer.Username} - ) - assert response.status_code == int(HTTPStatus.OK) + for keyword in [maintainer.Username, maintainer.Username.upper()]: + with client as request: + response = request.get("/packages", params={"SeB": "s", "K": keyword}) + assert response.status_code == int(HTTPStatus.OK) - root = parse_root(response.text) - rows = root.xpath('//table[@class="results"]/tbody/tr') - assert len(rows) == 1 + root = parse_root(response.text) + rows = root.xpath('//table[@class="results"]/tbody/tr') + assert len(rows) == 1 def test_packages_sort_by_name(client: TestClient, packages: list[Package]): diff --git a/test/test_packages_util.py b/test/test_packages_util.py index 649e7a99..1ff1c8bc 100644 --- a/test/test_packages_util.py +++ b/test/test_packages_util.py @@ -153,7 +153,7 @@ def test_pkg_required(package: Package): # We want to make sure "Package" data is included # to avoid lazy-loading the information for each dependency qry = util.pkg_required("test", list()) - assert "Packages_ID" in str(qry) + assert "packages_id" in str(qry).lower() # We should have 1 record assert qry.count() == 1 diff --git a/test/test_pkgbase_routes.py b/test/test_pkgbase_routes.py index a413fe8a..b17a371e 100644 --- a/test/test_pkgbase_routes.py +++ b/test/test_pkgbase_routes.py @@ -428,7 +428,7 @@ def test_pkgbase_comments( # create notification with db.begin(): - user.CommentNotify = 1 + user.CommentNotify = True db.create(PackageNotification, PackageBase=package.PackageBase, User=user) # post a comment diff --git a/test/test_util.py b/test/test_util.py index 1c3b51af..27c607ed 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -149,15 +149,15 @@ def assert_multiple_keys(pks): def test_hash_query(): # No conditions query = db.query(User) - assert util.hash_query(query) == "75e76026b7d576536e745ec22892cf8f5d7b5d62" + assert util.hash_query(query) == "ebbf077df70d97a1584f91d0dd6ec61e43aa101f" # With where clause query = db.query(User).filter(User.Username == "bla") - assert util.hash_query(query) == "4dca710f33b1344c27ec6a3c266970f4fa6a8a00" + assert util.hash_query(query) == "b51f2bfda67051f381a5c05b2946a1aa4d91e56d" # With where clause and sorting query = db.query(User).filter(User.Username == "bla").order_by(User.Username) - assert util.hash_query(query) == "ee2c7846fede430776e140f8dfe1d83cd21d2eed" + assert util.hash_query(query) == "8d458bfe1edfe8f78929fab590612e9e5d9db3a5" # With where clause, sorting and specific columns query = ( @@ -166,4 +166,4 @@ def test_hash_query(): .order_by(User.Username) .with_entities(User.Username) ) - assert util.hash_query(query) == "c1db751be61443d266cf643005eee7a884dac103" + assert util.hash_query(query) == "006811a386789f25d40a37496f6ac6651413c245"