[db] fix schema and migration for case insensitivity

Some of the columns that were changed still want to be
case insensitive. Good thing our tables have nice
separation.

Signed-off-by: Kevin Morris <kevr@0cost.org>

aurweb/schema.py

@@ -10,6 +10,10 @@ from sqlalchemy import CHAR, TIMESTAMP, Column, ForeignKey, Index, MetaData, Str
 from sqlalchemy.dialects.mysql import BIGINT, DECIMAL, INTEGER, TINYINT
 from sqlalchemy.ext.compiler import compiles
 
+import aurweb.config
+
+db_backend = aurweb.config.get("database", "backend")
+
 
 @compiles(TINYINT, 'sqlite')
 def compile_tinyint_sqlite(type_, compiler, **kw):
@@ -35,7 +39,9 @@ AccountTypes = Table(
     'AccountTypes', metadata,
     Column('ID', TINYINT(unsigned=True), primary_key=True),
     Column('AccountType', String(32), nullable=False, server_default=text("''")),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin'
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci'
 )
 
 
@@ -69,7 +75,9 @@ Users = Table(
     Column('OwnershipNotify', TINYINT(1), nullable=False, server_default=text("1")),
     Column('SSOAccountID', String(255), nullable=True, unique=True),
     Index('UsersAccountTypeID', 'AccountTypeID'),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -113,7 +121,9 @@ PackageBases = Table(
     Index('BasesNumVotes', 'NumVotes'),
     Index('BasesPackagerUID', 'PackagerUID'),
     Index('BasesSubmitterUID', 'SubmitterUID'),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -122,7 +132,9 @@ PackageKeywords = Table(
     'PackageKeywords', metadata,
     Column('PackageBaseID', ForeignKey('PackageBases.ID', ondelete='CASCADE'), primary_key=True, nullable=False),
     Column('Keyword', String(255), primary_key=True, nullable=False, server_default=text("''")),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -135,7 +147,9 @@ Packages = Table(
     Column('Version', String(255), nullable=False, server_default=text("''")),
     Column('Description', String(255)),
     Column('URL', String(8000)),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -144,7 +158,9 @@ Licenses = Table(
     'Licenses', metadata,
     Column('ID', INTEGER(unsigned=True), primary_key=True),
     Column('Name', String(255), nullable=False, unique=True),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -162,7 +178,9 @@ Groups = Table(
     'Groups', metadata,
     Column('ID', INTEGER(unsigned=True), primary_key=True),
     Column('Name', String(255), nullable=False, unique=True),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -180,7 +198,9 @@ DependencyTypes = Table(
     'DependencyTypes', metadata,
     Column('ID', TINYINT(unsigned=True), primary_key=True),
     Column('Name', String(32), nullable=False, server_default=text("''")),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -195,7 +215,9 @@ PackageDepends = Table(
     Column('DepArch', String(255)),
     Index('DependsDepName', 'DepName'),
     Index('DependsPackageID', 'PackageID'),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -204,7 +226,9 @@ RelationTypes = Table(
     'RelationTypes', metadata,
     Column('ID', TINYINT(unsigned=True), primary_key=True),
     Column('Name', String(32), nullable=False, server_default=text("''")),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -218,7 +242,9 @@ PackageRelations = Table(
     Column('RelArch', String(255)),
     Index('RelationsPackageID', 'PackageID'),
     Index('RelationsRelName', 'RelName'),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -229,7 +255,9 @@ PackageSources = Table(
     Column('Source', String(8000), nullable=False, server_default=text("'/dev/null'")),
     Column('SourceArch', String(255)),
     Index('SourcesPackageID', 'PackageID'),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -262,7 +290,9 @@ PackageComments = Table(
     Column('PinnedTS', BIGINT(unsigned=True), nullable=False, server_default=text("0")),
     Index('CommentsPackageBaseID', 'PackageBaseID'),
     Index('CommentsUsersID', 'UsersID'),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -293,7 +323,9 @@ PackageBlacklist = Table(
     'PackageBlacklist', metadata,
     Column('ID', INTEGER(unsigned=True), primary_key=True),
     Column('Name', String(64), nullable=False, unique=True),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -314,7 +346,9 @@ RequestTypes = Table(
     'RequestTypes', metadata,
     Column('ID', TINYINT(unsigned=True), primary_key=True),
     Column('Name', String(32), nullable=False, server_default=text("''")),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -335,7 +369,9 @@ PackageRequests = Table(
     Column('Status', TINYINT(unsigned=True), nullable=False, server_default=text("0")),
     Index('RequestsPackageBaseID', 'PackageBaseID'),
     Index('RequestsUsersID', 'UsersID'),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -371,7 +407,9 @@ Bans = Table(
     'Bans', metadata,
     Column('IPAddress', String(45), primary_key=True),
     Column('BanTS', TIMESTAMP, nullable=False),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -382,7 +420,9 @@ Terms = Table(
     Column('Description', String(255), nullable=False),
     Column('URL', String(8000), nullable=False),
     Column('Revision', INTEGER(unsigned=True), nullable=False, server_default=text("1")),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )
 
 
@@ -403,5 +443,7 @@ ApiRateLimit = Table(
     Column('Requests', INTEGER(11), nullable=False),
     Column('WindowStart', BIGINT(20), nullable=False),
     Index('ApiRateLimitWindowStart', 'WindowStart'),
-    mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
+    mysql_engine='InnoDB',
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_general_ci',
 )

migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py

@@ -7,6 +7,8 @@ Create Date: 2021-05-17 14:23:00.008479
 """
 from alembic import op
 
+import aurweb.config
+
 # revision identifiers, used by Alembic.
 revision = '56e2ce8e2ffa'
 down_revision = 'ef39fcd6e1cd'
@@ -14,10 +16,31 @@ branch_labels = None
 depends_on = None
 
 # Tables affected by charset/collate change
-tables = ['AccountTypes', 'ApiRateLimit', 'Bans', 'DependencyTypes', 'Groups', 'Licenses', 'OfficialProviders',
+tables = [
-          'PackageBases', 'PackageBlacklist', 'PackageComments', 'PackageDepends', 'PackageKeywords',
+    ('AccountTypes', 'utf8mb4', 'utf8mb4_general_ci'),
-          'PackageRelations', 'PackageRequests', 'PackageSources', 'Packages', 'RelationTypes', 'RequestTypes',
+    ('ApiRateLimit', 'utf8mb4', 'utf8mb4_general_ci'),
-          'SSHPubKeys', 'Sessions', 'TU_VoteInfo', 'Terms', 'Users']
+    ('Bans', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('DependencyTypes', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('Groups', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('Licenses', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('OfficialProviders', 'utf8mb4', 'utf8mb4_bin'),
+    ('PackageBases', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('PackageBlacklist', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('PackageComments', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('PackageDepends', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('PackageKeywords', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('PackageRelations', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('PackageRequests', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('PackageSources', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('Packages', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('RelationTypes', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('RequestTypes', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('SSHPubKeys', 'utf8mb4', 'utf8mb4_bin'),
+    ('Sessions', 'utf8mb4', 'utf8mb4_bin'),
+    ('TU_VoteInfo', 'utf8mb4', 'utf8mb4_bin'),
+    ('Terms', 'utf8mb4', 'utf8mb4_general_ci'),
+    ('Users', 'utf8mb4', 'utf8mb4_general_ci')
+]
 
 # Indexes affected by charset/collate change
 # Map of Unique Indexes key = index_name, value = [table_name, column1, column2]
@@ -27,9 +50,7 @@ indexes = {'ProviderNameProvides': ['OfficialProviders', 'Name', 'Provides']}
 src_charset = "utf8"
 src_collate = "utf8_general_ci"
 
-# Destination charset/collation, after this migration is run.
+db_backend = aurweb.config.get("database", "backend")
-dst_charset = "utf8mb4"
-dst_collate = "utf8mb4_bin"
 
 
 def rebuild_unique_indexes_with_str_cols():
@@ -53,11 +74,12 @@ def do_all(iterable, fn):
 
 
 def upgrade():
-    def op_execute(table):
+    def op_execute(table_meta):
+        table, charset, collate = table_meta
         sql = f"""
 ALTER TABLE {table} 
-CONVERT TO CHARACTER SET {dst_charset} 
+CONVERT TO CHARACTER SET {charset} 
-COLLATE {dst_collate}
+COLLATE {collate}
 """
         op.execute(sql)
 
@@ -66,7 +88,11 @@ COLLATE {dst_collate}
 
 
 def downgrade():
-    def op_execute(table):
+    if db_backend == "sqlite":
+        return None
+
+    def op_execute(table_meta):
+        table, charset, collate = table_meta
         sql = f"""
 ALTER TABLE {table} 
 CONVERT TO CHARACTER SET {src_charset}