diff --git a/aurweb/schema.py b/aurweb/schema.py index f76d4e06..f0162045 100644 --- a/aurweb/schema.py +++ b/aurweb/schema.py @@ -10,6 +10,10 @@ from sqlalchemy import CHAR, TIMESTAMP, Column, ForeignKey, Index, MetaData, Str from sqlalchemy.dialects.mysql import BIGINT, DECIMAL, INTEGER, TINYINT from sqlalchemy.ext.compiler import compiles +import aurweb.config + +db_backend = aurweb.config.get("database", "backend") + @compiles(TINYINT, 'sqlite') def compile_tinyint_sqlite(type_, compiler, **kw): @@ -35,7 +39,9 @@ AccountTypes = Table( 'AccountTypes', metadata, Column('ID', TINYINT(unsigned=True), primary_key=True), Column('AccountType', String(32), nullable=False, server_default=text("''")), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin' + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci' ) @@ -69,7 +75,9 @@ Users = Table( Column('OwnershipNotify', TINYINT(1), nullable=False, server_default=text("1")), Column('SSOAccountID', String(255), nullable=True, unique=True), Index('UsersAccountTypeID', 'AccountTypeID'), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -113,7 +121,9 @@ PackageBases = Table( Index('BasesNumVotes', 'NumVotes'), Index('BasesPackagerUID', 'PackagerUID'), Index('BasesSubmitterUID', 'SubmitterUID'), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -122,7 +132,9 @@ PackageKeywords = Table( 'PackageKeywords', metadata, Column('PackageBaseID', ForeignKey('PackageBases.ID', ondelete='CASCADE'), primary_key=True, nullable=False), Column('Keyword', String(255), primary_key=True, nullable=False, server_default=text("''")), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -135,7 +147,9 @@ Packages = Table( Column('Version', String(255), nullable=False, server_default=text("''")), Column('Description', String(255)), Column('URL', String(8000)), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -144,7 +158,9 @@ Licenses = Table( 'Licenses', metadata, Column('ID', INTEGER(unsigned=True), primary_key=True), Column('Name', String(255), nullable=False, unique=True), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -162,7 +178,9 @@ Groups = Table( 'Groups', metadata, Column('ID', INTEGER(unsigned=True), primary_key=True), Column('Name', String(255), nullable=False, unique=True), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -180,7 +198,9 @@ DependencyTypes = Table( 'DependencyTypes', metadata, Column('ID', TINYINT(unsigned=True), primary_key=True), Column('Name', String(32), nullable=False, server_default=text("''")), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -195,7 +215,9 @@ PackageDepends = Table( Column('DepArch', String(255)), Index('DependsDepName', 'DepName'), Index('DependsPackageID', 'PackageID'), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -204,7 +226,9 @@ RelationTypes = Table( 'RelationTypes', metadata, Column('ID', TINYINT(unsigned=True), primary_key=True), Column('Name', String(32), nullable=False, server_default=text("''")), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -218,7 +242,9 @@ PackageRelations = Table( Column('RelArch', String(255)), Index('RelationsPackageID', 'PackageID'), Index('RelationsRelName', 'RelName'), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -229,7 +255,9 @@ PackageSources = Table( Column('Source', String(8000), nullable=False, server_default=text("'/dev/null'")), Column('SourceArch', String(255)), Index('SourcesPackageID', 'PackageID'), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -262,7 +290,9 @@ PackageComments = Table( Column('PinnedTS', BIGINT(unsigned=True), nullable=False, server_default=text("0")), Index('CommentsPackageBaseID', 'PackageBaseID'), Index('CommentsUsersID', 'UsersID'), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -293,7 +323,9 @@ PackageBlacklist = Table( 'PackageBlacklist', metadata, Column('ID', INTEGER(unsigned=True), primary_key=True), Column('Name', String(64), nullable=False, unique=True), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -314,7 +346,9 @@ RequestTypes = Table( 'RequestTypes', metadata, Column('ID', TINYINT(unsigned=True), primary_key=True), Column('Name', String(32), nullable=False, server_default=text("''")), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -335,7 +369,9 @@ PackageRequests = Table( Column('Status', TINYINT(unsigned=True), nullable=False, server_default=text("0")), Index('RequestsPackageBaseID', 'PackageBaseID'), Index('RequestsUsersID', 'UsersID'), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -371,7 +407,9 @@ Bans = Table( 'Bans', metadata, Column('IPAddress', String(45), primary_key=True), Column('BanTS', TIMESTAMP, nullable=False), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -382,7 +420,9 @@ Terms = Table( Column('Description', String(255), nullable=False), Column('URL', String(8000), nullable=False), Column('Revision', INTEGER(unsigned=True), nullable=False, server_default=text("1")), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) @@ -403,5 +443,7 @@ ApiRateLimit = Table( Column('Requests', INTEGER(11), nullable=False), Column('WindowStart', BIGINT(20), nullable=False), Index('ApiRateLimitWindowStart', 'WindowStart'), - mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_general_ci', ) diff --git a/migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py b/migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py index ef7a903c..e198c34c 100644 --- a/migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py +++ b/migrations/versions/56e2ce8e2ffa_utf8mb4_charset_and_collation.py @@ -7,6 +7,8 @@ Create Date: 2021-05-17 14:23:00.008479 """ from alembic import op +import aurweb.config + # revision identifiers, used by Alembic. revision = '56e2ce8e2ffa' down_revision = 'ef39fcd6e1cd' @@ -14,10 +16,31 @@ branch_labels = None depends_on = None # Tables affected by charset/collate change -tables = ['AccountTypes', 'ApiRateLimit', 'Bans', 'DependencyTypes', 'Groups', 'Licenses', 'OfficialProviders', - 'PackageBases', 'PackageBlacklist', 'PackageComments', 'PackageDepends', 'PackageKeywords', - 'PackageRelations', 'PackageRequests', 'PackageSources', 'Packages', 'RelationTypes', 'RequestTypes', - 'SSHPubKeys', 'Sessions', 'TU_VoteInfo', 'Terms', 'Users'] +tables = [ + ('AccountTypes', 'utf8mb4', 'utf8mb4_general_ci'), + ('ApiRateLimit', 'utf8mb4', 'utf8mb4_general_ci'), + ('Bans', 'utf8mb4', 'utf8mb4_general_ci'), + ('DependencyTypes', 'utf8mb4', 'utf8mb4_general_ci'), + ('Groups', 'utf8mb4', 'utf8mb4_general_ci'), + ('Licenses', 'utf8mb4', 'utf8mb4_general_ci'), + ('OfficialProviders', 'utf8mb4', 'utf8mb4_bin'), + ('PackageBases', 'utf8mb4', 'utf8mb4_general_ci'), + ('PackageBlacklist', 'utf8mb4', 'utf8mb4_general_ci'), + ('PackageComments', 'utf8mb4', 'utf8mb4_general_ci'), + ('PackageDepends', 'utf8mb4', 'utf8mb4_general_ci'), + ('PackageKeywords', 'utf8mb4', 'utf8mb4_general_ci'), + ('PackageRelations', 'utf8mb4', 'utf8mb4_general_ci'), + ('PackageRequests', 'utf8mb4', 'utf8mb4_general_ci'), + ('PackageSources', 'utf8mb4', 'utf8mb4_general_ci'), + ('Packages', 'utf8mb4', 'utf8mb4_general_ci'), + ('RelationTypes', 'utf8mb4', 'utf8mb4_general_ci'), + ('RequestTypes', 'utf8mb4', 'utf8mb4_general_ci'), + ('SSHPubKeys', 'utf8mb4', 'utf8mb4_bin'), + ('Sessions', 'utf8mb4', 'utf8mb4_bin'), + ('TU_VoteInfo', 'utf8mb4', 'utf8mb4_bin'), + ('Terms', 'utf8mb4', 'utf8mb4_general_ci'), + ('Users', 'utf8mb4', 'utf8mb4_general_ci') +] # Indexes affected by charset/collate change # Map of Unique Indexes key = index_name, value = [table_name, column1, column2] @@ -27,9 +50,7 @@ indexes = {'ProviderNameProvides': ['OfficialProviders', 'Name', 'Provides']} src_charset = "utf8" src_collate = "utf8_general_ci" -# Destination charset/collation, after this migration is run. -dst_charset = "utf8mb4" -dst_collate = "utf8mb4_bin" +db_backend = aurweb.config.get("database", "backend") def rebuild_unique_indexes_with_str_cols(): @@ -53,11 +74,12 @@ def do_all(iterable, fn): def upgrade(): - def op_execute(table): + def op_execute(table_meta): + table, charset, collate = table_meta sql = f""" ALTER TABLE {table} -CONVERT TO CHARACTER SET {dst_charset} -COLLATE {dst_collate} +CONVERT TO CHARACTER SET {charset} +COLLATE {collate} """ op.execute(sql) @@ -66,7 +88,11 @@ COLLATE {dst_collate} def downgrade(): - def op_execute(table): + if db_backend == "sqlite": + return None + + def op_execute(table_meta): + table, charset, collate = table_meta sql = f""" ALTER TABLE {table} CONVERT TO CHARACTER SET {src_charset}