mirror of
https://gitlab.archlinux.org/archlinux/aurweb.git
synced 2025-02-03 10:43:03 +01:00
[db] fix schema and migration for case insensitivity
Some of the columns that were changed still want to be case insensitive. Good thing our tables have nice separation. Signed-off-by: Kevin Morris <kevr@0cost.org>
This commit is contained in:
Kevin Morris
parent
b1121dc6ca
commit
bac38edd48
2 changed files with 98 additions and 30 deletions
|
|
@ -10,6 +10,10 @@ from sqlalchemy import CHAR, TIMESTAMP, Column, ForeignKey, Index, MetaData, Str
|
|||
from sqlalchemy.dialects.mysql import BIGINT, DECIMAL, INTEGER, TINYINT
|
||||
from sqlalchemy.ext.compiler import compiles
|
||||
|
||||
import aurweb.config
|
||||
|
||||
db_backend = aurweb.config.get("database", "backend")
|
||||
|
||||
|
||||
@compiles(TINYINT, 'sqlite')
|
||||
def compile_tinyint_sqlite(type_, compiler, **kw):
|
||||
|
|
@ -35,7 +39,9 @@ AccountTypes = Table(
|
|||
'AccountTypes', metadata,
|
||||
Column('ID', TINYINT(unsigned=True), primary_key=True),
|
||||
Column('AccountType', String(32), nullable=False, server_default=text("''")),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin'
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci'
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -69,7 +75,9 @@ Users = Table(
|
|||
Column('OwnershipNotify', TINYINT(1), nullable=False, server_default=text("1")),
|
||||
Column('SSOAccountID', String(255), nullable=True, unique=True),
|
||||
Index('UsersAccountTypeID', 'AccountTypeID'),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -113,7 +121,9 @@ PackageBases = Table(
|
|||
Index('BasesNumVotes', 'NumVotes'),
|
||||
Index('BasesPackagerUID', 'PackagerUID'),
|
||||
Index('BasesSubmitterUID', 'SubmitterUID'),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -122,7 +132,9 @@ PackageKeywords = Table(
|
|||
'PackageKeywords', metadata,
|
||||
Column('PackageBaseID', ForeignKey('PackageBases.ID', ondelete='CASCADE'), primary_key=True, nullable=False),
|
||||
Column('Keyword', String(255), primary_key=True, nullable=False, server_default=text("''")),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -135,7 +147,9 @@ Packages = Table(
|
|||
Column('Version', String(255), nullable=False, server_default=text("''")),
|
||||
Column('Description', String(255)),
|
||||
Column('URL', String(8000)),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -144,7 +158,9 @@ Licenses = Table(
|
|||
'Licenses', metadata,
|
||||
Column('ID', INTEGER(unsigned=True), primary_key=True),
|
||||
Column('Name', String(255), nullable=False, unique=True),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -162,7 +178,9 @@ Groups = Table(
|
|||
'Groups', metadata,
|
||||
Column('ID', INTEGER(unsigned=True), primary_key=True),
|
||||
Column('Name', String(255), nullable=False, unique=True),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -180,7 +198,9 @@ DependencyTypes = Table(
|
|||
'DependencyTypes', metadata,
|
||||
Column('ID', TINYINT(unsigned=True), primary_key=True),
|
||||
Column('Name', String(32), nullable=False, server_default=text("''")),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -195,7 +215,9 @@ PackageDepends = Table(
|
|||
Column('DepArch', String(255)),
|
||||
Index('DependsDepName', 'DepName'),
|
||||
Index('DependsPackageID', 'PackageID'),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -204,7 +226,9 @@ RelationTypes = Table(
|
|||
'RelationTypes', metadata,
|
||||
Column('ID', TINYINT(unsigned=True), primary_key=True),
|
||||
Column('Name', String(32), nullable=False, server_default=text("''")),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -218,7 +242,9 @@ PackageRelations = Table(
|
|||
Column('RelArch', String(255)),
|
||||
Index('RelationsPackageID', 'PackageID'),
|
||||
Index('RelationsRelName', 'RelName'),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -229,7 +255,9 @@ PackageSources = Table(
|
|||
Column('Source', String(8000), nullable=False, server_default=text("'/dev/null'")),
|
||||
Column('SourceArch', String(255)),
|
||||
Index('SourcesPackageID', 'PackageID'),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -262,7 +290,9 @@ PackageComments = Table(
|
|||
Column('PinnedTS', BIGINT(unsigned=True), nullable=False, server_default=text("0")),
|
||||
Index('CommentsPackageBaseID', 'PackageBaseID'),
|
||||
Index('CommentsUsersID', 'UsersID'),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -293,7 +323,9 @@ PackageBlacklist = Table(
|
|||
'PackageBlacklist', metadata,
|
||||
Column('ID', INTEGER(unsigned=True), primary_key=True),
|
||||
Column('Name', String(64), nullable=False, unique=True),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -314,7 +346,9 @@ RequestTypes = Table(
|
|||
'RequestTypes', metadata,
|
||||
Column('ID', TINYINT(unsigned=True), primary_key=True),
|
||||
Column('Name', String(32), nullable=False, server_default=text("''")),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -335,7 +369,9 @@ PackageRequests = Table(
|
|||
Column('Status', TINYINT(unsigned=True), nullable=False, server_default=text("0")),
|
||||
Index('RequestsPackageBaseID', 'PackageBaseID'),
|
||||
Index('RequestsUsersID', 'UsersID'),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -371,7 +407,9 @@ Bans = Table(
|
|||
'Bans', metadata,
|
||||
Column('IPAddress', String(45), primary_key=True),
|
||||
Column('BanTS', TIMESTAMP, nullable=False),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -382,7 +420,9 @@ Terms = Table(
|
|||
Column('Description', String(255), nullable=False),
|
||||
Column('URL', String(8000), nullable=False),
|
||||
Column('Revision', INTEGER(unsigned=True), nullable=False, server_default=text("1")),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -403,5 +443,7 @@ ApiRateLimit = Table(
|
|||
Column('Requests', INTEGER(11), nullable=False),
|
||||
Column('WindowStart', BIGINT(20), nullable=False),
|
||||
Index('ApiRateLimitWindowStart', 'WindowStart'),
|
||||
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin',
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8mb4',
|
||||
mysql_collate='utf8mb4_general_ci',
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ Create Date: 2021-05-17 14:23:00.008479
|
|||
"""
|
||||
from alembic import op
|
||||
|
||||
import aurweb.config
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '56e2ce8e2ffa'
|
||||
down_revision = 'ef39fcd6e1cd'
|
||||
|
|
@ -14,10 +16,31 @@ branch_labels = None
|
|||
depends_on = None
|
||||
|
||||
# Tables affected by charset/collate change
|
||||
tables = ['AccountTypes', 'ApiRateLimit', 'Bans', 'DependencyTypes', 'Groups', 'Licenses', 'OfficialProviders',
|
||||
'PackageBases', 'PackageBlacklist', 'PackageComments', 'PackageDepends', 'PackageKeywords',
|
||||
'PackageRelations', 'PackageRequests', 'PackageSources', 'Packages', 'RelationTypes', 'RequestTypes',
|
||||
'SSHPubKeys', 'Sessions', 'TU_VoteInfo', 'Terms', 'Users']
|
||||
tables = [
|
||||
('AccountTypes', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('ApiRateLimit', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('Bans', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('DependencyTypes', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('Groups', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('Licenses', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('OfficialProviders', 'utf8mb4', 'utf8mb4_bin'),
|
||||
('PackageBases', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('PackageBlacklist', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('PackageComments', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('PackageDepends', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('PackageKeywords', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('PackageRelations', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('PackageRequests', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('PackageSources', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('Packages', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('RelationTypes', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('RequestTypes', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('SSHPubKeys', 'utf8mb4', 'utf8mb4_bin'),
|
||||
('Sessions', 'utf8mb4', 'utf8mb4_bin'),
|
||||
('TU_VoteInfo', 'utf8mb4', 'utf8mb4_bin'),
|
||||
('Terms', 'utf8mb4', 'utf8mb4_general_ci'),
|
||||
('Users', 'utf8mb4', 'utf8mb4_general_ci')
|
||||
]
|
||||
|
||||
# Indexes affected by charset/collate change
|
||||
# Map of Unique Indexes key = index_name, value = [table_name, column1, column2]
|
||||
|
|
@ -27,9 +50,7 @@ indexes = {'ProviderNameProvides': ['OfficialProviders', 'Name', 'Provides']}
|
|||
src_charset = "utf8"
|
||||
src_collate = "utf8_general_ci"
|
||||
|
||||
# Destination charset/collation, after this migration is run.
|
||||
dst_charset = "utf8mb4"
|
||||
dst_collate = "utf8mb4_bin"
|
||||
db_backend = aurweb.config.get("database", "backend")
|
||||
|
||||
|
||||
def rebuild_unique_indexes_with_str_cols():
|
||||
|
|
@ -53,11 +74,12 @@ def do_all(iterable, fn):
|
|||
|
||||
|
||||
def upgrade():
|
||||
def op_execute(table):
|
||||
def op_execute(table_meta):
|
||||
table, charset, collate = table_meta
|
||||
sql = f"""
|
||||
ALTER TABLE {table}
|
||||
CONVERT TO CHARACTER SET {dst_charset}
|
||||
COLLATE {dst_collate}
|
||||
CONVERT TO CHARACTER SET {charset}
|
||||
COLLATE {collate}
|
||||
"""
|
||||
op.execute(sql)
|
||||
|
||||
|
|
@ -66,7 +88,11 @@ COLLATE {dst_collate}
|
|||
|
||||
|
||||
def downgrade():
|
||||
def op_execute(table):
|
||||
if db_backend == "sqlite":
|
||||
return None
|
||||
|
||||
def op_execute(table_meta):
|
||||
table, charset, collate = table_meta
|
||||
sql = f"""
|
||||
ALTER TABLE {table}
|
||||
CONVERT TO CHARACTER SET {src_charset}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue