From 8387f325f69c4505cbd94c484577f047f06594e8 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Fri, 18 Feb 2022 13:02:28 -0800 Subject: [PATCH] fix: resolve null VoteTS columns via migration Somehow, many aur.al records of PackageVotes do not have a valid VoteTS value. This migration fixes that issue by setting all null VoteTS columns to the epoch timestamp. Signed-off-by: Kevin Morris --- aurweb/schema.py | 2 +- .../d64e5571bc8d_fix_pkgvote_votets.py | 37 +++++++++++++++++++ schema/gendummydata.py | 27 ++++++++------ 3 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 migrations/versions/d64e5571bc8d_fix_pkgvote_votets.py diff --git a/aurweb/schema.py b/aurweb/schema.py index 39550ff6..d2644541 100644 --- a/aurweb/schema.py +++ b/aurweb/schema.py @@ -269,7 +269,7 @@ PackageVotes = Table( 'PackageVotes', metadata, Column('UsersID', ForeignKey('Users.ID', ondelete='CASCADE'), nullable=False), Column('PackageBaseID', ForeignKey('PackageBases.ID', ondelete='CASCADE'), nullable=False), - Column('VoteTS', BIGINT(unsigned=True)), + Column('VoteTS', BIGINT(unsigned=True), nullable=False), Index('VoteUsersIDPackageID', 'UsersID', 'PackageBaseID', unique=True), Index('VotesPackageBaseID', 'PackageBaseID'), Index('VotesUsersID', 'UsersID'), diff --git a/migrations/versions/d64e5571bc8d_fix_pkgvote_votets.py b/migrations/versions/d64e5571bc8d_fix_pkgvote_votets.py new file mode 100644 index 00000000..a89d97ef --- /dev/null +++ b/migrations/versions/d64e5571bc8d_fix_pkgvote_votets.py @@ -0,0 +1,37 @@ +"""fix pkgvote votets + +Revision ID: d64e5571bc8d +Revises: be7adae47ac3 +Create Date: 2022-02-18 12:47:05.322766 + +""" +from datetime import datetime + +import sqlalchemy as sa + +from alembic import op + +from aurweb import db +from aurweb.models import PackageVote + +# revision identifiers, used by Alembic. +revision = 'd64e5571bc8d' +down_revision = 'be7adae47ac3' +branch_labels = None +depends_on = None + +table = PackageVote.__tablename__ +column = 'VoteTS' +epoch = datetime(1970, 1, 1) + + +def upgrade(): + with db.begin(): + records = db.query(PackageVote).filter(PackageVote.VoteTS.is_(None)) + for record in records: + record.VoteTS = epoch.timestamp() + op.alter_column(table, column, existing_type=sa.BIGINT(), nullable=False) + + +def downgrade(): + op.alter_column(table, column, existing_type=sa.BIGINT(), nullable=True) diff --git a/schema/gendummydata.py b/schema/gendummydata.py index 275b3601..aedfda7e 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -16,6 +16,8 @@ import random import sys import time +from datetime import datetime + import bcrypt LOG_LEVEL = logging.DEBUG # logging level. set to logging.INFO to reduce output @@ -203,7 +205,7 @@ for u in user_keys: log.debug("Number of developers: %d" % len(developers)) log.debug("Number of trusted users: %d" % len(trustedusers)) -log.debug("Number of users: %d" % (MAX_USERS-len(developers)-len(trustedusers))) +log.debug("Number of users: %d" % (MAX_USERS - len(developers) - len(trustedusers))) log.debug("Number of packages: %d" % MAX_PKGS) log.debug("Gathering text from fortune file...") @@ -244,26 +246,27 @@ for p in list(seen_pkgs.keys()): # num_comments = random.randrange(PKG_CMNTS[0], PKG_CMNTS[1]) for i in range(0, num_comments): - now = NOW + random.randrange(400, 86400*3) + now = NOW + random.randrange(400, 86400 * 3) s = ("INSERT INTO PackageComments (PackageBaseID, UsersID," " Comments, RenderedComment, CommentTS) VALUES (%d, %d, '%s', '', %d);\n") s = s % (seen_pkgs[p], genUID(), genFortune(), now) out.write(s) # Cast votes -# +utcnow = int(datetime.utcnow().timestamp()) + track_votes = {} log.debug("Casting votes for packages.") for u in user_keys: - num_votes = random.randrange(int(len(seen_pkgs)*VOTING[0]), - int(len(seen_pkgs)*VOTING[1])) + num_votes = random.randrange(int(len(seen_pkgs) * VOTING[0]), + int(len(seen_pkgs) * VOTING[1])) pkgvote = {} for v in range(num_votes): pkg = random.randrange(1, len(seen_pkgs) + 1) if pkg not in pkgvote: - s = ("INSERT INTO PackageVotes (UsersID, PackageBaseID)" - " VALUES (%d, %d);\n") - s = s % (seen_users[u], pkg) + s = ("INSERT INTO PackageVotes (UsersID, PackageBaseID, VoteTS)" + " VALUES (%d, %d, %d);\n") + s = s % (seen_users[u], pkg, utcnow) pkgvote[pkg] = 1 if pkg not in track_votes: track_votes[pkg] = 0 @@ -318,14 +321,14 @@ for p in seen_pkgs_keys: # log.debug("Creating SQL statements for trusted user proposals.") count = 0 -for t in range(0, OPEN_PROPOSALS+CLOSE_PROPOSALS): +for t in range(0, OPEN_PROPOSALS + CLOSE_PROPOSALS): now = int(time.time()) if count < CLOSE_PROPOSALS: - start = now - random.randrange(3600*24*7, 3600*24*21) - end = now - random.randrange(0, 3600*24*7) + start = now - random.randrange(3600 * 24 * 7, 3600 * 24 * 21) + end = now - random.randrange(0, 3600 * 24 * 7) else: start = now - end = now + random.randrange(3600*24, 3600*24*7) + end = now + random.randrange(3600 * 24, 3600 * 24 * 7) if count % 5 == 0: # Don't make the vote about anyone once in a while user = "" else: