From 356f7e2f41d15299d8781974dddbcaf809df36d9 Mon Sep 17 00:00:00 2001 From: Viet Than Date: Wed, 2 Aug 2023 17:36:16 -0500 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=8F=A6=20Database=20Change:=20convert?= =?UTF-8?q?=20created=20utc=20to=20datetimez=20for=20follows=20(#666)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- files/classes/follows.py | 4 +- files/routes/users.py | 4 +- ...bd8bc_change_to_datetimez_table_follows.py | 53 +++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 migrations/versions/2023_07_30_00_46_35_1ffc63ebd8bc_change_to_datetimez_table_follows.py diff --git a/files/classes/follows.py b/files/classes/follows.py index 8bffa1e9d..54325c75d 100644 --- a/files/classes/follows.py +++ b/files/classes/follows.py @@ -1,8 +1,8 @@ from sqlalchemy import * from sqlalchemy.orm import relationship -from files.classes.base import CreatedBase +from files.classes.base import CreatedDateTimeBase -class Follow(CreatedBase): +class Follow(CreatedDateTimeBase): __tablename__ = "follows" target_id = Column(Integer, ForeignKey("users.id"), primary_key=True) user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) diff --git a/files/routes/users.py b/files/routes/users.py index 88d1b3624..164273653 100644 --- a/files/routes/users.py +++ b/files/routes/users.py @@ -628,14 +628,14 @@ def redditor_moment_redirect(username:str): @auth_desired def followers(v, username): u = get_user(username, v=v) - users = g.db.query(User).join(Follow, Follow.target_id == u.id).filter(Follow.user_id == User.id).order_by(Follow.created_utc).all() + users = g.db.query(User).join(Follow, Follow.target_id == u.id).filter(Follow.user_id == User.id).order_by(Follow.created_datetimez).all() return render_template("followers.html", v=v, u=u, users=users) @app.get("/@/following") @auth_desired def following(v, username): u = get_user(username, v=v) - users = g.db.query(User).join(Follow, Follow.user_id == u.id).filter(Follow.target_id == User.id).order_by(Follow.created_utc).all() + users = g.db.query(User).join(Follow, Follow.user_id == u.id).filter(Follow.target_id == User.id).order_by(Follow.created_datetimez).all() return render_template("following.html", v=v, u=u, users=users) @app.get("/views") diff --git a/migrations/versions/2023_07_30_00_46_35_1ffc63ebd8bc_change_to_datetimez_table_follows.py b/migrations/versions/2023_07_30_00_46_35_1ffc63ebd8bc_change_to_datetimez_table_follows.py new file mode 100644 index 000000000..cd09cc7b7 --- /dev/null +++ b/migrations/versions/2023_07_30_00_46_35_1ffc63ebd8bc_change_to_datetimez_table_follows.py @@ -0,0 +1,53 @@ +"""Change to datetimez table follows + +Revision ID: 1ffc63ebd8bc +Revises: 7ae4658467d7 +Create Date: 2023-07-30 00:46:35.798549+00:00 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql.functions import now + + +# revision identifiers, used by Alembic. +revision = '1ffc63ebd8bc' +down_revision = 'bdb4b7a2e88b' +branch_labels = None +depends_on = None + +table_name = 'follows' +from_column = 'created_utc' +to_column = 'created_datetimez' + + +def upgrade(): + op.add_column(table_name, sa.Column(to_column, sa.DateTime(timezone=True), nullable=True, server_default=now())) + op.execute(f""" + UPDATE {table_name} + SET {to_column} = + CASE + WHEN {from_column} > 0 THEN + (timestamp 'epoch' + {from_column} * interval '1 second') at time zone 'utc' + ELSE NULL + END + """) + op.alter_column(table_name, to_column, nullable=False) + op.drop_column(table_name, from_column) + + +def downgrade(): + """ + Downgrade will truncate the milliseconds. + """ + op.add_column(table_name, sa.Column('created_utc', sa.Integer(), server_default=sa.text('0'), nullable=True)) + op.execute(f""" + UPDATE {table_name} + SET created_utc = + COALESCE( + EXTRACT(EPOCH FROM {to_column})::integer, + 0 + ) + """) + op.alter_column(table_name, from_column, nullable=False) + op.drop_column(table_name, to_column) From 34b328583ca908052079c1ab5a1ba77589ea8cd5 Mon Sep 17 00:00:00 2001 From: Viet Than Date: Wed, 2 Aug 2023 17:37:38 -0500 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=8F=A6=20Database=20Change:=20convert?= =?UTF-8?q?=20created=20utc=20to=20datetimez=20for=20modactions=20(#667)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- files/classes/mod_logs.py | 4 +- files/routes/admin/admin.py | 10 ++-- ...a8_change_to_datetimez_table_modactions.py | 52 +++++++++++++++++++ 3 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 migrations/versions/2023_07_30_01_08_22_84e02ade75a8_change_to_datetimez_table_modactions.py diff --git a/files/classes/mod_logs.py b/files/classes/mod_logs.py index 245bb98fc..f9e23a68b 100644 --- a/files/classes/mod_logs.py +++ b/files/classes/mod_logs.py @@ -4,12 +4,12 @@ from copy import deepcopy from sqlalchemy import * from sqlalchemy.orm import relationship -from files.classes.base import CreatedBase +from files.classes.base import CreatedDateTimeBase from files.helpers.config.const import * from files.helpers.lazy import lazy -class ModAction(CreatedBase): +class ModAction(CreatedDateTimeBase): __tablename__ = "modactions" id = Column(Integer, primary_key=True) user_id = Column(Integer, ForeignKey("users.id")) diff --git a/files/routes/admin/admin.py b/files/routes/admin/admin.py index 8fc7d3074..1dcca6815 100644 --- a/files/routes/admin/admin.py +++ b/files/routes/admin/admin.py @@ -1,6 +1,6 @@ import json import time -from datetime import datetime +from datetime import datetime, timedelta, timezone import requests @@ -128,12 +128,12 @@ def revert_actions(v, username): ) g.db.add(ma) - cutoff = int(time.time()) - 86400 + cutoff = datetime.now(tz=timezone.utc) - timedelta(1) - posts = [x[0] for x in g.db.query(ModAction.target_submission_id).filter(ModAction.user_id == user.id, ModAction.created_utc > cutoff, ModAction.kind == 'remove_post').all()] + posts = [x[0] for x in g.db.query(ModAction.target_submission_id).filter(ModAction.user_id == user.id, ModAction.created_datetimez > cutoff, ModAction.kind == 'remove_post').all()] posts = g.db.query(Submission).filter(Submission.id.in_(posts)).all() - comments = [x[0] for x in g.db.query(ModAction.target_comment_id).filter(ModAction.user_id == user.id, ModAction.created_utc > cutoff, ModAction.kind == 'remove_comment').all()] + comments = [x[0] for x in g.db.query(ModAction.target_comment_id).filter(ModAction.user_id == user.id, ModAction.created_datetimez > cutoff, ModAction.kind == 'remove_comment').all()] comments = g.db.query(Comment).filter(Comment.id.in_(comments)).all() for item in posts + comments: @@ -141,7 +141,7 @@ def revert_actions(v, username): item.state_mod_set_by = v.username g.db.add(item) - users = (x[0] for x in g.db.query(ModAction.target_user_id).filter(ModAction.user_id == user.id, ModAction.created_utc > cutoff, ModAction.kind.in_(('shadowban', 'ban_user'))).all()) + users = (x[0] for x in g.db.query(ModAction.target_user_id).filter(ModAction.user_id == user.id, ModAction.created_datetimez > cutoff, ModAction.kind.in_(('shadowban', 'ban_user'))).all()) users = g.db.query(User).filter(User.id.in_(users)).all() for user in users: diff --git a/migrations/versions/2023_07_30_01_08_22_84e02ade75a8_change_to_datetimez_table_modactions.py b/migrations/versions/2023_07_30_01_08_22_84e02ade75a8_change_to_datetimez_table_modactions.py new file mode 100644 index 000000000..9fcc6142f --- /dev/null +++ b/migrations/versions/2023_07_30_01_08_22_84e02ade75a8_change_to_datetimez_table_modactions.py @@ -0,0 +1,52 @@ +"""Change to datetimez table modactions + +Revision ID: 84e02ade75a8 +Revises: 1ffc63ebd8bc +Create Date: 2023-07-30 01:08:22.348150+00:00 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql.functions import now + + +# revision identifiers, used by Alembic. +revision = '84e02ade75a8' +down_revision = '1ffc63ebd8bc' +branch_labels = None +depends_on = None + +table_name = 'modactions' +from_column = 'created_utc' +to_column = 'created_datetimez' + +def upgrade(): + op.add_column(table_name, sa.Column(to_column, sa.DateTime(timezone=True), nullable=True, server_default=now())) + op.execute(f""" + UPDATE {table_name} + SET {to_column} = + CASE + WHEN {from_column} > 0 THEN + (timestamp 'epoch' + {from_column} * interval '1 second') at time zone 'utc' + ELSE NULL + END + """) + op.alter_column(table_name, to_column, nullable=False) + op.drop_column(table_name, from_column) + + +def downgrade(): + """ + Downgrade will truncate the milliseconds. + """ + op.add_column(table_name, sa.Column('created_utc', sa.Integer(), server_default=sa.text('0'), nullable=True)) + op.execute(f""" + UPDATE {table_name} + SET created_utc = + COALESCE( + EXTRACT(EPOCH FROM {to_column})::integer, + 0 + ) + """) + op.alter_column(table_name, from_column, nullable=False) + op.drop_column(table_name, to_column) From e1075eb722bbcdb5cd75cf7f45202e1d068cc720 Mon Sep 17 00:00:00 2001 From: Viet Than Date: Wed, 2 Aug 2023 21:35:05 -0500 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=8F=A6=20Database=20Change:=20convert?= =?UTF-8?q?=20created=20utc=20to=20datetimez=20for=20notifications=20(#668?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- files/classes/notifications.py | 4 +- files/routes/front.py | 6 +-- ...change_to_datetimez_table_notifications.py | 52 +++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 migrations/versions/2023_07_30_01_49_16_4376dcec1ad8_change_to_datetimez_table_notifications.py diff --git a/files/classes/notifications.py b/files/classes/notifications.py index b4b87510b..8fc8cbf77 100644 --- a/files/classes/notifications.py +++ b/files/classes/notifications.py @@ -1,8 +1,8 @@ from sqlalchemy import * from sqlalchemy.orm import relationship -from files.classes.base import CreatedBase +from files.classes.base import CreatedDateTimeBase -class Notification(CreatedBase): +class Notification(CreatedDateTimeBase): __tablename__ = "notifications" user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) diff --git a/files/routes/front.py b/files/routes/front.py index ce488f9af..2ce1a5b56 100644 --- a/files/routes/front.py +++ b/files/routes/front.py @@ -30,7 +30,7 @@ def unread(v): Comment.state_mod == StateMod.VISIBLE, Comment.state_user_deleted_utc == None, Comment.author_id != AUTOJANNY_ID, - ).order_by(Notification.created_utc.desc()).all() + ).order_by(Notification.created_datetimez.desc()).all() for n, c in listing: n.read = True @@ -52,7 +52,7 @@ def notifications_main(v: User): Comment.state_mod == StateMod.VISIBLE, Comment.state_user_deleted_utc == None, Comment.author_id != AUTOJANNY_ID, - ).order_by(Notification.created_utc.desc())) + ).order_by(Notification.created_datetimez.desc())) if not v.shadowbanned and v.admin_level < 3: comments = comments.join(Comment.author).filter(User.shadowbanned == None) @@ -96,7 +96,7 @@ def notifications_posts(v: User): notifications = (g.db.query(Notification, Comment) .join(Comment, Notification.comment_id == Comment.id) .filter(Notification.user_id == v.id, Comment.author_id == AUTOJANNY_ID) - .order_by(Notification.created_utc.desc()).offset(25 * (page - 1)).limit(101).all()) + .order_by(Notification.created_datetimez.desc()).offset(25 * (page - 1)).limit(101).all()) listing = [] diff --git a/migrations/versions/2023_07_30_01_49_16_4376dcec1ad8_change_to_datetimez_table_notifications.py b/migrations/versions/2023_07_30_01_49_16_4376dcec1ad8_change_to_datetimez_table_notifications.py new file mode 100644 index 000000000..504be2489 --- /dev/null +++ b/migrations/versions/2023_07_30_01_49_16_4376dcec1ad8_change_to_datetimez_table_notifications.py @@ -0,0 +1,52 @@ +"""Change to datetimez table notifications + +Revision ID: 4376dcec1ad8 +Revises: 84e02ade75a8 +Create Date: 2023-07-30 01:49:16.238320+00:00 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql.functions import now + + +# revision identifiers, used by Alembic. +revision = '4376dcec1ad8' +down_revision = '84e02ade75a8' +branch_labels = None +depends_on = None + +table_name = 'notifications' +from_column = 'created_utc' +to_column = 'created_datetimez' + +def upgrade(): + op.add_column(table_name, sa.Column(to_column, sa.DateTime(timezone=True), nullable=True, server_default=now())) + op.execute(f""" + UPDATE {table_name} + SET {to_column} = + CASE + WHEN {from_column} > 0 THEN + (timestamp 'epoch' + {from_column} * interval '1 second') at time zone 'utc' + ELSE NULL + END + """) + op.alter_column(table_name, to_column, nullable=False) + op.drop_column(table_name, from_column) + + +def downgrade(): + """ + Downgrade will truncate the milliseconds. + """ + op.add_column(table_name, sa.Column('created_utc', sa.Integer(), server_default=sa.text('0'), nullable=True)) + op.execute(f""" + UPDATE {table_name} + SET created_utc = + COALESCE( + EXTRACT(EPOCH FROM {to_column})::integer, + 0 + ) + """) + op.alter_column(table_name, from_column, nullable=False) + op.drop_column(table_name, to_column) From 53c78c4536921519cf8599101baa3a561b4931a7 Mon Sep 17 00:00:00 2001 From: Viet Than Date: Wed, 2 Aug 2023 21:46:48 -0500 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=8F=A6=20Database=20Change:=20convert?= =?UTF-8?q?=20created=20utc=20to=20datetimez=20for=20usernotes=20(#669)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- files/classes/usernotes.py | 4 +- files/commands/volunteer_janitor_recalc.py | 2 +- ...952_change_to_datetimez_table_usernotes.py | 54 +++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 migrations/versions/2023_07_30_02_06_08_7f0427506952_change_to_datetimez_table_usernotes.py diff --git a/files/classes/usernotes.py b/files/classes/usernotes.py index ee9f7937c..b5aeb8a5a 100644 --- a/files/classes/usernotes.py +++ b/files/classes/usernotes.py @@ -1,6 +1,6 @@ from sqlalchemy import * from sqlalchemy.orm import relationship -from files.classes.base import CreatedBase +from files.classes.base import CreatedDateTimeBase from files.helpers.config.const import * from enum import Enum from sqlalchemy import Enum as EnumType @@ -15,7 +15,7 @@ class UserTag(Enum): Spam = 6 Bot = 7 -class UserNote(CreatedBase): +class UserNote(CreatedDateTimeBase): __tablename__ = "usernotes" id = Column(Integer, primary_key=True) diff --git a/files/commands/volunteer_janitor_recalc.py b/files/commands/volunteer_janitor_recalc.py index 5941c6efb..10d3b8b44 100644 --- a/files/commands/volunteer_janitor_recalc.py +++ b/files/commands/volunteer_janitor_recalc.py @@ -169,7 +169,7 @@ def volunteer_janitor_recalc(db: Session, diagnostics: bool = False): usernotes_raw = db.query(UserNote) \ .where(UserNote.tag.in_([UserTag.Warning, UserTag.Tempban, UserTag.Permban, UserTag.Spam, UserTag.Bot])) \ - .options(sqlalchemy.orm.load_only('reference_user', 'created_utc', 'tag')) + .options(sqlalchemy.orm.load_only('reference_user', 'created_datetimez', 'tag')) # Here we're trying to figure out whether modhats are actually warnings/bans # We don't have a formal connection between "a comment is bad" and "the user got a warning", so we're kind of awkwardly trying to derive it from our database diff --git a/migrations/versions/2023_07_30_02_06_08_7f0427506952_change_to_datetimez_table_usernotes.py b/migrations/versions/2023_07_30_02_06_08_7f0427506952_change_to_datetimez_table_usernotes.py new file mode 100644 index 000000000..5b898a821 --- /dev/null +++ b/migrations/versions/2023_07_30_02_06_08_7f0427506952_change_to_datetimez_table_usernotes.py @@ -0,0 +1,54 @@ +"""Change to datetimez table usernotes + +Revision ID: 7f0427506952 +Revises: 4376dcec1ad8 +Create Date: 2023-07-30 02:06:08.372763+00:00 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql.functions import now + + +# revision identifiers, used by Alembic. +revision = '7f0427506952' +down_revision = '4376dcec1ad8' +branch_labels = None +depends_on = None + + +table_name = 'usernotes' +from_column = 'created_utc' +to_column = 'created_datetimez' + +def upgrade(): + op.add_column(table_name, sa.Column(to_column, sa.DateTime(timezone=True), nullable=True, server_default=now())) + op.execute(f""" + UPDATE {table_name} + SET {to_column} = + CASE + WHEN {from_column} > 0 THEN + (timestamp 'epoch' + {from_column} * interval '1 second') at time zone 'utc' + ELSE NULL + END + """) + op.alter_column(table_name, to_column, nullable=False) + op.drop_column(table_name, from_column) + + +def downgrade(): + """ + Downgrade will truncate the milliseconds. + """ + op.add_column(table_name, sa.Column('created_utc', sa.Integer(), server_default=sa.text('0'), nullable=True)) + op.execute(f""" + UPDATE {table_name} + SET created_utc = + COALESCE( + EXTRACT(EPOCH FROM {to_column})::integer, + 0 + ) + """) + op.alter_column(table_name, from_column, nullable=False) + op.drop_column(table_name, to_column) +