From 4892b58d1030fea1e6ba5f028328d8a391e781da Mon Sep 17 00:00:00 2001 From: faul-sname <95843830+faul-sname@users.noreply.github.com> Date: Tue, 17 May 2022 16:55:17 -0700 Subject: [PATCH] Add migrations using alembic. * #39 Add Flask-Migrate dep * #39 Make it such that flask db init can run https://github.com/miguelgrinberg/Flask-Migrate/issues/196#issuecomment-381343393 * Run flask db init, update migrations.env, commit artifacts * Set up a script such that you can `docker-compose exec files bash -c 'cd /service; FLASK_APP="files/cli:app" flask '` and have it do whatever flask thing you want * Fix circular dependency * import * is evil * Initial alembic migration, has issues with constraints and nullable columns * Bring alts table up to date with alembic autogenerate * Rerun flask db revision autogenerate * Bring award_relationships table up to date with alembic autogenerate * [#39/alembic] files/classes/__init__.py is evil but is at least explicitly evil now * #39 fix model in files/classes/badges.py * #39 fix model in files/classes/domains.py and files/classes/clients.py * #39 fix models: comment saves, comment flags * #39 fix models: comments * Few more imports * #39 columns that are not nullable should be flagged as not nullable * #39 Add missing indexes to model defs * [#39] add missing unique constraints to model defs * [#39] Temporarily undo any model changes which cause the sqlalchemy model to be out of sync with the actual dump * #39 Deforeignkeyify the correct column to make alembic happy * #39 flask db revision --autogenerate now creates an empty migration * #39 Migration format such that files are listed in creation order * #39 Better first revision * #39 Revert the model changes that were required to get to zero differences between db revision --autogenerate and the existing schema * #39 The first real migration * #39 Ensure that foreign key constraints are named in migration * #39 Alembic migrations for FK constraints, column defs * [#39] Run DB migrations before starting tests * [#39] New test to ensure migrations are up to date * [#39] More descriptive test failure message * Add -T flag to docker-compose exec * [#39] Run alembic migrations when starting the container --- files/__main__.py | 1 + files/classes/__init__.py | 117 ++++++++++++++---- files/classes/alts.py | 4 +- files/classes/award.py | 11 +- files/classes/badges.py | 8 +- files/classes/clients.py | 16 ++- files/classes/comment.py | 32 +++-- files/classes/domains.py | 8 +- files/classes/exiles.py | 5 +- files/classes/flags.py | 8 +- files/classes/follows.py | 4 +- files/classes/marsey.py | 10 +- files/classes/mod.py | 4 +- files/classes/mod_logs.py | 8 +- files/classes/notifications.py | 8 +- files/classes/saves.py | 4 + files/classes/sub.py | 2 + files/classes/sub_block.py | 2 + files/classes/submission.py | 48 ++++--- files/classes/subscriptions.py | 2 + files/classes/user.py | 114 ++++++++++------- files/classes/userblock.py | 2 + files/classes/views.py | 4 +- files/classes/votes.py | 18 ++- files/cli.py | 7 ++ files/tests/test_migrations_up_to_date.py | 78 ++++++++++++ migrations/README | 1 + migrations/alembic.ini | 51 ++++++++ migrations/env.py | 95 ++++++++++++++ migrations/script.py.mako | 24 ++++ ...aef77162269_create_empty_first_revision.py | 28 +++++ ..._update_db_to_match_models_at_fork_time.py | 38 ++++++ ..._16d6335dd9a3_add_usernotes_constraints.py | 46 +++++++ requirements.txt | 1 + run_tests.py | 11 +- supervisord.conf | 2 +- 36 files changed, 693 insertions(+), 129 deletions(-) create mode 100644 files/cli.py create mode 100644 files/tests/test_migrations_up_to_date.py create mode 100644 migrations/README create mode 100644 migrations/alembic.ini create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako create mode 100644 migrations/versions/2022_05_12_02_54_34_0aef77162269_create_empty_first_revision.py create mode 100644 migrations/versions/2022_05_12_03_08_32_4a1f7859151b_update_db_to_match_models_at_fork_time.py create mode 100644 migrations/versions/2022_05_16_19_42_28_16d6335dd9a3_add_usernotes_constraints.py diff --git a/files/__main__.py b/files/__main__.py index 8dbfe52c9..388ab2984 100644 --- a/files/__main__.py +++ b/files/__main__.py @@ -58,6 +58,7 @@ app.config['MAIL_USERNAME'] = environ.get("MAIL_USERNAME", "").strip() app.config['MAIL_PASSWORD'] = environ.get("MAIL_PASSWORD", "").strip() app.config['DESCRIPTION'] = environ.get("DESCRIPTION", "DESCRIPTION GOES HERE").strip() app.config['SETTINGS'] = {} +app.config['SQLALCHEMY_DATABASE_URI'] = app.config['DATABASE_URL'] r=redis.Redis(host=environ.get("REDIS_URL", "redis://localhost"), decode_responses=True, ssl_cert_reqs=None) diff --git a/files/classes/__init__.py b/files/classes/__init__.py index 37ac578fd..88c9fbec2 100644 --- a/files/classes/__init__.py +++ b/files/classes/__init__.py @@ -1,22 +1,95 @@ -from .alts import * -from .badges import * -from .clients import * -from .comment import * -from .domains import * -from .flags import * -from .user import * -from .userblock import * -from .usernotes import * -from .submission import * -from .votes import * -from .domains import * -from .subscriptions import * -from files.__main__ import app -from .mod_logs import * -from .award import * -from .marsey import * -from .sub_block import * -from .saves import * -from .views import * -from .notifications import * -from .follows import * +################################################################ +# WARNING! THIS FILE IS EVIL. # +################################################################ +# Previously, this file had a series of # +# from .alts import * # +# from .award import * # +# from .badges import * # +# and so on in that fashion. That means that anywhere that # +# from files.classes import * # +# (and there were a lot of places like that) got anything # +# was imported for any model imported. So if you, for example, # +# removed # +# from secrets import token_hex # +# from files/classes/user.py, the registration page would # +# break because files/routes/login.py did # +# from files.classes import * # +# in order to get the token_hex function rather than # +# importing it with something like # +# from secrets import token_hex # +# # +# Anyway, not fixing that right now, but in order to # +# what needed to be imported here such that # +# from files.classes import * # +# still imported the same stuff as before I ran the following: # +# $ find files/classes -type f -name '*.py' \ # +# -exec grep import '{}' ';' \ # +# | grep 'import' \ # +# | grep -v 'from [.]\|__init__\|from files.classes' \ # +# | sed 's/^[^:]*://g' \ # +# | sort \ # +# | uniq # +# and then reordered the list such that import * did not stomp # +# over stuff that had been explicitly imported. # +################################################################ + +# First the import * from places which don't go circular +from sqlalchemy import * +from flask import * + +# Then everything except what's in files.* +import pyotp +import random +import re +import time +from copy import deepcopy +from datetime import datetime +from flask import g +from flask import render_template +from json import loads +from math import floor +from os import environ +from os import environ, remove, path +from random import randint +from secrets import token_hex +from sqlalchemy.orm import deferred, aliased +from sqlalchemy.orm import relationship +from sqlalchemy.orm import relationship, deferred +from urllib.parse import urlencode, urlparse, parse_qs +from urllib.parse import urlparse + +# It is now safe to define the models +from .alts import Alt +from .award import AwardRelationship +from .badges import BadgeDef, Badge +from .clients import OauthApp, ClientAuth +from .comment import Comment +from .domains import BannedDomain +from .exiles import Exile +from .flags import Flag, CommentFlag +from .follows import Follow +from .marsey import Marsey +from .mod import Mod +from .mod_logs import ModAction +from .notifications import Notification +from .saves import SaveRelationship, CommentSaveRelationship +from .sub import Sub +from .sub_block import SubBlock +from .submission import Submission +from .subscriptions import Subscription +from .user import User +from .userblock import UserBlock +from .usernotes import UserTag, UserNote +from .views import ViewerRelationship +from .votes import Vote, CommentVote + +# Then the import * from files.* +from files.helpers.const import * +from files.helpers.images import * +from files.helpers.lazy import * +from files.helpers.security import * + +# Then the specific stuff we don't want stomped on +from files.helpers.discord import remove_user +from files.helpers.lazy import lazy +from files.__main__ import Base, app, cache diff --git a/files/classes/alts.py b/files/classes/alts.py index 5e9bf1d8a..d84a45403 100644 --- a/files/classes/alts.py +++ b/files/classes/alts.py @@ -7,7 +7,9 @@ class Alt(Base): user1 = Column(Integer, ForeignKey("users.id"), primary_key=True) user2 = Column(Integer, ForeignKey("users.id"), primary_key=True) - is_manual = Column(Boolean, default=False) + is_manual = Column(Boolean, nullable=False, default=False) + + Index('alts_user2_idx', user2) def __repr__(self): diff --git a/files/classes/award.py b/files/classes/award.py index f7fdf6e0d..b3c9a660e 100644 --- a/files/classes/award.py +++ b/files/classes/award.py @@ -8,17 +8,24 @@ from files.helpers.const import * class AwardRelationship(Base): __tablename__ = "award_relationships" + __table_args__ = ( + UniqueConstraint('user_id', 'submission_id', 'comment_id', name='award_constraint'), + ) id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey("users.id")) + user_id = Column(Integer, ForeignKey("users.id"), nullable=False) submission_id = Column(Integer, ForeignKey("submissions.id")) comment_id = Column(Integer, ForeignKey("comments.id")) - kind = Column(String) + kind = Column(String, nullable=False) user = relationship("User", primaryjoin="AwardRelationship.user_id==User.id", viewonly=True) post = relationship("Submission", primaryjoin="AwardRelationship.submission_id==Submission.id", viewonly=True) comment = relationship("Comment", primaryjoin="AwardRelationship.comment_id==Comment.id", viewonly=True) + Index('award_user_idx', user_id) + Index('award_post_idx', submission_id) + Index('award_comment_idx', comment_id) + @property @lazy diff --git a/files/classes/badges.py b/files/classes/badges.py index 58d3332b7..e25700b3e 100644 --- a/files/classes/badges.py +++ b/files/classes/badges.py @@ -9,15 +9,17 @@ from json import loads class BadgeDef(Base): __tablename__ = "badge_defs" + __table_args__ = ( + UniqueConstraint('name', name='badge_def_name_unique'), + ) id = Column(Integer, primary_key=True, autoincrement=True) - name = Column(String) + name = Column(String, nullable=False) description = Column(String) def __repr__(self): return f"" - class Badge(Base): __tablename__ = "badges" @@ -27,6 +29,8 @@ class Badge(Base): description = Column(String) url = Column(String) + Index('badges_badge_id_idx', badge_id) + user = relationship("User", viewonly=True) badge = relationship("BadgeDef", primaryjoin="foreign(Badge.badge_id) == remote(BadgeDef.id)", viewonly=True) diff --git a/files/classes/clients.py b/files/classes/clients.py index 6e0fed47d..ebb1d2670 100644 --- a/files/classes/clients.py +++ b/files/classes/clients.py @@ -11,13 +11,16 @@ import time class OauthApp(Base): __tablename__ = "oauth_apps" + __table_args__ = ( + UniqueConstraint('client_id', name='unique_id'), + ) id = Column(Integer, primary_key=True) client_id = Column(String) - app_name = Column(String) - redirect_uri = Column(String) - description = Column(String) - author_id = Column(Integer, ForeignKey("users.id")) + app_name = Column(String(length=50), nullable=False) + redirect_uri = Column(String(length=50), nullable=False) + description = Column(String(length=256), nullable=False) + author_id = Column(Integer, ForeignKey("users.id"), nullable=False) author = relationship("User", viewonly=True) @@ -65,10 +68,13 @@ class OauthApp(Base): class ClientAuth(Base): __tablename__ = "client_auths" + __table_args__ = ( + UniqueConstraint('access_token', name='unique_access'), + ) user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) oauth_client = Column(Integer, ForeignKey("oauth_apps.id"), primary_key=True) - access_token = Column(String) + access_token = Column(String, nullable=False) user = relationship("User", viewonly=True) application = relationship("OauthApp", viewonly=True) diff --git a/files/classes/comment.py b/files/classes/comment.py index b1f7f93b0..1d5cbbeed 100644 --- a/files/classes/comment.py +++ b/files/classes/comment.py @@ -19,28 +19,28 @@ class Comment(Base): __tablename__ = "comments" id = Column(Integer, primary_key=True) - author_id = Column(Integer, ForeignKey("users.id")) + author_id = Column(Integer, ForeignKey("users.id"), nullable=False) parent_submission = Column(Integer, ForeignKey("submissions.id")) - created_utc = Column(Integer) - edited_utc = Column(Integer, default=0) - is_banned = Column(Boolean, default=False) - ghost = Column(Boolean, default=False) + created_utc = Column(Integer, nullable=False) + edited_utc = Column(Integer, default=0, nullable=False) + is_banned = Column(Boolean, default=False, nullable=False) + ghost = Column(Boolean, default=False, nullable=False) bannedfor = Column(Boolean) - distinguish_level = Column(Integer, default=0) - deleted_utc = Column(Integer, default=0) + distinguish_level = Column(Integer, default=0, nullable=False) + deleted_utc = Column(Integer, default=0, nullable=False) is_approved = Column(Integer, ForeignKey("users.id")) - level = Column(Integer, default=1) + level = Column(Integer, default=1, nullable=False) parent_comment_id = Column(Integer, ForeignKey("comments.id")) top_comment_id = Column(Integer) - over_18 = Column(Boolean, default=False) - is_bot = Column(Boolean, default=False) + over_18 = Column(Boolean, default=False, nullable=False) + is_bot = Column(Boolean, default=False, nullable=False) is_pinned = Column(String) is_pinned_utc = Column(Integer) sentto = Column(Integer, ForeignKey("users.id")) app_id = Column(Integer, ForeignKey("oauth_apps.id")) - upvotes = Column(Integer, default=1) - downvotes = Column(Integer, default=0) - realupvotes = Column(Integer, default=1) + upvotes = Column(Integer, default=1, nullable=False) + downvotes = Column(Integer, default=0, nullable=False) + realupvotes = Column(Integer, default=1, nullable=False) body = Column(String) body_html = Column(String) ban_reason = Column(String) @@ -49,6 +49,12 @@ class Comment(Base): wordle_result = Column(String) treasure_amount = Column(String) + Index('comment_parent_index', parent_comment_id) + Index('comment_post_id_index', parent_submission) + Index('comments_user_index', author_id) + Index('fki_comment_approver_fkey', is_approved) + Index('fki_comment_sentto_fkey', sentto) + oauth_app = relationship("OauthApp", viewonly=True) post = relationship("Submission", viewonly=True) author = relationship("User", primaryjoin="User.id==Comment.author_id") diff --git a/files/classes/domains.py b/files/classes/domains.py index 776e3096a..f9acb7b32 100644 --- a/files/classes/domains.py +++ b/files/classes/domains.py @@ -5,4 +5,10 @@ class BannedDomain(Base): __tablename__ = "banneddomains" domain = Column(String, primary_key=True) - reason = Column(String) + reason = Column(String, nullable=False) + Index( + 'domains_domain_trgm_idx', + domain, + postgresql_using='gin', + postgresql_ops={'description':'gin_trgm_ops'} + ) diff --git a/files/classes/exiles.py b/files/classes/exiles.py index c30c068e6..117db21d2 100644 --- a/files/classes/exiles.py +++ b/files/classes/exiles.py @@ -7,7 +7,10 @@ class Exile(Base): __tablename__ = "exiles" user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) sub = Column(String, ForeignKey("subs.name"), primary_key=True) - exiler_id = Column(Integer, ForeignKey("users.id")) + exiler_id = Column(Integer, ForeignKey("users.id"), nullable=False) + + Index('fki_exile_exiler_fkey', exiler_id) + Index('fki_exile_sub_fkey', sub) exiler = relationship("User", primaryjoin="User.id==Exile.exiler_id", viewonly=True) diff --git a/files/classes/flags.py b/files/classes/flags.py index 5b2564d27..27401bad3 100644 --- a/files/classes/flags.py +++ b/files/classes/flags.py @@ -12,7 +12,9 @@ class Flag(Base): post_id = Column(Integer, ForeignKey("submissions.id"), primary_key=True) user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) reason = Column(String) - created_utc = Column(Integer) + created_utc = Column(Integer, nullable=False) + + Index('flag_user_idx', user_id) user = relationship("User", primaryjoin = "Flag.user_id == User.id", uselist = False, viewonly=True) @@ -45,7 +47,9 @@ class CommentFlag(Base): comment_id = Column(Integer, ForeignKey("comments.id"), primary_key=True) user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) reason = Column(String) - created_utc = Column(Integer) + created_utc = Column(Integer, nullable=False) + + Index('cflag_user_idx', user_id) user = relationship("User", primaryjoin = "CommentFlag.user_id == User.id", uselist = False, viewonly=True) diff --git a/files/classes/follows.py b/files/classes/follows.py index cacbd8130..f9a522b09 100644 --- a/files/classes/follows.py +++ b/files/classes/follows.py @@ -7,7 +7,9 @@ class Follow(Base): __tablename__ = "follows" target_id = Column(Integer, ForeignKey("users.id"), primary_key=True) user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) - created_utc = Column(Integer) + created_utc = Column(Integer, nullable=False) + + Index('follow_user_id_index', user_id) user = relationship("User", uselist=False, primaryjoin="User.id==Follow.user_id", viewonly=True) target = relationship("User", primaryjoin="User.id==Follow.target_id", viewonly=True) diff --git a/files/classes/marsey.py b/files/classes/marsey.py index 40e8e3ecf..b29c32ff4 100644 --- a/files/classes/marsey.py +++ b/files/classes/marsey.py @@ -5,9 +5,13 @@ class Marsey(Base): __tablename__ = "marseys" name = Column(String, primary_key=True) - author_id = Column(Integer, ForeignKey("users.id")) - tags = Column(String) - count = Column(Integer, default=0) + author_id = Column(Integer, ForeignKey("users.id"), nullable=False) + tags = Column(String(length=200), nullable=False) + count = Column(Integer, default=0, nullable=False) + + Index('marseys_idx2', author_id) + Index('marseys_idx3', count.desc()) + Index('marseys_idx', name) def __repr__(self): return f"" diff --git a/files/classes/mod.py b/files/classes/mod.py index 23f5ca552..0877cda2f 100644 --- a/files/classes/mod.py +++ b/files/classes/mod.py @@ -9,7 +9,9 @@ class Mod(Base): __tablename__ = "mods" user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) sub = Column(String, ForeignKey("subs.name"), primary_key=True) - created_utc = Column(Integer) + created_utc = Column(Integer, nullable=False) + + Index('fki_mod_sub_fkey', sub) def __init__(self, *args, **kwargs): if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time()) diff --git a/files/classes/mod_logs.py b/files/classes/mod_logs.py index 023e42792..97c8df133 100644 --- a/files/classes/mod_logs.py +++ b/files/classes/mod_logs.py @@ -16,7 +16,13 @@ class ModAction(Base): target_submission_id = Column(Integer, ForeignKey("submissions.id")) target_comment_id = Column(Integer, ForeignKey("comments.id")) _note=Column(String) - created_utc = Column(Integer) + created_utc = Column(Integer, nullable=False) + + Index('fki_modactions_user_fkey', target_user_id) + Index('modaction_action_idx', kind) + Index('modaction_cid_idx', target_comment_id) + Index('modaction_id_idx', id.desc()) + Index('modaction_pid_idx', target_submission_id) user = relationship("User", primaryjoin="User.id==ModAction.user_id", viewonly=True) target_user = relationship("User", primaryjoin="User.id==ModAction.target_user_id", viewonly=True) diff --git a/files/classes/notifications.py b/files/classes/notifications.py index 4edb47df8..0e2376f52 100644 --- a/files/classes/notifications.py +++ b/files/classes/notifications.py @@ -9,8 +9,12 @@ class Notification(Base): user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) comment_id = Column(Integer, ForeignKey("comments.id"), primary_key=True) - read = Column(Boolean, default=False) - created_utc = Column(Integer) + read = Column(Boolean, default=False, nullable=False) + created_utc = Column(Integer, nullable=False) + + Index('notification_read_idx', read) + Index('notifications_comment_idx', comment_id) + Index('notifs_user_read_idx', user_id, read) comment = relationship("Comment", viewonly=True) user = relationship("User", viewonly=True) diff --git a/files/classes/saves.py b/files/classes/saves.py index 50f9ef5d9..1e89898b6 100644 --- a/files/classes/saves.py +++ b/files/classes/saves.py @@ -10,6 +10,8 @@ class SaveRelationship(Base): user_id=Column(Integer, ForeignKey("users.id"), primary_key=True) submission_id=Column(Integer, ForeignKey("submissions.id"), primary_key=True) + Index('fki_save_relationship_submission_fkey', submission_id) + class CommentSaveRelationship(Base): @@ -18,3 +20,5 @@ class CommentSaveRelationship(Base): user_id=Column(Integer, ForeignKey("users.id"), primary_key=True) comment_id=Column(Integer, ForeignKey("comments.id"), primary_key=True) + + Index('fki_comment_save_relationship_comment_fkey', comment_id) diff --git a/files/classes/sub.py b/files/classes/sub.py index 9c7037822..19ab92b12 100644 --- a/files/classes/sub.py +++ b/files/classes/sub.py @@ -20,6 +20,8 @@ class Sub(Base): bannerurl = Column(String) css = Column(String) + Index('subs_idx', name) + blocks = relationship("SubBlock", lazy="dynamic", primaryjoin="SubBlock.sub==Sub.name", viewonly=True) diff --git a/files/classes/sub_block.py b/files/classes/sub_block.py index 92c7ad288..3707a63b7 100644 --- a/files/classes/sub_block.py +++ b/files/classes/sub_block.py @@ -8,5 +8,7 @@ class SubBlock(Base): user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) sub = Column(String, ForeignKey("subs.name"), primary_key=True) + Index('fki_sub_blocks_sub_fkey', sub) + def __repr__(self): return f"" diff --git a/files/classes/submission.py b/files/classes/submission.py index c46aa6683..3fd9e532e 100644 --- a/files/classes/submission.py +++ b/files/classes/submission.py @@ -19,32 +19,32 @@ class Submission(Base): __tablename__ = "submissions" id = Column(Integer, primary_key=True) - author_id = Column(Integer, ForeignKey("users.id")) - edited_utc = Column(Integer, default=0) - created_utc = Column(Integer) + author_id = Column(Integer, ForeignKey("users.id"), nullable=False) + edited_utc = Column(Integer, default=0, nullable=False) + created_utc = Column(Integer, nullable=False) thumburl = Column(String) - is_banned = Column(Boolean, default=False) + is_banned = Column(Boolean, default=False, nullable=False) bannedfor = Column(Boolean) - ghost = Column(Boolean, default=False) - views = Column(Integer, default=0) - deleted_utc = Column(Integer, default=0) - distinguish_level = Column(Integer, default=0) + ghost = Column(Boolean, default=False, nullable=False) + views = Column(Integer, default=0, nullable=False) + deleted_utc = Column(Integer, default=0, nullable=False) + distinguish_level = Column(Integer, default=0, nullable=False) stickied = Column(String) stickied_utc = Column(Integer) sub = Column(String, ForeignKey("subs.name")) - is_pinned = Column(Boolean, default=False) - private = Column(Boolean, default=False) - club = Column(Boolean, default=False) - comment_count = Column(Integer, default=0) + is_pinned = Column(Boolean, default=False, nullable=False) + private = Column(Boolean, default=False, nullable=False) + club = Column(Boolean, default=False, nullable=False) + comment_count = Column(Integer, default=0, nullable=False) is_approved = Column(Integer, ForeignKey("users.id")) - over_18 = Column(Boolean, default=False) - is_bot = Column(Boolean, default=False) - upvotes = Column(Integer, default=1) - downvotes = Column(Integer, default=0) + over_18 = Column(Boolean, default=False, nullable=False) + is_bot = Column(Boolean, default=False, nullable=False) + upvotes = Column(Integer, default=1, nullable=False) + downvotes = Column(Integer, default=0, nullable=False) realupvotes = Column(Integer, default=1) app_id=Column(Integer, ForeignKey("oauth_apps.id")) - title = Column(String) - title_html = Column(String) + title = Column(String, nullable=False) + title_html = Column(String, nullable=False) url = Column(String) body = Column(String) body_html = Column(String) @@ -52,6 +52,18 @@ class Submission(Base): ban_reason = Column(String) embed_url = Column(String) + Index('fki_submissions_approver_fkey', is_approved) + Index('post_app_id_idx', app_id) + Index('subimssion_binary_group_idx', is_banned, deleted_utc, over_18) + Index('submission_isbanned_idx', is_banned) + Index('submission_isdeleted_idx', deleted_utc) + Index('submission_new_sort_idx', is_banned, deleted_utc, created_utc.desc(), over_18) + Index('submission_pinned_idx', is_pinned) + Index('submissions_author_index', author_id) + Index('submissions_created_utc_asc_idx', created_utc.nullsfirst()) + Index('submissions_created_utc_desc_idx', created_utc.desc()) + Index('submissions_over18_index', over_18) + author = relationship("User", primaryjoin="Submission.author_id==User.id") oauth_app = relationship("OauthApp", viewonly=True) approved_by = relationship("User", uselist=False, primaryjoin="Submission.is_approved==User.id", viewonly=True) diff --git a/files/classes/subscriptions.py b/files/classes/subscriptions.py index 55baa4e08..a1b04e068 100644 --- a/files/classes/subscriptions.py +++ b/files/classes/subscriptions.py @@ -7,6 +7,8 @@ class Subscription(Base): user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) submission_id = Column(Integer, ForeignKey("submissions.id"), primary_key=True) + Index('subscription_user_index', user_id) + user = relationship("User", uselist=False, viewonly=True) def __init__(self, *args, **kwargs): diff --git a/files/classes/user.py b/files/classes/user.py index 607f37693..1d044b687 100644 --- a/files/classes/user.py +++ b/files/classes/user.py @@ -29,61 +29,68 @@ cardview = bool(int(environ.get("CARD_VIEW", 1))) class User(Base): __tablename__ = "users" + __table_args__ = ( + UniqueConstraint('bannerurl', name='one_banner'), + UniqueConstraint('discord_id', name='one_discord_account'), + UniqueConstraint('id', name='uid_unique'), + UniqueConstraint('original_username', name='users_original_username_key'), + UniqueConstraint('username', name='users_username_key'), + ) id = Column(Integer, primary_key=True) - username = Column(String) - namecolor = Column(String, default=DEFAULT_COLOR) + username = Column(String(length=255), nullable=False) + namecolor = Column(String(length=6), default=DEFAULT_COLOR, nullable=False) background = Column(String) customtitle = Column(String) customtitleplain = deferred(Column(String)) - titlecolor = Column(String, default=DEFAULT_COLOR) - theme = Column(String, default=defaulttheme) - themecolor = Column(String, default=DEFAULT_COLOR) - cardview = Column(Boolean, default=cardview) + titlecolor = Column(String(length=6), default=DEFAULT_COLOR, nullable=False) + theme = Column(String, default=defaulttheme, nullable=False) + themecolor = Column(String, default=DEFAULT_COLOR, nullable=False) + cardview = Column(Boolean, default=cardview, nullable=False) song = Column(String) highres = Column(String) profileurl = Column(String) bannerurl = Column(String) house = Column(String) - patron = Column(Integer, default=0) - patron_utc = Column(Integer, default=0) + patron = Column(Integer, default=0, nullable=False) + patron_utc = Column(Integer, default=0, nullable=False) verified = Column(String) verifiedcolor = Column(String) marseyawarded = Column(Integer) rehab = Column(Integer) longpost = Column(Integer) - winnings = Column(Integer, default=0) + winnings = Column(Integer, default=0, nullable=False) unblockable = Column(Boolean) bird = Column(Integer) email = deferred(Column(String)) css = deferred(Column(String)) profilecss = deferred(Column(String)) - passhash = deferred(Column(String)) - post_count = Column(Integer, default=0) - comment_count = Column(Integer, default=0) - received_award_count = Column(Integer, default=0) - created_utc = Column(Integer) - admin_level = Column(Integer, default=0) - coins_spent = Column(Integer, default=0) - lootboxes_bought = Column(Integer, default=0) - agendaposter = Column(Integer, default=0) - changelogsub = Column(Boolean, default=False) - is_activated = Column(Boolean, default=False) + passhash = deferred(Column(String, nullable=False)) + post_count = Column(Integer, default=0, nullable=False) + comment_count = Column(Integer, default=0, nullable=False) + received_award_count = Column(Integer, default=0, nullable=False) + created_utc = Column(Integer, nullable=False) + admin_level = Column(Integer, default=0, nullable=False) + coins_spent = Column(Integer, default=0, nullable=False) + lootboxes_bought = Column(Integer, default=0, nullable=False) + agendaposter = Column(Integer, default=0, nullable=False) + changelogsub = Column(Boolean, default=False, nullable=False) + is_activated = Column(Boolean, default=False, nullable=False) shadowbanned = Column(String) - over_18 = Column(Boolean, default=False) - hidevotedon = Column(Boolean, default=False) - highlightcomments = Column(Boolean, default=True) - slurreplacer = Column(Boolean, default=True) + over_18 = Column(Boolean, default=False, nullable=False) + hidevotedon = Column(Boolean, default=False, nullable=False) + highlightcomments = Column(Boolean, default=True, nullable=False) + slurreplacer = Column(Boolean, default=True, nullable=False) flairchanged = Column(Integer) - newtab = Column(Boolean, default=False) - newtabexternal = Column(Boolean, default=True) - reddit = Column(String, default='old.reddit.com') + newtab = Column(Boolean, default=False, nullable=False) + newtabexternal = Column(Boolean, default=True, nullable=False) + reddit = Column(String, default='old.reddit.com', nullable=False) nitter = Column(Boolean) mute = Column(Boolean) unmutable = Column(Boolean) eye = Column(Boolean) alt = Column(Boolean) - frontsize = Column(Integer, default=25) - controversial = Column(Boolean, default=False) + frontsize = Column(Integer, default=25, nullable=False) + controversial = Column(Boolean, default=False, nullable=False) bio = deferred(Column(String)) bio_html = Column(String) sig = deferred(Column(String)) @@ -97,28 +104,49 @@ class User(Base): friends_html = deferred(Column(String)) enemies = deferred(Column(String)) enemies_html = deferred(Column(String)) - is_banned = Column(Integer, default=0) - unban_utc = Column(Integer, default=0) + is_banned = Column(Integer, default=0, nullable=False) + unban_utc = Column(Integer, default=0, nullable=False) ban_reason = deferred(Column(String)) club_allowed = Column(Boolean) - login_nonce = Column(Integer, default=0) + login_nonce = Column(Integer, default=0, nullable=False) reserved = deferred(Column(String)) - coins = Column(Integer, default=0) - truecoins = Column(Integer, default=0) - procoins = Column(Integer, default=0) + coins = Column(Integer, default=0, nullable=False) + truecoins = Column(Integer, default=0, nullable=False) + procoins = Column(Integer, default=0, nullable=False) mfa_secret = deferred(Column(String)) - is_private = Column(Boolean, default=False) - stored_subscriber_count = Column(Integer, default=0) - defaultsortingcomments = Column(String, default="new") - defaultsorting = Column(String, default="new") - defaulttime = Column(String, default=defaulttimefilter) - is_nofollow = Column(Boolean, default=False) + is_private = Column(Boolean, default=False, nullable=False) + stored_subscriber_count = Column(Integer, default=0, nullable=False) + defaultsortingcomments = Column(String, default="new", nullable=False) + defaultsorting = Column(String, default="new", nullable=False) + defaulttime = Column(String, default=defaulttimefilter, nullable=False) + is_nofollow = Column(Boolean, default=False, nullable=False) custom_filter_list = Column(String) discord_id = Column(String) - ban_evade = Column(Integer, default=0) + ban_evade = Column(Integer, default=0, nullable=False) original_username = deferred(Column(String)) referred_by = Column(Integer, ForeignKey("users.id")) - subs_created = Column(Integer, default=0) + subs_created = Column(Integer, default=0, nullable=False) + + Index( + 'users_original_username_trgm_idx', + original_username, + postgresql_using='gin', + postgresql_ops={'description':'gin_trgm_ops'} + ) + Index( + 'users_username_trgm_idx', + username, + postgresql_using='gin', + postgresql_ops={'description':'gin_trgm_ops'} + ) + + Index('discord_id_idx', discord_id) + Index('fki_user_referrer_fkey', referred_by) + Index('user_banned_idx', is_banned) + Index('user_private_idx', is_private) + Index('users_created_utc_index', created_utc) + Index('users_subs_idx', stored_subscriber_count) + Index('users_unbanutc_idx', unban_utc.desc()) badges = relationship("Badge", viewonly=True) subscriptions = relationship("Subscription", viewonly=True) diff --git a/files/classes/userblock.py b/files/classes/userblock.py index a9565c4a4..c694d0003 100644 --- a/files/classes/userblock.py +++ b/files/classes/userblock.py @@ -8,6 +8,8 @@ class UserBlock(Base): user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) target_id = Column(Integer, ForeignKey("users.id"), primary_key=True) + Index('block_target_idx', target_id) + user = relationship("User", primaryjoin="User.id==UserBlock.user_id", viewonly=True) target = relationship("User", primaryjoin="User.id==UserBlock.target_id", viewonly=True) diff --git a/files/classes/views.py b/files/classes/views.py index e91d094ea..29a206577 100644 --- a/files/classes/views.py +++ b/files/classes/views.py @@ -10,7 +10,9 @@ class ViewerRelationship(Base): user_id = Column(Integer, ForeignKey('users.id'), primary_key=True) viewer_id = Column(Integer, ForeignKey('users.id'), primary_key=True) - last_view_utc = Column(Integer) + last_view_utc = Column(Integer, nullable=False) + + Index('fki_view_viewer_fkey', viewer_id) viewer = relationship("User", primaryjoin="ViewerRelationship.viewer_id == User.id", viewonly=True) diff --git a/files/classes/votes.py b/files/classes/votes.py index f2af7b524..6e83b6f2c 100644 --- a/files/classes/votes.py +++ b/files/classes/votes.py @@ -11,10 +11,13 @@ class Vote(Base): submission_id = Column(Integer, ForeignKey("submissions.id"), primary_key=True) user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) - vote_type = Column(Integer) + vote_type = Column(Integer, nullable=False) app_id = Column(Integer, ForeignKey("oauth_apps.id")) - real = Column(Boolean, default=True) - created_utc = Column(Integer) + real = Column(Boolean, default=True, nullable=False) + created_utc = Column(Integer, nullable=False) + + Index('votes_type_index', vote_type) + Index('vote_user_index', user_id) user = relationship("User", lazy="subquery", viewonly=True) post = relationship("Submission", lazy="subquery", viewonly=True) @@ -52,10 +55,13 @@ class CommentVote(Base): comment_id = Column(Integer, ForeignKey("comments.id"), primary_key=True) user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) - vote_type = Column(Integer) + vote_type = Column(Integer, nullable=False) app_id = Column(Integer, ForeignKey("oauth_apps.id")) - real = Column(Boolean, default=True) - created_utc = Column(Integer) + real = Column(Boolean, default=True, nullable=False) + created_utc = Column(Integer, nullable=False) + + Index('cvote_user_index', user_id) + Index('commentvotes_comments_type_index', vote_type) user = relationship("User", lazy="subquery") comment = relationship("Comment", lazy="subquery", viewonly=True) diff --git a/files/cli.py b/files/cli.py new file mode 100644 index 000000000..ecba58ec9 --- /dev/null +++ b/files/cli.py @@ -0,0 +1,7 @@ +from flask_migrate import Migrate +from flask_sqlalchemy import SQLAlchemy +from .__main__ import app +import files.classes + +db = SQLAlchemy(app) +migrate = Migrate(app, db) diff --git a/files/tests/test_migrations_up_to_date.py b/files/tests/test_migrations_up_to_date.py new file mode 100644 index 000000000..3d22bc8b5 --- /dev/null +++ b/files/tests/test_migrations_up_to_date.py @@ -0,0 +1,78 @@ +import inspect +import migrations.versions +import os +import subprocess + +from files.__main__ import app + +APP_PATH = app.root_path +BASE_PATH = os.path.join(*os.path.split(APP_PATH)[:-1]) +VERSIONS_PATH = migrations.versions.__path__._path[0]; + +def test_migrations_up_to_date(): + def get_versions(): + all_versions = [f.path for f in os.scandir(VERSIONS_PATH)] + filtered_versions = [] + for entry in all_versions: + if not os.path.isfile(entry): + continue + *dir_parts, filename = os.path.split(entry) + base, ext = os.path.splitext(filename) + if ext == '.py': + filtered_versions.append(entry) + return filtered_versions + + def get_method_body_lines(method): + method_lines, _ = inspect.getsourcelines(method) + return [l.strip() for l in method_lines if not l.strip().startswith('#')][1:] + + versions_before = get_versions() + result = subprocess.run( + [ + 'python3', + '-m', + 'flask', + 'db', + 'revision', + '--autogenerate', + '--rev-id=ci_verify_empty_revision', + '--message=should_be_empty', + ], + cwd=BASE_PATH, + env={ + **os.environ, + 'FLASK_APP': 'files/cli:app', + }, + capture_output=True, + text=True, + check=True + ) + versions_after = get_versions() + new_versions = [v for v in versions_after if v not in versions_before] + try: + for version in new_versions: + filename = os.path.split(version)[-1] + base, ext = os.path.splitext(filename) + __import__(f'migrations.versions.{base}') + migration = getattr(migrations.versions, base) + upgrade_lines = get_method_body_lines(migration.upgrade) + assert ["pass"] == upgrade_lines, "\n".join([ + "", + "Expected upgrade script to be empty (pass) but got", + *[f"\t>\t{l}" for l in upgrade_lines], + "To fix this issue, please run", + "\t$ flask db revision --autogenerate --message='short description of schema changes'", + "to generate a candidate migration, and make any necessary changes to that candidate migration (e.g. naming foreign key constraints)", + ]) + downgrade_lines = get_method_body_lines(migration.downgrade) + assert ["pass"] == downgrade_lines, "\n".join([ + "", + "Expected downgrade script to be empty (pass) but got", + *[f"\t>{l}" for l in downgrade_lines], + "To fix this issue, please run", + "\tflask db revision --autogenerate --message='short description of schema changes'", + "to generate a candidate migration, and make any necessary changes to that candidate migration (e.g. naming foreign key constraints)", + ]) + finally: + for version in new_versions: + os.remove(version) diff --git a/migrations/README b/migrations/README new file mode 100644 index 000000000..0e0484415 --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Single-database configuration for Flask. diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 000000000..4e7657691 --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,51 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d_%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s +timezone = UTC + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic,flask_migrate + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[logger_flask_migrate] +level = INFO +handlers = +qualname = flask_migrate + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 000000000..9bd7c3c00 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,95 @@ +from __future__ import with_statement + +from files.classes import * +import logging +from logging.config import fileConfig + +from flask import current_app + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +connection_string = current_app.config.get('DATABASE_URL') + +config = context.config +config.set_main_option('sqlalchemy.url', connection_string) + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + +# add your model's MetaData object here +# for 'autogenerate' support +from files.__main__ import Base +target_metadata = Base.metadata + +config.set_main_option( + 'sqlalchemy.url', + str(current_app.extensions['migrate'].db.get_engine().url).replace( + '%', '%%')) + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + connectable = current_app.extensions['migrate'].db.get_engine() + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + process_revision_directives=process_revision_directives, + **current_app.extensions['migrate'].configure_args + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 000000000..2c0156303 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/2022_05_12_02_54_34_0aef77162269_create_empty_first_revision.py b/migrations/versions/2022_05_12_02_54_34_0aef77162269_create_empty_first_revision.py new file mode 100644 index 000000000..d38208d22 --- /dev/null +++ b/migrations/versions/2022_05_12_02_54_34_0aef77162269_create_empty_first_revision.py @@ -0,0 +1,28 @@ +"""create empty first revision + +Revision ID: 0aef77162269 +Revises: +Create Date: 2022-05-12 02:54:34.564536+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '0aef77162269' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/migrations/versions/2022_05_12_03_08_32_4a1f7859151b_update_db_to_match_models_at_fork_time.py b/migrations/versions/2022_05_12_03_08_32_4a1f7859151b_update_db_to_match_models_at_fork_time.py new file mode 100644 index 000000000..2ec5a8820 --- /dev/null +++ b/migrations/versions/2022_05_12_03_08_32_4a1f7859151b_update_db_to_match_models_at_fork_time.py @@ -0,0 +1,38 @@ +"""update db to match models at fork time + +Revision ID: 4a1f7859151b +Revises: 0aef77162269 +Create Date: 2022-05-12 03:08:32.417479+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '4a1f7859151b' +down_revision = '0aef77162269' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('submissions', sa.Column('bump_utc', sa.Integer(), server_default=sa.schema.FetchedValue(), nullable=True)) + op.create_foreign_key('comments_app_id_fkey', 'comments', 'oauth_apps', ['app_id'], ['id']) + op.create_foreign_key('commentvotes_app_id_fkey', 'commentvotes', 'oauth_apps', ['app_id'], ['id']) + op.create_foreign_key('modactions_user_id_fkey', 'modactions', 'users', ['user_id'], ['id']) + op.create_foreign_key('submissions_app_id_fkey', 'submissions', 'oauth_apps', ['app_id'], ['id']) + op.create_foreign_key('votes_app_id_fkey', 'votes', 'oauth_apps', ['app_id'], ['id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('votes_app_id_fkey', 'votes', type_='foreignkey') + op.drop_constraint('submissions_app_id_fkey', 'submissions', type_='foreignkey') + op.drop_constraint('modactions_user_id_fkey', 'modactions', type_='foreignkey') + op.drop_constraint('commentvotes_app_id_fkey', 'commentvotes', type_='foreignkey') + op.drop_constraint('comments_app_id_fkey', 'comments', type_='foreignkey') + op.drop_column('submissions', 'bump_utc') + # ### end Alembic commands ### diff --git a/migrations/versions/2022_05_16_19_42_28_16d6335dd9a3_add_usernotes_constraints.py b/migrations/versions/2022_05_16_19_42_28_16d6335dd9a3_add_usernotes_constraints.py new file mode 100644 index 000000000..1fba05225 --- /dev/null +++ b/migrations/versions/2022_05_16_19_42_28_16d6335dd9a3_add_usernotes_constraints.py @@ -0,0 +1,46 @@ +"""add usernotes constraints + +Revision ID: 16d6335dd9a3 +Revises: 4a1f7859151b +Create Date: 2022-05-16 19:42:28.708577+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '16d6335dd9a3' +down_revision = '4a1f7859151b' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('usernotes', 'note', + existing_type=sa.VARCHAR(length=10000), + nullable=False) + op.alter_column('usernotes', 'tag', + existing_type=sa.VARCHAR(length=10), + nullable=False) + op.create_foreign_key('usernotes_author_id_fkey', 'usernotes', 'users', ['author_id'], ['id']) + op.create_foreign_key('usernotes_reference_comment_fkey', 'usernotes', 'comments', ['reference_comment'], ['id'], ondelete='SET NULL') + op.create_foreign_key('usernotes_reference_post_fkey', 'usernotes', 'submissions', ['reference_post'], ['id'], ondelete='SET NULL') + op.create_foreign_key('usernotes_reference_user_fkey', 'usernotes', 'users', ['reference_user'], ['id'], ondelete='CASCADE') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('usernotes_reference_user_fkey', 'usernotes', type_='foreignkey') + op.drop_constraint('usernotes_reference_post_fkey', 'usernotes', type_='foreignkey') + op.drop_constraint('usernotes_reference_comment_fkey', 'usernotes', type_='foreignkey') + op.drop_constraint('usernotes_author_id_fkey', 'usernotes', type_='foreignkey') + op.alter_column('usernotes', 'tag', + existing_type=sa.VARCHAR(length=10), + nullable=True) + op.alter_column('usernotes', 'note', + existing_type=sa.VARCHAR(length=10000), + nullable=True) + # ### end Alembic commands ### diff --git a/requirements.txt b/requirements.txt index 7f1319eff..271b48610 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ Flask-Caching Flask-Compress Flask-Limiter Flask-Mail +Flask-Migrate Flask-Socketio gevent gevent-websocket diff --git a/run_tests.py b/run_tests.py index 1cabb5029..e6259434a 100755 --- a/run_tests.py +++ b/run_tests.py @@ -33,10 +33,15 @@ subprocess.run([ # run the test print("Running test . . .") result = subprocess.run([ - "docker", + "docker-compose", "exec", - "themotte", - "bash", "-c", "cd service && python3 -m pytest -s" + '-T', + "files", + "bash", "-c", ' && '.join([ + "cd service", + "FLASK_APP=files/cli:app python3 -m flask db upgrade", + "python3 -m pytest -s", + ]) ]) if not was_running: diff --git a/supervisord.conf b/supervisord.conf index c9ecbc935..e07910100 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -5,7 +5,7 @@ logfile=/tmp/supervisord.log [program:service] directory=/service -command=gunicorn files.__main__:app -k gevent -w 1 --reload -b 0.0.0.0:80 --max-requests 1000 --max-requests-jitter 500 +command=sh -c 'FLASK_APP=files/cli:app python3 -m flask db upgrade && gunicorn files.__main__:app -k gevent -w 1 --reload -b 0.0.0.0:80 --max-requests 1000 --max-requests-jitter 500' stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr