Add migrations using alembic.

* #39 Add Flask-Migrate dep

* #39 Make it such that flask db init can run

https://github.com/miguelgrinberg/Flask-Migrate/issues/196#issuecomment-381343393

* Run flask db init, update migrations.env, commit artifacts

* Set up a script such that you can `docker-compose exec files bash -c 'cd /service; FLASK_APP="files/cli:app" flask '` and have it do whatever flask thing you want

* Fix circular dependency

* import * is evil

* Initial alembic migration, has issues with constraints and nullable columns

* Bring alts table up to date with alembic autogenerate

* Rerun flask db revision autogenerate

* Bring award_relationships table up to date with alembic autogenerate

* [#39/alembic] files/classes/__init__.py is evil but is at least explicitly evil now

* #39 fix model in files/classes/badges.py

* #39 fix model in files/classes/domains.py and files/classes/clients.py

* #39 fix models: comment saves, comment flags

* #39 fix models: comments

* Few more imports

* #39 columns that are not nullable should be flagged as not nullable

* #39 Add missing indexes to model defs

* [#39] add missing unique constraints to model defs

* [#39] Temporarily undo any model changes which cause the sqlalchemy model to be out of sync with the actual dump

* #39 Deforeignkeyify the correct column to make alembic happy

* #39 flask db revision --autogenerate now creates an empty migration

* #39 Migration format such that files are listed in creation order

* #39 Better first revision

* #39 Revert the model changes that were required to get to zero differences between db revision --autogenerate and the existing schema

* #39 The first real migration

* #39 Ensure that foreign key constraints are named in migration

* #39 Alembic migrations for FK constraints, column defs

* [#39] Run DB migrations before starting tests

* [#39] New test to ensure migrations are up to date

* [#39] More descriptive test failure message

* Add -T flag to docker-compose exec

* [#39] Run alembic migrations when starting the container
This commit is contained in:
faul-sname 2022-05-17 16:55:17 -07:00 committed by GitHub
parent 19903cccb5
commit 4892b58d10
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 693 additions and 129 deletions

View file

@ -58,6 +58,7 @@ app.config['MAIL_USERNAME'] = environ.get("MAIL_USERNAME", "").strip()
app.config['MAIL_PASSWORD'] = environ.get("MAIL_PASSWORD", "").strip()
app.config['DESCRIPTION'] = environ.get("DESCRIPTION", "DESCRIPTION GOES HERE").strip()
app.config['SETTINGS'] = {}
app.config['SQLALCHEMY_DATABASE_URI'] = app.config['DATABASE_URL']
r=redis.Redis(host=environ.get("REDIS_URL", "redis://localhost"), decode_responses=True, ssl_cert_reqs=None)

View file

@ -1,22 +1,95 @@
from .alts import *
from .badges import *
from .clients import *
from .comment import *
from .domains import *
from .flags import *
from .user import *
from .userblock import *
from .usernotes import *
from .submission import *
from .votes import *
from .domains import *
from .subscriptions import *
from files.__main__ import app
from .mod_logs import *
from .award import *
from .marsey import *
from .sub_block import *
from .saves import *
from .views import *
from .notifications import *
from .follows import *
################################################################
# WARNING! THIS FILE IS EVIL. #
################################################################
# Previously, this file had a series of #
# from .alts import * #
# from .award import * #
# from .badges import * #
# and so on in that fashion. That means that anywhere that #
# from files.classes import * #
# (and there were a lot of places like that) got anything #
# was imported for any model imported. So if you, for example, #
# removed #
# from secrets import token_hex #
# from files/classes/user.py, the registration page would #
# break because files/routes/login.py did #
# from files.classes import * #
# in order to get the token_hex function rather than #
# importing it with something like #
# from secrets import token_hex #
# #
# Anyway, not fixing that right now, but in order to #
# what needed to be imported here such that #
# from files.classes import * #
# still imported the same stuff as before I ran the following: #
# $ find files/classes -type f -name '*.py' \ #
# -exec grep import '{}' ';' \ #
# | grep 'import' \ #
# | grep -v 'from [.]\|__init__\|from files.classes' \ #
# | sed 's/^[^:]*://g' \ #
# | sort \ #
# | uniq #
# and then reordered the list such that import * did not stomp #
# over stuff that had been explicitly imported. #
################################################################
# First the import * from places which don't go circular
from sqlalchemy import *
from flask import *
# Then everything except what's in files.*
import pyotp
import random
import re
import time
from copy import deepcopy
from datetime import datetime
from flask import g
from flask import render_template
from json import loads
from math import floor
from os import environ
from os import environ, remove, path
from random import randint
from secrets import token_hex
from sqlalchemy.orm import deferred, aliased
from sqlalchemy.orm import relationship
from sqlalchemy.orm import relationship, deferred
from urllib.parse import urlencode, urlparse, parse_qs
from urllib.parse import urlparse
# It is now safe to define the models
from .alts import Alt
from .award import AwardRelationship
from .badges import BadgeDef, Badge
from .clients import OauthApp, ClientAuth
from .comment import Comment
from .domains import BannedDomain
from .exiles import Exile
from .flags import Flag, CommentFlag
from .follows import Follow
from .marsey import Marsey
from .mod import Mod
from .mod_logs import ModAction
from .notifications import Notification
from .saves import SaveRelationship, CommentSaveRelationship
from .sub import Sub
from .sub_block import SubBlock
from .submission import Submission
from .subscriptions import Subscription
from .user import User
from .userblock import UserBlock
from .usernotes import UserTag, UserNote
from .views import ViewerRelationship
from .votes import Vote, CommentVote
# Then the import * from files.*
from files.helpers.const import *
from files.helpers.images import *
from files.helpers.lazy import *
from files.helpers.security import *
# Then the specific stuff we don't want stomped on
from files.helpers.discord import remove_user
from files.helpers.lazy import lazy
from files.__main__ import Base, app, cache

View file

@ -7,7 +7,9 @@ class Alt(Base):
user1 = Column(Integer, ForeignKey("users.id"), primary_key=True)
user2 = Column(Integer, ForeignKey("users.id"), primary_key=True)
is_manual = Column(Boolean, default=False)
is_manual = Column(Boolean, nullable=False, default=False)
Index('alts_user2_idx', user2)
def __repr__(self):

View file

@ -8,17 +8,24 @@ from files.helpers.const import *
class AwardRelationship(Base):
__tablename__ = "award_relationships"
__table_args__ = (
UniqueConstraint('user_id', 'submission_id', 'comment_id', name='award_constraint'),
)
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey("users.id"))
user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
submission_id = Column(Integer, ForeignKey("submissions.id"))
comment_id = Column(Integer, ForeignKey("comments.id"))
kind = Column(String)
kind = Column(String, nullable=False)
user = relationship("User", primaryjoin="AwardRelationship.user_id==User.id", viewonly=True)
post = relationship("Submission", primaryjoin="AwardRelationship.submission_id==Submission.id", viewonly=True)
comment = relationship("Comment", primaryjoin="AwardRelationship.comment_id==Comment.id", viewonly=True)
Index('award_user_idx', user_id)
Index('award_post_idx', submission_id)
Index('award_comment_idx', comment_id)
@property
@lazy

View file

@ -9,15 +9,17 @@ from json import loads
class BadgeDef(Base):
__tablename__ = "badge_defs"
__table_args__ = (
UniqueConstraint('name', name='badge_def_name_unique'),
)
id = Column(Integer, primary_key=True, autoincrement=True)
name = Column(String)
name = Column(String, nullable=False)
description = Column(String)
def __repr__(self):
return f"<BadgeDef(id={self.id})>"
class Badge(Base):
__tablename__ = "badges"
@ -27,6 +29,8 @@ class Badge(Base):
description = Column(String)
url = Column(String)
Index('badges_badge_id_idx', badge_id)
user = relationship("User", viewonly=True)
badge = relationship("BadgeDef", primaryjoin="foreign(Badge.badge_id) == remote(BadgeDef.id)", viewonly=True)

View file

@ -11,13 +11,16 @@ import time
class OauthApp(Base):
__tablename__ = "oauth_apps"
__table_args__ = (
UniqueConstraint('client_id', name='unique_id'),
)
id = Column(Integer, primary_key=True)
client_id = Column(String)
app_name = Column(String)
redirect_uri = Column(String)
description = Column(String)
author_id = Column(Integer, ForeignKey("users.id"))
app_name = Column(String(length=50), nullable=False)
redirect_uri = Column(String(length=50), nullable=False)
description = Column(String(length=256), nullable=False)
author_id = Column(Integer, ForeignKey("users.id"), nullable=False)
author = relationship("User", viewonly=True)
@ -65,10 +68,13 @@ class OauthApp(Base):
class ClientAuth(Base):
__tablename__ = "client_auths"
__table_args__ = (
UniqueConstraint('access_token', name='unique_access'),
)
user_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
oauth_client = Column(Integer, ForeignKey("oauth_apps.id"), primary_key=True)
access_token = Column(String)
access_token = Column(String, nullable=False)
user = relationship("User", viewonly=True)
application = relationship("OauthApp", viewonly=True)

View file

@ -19,28 +19,28 @@ class Comment(Base):
__tablename__ = "comments"
id = Column(Integer, primary_key=True)
author_id = Column(Integer, ForeignKey("users.id"))
author_id = Column(Integer, ForeignKey("users.id"), nullable=False)
parent_submission = Column(Integer, ForeignKey("submissions.id"))
created_utc = Column(Integer)
edited_utc = Column(Integer, default=0)
is_banned = Column(Boolean, default=False)
ghost = Column(Boolean, default=False)
created_utc = Column(Integer, nullable=False)
edited_utc = Column(Integer, default=0, nullable=False)
is_banned = Column(Boolean, default=False, nullable=False)
ghost = Column(Boolean, default=False, nullable=False)
bannedfor = Column(Boolean)
distinguish_level = Column(Integer, default=0)
deleted_utc = Column(Integer, default=0)
distinguish_level = Column(Integer, default=0, nullable=False)
deleted_utc = Column(Integer, default=0, nullable=False)
is_approved = Column(Integer, ForeignKey("users.id"))
level = Column(Integer, default=1)
level = Column(Integer, default=1, nullable=False)
parent_comment_id = Column(Integer, ForeignKey("comments.id"))
top_comment_id = Column(Integer)
over_18 = Column(Boolean, default=False)
is_bot = Column(Boolean, default=False)
over_18 = Column(Boolean, default=False, nullable=False)
is_bot = Column(Boolean, default=False, nullable=False)
is_pinned = Column(String)
is_pinned_utc = Column(Integer)
sentto = Column(Integer, ForeignKey("users.id"))
app_id = Column(Integer, ForeignKey("oauth_apps.id"))
upvotes = Column(Integer, default=1)
downvotes = Column(Integer, default=0)
realupvotes = Column(Integer, default=1)
upvotes = Column(Integer, default=1, nullable=False)
downvotes = Column(Integer, default=0, nullable=False)
realupvotes = Column(Integer, default=1, nullable=False)
body = Column(String)
body_html = Column(String)
ban_reason = Column(String)
@ -49,6 +49,12 @@ class Comment(Base):
wordle_result = Column(String)
treasure_amount = Column(String)
Index('comment_parent_index', parent_comment_id)
Index('comment_post_id_index', parent_submission)
Index('comments_user_index', author_id)
Index('fki_comment_approver_fkey', is_approved)
Index('fki_comment_sentto_fkey', sentto)
oauth_app = relationship("OauthApp", viewonly=True)
post = relationship("Submission", viewonly=True)
author = relationship("User", primaryjoin="User.id==Comment.author_id")

View file

@ -5,4 +5,10 @@ class BannedDomain(Base):
__tablename__ = "banneddomains"
domain = Column(String, primary_key=True)
reason = Column(String)
reason = Column(String, nullable=False)
Index(
'domains_domain_trgm_idx',
domain,
postgresql_using='gin',
postgresql_ops={'description':'gin_trgm_ops'}
)

View file

@ -7,7 +7,10 @@ class Exile(Base):
__tablename__ = "exiles"
user_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
sub = Column(String, ForeignKey("subs.name"), primary_key=True)
exiler_id = Column(Integer, ForeignKey("users.id"))
exiler_id = Column(Integer, ForeignKey("users.id"), nullable=False)
Index('fki_exile_exiler_fkey', exiler_id)
Index('fki_exile_sub_fkey', sub)
exiler = relationship("User", primaryjoin="User.id==Exile.exiler_id", viewonly=True)

View file

@ -12,7 +12,9 @@ class Flag(Base):
post_id = Column(Integer, ForeignKey("submissions.id"), primary_key=True)
user_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
reason = Column(String)
created_utc = Column(Integer)
created_utc = Column(Integer, nullable=False)
Index('flag_user_idx', user_id)
user = relationship("User", primaryjoin = "Flag.user_id == User.id", uselist = False, viewonly=True)
@ -45,7 +47,9 @@ class CommentFlag(Base):
comment_id = Column(Integer, ForeignKey("comments.id"), primary_key=True)
user_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
reason = Column(String)
created_utc = Column(Integer)
created_utc = Column(Integer, nullable=False)
Index('cflag_user_idx', user_id)
user = relationship("User", primaryjoin = "CommentFlag.user_id == User.id", uselist = False, viewonly=True)

View file

@ -7,7 +7,9 @@ class Follow(Base):
__tablename__ = "follows"
target_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
user_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
created_utc = Column(Integer)
created_utc = Column(Integer, nullable=False)
Index('follow_user_id_index', user_id)
user = relationship("User", uselist=False, primaryjoin="User.id==Follow.user_id", viewonly=True)
target = relationship("User", primaryjoin="User.id==Follow.target_id", viewonly=True)

View file

@ -5,9 +5,13 @@ class Marsey(Base):
__tablename__ = "marseys"
name = Column(String, primary_key=True)
author_id = Column(Integer, ForeignKey("users.id"))
tags = Column(String)
count = Column(Integer, default=0)
author_id = Column(Integer, ForeignKey("users.id"), nullable=False)
tags = Column(String(length=200), nullable=False)
count = Column(Integer, default=0, nullable=False)
Index('marseys_idx2', author_id)
Index('marseys_idx3', count.desc())
Index('marseys_idx', name)
def __repr__(self):
return f"<Marsey(name={self.name})>"

View file

@ -9,7 +9,9 @@ class Mod(Base):
__tablename__ = "mods"
user_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
sub = Column(String, ForeignKey("subs.name"), primary_key=True)
created_utc = Column(Integer)
created_utc = Column(Integer, nullable=False)
Index('fki_mod_sub_fkey', sub)
def __init__(self, *args, **kwargs):
if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time())

View file

@ -16,7 +16,13 @@ class ModAction(Base):
target_submission_id = Column(Integer, ForeignKey("submissions.id"))
target_comment_id = Column(Integer, ForeignKey("comments.id"))
_note=Column(String)
created_utc = Column(Integer)
created_utc = Column(Integer, nullable=False)
Index('fki_modactions_user_fkey', target_user_id)
Index('modaction_action_idx', kind)
Index('modaction_cid_idx', target_comment_id)
Index('modaction_id_idx', id.desc())
Index('modaction_pid_idx', target_submission_id)
user = relationship("User", primaryjoin="User.id==ModAction.user_id", viewonly=True)
target_user = relationship("User", primaryjoin="User.id==ModAction.target_user_id", viewonly=True)

View file

@ -9,8 +9,12 @@ class Notification(Base):
user_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
comment_id = Column(Integer, ForeignKey("comments.id"), primary_key=True)
read = Column(Boolean, default=False)
created_utc = Column(Integer)
read = Column(Boolean, default=False, nullable=False)
created_utc = Column(Integer, nullable=False)
Index('notification_read_idx', read)
Index('notifications_comment_idx', comment_id)
Index('notifs_user_read_idx', user_id, read)
comment = relationship("Comment", viewonly=True)
user = relationship("User", viewonly=True)

View file

@ -10,6 +10,8 @@ class SaveRelationship(Base):
user_id=Column(Integer, ForeignKey("users.id"), primary_key=True)
submission_id=Column(Integer, ForeignKey("submissions.id"), primary_key=True)
Index('fki_save_relationship_submission_fkey', submission_id)
class CommentSaveRelationship(Base):
@ -18,3 +20,5 @@ class CommentSaveRelationship(Base):
user_id=Column(Integer, ForeignKey("users.id"), primary_key=True)
comment_id=Column(Integer, ForeignKey("comments.id"), primary_key=True)
Index('fki_comment_save_relationship_comment_fkey', comment_id)

View file

@ -20,6 +20,8 @@ class Sub(Base):
bannerurl = Column(String)
css = Column(String)
Index('subs_idx', name)
blocks = relationship("SubBlock", lazy="dynamic", primaryjoin="SubBlock.sub==Sub.name", viewonly=True)

View file

@ -8,5 +8,7 @@ class SubBlock(Base):
user_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
sub = Column(String, ForeignKey("subs.name"), primary_key=True)
Index('fki_sub_blocks_sub_fkey', sub)
def __repr__(self):
return f"<SubBlock(user_id={self.user_id}, sub={self.sub})>"

View file

@ -19,32 +19,32 @@ class Submission(Base):
__tablename__ = "submissions"
id = Column(Integer, primary_key=True)
author_id = Column(Integer, ForeignKey("users.id"))
edited_utc = Column(Integer, default=0)
created_utc = Column(Integer)
author_id = Column(Integer, ForeignKey("users.id"), nullable=False)
edited_utc = Column(Integer, default=0, nullable=False)
created_utc = Column(Integer, nullable=False)
thumburl = Column(String)
is_banned = Column(Boolean, default=False)
is_banned = Column(Boolean, default=False, nullable=False)
bannedfor = Column(Boolean)
ghost = Column(Boolean, default=False)
views = Column(Integer, default=0)
deleted_utc = Column(Integer, default=0)
distinguish_level = Column(Integer, default=0)
ghost = Column(Boolean, default=False, nullable=False)
views = Column(Integer, default=0, nullable=False)
deleted_utc = Column(Integer, default=0, nullable=False)
distinguish_level = Column(Integer, default=0, nullable=False)
stickied = Column(String)
stickied_utc = Column(Integer)
sub = Column(String, ForeignKey("subs.name"))
is_pinned = Column(Boolean, default=False)
private = Column(Boolean, default=False)
club = Column(Boolean, default=False)
comment_count = Column(Integer, default=0)
is_pinned = Column(Boolean, default=False, nullable=False)
private = Column(Boolean, default=False, nullable=False)
club = Column(Boolean, default=False, nullable=False)
comment_count = Column(Integer, default=0, nullable=False)
is_approved = Column(Integer, ForeignKey("users.id"))
over_18 = Column(Boolean, default=False)
is_bot = Column(Boolean, default=False)
upvotes = Column(Integer, default=1)
downvotes = Column(Integer, default=0)
over_18 = Column(Boolean, default=False, nullable=False)
is_bot = Column(Boolean, default=False, nullable=False)
upvotes = Column(Integer, default=1, nullable=False)
downvotes = Column(Integer, default=0, nullable=False)
realupvotes = Column(Integer, default=1)
app_id=Column(Integer, ForeignKey("oauth_apps.id"))
title = Column(String)
title_html = Column(String)
title = Column(String, nullable=False)
title_html = Column(String, nullable=False)
url = Column(String)
body = Column(String)
body_html = Column(String)
@ -52,6 +52,18 @@ class Submission(Base):
ban_reason = Column(String)
embed_url = Column(String)
Index('fki_submissions_approver_fkey', is_approved)
Index('post_app_id_idx', app_id)
Index('subimssion_binary_group_idx', is_banned, deleted_utc, over_18)
Index('submission_isbanned_idx', is_banned)
Index('submission_isdeleted_idx', deleted_utc)
Index('submission_new_sort_idx', is_banned, deleted_utc, created_utc.desc(), over_18)
Index('submission_pinned_idx', is_pinned)
Index('submissions_author_index', author_id)
Index('submissions_created_utc_asc_idx', created_utc.nullsfirst())
Index('submissions_created_utc_desc_idx', created_utc.desc())
Index('submissions_over18_index', over_18)
author = relationship("User", primaryjoin="Submission.author_id==User.id")
oauth_app = relationship("OauthApp", viewonly=True)
approved_by = relationship("User", uselist=False, primaryjoin="Submission.is_approved==User.id", viewonly=True)

View file

@ -7,6 +7,8 @@ class Subscription(Base):
user_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
submission_id = Column(Integer, ForeignKey("submissions.id"), primary_key=True)
Index('subscription_user_index', user_id)
user = relationship("User", uselist=False, viewonly=True)
def __init__(self, *args, **kwargs):

View file

@ -29,61 +29,68 @@ cardview = bool(int(environ.get("CARD_VIEW", 1)))
class User(Base):
__tablename__ = "users"
__table_args__ = (
UniqueConstraint('bannerurl', name='one_banner'),
UniqueConstraint('discord_id', name='one_discord_account'),
UniqueConstraint('id', name='uid_unique'),
UniqueConstraint('original_username', name='users_original_username_key'),
UniqueConstraint('username', name='users_username_key'),
)
id = Column(Integer, primary_key=True)
username = Column(String)
namecolor = Column(String, default=DEFAULT_COLOR)
username = Column(String(length=255), nullable=False)
namecolor = Column(String(length=6), default=DEFAULT_COLOR, nullable=False)
background = Column(String)
customtitle = Column(String)
customtitleplain = deferred(Column(String))
titlecolor = Column(String, default=DEFAULT_COLOR)
theme = Column(String, default=defaulttheme)
themecolor = Column(String, default=DEFAULT_COLOR)
cardview = Column(Boolean, default=cardview)
titlecolor = Column(String(length=6), default=DEFAULT_COLOR, nullable=False)
theme = Column(String, default=defaulttheme, nullable=False)
themecolor = Column(String, default=DEFAULT_COLOR, nullable=False)
cardview = Column(Boolean, default=cardview, nullable=False)
song = Column(String)
highres = Column(String)
profileurl = Column(String)
bannerurl = Column(String)
house = Column(String)
patron = Column(Integer, default=0)
patron_utc = Column(Integer, default=0)
patron = Column(Integer, default=0, nullable=False)
patron_utc = Column(Integer, default=0, nullable=False)
verified = Column(String)
verifiedcolor = Column(String)
marseyawarded = Column(Integer)
rehab = Column(Integer)
longpost = Column(Integer)
winnings = Column(Integer, default=0)
winnings = Column(Integer, default=0, nullable=False)
unblockable = Column(Boolean)
bird = Column(Integer)
email = deferred(Column(String))
css = deferred(Column(String))
profilecss = deferred(Column(String))
passhash = deferred(Column(String))
post_count = Column(Integer, default=0)
comment_count = Column(Integer, default=0)
received_award_count = Column(Integer, default=0)
created_utc = Column(Integer)
admin_level = Column(Integer, default=0)
coins_spent = Column(Integer, default=0)
lootboxes_bought = Column(Integer, default=0)
agendaposter = Column(Integer, default=0)
changelogsub = Column(Boolean, default=False)
is_activated = Column(Boolean, default=False)
passhash = deferred(Column(String, nullable=False))
post_count = Column(Integer, default=0, nullable=False)
comment_count = Column(Integer, default=0, nullable=False)
received_award_count = Column(Integer, default=0, nullable=False)
created_utc = Column(Integer, nullable=False)
admin_level = Column(Integer, default=0, nullable=False)
coins_spent = Column(Integer, default=0, nullable=False)
lootboxes_bought = Column(Integer, default=0, nullable=False)
agendaposter = Column(Integer, default=0, nullable=False)
changelogsub = Column(Boolean, default=False, nullable=False)
is_activated = Column(Boolean, default=False, nullable=False)
shadowbanned = Column(String)
over_18 = Column(Boolean, default=False)
hidevotedon = Column(Boolean, default=False)
highlightcomments = Column(Boolean, default=True)
slurreplacer = Column(Boolean, default=True)
over_18 = Column(Boolean, default=False, nullable=False)
hidevotedon = Column(Boolean, default=False, nullable=False)
highlightcomments = Column(Boolean, default=True, nullable=False)
slurreplacer = Column(Boolean, default=True, nullable=False)
flairchanged = Column(Integer)
newtab = Column(Boolean, default=False)
newtabexternal = Column(Boolean, default=True)
reddit = Column(String, default='old.reddit.com')
newtab = Column(Boolean, default=False, nullable=False)
newtabexternal = Column(Boolean, default=True, nullable=False)
reddit = Column(String, default='old.reddit.com', nullable=False)
nitter = Column(Boolean)
mute = Column(Boolean)
unmutable = Column(Boolean)
eye = Column(Boolean)
alt = Column(Boolean)
frontsize = Column(Integer, default=25)
controversial = Column(Boolean, default=False)
frontsize = Column(Integer, default=25, nullable=False)
controversial = Column(Boolean, default=False, nullable=False)
bio = deferred(Column(String))
bio_html = Column(String)
sig = deferred(Column(String))
@ -97,28 +104,49 @@ class User(Base):
friends_html = deferred(Column(String))
enemies = deferred(Column(String))
enemies_html = deferred(Column(String))
is_banned = Column(Integer, default=0)
unban_utc = Column(Integer, default=0)
is_banned = Column(Integer, default=0, nullable=False)
unban_utc = Column(Integer, default=0, nullable=False)
ban_reason = deferred(Column(String))
club_allowed = Column(Boolean)
login_nonce = Column(Integer, default=0)
login_nonce = Column(Integer, default=0, nullable=False)
reserved = deferred(Column(String))
coins = Column(Integer, default=0)
truecoins = Column(Integer, default=0)
procoins = Column(Integer, default=0)
coins = Column(Integer, default=0, nullable=False)
truecoins = Column(Integer, default=0, nullable=False)
procoins = Column(Integer, default=0, nullable=False)
mfa_secret = deferred(Column(String))
is_private = Column(Boolean, default=False)
stored_subscriber_count = Column(Integer, default=0)
defaultsortingcomments = Column(String, default="new")
defaultsorting = Column(String, default="new")
defaulttime = Column(String, default=defaulttimefilter)
is_nofollow = Column(Boolean, default=False)
is_private = Column(Boolean, default=False, nullable=False)
stored_subscriber_count = Column(Integer, default=0, nullable=False)
defaultsortingcomments = Column(String, default="new", nullable=False)
defaultsorting = Column(String, default="new", nullable=False)
defaulttime = Column(String, default=defaulttimefilter, nullable=False)
is_nofollow = Column(Boolean, default=False, nullable=False)
custom_filter_list = Column(String)
discord_id = Column(String)
ban_evade = Column(Integer, default=0)
ban_evade = Column(Integer, default=0, nullable=False)
original_username = deferred(Column(String))
referred_by = Column(Integer, ForeignKey("users.id"))
subs_created = Column(Integer, default=0)
subs_created = Column(Integer, default=0, nullable=False)
Index(
'users_original_username_trgm_idx',
original_username,
postgresql_using='gin',
postgresql_ops={'description':'gin_trgm_ops'}
)
Index(
'users_username_trgm_idx',
username,
postgresql_using='gin',
postgresql_ops={'description':'gin_trgm_ops'}
)
Index('discord_id_idx', discord_id)
Index('fki_user_referrer_fkey', referred_by)
Index('user_banned_idx', is_banned)
Index('user_private_idx', is_private)
Index('users_created_utc_index', created_utc)
Index('users_subs_idx', stored_subscriber_count)
Index('users_unbanutc_idx', unban_utc.desc())
badges = relationship("Badge", viewonly=True)
subscriptions = relationship("Subscription", viewonly=True)

View file

@ -8,6 +8,8 @@ class UserBlock(Base):
user_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
target_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
Index('block_target_idx', target_id)
user = relationship("User", primaryjoin="User.id==UserBlock.user_id", viewonly=True)
target = relationship("User", primaryjoin="User.id==UserBlock.target_id", viewonly=True)

View file

@ -10,7 +10,9 @@ class ViewerRelationship(Base):
user_id = Column(Integer, ForeignKey('users.id'), primary_key=True)
viewer_id = Column(Integer, ForeignKey('users.id'), primary_key=True)
last_view_utc = Column(Integer)
last_view_utc = Column(Integer, nullable=False)
Index('fki_view_viewer_fkey', viewer_id)
viewer = relationship("User", primaryjoin="ViewerRelationship.viewer_id == User.id", viewonly=True)

View file

@ -11,10 +11,13 @@ class Vote(Base):
submission_id = Column(Integer, ForeignKey("submissions.id"), primary_key=True)
user_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
vote_type = Column(Integer)
vote_type = Column(Integer, nullable=False)
app_id = Column(Integer, ForeignKey("oauth_apps.id"))
real = Column(Boolean, default=True)
created_utc = Column(Integer)
real = Column(Boolean, default=True, nullable=False)
created_utc = Column(Integer, nullable=False)
Index('votes_type_index', vote_type)
Index('vote_user_index', user_id)
user = relationship("User", lazy="subquery", viewonly=True)
post = relationship("Submission", lazy="subquery", viewonly=True)
@ -52,10 +55,13 @@ class CommentVote(Base):
comment_id = Column(Integer, ForeignKey("comments.id"), primary_key=True)
user_id = Column(Integer, ForeignKey("users.id"), primary_key=True)
vote_type = Column(Integer)
vote_type = Column(Integer, nullable=False)
app_id = Column(Integer, ForeignKey("oauth_apps.id"))
real = Column(Boolean, default=True)
created_utc = Column(Integer)
real = Column(Boolean, default=True, nullable=False)
created_utc = Column(Integer, nullable=False)
Index('cvote_user_index', user_id)
Index('commentvotes_comments_type_index', vote_type)
user = relationship("User", lazy="subquery")
comment = relationship("Comment", lazy="subquery", viewonly=True)

7
files/cli.py Normal file
View file

@ -0,0 +1,7 @@
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
from .__main__ import app
import files.classes
db = SQLAlchemy(app)
migrate = Migrate(app, db)

View file

@ -0,0 +1,78 @@
import inspect
import migrations.versions
import os
import subprocess
from files.__main__ import app
APP_PATH = app.root_path
BASE_PATH = os.path.join(*os.path.split(APP_PATH)[:-1])
VERSIONS_PATH = migrations.versions.__path__._path[0];
def test_migrations_up_to_date():
def get_versions():
all_versions = [f.path for f in os.scandir(VERSIONS_PATH)]
filtered_versions = []
for entry in all_versions:
if not os.path.isfile(entry):
continue
*dir_parts, filename = os.path.split(entry)
base, ext = os.path.splitext(filename)
if ext == '.py':
filtered_versions.append(entry)
return filtered_versions
def get_method_body_lines(method):
method_lines, _ = inspect.getsourcelines(method)
return [l.strip() for l in method_lines if not l.strip().startswith('#')][1:]
versions_before = get_versions()
result = subprocess.run(
[
'python3',
'-m',
'flask',
'db',
'revision',
'--autogenerate',
'--rev-id=ci_verify_empty_revision',
'--message=should_be_empty',
],
cwd=BASE_PATH,
env={
**os.environ,
'FLASK_APP': 'files/cli:app',
},
capture_output=True,
text=True,
check=True
)
versions_after = get_versions()
new_versions = [v for v in versions_after if v not in versions_before]
try:
for version in new_versions:
filename = os.path.split(version)[-1]
base, ext = os.path.splitext(filename)
__import__(f'migrations.versions.{base}')
migration = getattr(migrations.versions, base)
upgrade_lines = get_method_body_lines(migration.upgrade)
assert ["pass"] == upgrade_lines, "\n".join([
"",
"Expected upgrade script to be empty (pass) but got",
*[f"\t>\t{l}" for l in upgrade_lines],
"To fix this issue, please run",
"\t$ flask db revision --autogenerate --message='short description of schema changes'",
"to generate a candidate migration, and make any necessary changes to that candidate migration (e.g. naming foreign key constraints)",
])
downgrade_lines = get_method_body_lines(migration.downgrade)
assert ["pass"] == downgrade_lines, "\n".join([
"",
"Expected downgrade script to be empty (pass) but got",
*[f"\t>{l}" for l in downgrade_lines],
"To fix this issue, please run",
"\tflask db revision --autogenerate --message='short description of schema changes'",
"to generate a candidate migration, and make any necessary changes to that candidate migration (e.g. naming foreign key constraints)",
])
finally:
for version in new_versions:
os.remove(version)

1
migrations/README Normal file
View file

@ -0,0 +1 @@
Single-database configuration for Flask.

51
migrations/alembic.ini Normal file
View file

@ -0,0 +1,51 @@
# A generic, single database configuration.
[alembic]
# template used to generate migration files
file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d_%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s
timezone = UTC
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic,flask_migrate
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[logger_flask_migrate]
level = INFO
handlers =
qualname = flask_migrate
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

95
migrations/env.py Normal file
View file

@ -0,0 +1,95 @@
from __future__ import with_statement
from files.classes import *
import logging
from logging.config import fileConfig
from flask import current_app
from alembic import context
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
connection_string = current_app.config.get('DATABASE_URL')
config = context.config
config.set_main_option('sqlalchemy.url', connection_string)
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
logger = logging.getLogger('alembic.env')
# add your model's MetaData object here
# for 'autogenerate' support
from files.__main__ import Base
target_metadata = Base.metadata
config.set_main_option(
'sqlalchemy.url',
str(current_app.extensions['migrate'].db.get_engine().url).replace(
'%', '%%'))
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url, target_metadata=target_metadata, literal_binds=True
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
connectable = current_app.extensions['migrate'].db.get_engine()
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

24
migrations/script.py.mako Normal file
View file

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View file

@ -0,0 +1,28 @@
"""create empty first revision
Revision ID: 0aef77162269
Revises:
Create Date: 2022-05-12 02:54:34.564536+00:00
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '0aef77162269'
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###

View file

@ -0,0 +1,38 @@
"""update db to match models at fork time
Revision ID: 4a1f7859151b
Revises: 0aef77162269
Create Date: 2022-05-12 03:08:32.417479+00:00
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '4a1f7859151b'
down_revision = '0aef77162269'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('submissions', sa.Column('bump_utc', sa.Integer(), server_default=sa.schema.FetchedValue(), nullable=True))
op.create_foreign_key('comments_app_id_fkey', 'comments', 'oauth_apps', ['app_id'], ['id'])
op.create_foreign_key('commentvotes_app_id_fkey', 'commentvotes', 'oauth_apps', ['app_id'], ['id'])
op.create_foreign_key('modactions_user_id_fkey', 'modactions', 'users', ['user_id'], ['id'])
op.create_foreign_key('submissions_app_id_fkey', 'submissions', 'oauth_apps', ['app_id'], ['id'])
op.create_foreign_key('votes_app_id_fkey', 'votes', 'oauth_apps', ['app_id'], ['id'])
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint('votes_app_id_fkey', 'votes', type_='foreignkey')
op.drop_constraint('submissions_app_id_fkey', 'submissions', type_='foreignkey')
op.drop_constraint('modactions_user_id_fkey', 'modactions', type_='foreignkey')
op.drop_constraint('commentvotes_app_id_fkey', 'commentvotes', type_='foreignkey')
op.drop_constraint('comments_app_id_fkey', 'comments', type_='foreignkey')
op.drop_column('submissions', 'bump_utc')
# ### end Alembic commands ###

View file

@ -0,0 +1,46 @@
"""add usernotes constraints
Revision ID: 16d6335dd9a3
Revises: 4a1f7859151b
Create Date: 2022-05-16 19:42:28.708577+00:00
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '16d6335dd9a3'
down_revision = '4a1f7859151b'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('usernotes', 'note',
existing_type=sa.VARCHAR(length=10000),
nullable=False)
op.alter_column('usernotes', 'tag',
existing_type=sa.VARCHAR(length=10),
nullable=False)
op.create_foreign_key('usernotes_author_id_fkey', 'usernotes', 'users', ['author_id'], ['id'])
op.create_foreign_key('usernotes_reference_comment_fkey', 'usernotes', 'comments', ['reference_comment'], ['id'], ondelete='SET NULL')
op.create_foreign_key('usernotes_reference_post_fkey', 'usernotes', 'submissions', ['reference_post'], ['id'], ondelete='SET NULL')
op.create_foreign_key('usernotes_reference_user_fkey', 'usernotes', 'users', ['reference_user'], ['id'], ondelete='CASCADE')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint('usernotes_reference_user_fkey', 'usernotes', type_='foreignkey')
op.drop_constraint('usernotes_reference_post_fkey', 'usernotes', type_='foreignkey')
op.drop_constraint('usernotes_reference_comment_fkey', 'usernotes', type_='foreignkey')
op.drop_constraint('usernotes_author_id_fkey', 'usernotes', type_='foreignkey')
op.alter_column('usernotes', 'tag',
existing_type=sa.VARCHAR(length=10),
nullable=True)
op.alter_column('usernotes', 'note',
existing_type=sa.VARCHAR(length=10000),
nullable=True)
# ### end Alembic commands ###

View file

@ -5,6 +5,7 @@ Flask-Caching
Flask-Compress
Flask-Limiter
Flask-Mail
Flask-Migrate
Flask-Socketio
gevent
gevent-websocket

View file

@ -33,10 +33,15 @@ subprocess.run([
# run the test
print("Running test . . .")
result = subprocess.run([
"docker",
"docker-compose",
"exec",
"themotte",
"bash", "-c", "cd service && python3 -m pytest -s"
'-T',
"files",
"bash", "-c", ' && '.join([
"cd service",
"FLASK_APP=files/cli:app python3 -m flask db upgrade",
"python3 -m pytest -s",
])
])
if not was_running:

View file

@ -5,7 +5,7 @@ logfile=/tmp/supervisord.log
[program:service]
directory=/service
command=gunicorn files.__main__:app -k gevent -w 1 --reload -b 0.0.0.0:80 --max-requests 1000 --max-requests-jitter 500
command=sh -c 'FLASK_APP=files/cli:app python3 -m flask db upgrade && gunicorn files.__main__:app -k gevent -w 1 --reload -b 0.0.0.0:80 --max-requests 1000 --max-requests-jitter 500'
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr