Merge pull request #439 from TLSM/pr-eager-content

Eager loading comments for posts, submission_listings, supporting changes.
This commit is contained in:
Ben Rog-Wilhelm 2022-12-17 15:29:44 -08:00 committed by GitHub
commit 23a8fb9663
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 419 additions and 423 deletions

View file

@ -33,7 +33,9 @@ class Badge(Base):
Index('badges_badge_id_idx', badge_id)
user = relationship("User", viewonly=True)
badge = relationship("BadgeDef", primaryjoin="foreign(Badge.badge_id) == remote(BadgeDef.id)", viewonly=True)
badge = relationship("BadgeDef",
primaryjoin="foreign(Badge.badge_id) == remote(BadgeDef.id)",
lazy="joined", innerjoin=True, viewonly=True)
def __repr__(self):
return f"<Badge(user_id={self.user_id}, badge_id={self.badge_id})>"

View file

@ -58,8 +58,13 @@ class Comment(Base):
senttouser = relationship("User", primaryjoin="User.id==Comment.sentto", viewonly=True)
parent_comment = relationship("Comment", remote_side=[id], viewonly=True)
child_comments = relationship("Comment", lazy="dynamic", remote_side=[parent_comment_id], viewonly=True)
awards = relationship("AwardRelationship", viewonly=True)
reports = relationship("CommentFlag", viewonly=True)
awards = relationship("AwardRelationship",
primaryjoin="AwardRelationship.comment_id == Comment.id",
viewonly=True)
reports = relationship("CommentFlag",
primaryjoin="CommentFlag.comment_id == Comment.id",
order_by="CommentFlag.created_utc",
viewonly=True)
notes = relationship("UserNote", back_populates="comment")
def __init__(self, *args, **kwargs):
@ -70,7 +75,6 @@ class Comment(Base):
super().__init__(*args, **kwargs)
def __repr__(self):
return f"<Comment(id={self.id})>"
@property
@ -87,7 +91,7 @@ class Comment(Base):
@lazy
def flags(self, v):
flags = g.db.query(CommentFlag).filter_by(comment_id=self.id).order_by(CommentFlag.created_utc).all()
flags = self.reports
if not (v and (v.shadowbanned or v.admin_level > 2)):
for flag in flags:
if flag.user.shadowbanned:

View file

@ -19,6 +19,7 @@ from .sub_block import *
from files.__main__ import app, Base, cache
from files.helpers.security import *
from files.helpers.assetcache import assetcache_path
from files.helpers.contentsorting import apply_time_filter, sort_objects
import random
from datetime import datetime
from os import environ, remove, path
@ -293,33 +294,8 @@ class User(Base):
if not (v and (v.admin_level > 1 or v.id == self.id)):
posts = posts.filter_by(deleted_utc=0, is_banned=False, private=False, ghost=False)
now = int(time.time())
if t == 'hour':
cutoff = now - 3600
elif t == 'day':
cutoff = now - 86400
elif t == 'week':
cutoff = now - 604800
elif t == 'month':
cutoff = now - 2592000
elif t == 'year':
cutoff = now - 31536000
else:
cutoff = 0
posts = posts.filter(Submission.created_utc >= cutoff)
if sort == "new":
posts = posts.order_by(Submission.created_utc.desc())
elif sort == "old":
posts = posts.order_by(Submission.created_utc)
elif sort == "controversial":
posts = posts.order_by((Submission.upvotes+1)/(Submission.downvotes+1) + (Submission.downvotes+1)/(Submission.upvotes+1), Submission.downvotes.desc())
elif sort == "top":
posts = posts.order_by(Submission.downvotes - Submission.upvotes)
elif sort == "bottom":
posts = posts.order_by(Submission.upvotes - Submission.downvotes)
elif sort == "comments":
posts = posts.order_by(Submission.comment_count.desc())
posts = apply_time_filter(posts, t, Submission)
posts = sort_objects(posts, sort, Submission)
posts = posts.offset(25 * (page - 1)).limit(26).all()

View file

@ -130,8 +130,12 @@ def seed_db():
db.session.add(comment)
comments.append(comment)
db.session.commit()
db.session.flush()
for c in comments:
c.top_comment_id = c.id
db.session.add(c)
db.session.commit()
print(f"Creating {NUM_REPLY_COMMENTS} reply comments")
for i in range(NUM_REPLY_COMMENTS):
@ -143,6 +147,7 @@ def seed_db():
author_id=user.id,
parent_submission=str(parent.post.id),
parent_comment_id=parent.id,
top_comment_id=parent.top_comment_id,
level=parent.level + 1,
over_18=False,
is_bot=False,
@ -155,7 +160,6 @@ def seed_db():
comments.append(comment)
db.session.commit()
db.session.flush()
print("Updating comment counts for all posts")
for post in posts:

View file

@ -0,0 +1,87 @@
import time
from sqlalchemy.sql import func
from files.helpers.const import *
def apply_time_filter(objects, t, cls):
now = int(time.time())
if t == 'hour':
cutoff = now - (60 * 60)
elif t == 'day':
cutoff = now - (24 * 60 * 60)
elif t == 'week':
cutoff = now - (7 * 24 * 60 * 60)
elif t == 'month':
cutoff = now - (30 * 24 * 60 * 60)
elif t == 'year':
cutoff = now - (365 * 24 * 60 * 60)
else:
cutoff = 0
return objects.filter(cls.created_utc >= cutoff)
def sort_objects(objects, sort, cls):
if sort == 'hot':
ti = int(time.time()) + 3600
return objects.order_by(
-100000
* (cls.upvotes + 1)
/ (func.power((ti - cls.created_utc) / 1000, 1.23)),
cls.created_utc.desc())
elif sort == 'bump' and cls.__name__ == 'Submission':
return objects.filter(cls.comment_count > 1).order_by(
cls.bump_utc.desc(), cls.created_utc.desc())
elif sort == 'comments' and cls.__name__ == 'Submission':
return objects.order_by(
cls.comment_count.desc(), cls.created_utc.desc())
elif sort == 'controversial':
return objects.order_by(
(cls.upvotes + 1) / (cls.downvotes + 1)
+ (cls.downvotes + 1) / (cls.upvotes + 1),
cls.downvotes.desc(), cls.created_utc.desc())
elif sort == 'top':
return objects.order_by(
cls.downvotes - cls.upvotes, cls.created_utc.desc())
elif sort == 'bottom':
return objects.order_by(
cls.upvotes - cls.downvotes, cls.created_utc.desc())
elif sort == 'old':
return objects.order_by(cls.created_utc)
else: # default, or sort == 'new'
return objects.order_by(cls.created_utc.desc())
# Presently designed around files.helpers.get.get_comment_trees_eager
# Behavior should parallel that of sort_objects above. TODO: Unify someday?
def sort_comment_results(comments, sort):
DESC = (2 << 30) - 1 # descending sorts, Y2038 problem, change before then
if sort == 'hot':
ti = int(time.time()) + 3600
key_func = lambda c: (
-100000
* (c.upvotes + 1)
/ (pow(((ti - c.created_utc) / 1000), 1.23)),
DESC - c.created_utc
)
elif sort == 'controversial':
key_func = lambda c: (
(c.upvotes + 1) / (c.downvotes + 1)
+ (c.downvotes + 1) / (c.upvotes + 1),
DESC - c.downvotes,
DESC - c.created_utc
)
elif sort == 'top':
key_func = lambda c: (c.downvotes - c.upvotes, DESC - c.created_utc)
elif sort == 'bottom':
key_func = lambda c: (c.upvotes - c.downvotes, DESC - c.created_utc)
elif sort == 'old':
key_func = lambda c: c.created_utc
else: # default, or sort == 'new'
key_func = lambda c: DESC - c.created_utc
key_func_pinned = lambda c: (
(c.is_pinned is None, c.is_pinned == '', c.is_pinned), # sort None last
key_func(c))
return sorted(comments, key=key_func_pinned)

View file

@ -1,15 +1,22 @@
from files.classes import *
from files.helpers.strings import sql_ilike_clean
from collections import defaultdict
from typing import Iterable, List, Optional, Type, Union
from flask import g
from sqlalchemy import and_, any_, or_
from sqlalchemy.orm import selectinload
from files.classes import *
from files.helpers.const import AUTOJANNY_ID
from files.helpers.contentsorting import sort_comment_results
from files.helpers.strings import sql_ilike_clean
def get_id(username, v=None, graceful=False):
def get_id(
username:str,
graceful:bool=False) -> Optional[int]:
username = sql_ilike_clean(username)
user = g.db.query(
User.id
).filter(
user = g.db.query(User.id).filter(
or_(
User.username.ilike(username),
User.original_username.ilike(username)
@ -17,25 +24,23 @@ def get_id(username, v=None, graceful=False):
).one_or_none()
if not user:
if not graceful:
abort(404)
else:
return None
if graceful: return None
abort(404)
return user[0]
def get_user(username, v=None, graceful=False):
if not username:
if not graceful: abort(404)
else: return None
def get_user(
username:Optional[str],
v:Optional[User]=None,
graceful:bool=False,
include_blocks:bool=False) -> Optional[User]:
username = sql_ilike_clean(username)
if not username:
if graceful: return None
abort(404)
user = g.db.query(
User
).filter(
user = g.db.query(User).filter(
or_(
User.username.ilike(username),
User.original_username.ilike(username)
@ -43,34 +48,23 @@ def get_user(username, v=None, graceful=False):
).one_or_none()
if not user:
if not graceful: abort(404)
else: return None
if graceful: return None
abort(404)
if v:
block = g.db.query(UserBlock).filter(
or_(
and_(
UserBlock.user_id == v.id,
UserBlock.target_id == user.id
),
and_(UserBlock.user_id == user.id,
UserBlock.target_id == v.id
)
)
).first()
user.is_blocking = block and block.user_id == v.id
user.is_blocked = block and block.target_id == v.id
if v and include_blocks:
user = _add_block_props(user, v)
return user
def get_users(usernames, v=None, graceful=False):
if not usernames:
if not graceful: abort(404)
else: return []
def get_users(
usernames:Iterable[str],
graceful:bool=False) -> List[User]:
if not usernames: return []
usernames = [ sql_ilike_clean(n) for n in usernames ]
if not any(usernames):
if graceful and len(usernames) == 0: return []
abort(404)
users = g.db.query(User).filter(
or_(
User.username == any_(usernames),
@ -78,96 +72,91 @@ def get_users(usernames, v=None, graceful=False):
)
).all()
if not users:
if not graceful: abort(404)
else: return []
if len(users) != len(usernames) and not graceful:
abort(404)
return users
def get_account(id, v=None):
try: id = int(id)
except: abort(404)
def get_account(
id:Union[str,int],
v:Optional[User]=None,
graceful:bool=False,
include_blocks:bool=False) -> Optional[User]:
try:
id = int(id)
except:
if graceful: return None
abort(404)
user = g.db.query(User).filter_by(id = id).one_or_none()
if not user: abort(404)
user = g.db.get(User, id)
if not user:
if graceful: return None
abort(404)
if v:
block = g.db.query(UserBlock).filter(
or_(
and_(
UserBlock.user_id == v.id,
UserBlock.target_id == user.id
),
and_(UserBlock.user_id == user.id,
UserBlock.target_id == v.id
)
)
).first()
user.is_blocking = block and block.user_id == v.id
user.is_blocked = block and block.target_id == v.id
if v and include_blocks:
user = _add_block_props(user, v)
return user
def get_post(i, v=None, graceful=False):
def get_post(
i:Union[str,int],
v:Optional[User]=None,
graceful:bool=False) -> Optional[Submission]:
try: i = int(i)
except:
if graceful: return None
else: abort(404)
abort(404)
if v:
vt = g.db.query(Vote).filter_by(
user_id=v.id, submission_id=i).subquery()
blocking = v.blocking.subquery()
items = g.db.query(
post = g.db.query(
Submission,
vt.c.vote_type,
blocking.c.target_id,
)
items=items.filter(Submission.id == i
post = post.filter(Submission.id == i
).join(
vt,
vt.c.submission_id == Submission.id,
vt,
vt.c.submission_id == Submission.id,
isouter=True
).join(
blocking,
blocking.c.target_id == Submission.author_id,
blocking,
blocking.c.target_id == Submission.author_id,
isouter=True
)
items=items.one_or_none()
post = post.one_or_none()
if not items:
if not post:
if graceful: return None
else: abort(404)
x = items[0]
x.voted = items[1] or 0
x.is_blocking = items[2] or 0
x = post[0]
x.voted = post[1] or 0
x.is_blocking = post[2] or 0
else:
items = g.db.query(
Submission
).filter(Submission.id == i).one_or_none()
if not items:
post = g.db.get(Submission, i)
if not post:
if graceful: return None
else: abort(404)
x=items
x = post
return x
def get_posts(pids, v=None):
if not pids:
return []
def get_posts(
pids:Iterable[int],
v:Optional[User]=None,
eager:bool=False) -> List[Submission]:
if not pids: return []
if v:
vt = g.db.query(Vote).filter(
vt = g.db.query(Vote.vote_type, Vote.submission_id).filter(
Vote.submission_id.in_(pids),
Vote.user_id==v.id
).subquery()
@ -183,67 +172,62 @@ def get_posts(pids, v=None):
).filter(
Submission.id.in_(pids)
).join(
vt, vt.c.submission_id==Submission.id, isouter=True
vt, vt.c.submission_id == Submission.id, isouter=True
).join(
blocking,
blocking.c.target_id == Submission.author_id,
isouter=True
blocking, blocking.c.target_id == Submission.author_id, isouter=True
).join(
blocked,
blocked.c.user_id == Submission.author_id,
isouter=True
).all()
output = [p[0] for p in query]
for i in range(len(output)):
output[i].voted = query[i][1] or 0
output[i].is_blocking = query[i][2] or 0
output[i].is_blocked = query[i][3] or 0
blocked, blocked.c.user_id == Submission.author_id, isouter=True
)
else:
output = g.db.query(Submission,).filter(Submission.id.in_(pids)).all()
query = g.db.query(Submission).filter(Submission.id.in_(pids))
if eager:
query = query.options(
selectinload(Submission.author).options(
selectinload(User.badges),
selectinload(User.notes),
),
selectinload(Submission.reports),
selectinload(Submission.awards),
)
results = query.all()
if v:
output = [p[0] for p in results]
for i in range(len(output)):
output[i].voted = results[i][1] or 0
output[i].is_blocking = results[i][2] or 0
output[i].is_blocked = results[i][3] or 0
else:
output = results
return sorted(output, key=lambda x: pids.index(x.id))
def get_comment(i, v=None, graceful=False):
def get_comment(
i:Union[str,int],
v:Optional[User]=None,
graceful:bool=False) -> Optional[Comment]:
try: i = int(i)
except:
if graceful: return None
abort(404)
if not i:
if graceful: return None
else: abort(404)
if v:
comment = g.db.get(Comment, i)
if not comment:
if graceful: return None
else: abort(404)
comment=g.db.query(Comment).filter(Comment.id == i).one_or_none()
if not comment and not graceful: abort(404)
block = g.db.query(UserBlock).filter(
or_(
and_(
UserBlock.user_id == v.id,
UserBlock.target_id == comment.author_id
),
and_(
UserBlock.user_id == comment.author_id,
UserBlock.target_id == v.id
)
)
).first()
vts = g.db.query(CommentVote).filter_by(user_id=v.id, comment_id=comment.id)
vt = g.db.query(CommentVote).filter_by(user_id=v.id, comment_id=comment.id).one_or_none()
comment.is_blocking = block and block.user_id == v.id
comment.is_blocked = block and block.target_id == v.id
comment.voted = vt.vote_type if vt else 0
else:
comment = g.db.query(Comment).filter(Comment.id == i).one_or_none()
if not comment and not graceful:abort(404)
return comment
return _add_vote_and_block_props(comment, v, CommentVote)
def get_comments(cids, v=None, load_parent=False):
def get_comments(
cids:Iterable[int],
v:Optional[User]=None) -> List[Comment]:
if not cids: return []
if v:
@ -261,7 +245,8 @@ def get_comments(cids, v=None, load_parent=False):
).filter(Comment.id.in_(cids))
if not (v and (v.shadowbanned or v.admin_level > 1)):
comments = comments.join(User, User.id == Comment.author_id).filter(User.shadowbanned == None)
comments = comments.join(User, User.id == Comment.author_id) \
.filter(User.shadowbanned == None)
comments = comments.join(
votes,
@ -284,21 +269,88 @@ def get_comments(cids, v=None, load_parent=False):
comment.is_blocking = c[2] or 0
comment.is_blocked = c[3] or 0
output.append(comment)
else:
output = g.db.query(Comment).join(User, User.id == Comment.author_id).filter(User.shadowbanned == None, Comment.id.in_(cids)).all()
if load_parent:
parents = [x.parent_comment_id for x in output if x.parent_comment_id]
parents = get_comments(parents, v=v)
parents = {x.id: x for x in parents}
for c in output: c.sex = parents.get(c.parent_comment_id)
output = g.db.query(Comment) \
.join(User, User.id == Comment.author_id) \
.filter(User.shadowbanned == None, Comment.id.in_(cids)) \
.all()
return sorted(output, key=lambda x: cids.index(x.id))
def get_domain(s):
# TODO: There is probably some way to unify this with get_comments. However, in
# the interim, it's a hot path and benefits from having tailored code.
def get_comment_trees_eager(
top_comment_ids:Iterable[int],
sort:str="old",
v:Optional[User]=None) -> List[Comment]:
if v:
votes = g.db.query(CommentVote).filter_by(user_id=v.id).subquery()
blocking = v.blocking.subquery()
blocked = v.blocked.subquery()
query = g.db.query(
Comment,
votes.c.vote_type,
blocking.c.target_id,
blocked.c.target_id,
).join(
votes, votes.c.comment_id==Comment.id, isouter=True
).join(
blocking,
blocking.c.target_id == Comment.author_id,
isouter=True
).join(
blocked,
blocked.c.user_id == Comment.author_id,
isouter=True
)
else:
query = g.db.query(Comment)
query = query.filter(Comment.top_comment_id.in_(top_comment_ids))
query = query.options(
selectinload(Comment.author).options(
selectinload(User.badges),
selectinload(User.notes),
),
selectinload(Comment.reports).options(
selectinload(CommentFlag.user),
),
selectinload(Comment.awards),
)
results = query.all()
if v:
comments = [c[0] for c in results]
for i in range(len(comments)):
comments[i].voted = results[i][1] or 0
comments[i].is_blocking = results[i][2] or 0
comments[i].is_blocked = results[i][3] or 0
else:
comments = results
comments_map = {}
comments_map_parent = defaultdict(lambda: list())
for c in comments:
c.replies2 = []
comments_map[c.id] = c
comments_map_parent[c.parent_comment_id].append(c)
for parent_id in comments_map_parent:
if parent_id is None: continue
comments_map_parent[parent_id] = sort_comment_results(
comments_map_parent[parent_id], sort)
comments_map[parent_id].replies2 = comments_map_parent[parent_id]
return [comments_map[tcid] for tcid in top_comment_ids]
# TODO: This function was concisely inlined into posts.py in upstream.
# Think it involved adding `tldextract` as a dependency.
def get_domain(s:str) -> Optional[BannedDomain]:
parts = s.split(".")
domain_list = set()
for i in range(len(parts)):
@ -308,7 +360,9 @@ def get_domain(s):
domain_list.add(new_domain)
doms = [x for x in g.db.query(BannedDomain).filter(BannedDomain.domain.in_(domain_list)).all()]
doms = g.db.query(BannedDomain) \
.filter(BannedDomain.domain.in_(domain_list)).all()
doms = [x for x in doms]
if not doms:
return None
@ -316,3 +370,70 @@ def get_domain(s):
doms = sorted(doms, key=lambda x: len(x.domain), reverse=True)
return doms[0]
def _add_block_props(
target:Union[Submission, Comment, User],
v:Optional[User]):
if not v: return target
id = None
if any(isinstance(target, cls) for cls in [Submission, Comment]):
id = target.author_id
elif isinstance(target, User):
id = target.id
else:
raise TypeError("add_block_props only supports non-None "
"submissions, comments, and users")
if hasattr(target, 'is_blocking') and hasattr(target, 'is_blocked'):
return target
# users can't block or be blocked by themselves or AutoJanny
if v.id == id or id == AUTOJANNY_ID:
target.is_blocking = False
target.is_blocked = False
return target
block = g.db.query(UserBlock).filter(
or_(
and_(
UserBlock.user_id == v.id,
UserBlock.target_id == id
),
and_(
UserBlock.user_id == id,
UserBlock.target_id == v.id
)
)
).first()
target.is_blocking = block and block.user_id == v.id
target.is_blocked = block and block.target_id == v.id
return target
def _add_vote_props(
target:Union[Submission, Comment],
v:Optional[User],
vote_cls:Union[Type[Vote], Type[CommentVote], None]):
if hasattr(target, 'voted'): return target
vt = g.db.query(vote_cls.vote_type).filter_by(user_id=v.id)
if vote_cls is Vote:
vt = vt.filter_by(submission_id=target.id)
elif vote_cls is CommentVote:
vt = vt.filter_by(comment_id=target.id)
else:
vt = None
if vt: vt = vt.one_or_none()
target.voted = vt.vote_type if vt else 0
return target
def _add_vote_and_block_props(
target:Union[Submission, Comment],
v:Optional[User],
vote_cls:Union[Type[Vote], Type[CommentVote], None]):
if not v: return target
target = _add_block_props(target, v)
return _add_vote_props(target, v, vote_cls)

View file

@ -1,8 +1,9 @@
import typing
# clean strings for searching
def sql_ilike_clean(my_str):
if my_str is None:
return None
return my_str.replace(r'\\', '').replace('_', r'\_').replace('%', '').strip()
# this will also just return a bool verbatim

View file

@ -2,6 +2,7 @@ from files.helpers.wrappers import *
from files.helpers.get import *
from files.__main__ import app, cache, limiter
from files.classes.submission import Submission
from files.helpers.contentsorting import apply_time_filter, sort_objects
defaulttimefilter = environ.get("DEFAULT_TIME_FILTER", "all").strip()
@ -204,7 +205,7 @@ def front_all(v, sub=None, subdomain=None):
site=SITE
)
posts = get_posts(ids, v=v)
posts = get_posts(ids, v=v, eager=True)
if v:
if v.hidevotedon: posts = [x for x in posts if not hasattr(x, 'voted') or not x.voted]
@ -250,15 +251,7 @@ def frontlist(v=None, sort='new', page=1, t="all", ids_only=True, ccmode="false"
if lt: posts = posts.filter(Submission.created_utc < lt)
if not gt and not lt:
if t == 'all': cutoff = 0
else:
now = int(time.time())
if t == 'hour': cutoff = now - 3600
elif t == 'week': cutoff = now - 604800
elif t == 'month': cutoff = now - 2592000
elif t == 'year': cutoff = now - 31536000
else: cutoff = now - 86400
posts = posts.filter(Submission.created_utc >= cutoff)
posts = apply_time_filter(posts, t, Submission)
if (ccmode == "true"):
posts = posts.filter(Submission.club == True)
@ -282,27 +275,7 @@ def frontlist(v=None, sort='new', page=1, t="all", ids_only=True, ccmode="false"
if not (v and v.shadowbanned):
posts = posts.join(User, User.id == Submission.author_id).filter(User.shadowbanned == None)
if sort == "hot":
ti = int(time.time()) + 3600
posts = posts.order_by(
-100000
* (Submission.realupvotes + 1 + Submission.comment_count / 10)
/ (func.power((ti - Submission.created_utc) / 1000, 1.23)),
Submission.created_utc.desc())
elif sort == "bump":
posts = posts.filter(Submission.comment_count > 1).order_by(Submission.bump_utc.desc(), Submission.created_utc.desc())
elif sort == "new":
posts = posts.order_by(Submission.created_utc.desc())
elif sort == "old":
posts = posts.order_by(Submission.created_utc)
elif sort == "controversial":
posts = posts.order_by((Submission.upvotes+1)/(Submission.downvotes+1) + (Submission.downvotes+1)/(Submission.upvotes+1), Submission.downvotes.desc(), Submission.created_utc.desc())
elif sort == "top":
posts = posts.order_by(Submission.downvotes - Submission.upvotes, Submission.created_utc.desc())
elif sort == "bottom":
posts = posts.order_by(Submission.upvotes - Submission.downvotes, Submission.created_utc.desc())
elif sort == "comments":
posts = posts.order_by(Submission.comment_count.desc(), Submission.created_utc.desc())
posts = sort_objects(posts, sort, Submission)
if v: size = v.frontsize or 0
else: size = 25
@ -376,32 +349,8 @@ def changeloglist(v=None, sort="new", page=1, t="all", site=None):
posts = posts.filter(Submission.title.ilike('_changelog%'), Submission.author_id.in_(admins))
if t != 'all':
cutoff = 0
now = int(time.time())
if t == 'hour':
cutoff = now - 3600
elif t == 'day':
cutoff = now - 86400
elif t == 'week':
cutoff = now - 604800
elif t == 'month':
cutoff = now - 2592000
elif t == 'year':
cutoff = now - 31536000
posts = posts.filter(Submission.created_utc >= cutoff)
if sort == "new":
posts = posts.order_by(Submission.created_utc.desc())
elif sort == "old":
posts = posts.order_by(Submission.created_utc)
elif sort == "controversial":
posts = posts.order_by((Submission.upvotes+1)/(Submission.downvotes+1) + (Submission.downvotes+1)/(Submission.upvotes+1), Submission.downvotes.desc())
elif sort == "top":
posts = posts.order_by(Submission.downvotes - Submission.upvotes)
elif sort == "bottom":
posts = posts.order_by(Submission.upvotes - Submission.downvotes)
elif sort == "comments":
posts = posts.order_by(Submission.comment_count.desc())
posts = apply_time_filter(posts, t, Submission)
posts = sort_objects(posts, sort, Submission)
posts = posts.offset(25 * (page - 1)).limit(26).all()
@ -477,31 +426,8 @@ def get_comments_idlist(page=1, v=None, sort="new", t="all", gt=0, lt=0):
if lt: comments = comments.filter(Comment.created_utc < lt)
if not gt and not lt:
now = int(time.time())
if t == 'hour':
cutoff = now - 3600
elif t == 'day':
cutoff = now - 86400
elif t == 'week':
cutoff = now - 604800
elif t == 'month':
cutoff = now - 2592000
elif t == 'year':
cutoff = now - 31536000
else:
cutoff = 0
comments = comments.filter(Comment.created_utc >= cutoff)
if sort == "new":
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.downvotes - Comment.upvotes)
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = apply_time_filter(comments, t, Comment)
comments = sort_objects(comments, sort, Comment)
comments = comments.offset(25 * (page - 1)).limit(26).all()
return [x[0] for x in comments]

View file

@ -4,6 +4,7 @@ from files.helpers.wrappers import *
from files.helpers.sanitize import *
from files.helpers.strings import sql_ilike_clean
from files.helpers.alerts import *
from files.helpers.contentsorting import sort_objects
from files.helpers.const import *
from files.classes import *
from flask import *
@ -184,34 +185,13 @@ def post_id(pid, anything=None, v=None, sub=None):
pinned = [c[0] for c in comments.filter(Comment.is_pinned != None).all()]
comments = comments.filter(Comment.level == 1, Comment.is_pinned == None)
if sort == "new":
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.realupvotes.desc())
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = sort_objects(comments, sort, Comment)
comments = [c[0] for c in comments.all()]
else:
pinned = g.db.query(Comment).filter(Comment.parent_submission == post.id, Comment.is_pinned != None).all()
comments = g.db.query(Comment).join(User, User.id == Comment.author_id).filter(User.shadowbanned == None, Comment.parent_submission == post.id, Comment.level == 1, Comment.is_pinned == None)
if sort == "new":
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.realupvotes.desc())
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = sort_objects(comments, sort, Comment)
filter_clause = (Comment.filter_state != 'filtered') & (Comment.filter_state != 'removed')
comments = comments.filter(filter_clause)
@ -250,11 +230,14 @@ def post_id(pid, anything=None, v=None, sub=None):
g.db.add(pin)
pinned.remove(pin)
post.replies = pinned + comments
top_comments = pinned + comments
top_comment_ids = [c.id for c in top_comments]
post.replies = get_comment_trees_eager(top_comment_ids, sort, v)
post.views += 1
g.db.add(post)
g.db.commit()
g.db.flush()
if request.headers.get("Authorization"): return post.json
else:
if post.is_banned and not (v and (v.admin_level > 1 or post.author_id == v.id)): template = "submission_banned.html"
@ -324,15 +307,7 @@ def viewmore(v, pid, sort, offset):
if sort == "new":
comments = comments.filter(Comment.created_utc < newest.created_utc)
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.realupvotes.desc())
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = sort_objects(comments, sort, Comment)
comments = [c[0] for c in comments.all()]
else:
@ -340,15 +315,7 @@ def viewmore(v, pid, sort, offset):
if sort == "new":
comments = comments.filter(Comment.created_utc < newest.created_utc)
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.realupvotes.desc())
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = sort_objects(comments, sort, Comment)
comments = comments.all()
comments = comments[offset:]

View file

@ -3,6 +3,7 @@ import re
from sqlalchemy import *
from flask import *
from files.__main__ import app
from files.helpers.contentsorting import apply_time_filter, sort_objects
from files.helpers.strings import sql_ilike_clean
@ -13,8 +14,6 @@ valid_params=[
]
def searchparse(text):
criteria = {x[0]:x[1] for x in query_regex.findall(text)}
for x in criteria:
@ -29,13 +28,9 @@ def searchparse(text):
return criteria
@app.get("/search/posts")
@auth_desired
def searchposts(v):
query = request.values.get("q", '').strip()
page = max(1, int(request.values.get("page", 1)))
@ -45,17 +40,6 @@ def searchposts(v):
criteria=searchparse(query)
posts = g.db.query(Submission.id)
if not (v and v.paid_dues): posts = posts.filter_by(club=False)
@ -117,35 +101,9 @@ def searchposts(v):
)
)
if t:
now = int(time.time())
if t == 'hour':
cutoff = now - 3600
elif t == 'day':
cutoff = now - 86400
elif t == 'week':
cutoff = now - 604800
elif t == 'month':
cutoff = now - 2592000
elif t == 'year':
cutoff = now - 31536000
else:
cutoff = 0
posts = posts.filter(Submission.created_utc >= cutoff)
if sort == "new":
posts = posts.order_by(Submission.created_utc.desc())
elif sort == "old":
posts = posts.order_by(Submission.created_utc)
elif sort == "controversial":
posts = posts.order_by((Submission.upvotes+1)/(Submission.downvotes+1) + (Submission.downvotes+1)/(Submission.upvotes+1), Submission.downvotes.desc())
elif sort == "top":
posts = posts.order_by(Submission.downvotes - Submission.upvotes)
elif sort == "bottom":
posts = posts.order_by(Submission.upvotes - Submission.downvotes)
elif sort == "comments":
posts = posts.order_by(Submission.comment_count.desc())
posts = apply_time_filter(posts, t, Submission)
posts = sort_objects(posts, sort, Submission)
total = posts.count()
@ -154,12 +112,10 @@ def searchposts(v):
ids = [x[0] for x in posts]
next_exists = (len(ids) > 25)
ids = ids[:25]
posts = get_posts(ids, v=v)
posts = get_posts(ids, v=v, eager=True)
if request.headers.get("Authorization"): return {"total":total, "data":[x.json for x in posts]}
@ -174,11 +130,10 @@ def searchposts(v):
next_exists=next_exists
)
@app.get("/search/comments")
@auth_desired
def searchcomments(v):
query = request.values.get("q", '').strip()
try: page = max(1, int(request.values.get("page", 1)))
@ -211,21 +166,7 @@ def searchcomments(v):
if 'over18' in criteria: comments = comments.filter(Comment.over_18 == True)
if t:
now = int(time.time())
if t == 'hour':
cutoff = now - 3600
elif t == 'day':
cutoff = now - 86400
elif t == 'week':
cutoff = now - 604800
elif t == 'month':
cutoff = now - 2592000
elif t == 'year':
cutoff = now - 31536000
else:
cutoff = 0
comments = comments.filter(Comment.created_utc >= cutoff)
comments = apply_time_filter(comments, t, Comment)
if v and v.admin_level < 2:
private = [x[0] for x in g.db.query(Submission.id).filter(Submission.private == True).all()]
@ -239,17 +180,7 @@ def searchcomments(v):
club = [x[0] for x in g.db.query(Submission.id).filter(Submission.club == True).all()]
comments = comments.filter(Comment.parent_submission.notin_(club))
if sort == "new":
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.downvotes - Comment.upvotes)
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = sort_objects(comments, sort, Comment)
total = comments.count()
@ -269,7 +200,6 @@ def searchcomments(v):
@app.get("/search/users")
@auth_desired
def searchusers(v):
query = request.values.get("q", '').strip()
page = max(1, int(request.values.get("page", 1)))

View file

@ -8,6 +8,7 @@ from files.helpers.sanitize import *
from files.helpers.strings import sql_ilike_clean
from files.helpers.const import *
from files.helpers.assetcache import assetcache_path
from files.helpers.contentsorting import apply_time_filter, sort_objects
from files.mail import *
from flask import *
from files.__main__ import app, limiter, db_session
@ -90,7 +91,7 @@ def upvoters_posts(v, username, uid):
next_exists = len(listing) > 25
listing = listing[:25]
listing = get_posts(listing, v=v)
listing = get_posts(listing, v=v, eager=True)
return render_template("voted_posts.html", next_exists=next_exists, listing=listing, page=page, v=v)
@ -132,7 +133,7 @@ def downvoters_posts(v, username, uid):
next_exists = len(listing) > 25
listing = listing[:25]
listing = get_posts(listing, v=v)
listing = get_posts(listing, v=v, eager=True)
return render_template("voted_posts.html", next_exists=next_exists, listing=listing, page=page, v=v)
@ -173,7 +174,7 @@ def upvoting_posts(v, username, uid):
next_exists = len(listing) > 25
listing = listing[:25]
listing = get_posts(listing, v=v)
listing = get_posts(listing, v=v, eager=True)
return render_template("voted_posts.html", next_exists=next_exists, listing=listing, page=page, v=v)
@ -215,7 +216,7 @@ def downvoting_posts(v, username, uid):
next_exists = len(listing) > 25
listing = listing[:25]
listing = get_posts(listing, v=v)
listing = get_posts(listing, v=v, eager=True)
return render_template("voted_posts.html", next_exists=next_exists, listing=listing, page=page, v=v)
@ -540,7 +541,7 @@ def message2(v, username):
abort(403, "You have been permabanned and cannot send messages; " + \
"contact modmail if you think this decision was incorrect.")
user = get_user(username, v=v)
user = get_user(username, v=v, include_blocks=True)
if hasattr(user, 'is_blocking') and user.is_blocking: abort(403, "You're blocking this user.")
if v.admin_level <= 1 and hasattr(user, 'is_blocked') and user.is_blocked:
@ -772,7 +773,8 @@ def visitors(v):
@app.get("/@<username>")
@auth_desired
def u_username(username, v=None):
u = get_user(username, v=v)
u = get_user(username, v=v, include_blocks=True)
if username != u.username:
return redirect(SITE_FULL + request.full_path.replace(username, u.username)[:-1])
@ -816,7 +818,7 @@ def u_username(username, v=None):
for p in sticky:
ids = [p.id] + ids
listing = get_posts(ids, v=v)
listing = get_posts(ids, v=v, eager=True)
if u.unban_utc:
if request.headers.get("Authorization"): {"data": [x.json for x in listing]}
@ -848,7 +850,8 @@ def u_username(username, v=None):
@app.get("/@<username>/comments")
@auth_desired
def u_username_comments(username, v=None):
user = get_user(username, v=v)
user = get_user(username, v=v, include_blocks=True)
if username != user.username: return redirect(f'/@{user.username}/comments')
u = user
@ -884,31 +887,8 @@ def u_username_comments(username, v=None):
(Comment.filter_state != 'filtered') & (Comment.filter_state != 'removed')
)
now = int(time.time())
if t == 'hour':
cutoff = now - 3600
elif t == 'day':
cutoff = now - 86400
elif t == 'week':
cutoff = now - 604800
elif t == 'month':
cutoff = now - 2592000
elif t == 'year':
cutoff = now - 31536000
else:
cutoff = 0
comments = comments.filter(Comment.created_utc >= cutoff)
if sort == "new":
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.downvotes - Comment.upvotes)
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = apply_time_filter(comments, t, Comment)
comments = sort_objects(comments, sort, Comment)
comments = comments.offset(25 * (page - 1)).limit(26).all()
ids = [x.id for x in comments]
@ -927,8 +907,7 @@ def u_username_comments(username, v=None):
@app.get("/@<username>/info")
@auth_required
def u_username_info(username, v=None):
user=get_user(username, v=v)
user = get_user(username, v=v, include_blocks=True)
if hasattr(user, 'is_blocking') and user.is_blocking:
abort(401, "You're blocking this user.")
@ -940,8 +919,7 @@ def u_username_info(username, v=None):
@app.get("/<id>/info")
@auth_required
def u_user_id_info(id, v=None):
user=get_account(id, v=v)
user = get_account(id, v=v, include_blocks=True)
if hasattr(user, 'is_blocking') and user.is_blocking:
abort(401, "You're blocking this user.")
@ -1092,7 +1070,7 @@ def saved_posts(v, username):
ids=ids[:25]
listing = get_posts(ids, v=v)
listing = get_posts(ids, v=v, eager=True)
if request.headers.get("Authorization"): return {"data": [x.json for x in listing]}
return render_template("userpage.html",