From 5aaef144cf751a0668a2e746f63840132ce20912 Mon Sep 17 00:00:00 2001 From: TLSM Date: Mon, 28 Nov 2022 14:33:24 -0500 Subject: [PATCH] Deduplicate post/comment sorting & time filtering. Ported in from upstream with adjustments for TheMotte, most notably universal default to 'new' and fixes to 'hot'. Lumped into this PR because eager comment loading uses it. --- files/classes/user.py | 30 ++--------- files/helpers/contentsorting.py | 53 ++++++++++++++++++++ files/routes/front.py | 88 +++------------------------------ files/routes/posts.py | 46 ++--------------- files/routes/search.py | 82 +++--------------------------- files/routes/users.py | 28 ++--------- 6 files changed, 77 insertions(+), 250 deletions(-) create mode 100644 files/helpers/contentsorting.py diff --git a/files/classes/user.py b/files/classes/user.py index 4efbf44ee..ee65b14ed 100644 --- a/files/classes/user.py +++ b/files/classes/user.py @@ -19,6 +19,7 @@ from .sub_block import * from files.__main__ import app, Base, cache from files.helpers.security import * from files.helpers.assetcache import assetcache_path +from files.helpers.contentsorting import apply_time_filter, sort_objects import random from datetime import datetime from os import environ, remove, path @@ -292,33 +293,8 @@ class User(Base): if not (v and (v.admin_level > 1 or v.id == self.id)): posts = posts.filter_by(deleted_utc=0, is_banned=False, private=False, ghost=False) - now = int(time.time()) - if t == 'hour': - cutoff = now - 3600 - elif t == 'day': - cutoff = now - 86400 - elif t == 'week': - cutoff = now - 604800 - elif t == 'month': - cutoff = now - 2592000 - elif t == 'year': - cutoff = now - 31536000 - else: - cutoff = 0 - posts = posts.filter(Submission.created_utc >= cutoff) - - if sort == "new": - posts = posts.order_by(Submission.created_utc.desc()) - elif sort == "old": - posts = posts.order_by(Submission.created_utc) - elif sort == "controversial": - posts = posts.order_by((Submission.upvotes+1)/(Submission.downvotes+1) + (Submission.downvotes+1)/(Submission.upvotes+1), Submission.downvotes.desc()) - elif sort == "top": - posts = posts.order_by(Submission.downvotes - Submission.upvotes) - elif sort == "bottom": - posts = posts.order_by(Submission.upvotes - Submission.downvotes) - elif sort == "comments": - posts = posts.order_by(Submission.comment_count.desc()) + posts = apply_time_filter(posts, t, Submission) + posts = sort_objects(posts, sort, Submission) posts = posts.offset(25 * (page - 1)).limit(26).all() diff --git a/files/helpers/contentsorting.py b/files/helpers/contentsorting.py new file mode 100644 index 000000000..ec3bfb513 --- /dev/null +++ b/files/helpers/contentsorting.py @@ -0,0 +1,53 @@ +import time + +from sqlalchemy.sql import func + +from files.helpers.const import * + + +def apply_time_filter(objects, t, cls): + now = int(time.time()) + if t == 'hour': + cutoff = now - (60 * 60) + elif t == 'day': + cutoff = now - (24 * 60 * 60) + elif t == 'week': + cutoff = now - (7 * 24 * 60 * 60) + elif t == 'month': + cutoff = now - (30 * 24 * 60 * 60) + elif t == 'year': + cutoff = now - (365 * 24 * 60 * 60) + else: + cutoff = 0 + return objects.filter(cls.created_utc >= cutoff) + + +def sort_objects(objects, sort, cls): + if sort == 'hot': + ti = int(time.time()) + 3600 + return objects.order_by( + -100000 + * (cls.upvotes + 1) + / (func.power((ti - cls.created_utc) / 1000, 1.23)), + cls.created_utc.desc()) + elif sort == 'bump' and cls.__name__ == 'Submission': + return objects.filter(cls.comment_count > 1).order_by( + cls.bump_utc.desc(), cls.created_utc.desc()) + elif sort == 'comments' and cls.__name__ == 'Submission': + return objects.order_by( + cls.comment_count.desc(), cls.created_utc.desc()) + elif sort == 'controversial': + return objects.order_by( + (cls.upvotes + 1) / (cls.downvotes + 1) + + (cls.downvotes + 1) / (cls.upvotes + 1), + cls.downvotes.desc(), cls.created_utc.desc()) + elif sort == 'top': + return objects.order_by( + cls.downvotes - cls.upvotes, cls.created_utc.desc()) + elif sort == 'bottom': + return objects.order_by( + cls.upvotes - cls.downvotes, cls.created_utc.desc()) + elif sort == 'old': + return objects.order_by(cls.created_utc) + else: # default, or sort == 'new' + return objects.order_by(cls.created_utc.desc()) diff --git a/files/routes/front.py b/files/routes/front.py index 8a14c3126..57632a66a 100644 --- a/files/routes/front.py +++ b/files/routes/front.py @@ -2,6 +2,7 @@ from files.helpers.wrappers import * from files.helpers.get import * from files.__main__ import app, cache, limiter from files.classes.submission import Submission +from files.helpers.contentsorting import apply_time_filter, sort_objects defaulttimefilter = environ.get("DEFAULT_TIME_FILTER", "all").strip() @@ -250,15 +251,7 @@ def frontlist(v=None, sort='new', page=1, t="all", ids_only=True, ccmode="false" if lt: posts = posts.filter(Submission.created_utc < lt) if not gt and not lt: - if t == 'all': cutoff = 0 - else: - now = int(time.time()) - if t == 'hour': cutoff = now - 3600 - elif t == 'week': cutoff = now - 604800 - elif t == 'month': cutoff = now - 2592000 - elif t == 'year': cutoff = now - 31536000 - else: cutoff = now - 86400 - posts = posts.filter(Submission.created_utc >= cutoff) + posts = apply_time_filter(posts, t, Submission) if (ccmode == "true"): posts = posts.filter(Submission.club == True) @@ -282,27 +275,7 @@ def frontlist(v=None, sort='new', page=1, t="all", ids_only=True, ccmode="false" if not (v and v.shadowbanned): posts = posts.join(User, User.id == Submission.author_id).filter(User.shadowbanned == None) - if sort == "hot": - ti = int(time.time()) + 3600 - posts = posts.order_by( - -100000 - * (Submission.realupvotes + 1 + Submission.comment_count / 10) - / (func.power((ti - Submission.created_utc) / 1000, 1.23)), - Submission.created_utc.desc()) - elif sort == "bump": - posts = posts.filter(Submission.comment_count > 1).order_by(Submission.bump_utc.desc(), Submission.created_utc.desc()) - elif sort == "new": - posts = posts.order_by(Submission.created_utc.desc()) - elif sort == "old": - posts = posts.order_by(Submission.created_utc) - elif sort == "controversial": - posts = posts.order_by((Submission.upvotes+1)/(Submission.downvotes+1) + (Submission.downvotes+1)/(Submission.upvotes+1), Submission.downvotes.desc(), Submission.created_utc.desc()) - elif sort == "top": - posts = posts.order_by(Submission.downvotes - Submission.upvotes, Submission.created_utc.desc()) - elif sort == "bottom": - posts = posts.order_by(Submission.upvotes - Submission.downvotes, Submission.created_utc.desc()) - elif sort == "comments": - posts = posts.order_by(Submission.comment_count.desc(), Submission.created_utc.desc()) + posts = sort_objects(posts, sort, Submission) if v: size = v.frontsize or 0 else: size = 25 @@ -376,32 +349,8 @@ def changeloglist(v=None, sort="new", page=1, t="all", site=None): posts = posts.filter(Submission.title.ilike('_changelog%'), Submission.author_id.in_(admins)) if t != 'all': - cutoff = 0 - now = int(time.time()) - if t == 'hour': - cutoff = now - 3600 - elif t == 'day': - cutoff = now - 86400 - elif t == 'week': - cutoff = now - 604800 - elif t == 'month': - cutoff = now - 2592000 - elif t == 'year': - cutoff = now - 31536000 - posts = posts.filter(Submission.created_utc >= cutoff) - - if sort == "new": - posts = posts.order_by(Submission.created_utc.desc()) - elif sort == "old": - posts = posts.order_by(Submission.created_utc) - elif sort == "controversial": - posts = posts.order_by((Submission.upvotes+1)/(Submission.downvotes+1) + (Submission.downvotes+1)/(Submission.upvotes+1), Submission.downvotes.desc()) - elif sort == "top": - posts = posts.order_by(Submission.downvotes - Submission.upvotes) - elif sort == "bottom": - posts = posts.order_by(Submission.upvotes - Submission.downvotes) - elif sort == "comments": - posts = posts.order_by(Submission.comment_count.desc()) + posts = apply_time_filter(posts, t, Submission) + posts = sort_objects(posts, sort, Submission) posts = posts.offset(25 * (page - 1)).limit(26).all() @@ -477,31 +426,8 @@ def get_comments_idlist(page=1, v=None, sort="new", t="all", gt=0, lt=0): if lt: comments = comments.filter(Comment.created_utc < lt) if not gt and not lt: - now = int(time.time()) - if t == 'hour': - cutoff = now - 3600 - elif t == 'day': - cutoff = now - 86400 - elif t == 'week': - cutoff = now - 604800 - elif t == 'month': - cutoff = now - 2592000 - elif t == 'year': - cutoff = now - 31536000 - else: - cutoff = 0 - comments = comments.filter(Comment.created_utc >= cutoff) - - if sort == "new": - comments = comments.order_by(Comment.created_utc.desc()) - elif sort == "old": - comments = comments.order_by(Comment.created_utc) - elif sort == "controversial": - comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc()) - elif sort == "top": - comments = comments.order_by(Comment.downvotes - Comment.upvotes) - elif sort == "bottom": - comments = comments.order_by(Comment.upvotes - Comment.downvotes) + comments = apply_time_filter(comments, t, Comment) + comments = sort_objects(comments, sort, Comment) comments = comments.offset(25 * (page - 1)).limit(26).all() return [x[0] for x in comments] diff --git a/files/routes/posts.py b/files/routes/posts.py index 13d5ebd1c..c066383fa 100644 --- a/files/routes/posts.py +++ b/files/routes/posts.py @@ -4,6 +4,7 @@ from files.helpers.wrappers import * from files.helpers.sanitize import * from files.helpers.strings import sql_ilike_clean from files.helpers.alerts import * +from files.helpers.contentsorting import sort_objects from files.helpers.const import * from files.classes import * from flask import * @@ -185,34 +186,13 @@ def post_id(pid, anything=None, v=None, sub=None): pinned = [c[0] for c in comments.filter(Comment.is_pinned != None).all()] comments = comments.filter(Comment.level == 1, Comment.is_pinned == None) - - if sort == "new": - comments = comments.order_by(Comment.created_utc.desc()) - elif sort == "old": - comments = comments.order_by(Comment.created_utc) - elif sort == "controversial": - comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc()) - elif sort == "top": - comments = comments.order_by(Comment.realupvotes.desc()) - elif sort == "bottom": - comments = comments.order_by(Comment.upvotes - Comment.downvotes) - + comments = sort_objects(comments, sort, Comment) comments = [c[0] for c in comments.all()] else: pinned = g.db.query(Comment).filter(Comment.parent_submission == post.id, Comment.is_pinned != None).all() comments = g.db.query(Comment).join(User, User.id == Comment.author_id).filter(User.shadowbanned == None, Comment.parent_submission == post.id, Comment.level == 1, Comment.is_pinned == None) - - if sort == "new": - comments = comments.order_by(Comment.created_utc.desc()) - elif sort == "old": - comments = comments.order_by(Comment.created_utc) - elif sort == "controversial": - comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc()) - elif sort == "top": - comments = comments.order_by(Comment.realupvotes.desc()) - elif sort == "bottom": - comments = comments.order_by(Comment.upvotes - Comment.downvotes) + comments = sort_objects(comments, sort, Comment) filter_clause = (Comment.filter_state != 'filtered') & (Comment.filter_state != 'removed') comments = comments.filter(filter_clause) @@ -325,15 +305,7 @@ def viewmore(v, pid, sort, offset): if sort == "new": comments = comments.filter(Comment.created_utc < newest.created_utc) - comments = comments.order_by(Comment.created_utc.desc()) - elif sort == "old": - comments = comments.order_by(Comment.created_utc) - elif sort == "controversial": - comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc()) - elif sort == "top": - comments = comments.order_by(Comment.realupvotes.desc()) - elif sort == "bottom": - comments = comments.order_by(Comment.upvotes - Comment.downvotes) + comments = sort_objects(comments, sort, Comment) comments = [c[0] for c in comments.all()] else: @@ -341,15 +313,7 @@ def viewmore(v, pid, sort, offset): if sort == "new": comments = comments.filter(Comment.created_utc < newest.created_utc) - comments = comments.order_by(Comment.created_utc.desc()) - elif sort == "old": - comments = comments.order_by(Comment.created_utc) - elif sort == "controversial": - comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc()) - elif sort == "top": - comments = comments.order_by(Comment.realupvotes.desc()) - elif sort == "bottom": - comments = comments.order_by(Comment.upvotes - Comment.downvotes) + comments = sort_objects(comments, sort, Comment) comments = comments.all() comments = comments[offset:] diff --git a/files/routes/search.py b/files/routes/search.py index d67227084..850e81589 100644 --- a/files/routes/search.py +++ b/files/routes/search.py @@ -3,6 +3,7 @@ import re from sqlalchemy import * from flask import * from files.__main__ import app +from files.helpers.contentsorting import apply_time_filter, sort_objects from files.helpers.strings import sql_ilike_clean @@ -13,8 +14,6 @@ valid_params=[ ] def searchparse(text): - - criteria = {x[0]:x[1] for x in query_regex.findall(text)} for x in criteria: @@ -29,13 +28,9 @@ def searchparse(text): return criteria - - - @app.get("/search/posts") @auth_desired def searchposts(v): - query = request.values.get("q", '').strip() page = max(1, int(request.values.get("page", 1))) @@ -45,17 +40,6 @@ def searchposts(v): criteria=searchparse(query) - - - - - - - - - - - posts = g.db.query(Submission.id) if not (v and v.paid_dues): posts = posts.filter_by(club=False) @@ -118,35 +102,9 @@ def searchposts(v): ) ) - if t: - now = int(time.time()) - if t == 'hour': - cutoff = now - 3600 - elif t == 'day': - cutoff = now - 86400 - elif t == 'week': - cutoff = now - 604800 - elif t == 'month': - cutoff = now - 2592000 - elif t == 'year': - cutoff = now - 31536000 - else: - cutoff = 0 - posts = posts.filter(Submission.created_utc >= cutoff) - - if sort == "new": - posts = posts.order_by(Submission.created_utc.desc()) - elif sort == "old": - posts = posts.order_by(Submission.created_utc) - elif sort == "controversial": - posts = posts.order_by((Submission.upvotes+1)/(Submission.downvotes+1) + (Submission.downvotes+1)/(Submission.upvotes+1), Submission.downvotes.desc()) - elif sort == "top": - posts = posts.order_by(Submission.downvotes - Submission.upvotes) - elif sort == "bottom": - posts = posts.order_by(Submission.upvotes - Submission.downvotes) - elif sort == "comments": - posts = posts.order_by(Submission.comment_count.desc()) + posts = apply_time_filter(posts, t, Submission) + posts = sort_objects(posts, sort, Submission) total = posts.count() @@ -155,8 +113,6 @@ def searchposts(v): ids = [x[0] for x in posts] - - next_exists = (len(ids) > 25) ids = ids[:25] @@ -175,11 +131,10 @@ def searchposts(v): next_exists=next_exists ) + @app.get("/search/comments") @auth_desired def searchcomments(v): - - query = request.values.get("q", '').strip() try: page = max(1, int(request.values.get("page", 1))) @@ -213,21 +168,7 @@ def searchcomments(v): if 'over18' in criteria: comments = comments.filter(Comment.over_18 == True) if t: - now = int(time.time()) - if t == 'hour': - cutoff = now - 3600 - elif t == 'day': - cutoff = now - 86400 - elif t == 'week': - cutoff = now - 604800 - elif t == 'month': - cutoff = now - 2592000 - elif t == 'year': - cutoff = now - 31536000 - else: - cutoff = 0 - comments = comments.filter(Comment.created_utc >= cutoff) - + comments = apply_time_filter(comments, t, Comment) if v and v.admin_level < 2: private = [x[0] for x in g.db.query(Submission.id).filter(Submission.private == True).all()] @@ -241,17 +182,7 @@ def searchcomments(v): club = [x[0] for x in g.db.query(Submission.id).filter(Submission.club == True).all()] comments = comments.filter(Comment.parent_submission.notin_(club)) - - if sort == "new": - comments = comments.order_by(Comment.created_utc.desc()) - elif sort == "old": - comments = comments.order_by(Comment.created_utc) - elif sort == "controversial": - comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc()) - elif sort == "top": - comments = comments.order_by(Comment.downvotes - Comment.upvotes) - elif sort == "bottom": - comments = comments.order_by(Comment.upvotes - Comment.downvotes) + comments = sort_objects(comments, sort, Comment) total = comments.count() @@ -271,7 +202,6 @@ def searchcomments(v): @app.get("/search/users") @auth_desired def searchusers(v): - query = request.values.get("q", '').strip() page = max(1, int(request.values.get("page", 1))) diff --git a/files/routes/users.py b/files/routes/users.py index e550f9ffc..9303d26a2 100644 --- a/files/routes/users.py +++ b/files/routes/users.py @@ -8,6 +8,7 @@ from files.helpers.sanitize import * from files.helpers.strings import sql_ilike_clean from files.helpers.const import * from files.helpers.assetcache import assetcache_path +from files.helpers.contentsorting import apply_time_filter, sort_objects from files.mail import * from flask import * from files.__main__ import app, limiter, db_session @@ -899,31 +900,8 @@ def u_username_comments(username, v=None): (Comment.filter_state != 'filtered') & (Comment.filter_state != 'removed') ) - now = int(time.time()) - if t == 'hour': - cutoff = now - 3600 - elif t == 'day': - cutoff = now - 86400 - elif t == 'week': - cutoff = now - 604800 - elif t == 'month': - cutoff = now - 2592000 - elif t == 'year': - cutoff = now - 31536000 - else: - cutoff = 0 - comments = comments.filter(Comment.created_utc >= cutoff) - - if sort == "new": - comments = comments.order_by(Comment.created_utc.desc()) - elif sort == "old": - comments = comments.order_by(Comment.created_utc) - elif sort == "controversial": - comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc()) - elif sort == "top": - comments = comments.order_by(Comment.downvotes - Comment.upvotes) - elif sort == "bottom": - comments = comments.order_by(Comment.upvotes - Comment.downvotes) + comments = apply_time_filter(comments, t, Comment) + comments = sort_objects(comments, sort, Comment) comments = comments.offset(25 * (page - 1)).limit(26).all() ids = [x.id for x in comments]