skorotkiewicz/rDrama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Deduplicate post/comment sorting & time filtering.

Browse source 
Ported in from upstream with adjustments for TheMotte, most notably
universal default to 'new' and fixes to 'hot'. Lumped into this PR
because eager comment loading uses it.
This commit is contained in:
TLSM 2022-11-28 14:33:24 -05:00
parent 4d22d9bce2
commit 5aaef144cf
No known key found for this signature in database
GPG key ID: E745A82778055C7E
6 changed files with 77 additions and 250 deletions
Download patch file Download diff file Expand all files Collapse all files

30
files/classes/user.py
View file

@ -19,6 +19,7 @@ from .sub_block import *
from files.__main__ import app, Base, cache
from files.helpers.security import *
from files.helpers.assetcache import assetcache_path
from files.helpers.contentsorting import apply_time_filter, sort_objects
import random
from datetime import datetime
from os import environ, remove, path
@ -292,33 +293,8 @@ class User(Base):
if not (v and (v.admin_level > 1 or v.id == self.id)):
posts = posts.filter_by(deleted_utc=0, is_banned=False, private=False, ghost=False)
now = int(time.time())
if t == 'hour':
cutoff = now - 3600
elif t == 'day':
cutoff = now - 86400
elif t == 'week':
cutoff = now - 604800
elif t == 'month':
cutoff = now - 2592000
elif t == 'year':
cutoff = now - 31536000
else:
cutoff = 0
posts = posts.filter(Submission.created_utc >= cutoff)
if sort == "new":
posts = posts.order_by(Submission.created_utc.desc())
elif sort == "old":
posts = posts.order_by(Submission.created_utc)
elif sort == "controversial":
posts = posts.order_by((Submission.upvotes+1)/(Submission.downvotes+1) + (Submission.downvotes+1)/(Submission.upvotes+1), Submission.downvotes.desc())
elif sort == "top":
posts = posts.order_by(Submission.downvotes - Submission.upvotes)
elif sort == "bottom":
posts = posts.order_by(Submission.upvotes - Submission.downvotes)
elif sort == "comments":
posts = posts.order_by(Submission.comment_count.desc())
posts = apply_time_filter(posts, t, Submission)
posts = sort_objects(posts, sort, Submission)
posts = posts.offset(25 * (page - 1)).limit(26).all()

53
files/helpers/contentsorting.py Normal file
View file

@ -0,0 +1,53 @@
import time
from sqlalchemy.sql import func
from files.helpers.const import *
def apply_time_filter(objects, t, cls):
now = int(time.time())
if t == 'hour':
cutoff = now - (60 * 60)
elif t == 'day':
cutoff = now - (24 * 60 * 60)
elif t == 'week':
cutoff = now - (7 * 24 * 60 * 60)
elif t == 'month':
cutoff = now - (30 * 24 * 60 * 60)
elif t == 'year':
cutoff = now - (365 * 24 * 60 * 60)
else:
cutoff = 0
return objects.filter(cls.created_utc >= cutoff)
def sort_objects(objects, sort, cls):
if sort == 'hot':
ti = int(time.time()) + 3600
return objects.order_by(
-100000
* (cls.upvotes + 1)
/ (func.power((ti - cls.created_utc) / 1000, 1.23)),
cls.created_utc.desc())
elif sort == 'bump' and cls.__name__ == 'Submission':
return objects.filter(cls.comment_count > 1).order_by(
cls.bump_utc.desc(), cls.created_utc.desc())
elif sort == 'comments' and cls.__name__ == 'Submission':
return objects.order_by(
cls.comment_count.desc(), cls.created_utc.desc())
elif sort == 'controversial':
return objects.order_by(
(cls.upvotes + 1) / (cls.downvotes + 1)
+ (cls.downvotes + 1) / (cls.upvotes + 1),
cls.downvotes.desc(), cls.created_utc.desc())
elif sort == 'top':
return objects.order_by(
cls.downvotes - cls.upvotes, cls.created_utc.desc())
elif sort == 'bottom':
return objects.order_by(
cls.upvotes - cls.downvotes, cls.created_utc.desc())
elif sort == 'old':
return objects.order_by(cls.created_utc)
else: # default, or sort == 'new'
return objects.order_by(cls.created_utc.desc())

88
files/routes/front.py
View file

@ -2,6 +2,7 @@ from files.helpers.wrappers import *
from files.helpers.get import *
from files.__main__ import app, cache, limiter
from files.classes.submission import Submission
from files.helpers.contentsorting import apply_time_filter, sort_objects
defaulttimefilter = environ.get("DEFAULT_TIME_FILTER", "all").strip()
@ -250,15 +251,7 @@ def frontlist(v=None, sort='new', page=1, t="all", ids_only=True, ccmode="false"
if lt: posts = posts.filter(Submission.created_utc < lt)
if not gt and not lt:
if t == 'all': cutoff = 0
else:
now = int(time.time())
if t == 'hour': cutoff = now - 3600
elif t == 'week': cutoff = now - 604800
elif t == 'month': cutoff = now - 2592000
elif t == 'year': cutoff = now - 31536000
else: cutoff = now - 86400
posts = posts.filter(Submission.created_utc >= cutoff)
posts = apply_time_filter(posts, t, Submission)
if (ccmode == "true"):
posts = posts.filter(Submission.club == True)
@ -282,27 +275,7 @@ def frontlist(v=None, sort='new', page=1, t="all", ids_only=True, ccmode="false"
if not (v and v.shadowbanned):
posts = posts.join(User, User.id == Submission.author_id).filter(User.shadowbanned == None)
if sort == "hot":
ti = int(time.time()) + 3600
posts = posts.order_by(
-100000
* (Submission.realupvotes + 1 + Submission.comment_count / 10)
/ (func.power((ti - Submission.created_utc) / 1000, 1.23)),
Submission.created_utc.desc())
elif sort == "bump":
posts = posts.filter(Submission.comment_count > 1).order_by(Submission.bump_utc.desc(), Submission.created_utc.desc())
elif sort == "new":
posts = posts.order_by(Submission.created_utc.desc())
elif sort == "old":
posts = posts.order_by(Submission.created_utc)
elif sort == "controversial":
posts = posts.order_by((Submission.upvotes+1)/(Submission.downvotes+1) + (Submission.downvotes+1)/(Submission.upvotes+1), Submission.downvotes.desc(), Submission.created_utc.desc())
elif sort == "top":
posts = posts.order_by(Submission.downvotes - Submission.upvotes, Submission.created_utc.desc())
elif sort == "bottom":
posts = posts.order_by(Submission.upvotes - Submission.downvotes, Submission.created_utc.desc())
elif sort == "comments":
posts = posts.order_by(Submission.comment_count.desc(), Submission.created_utc.desc())
posts = sort_objects(posts, sort, Submission)
if v: size = v.frontsize or 0
else: size = 25
@ -376,32 +349,8 @@ def changeloglist(v=None, sort="new", page=1, t="all", site=None):
posts = posts.filter(Submission.title.ilike('_changelog%'), Submission.author_id.in_(admins))
if t != 'all':
cutoff = 0
now = int(time.time())
if t == 'hour':
cutoff = now - 3600
elif t == 'day':
cutoff = now - 86400
elif t == 'week':
cutoff = now - 604800
elif t == 'month':
cutoff = now - 2592000
elif t == 'year':
cutoff = now - 31536000
posts = posts.filter(Submission.created_utc >= cutoff)
if sort == "new":
posts = posts.order_by(Submission.created_utc.desc())
elif sort == "old":
posts = posts.order_by(Submission.created_utc)
elif sort == "controversial":
posts = posts.order_by((Submission.upvotes+1)/(Submission.downvotes+1) + (Submission.downvotes+1)/(Submission.upvotes+1), Submission.downvotes.desc())
elif sort == "top":
posts = posts.order_by(Submission.downvotes - Submission.upvotes)
elif sort == "bottom":
posts = posts.order_by(Submission.upvotes - Submission.downvotes)
elif sort == "comments":
posts = posts.order_by(Submission.comment_count.desc())
posts = apply_time_filter(posts, t, Submission)
posts = sort_objects(posts, sort, Submission)
posts = posts.offset(25 * (page - 1)).limit(26).all()
@ -477,31 +426,8 @@ def get_comments_idlist(page=1, v=None, sort="new", t="all", gt=0, lt=0):
if lt: comments = comments.filter(Comment.created_utc < lt)
if not gt and not lt:
now = int(time.time())
if t == 'hour':
cutoff = now - 3600
elif t == 'day':
cutoff = now - 86400
elif t == 'week':
cutoff = now - 604800
elif t == 'month':
cutoff = now - 2592000
elif t == 'year':
cutoff = now - 31536000
else:
cutoff = 0
comments = comments.filter(Comment.created_utc >= cutoff)
if sort == "new":
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.downvotes - Comment.upvotes)
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = apply_time_filter(comments, t, Comment)
comments = sort_objects(comments, sort, Comment)
comments = comments.offset(25 * (page - 1)).limit(26).all()
return [x[0] for x in comments]

46
files/routes/posts.py
View file

@ -4,6 +4,7 @@ from files.helpers.wrappers import *
from files.helpers.sanitize import *
from files.helpers.strings import sql_ilike_clean
from files.helpers.alerts import *
from files.helpers.contentsorting import sort_objects
from files.helpers.const import *
from files.classes import *
from flask import *
@ -185,34 +186,13 @@ def post_id(pid, anything=None, v=None, sub=None):
pinned = [c[0] for c in comments.filter(Comment.is_pinned != None).all()]
comments = comments.filter(Comment.level == 1, Comment.is_pinned == None)
if sort == "new":
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.realupvotes.desc())
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = sort_objects(comments, sort, Comment)
comments = [c[0] for c in comments.all()]
else:
pinned = g.db.query(Comment).filter(Comment.parent_submission == post.id, Comment.is_pinned != None).all()
comments = g.db.query(Comment).join(User, User.id == Comment.author_id).filter(User.shadowbanned == None, Comment.parent_submission == post.id, Comment.level == 1, Comment.is_pinned == None)
if sort == "new":
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.realupvotes.desc())
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = sort_objects(comments, sort, Comment)
filter_clause = (Comment.filter_state != 'filtered') & (Comment.filter_state != 'removed')
comments = comments.filter(filter_clause)
@ -325,15 +305,7 @@ def viewmore(v, pid, sort, offset):
if sort == "new":
comments = comments.filter(Comment.created_utc < newest.created_utc)
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.realupvotes.desc())
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = sort_objects(comments, sort, Comment)
comments = [c[0] for c in comments.all()]
else:
@ -341,15 +313,7 @@ def viewmore(v, pid, sort, offset):
if sort == "new":
comments = comments.filter(Comment.created_utc < newest.created_utc)
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.realupvotes.desc())
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = sort_objects(comments, sort, Comment)
comments = comments.all()
comments = comments[offset:]

82
files/routes/search.py
View file

@ -3,6 +3,7 @@ import re
from sqlalchemy import *
from flask import *
from files.__main__ import app
from files.helpers.contentsorting import apply_time_filter, sort_objects
from files.helpers.strings import sql_ilike_clean
@ -13,8 +14,6 @@ valid_params=[
]
def searchparse(text):
criteria = {x[0]:x[1] for x in query_regex.findall(text)}
for x in criteria:
@ -29,13 +28,9 @@ def searchparse(text):
return criteria
@app.get("/search/posts")
@auth_desired
def searchposts(v):
query = request.values.get("q", '').strip()
page = max(1, int(request.values.get("page", 1)))
@ -45,17 +40,6 @@ def searchposts(v):
criteria=searchparse(query)
posts = g.db.query(Submission.id)
if not (v and v.paid_dues): posts = posts.filter_by(club=False)
@ -118,35 +102,9 @@ def searchposts(v):
)
)
if t:
now = int(time.time())
if t == 'hour':
cutoff = now - 3600
elif t == 'day':
cutoff = now - 86400
elif t == 'week':
cutoff = now - 604800
elif t == 'month':
cutoff = now - 2592000
elif t == 'year':
cutoff = now - 31536000
else:
cutoff = 0
posts = posts.filter(Submission.created_utc >= cutoff)
if sort == "new":
posts = posts.order_by(Submission.created_utc.desc())
elif sort == "old":
posts = posts.order_by(Submission.created_utc)
elif sort == "controversial":
posts = posts.order_by((Submission.upvotes+1)/(Submission.downvotes+1) + (Submission.downvotes+1)/(Submission.upvotes+1), Submission.downvotes.desc())
elif sort == "top":
posts = posts.order_by(Submission.downvotes - Submission.upvotes)
elif sort == "bottom":
posts = posts.order_by(Submission.upvotes - Submission.downvotes)
elif sort == "comments":
posts = posts.order_by(Submission.comment_count.desc())
posts = apply_time_filter(posts, t, Submission)
posts = sort_objects(posts, sort, Submission)
total = posts.count()
@ -155,8 +113,6 @@ def searchposts(v):
ids = [x[0] for x in posts]
next_exists = (len(ids) > 25)
ids = ids[:25]
@ -175,11 +131,10 @@ def searchposts(v):
next_exists=next_exists
)
@app.get("/search/comments")
@auth_desired
def searchcomments(v):
query = request.values.get("q", '').strip()
try: page = max(1, int(request.values.get("page", 1)))
@ -213,21 +168,7 @@ def searchcomments(v):
if 'over18' in criteria: comments = comments.filter(Comment.over_18 == True)
if t:
now = int(time.time())
if t == 'hour':
cutoff = now - 3600
elif t == 'day':
cutoff = now - 86400
elif t == 'week':
cutoff = now - 604800
elif t == 'month':
cutoff = now - 2592000
elif t == 'year':
cutoff = now - 31536000
else:
cutoff = 0
comments = comments.filter(Comment.created_utc >= cutoff)
comments = apply_time_filter(comments, t, Comment)
if v and v.admin_level < 2:
private = [x[0] for x in g.db.query(Submission.id).filter(Submission.private == True).all()]
@ -241,17 +182,7 @@ def searchcomments(v):
club = [x[0] for x in g.db.query(Submission.id).filter(Submission.club == True).all()]
comments = comments.filter(Comment.parent_submission.notin_(club))
if sort == "new":
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.downvotes - Comment.upvotes)
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = sort_objects(comments, sort, Comment)
total = comments.count()
@ -271,7 +202,6 @@ def searchcomments(v):
@app.get("/search/users")
@auth_desired
def searchusers(v):
query = request.values.get("q", '').strip()
page = max(1, int(request.values.get("page", 1)))

28
files/routes/users.py
View file

@ -8,6 +8,7 @@ from files.helpers.sanitize import *
from files.helpers.strings import sql_ilike_clean
from files.helpers.const import *
from files.helpers.assetcache import assetcache_path
from files.helpers.contentsorting import apply_time_filter, sort_objects
from files.mail import *
from flask import *
from files.__main__ import app, limiter, db_session
@ -899,31 +900,8 @@ def u_username_comments(username, v=None):
(Comment.filter_state != 'filtered') & (Comment.filter_state != 'removed')
)
now = int(time.time())
if t == 'hour':
cutoff = now - 3600
elif t == 'day':
cutoff = now - 86400
elif t == 'week':
cutoff = now - 604800
elif t == 'month':
cutoff = now - 2592000
elif t == 'year':
cutoff = now - 31536000
else:
cutoff = 0
comments = comments.filter(Comment.created_utc >= cutoff)
if sort == "new":
comments = comments.order_by(Comment.created_utc.desc())
elif sort == "old":
comments = comments.order_by(Comment.created_utc)
elif sort == "controversial":
comments = comments.order_by((Comment.upvotes+1)/(Comment.downvotes+1) + (Comment.downvotes+1)/(Comment.upvotes+1), Comment.downvotes.desc())
elif sort == "top":
comments = comments.order_by(Comment.downvotes - Comment.upvotes)
elif sort == "bottom":
comments = comments.order_by(Comment.upvotes - Comment.downvotes)
comments = apply_time_filter(comments, t, Comment)
comments = sort_objects(comments, sort, Comment)
comments = comments.offset(25 * (page - 1)).limit(26).all()
ids = [x.id for x in comments]