Merge branch 'frost' into feature-award-feature-flag

This commit is contained in:
Snakes 2023-02-11 23:41:41 -05:00 committed by GitHub
commit 84e5c7c651
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 248 additions and 526 deletions

View file

@ -4,7 +4,7 @@ FROM python:3.10 AS base
ARG DEBIAN_FRONTEND=noninteractive
RUN apt update && apt -y upgrade && apt install -y supervisor ffmpeg
RUN apt update && apt -y upgrade && apt install -y supervisor
# we'll end up blowing away this directory via docker-compose
WORKDIR /service
@ -13,7 +13,7 @@ COPY poetry.lock .
RUN pip install 'poetry==1.2.2'
RUN poetry config virtualenvs.create false && poetry install
RUN mkdir /images && mkdir /songs
RUN mkdir /images
EXPOSE 80/tcp

1
env
View file

@ -31,6 +31,7 @@ MENTION_LIMIT=100
MULTIMEDIA_EMBEDDING_ENABLED=False
RESULTS_PER_PAGE_COMMENTS=200
SCORE_HIDING_TIME_HOURS=24
SQLALCHEMY_WARN_20=0
# Profiling system; uncomment to enable
# Stores and exposes sensitive data!

View file

@ -141,8 +141,8 @@ app.config['RATE_LIMITER_ENABLED'] = not bool_from_string(environ.get('DBG_LIMIT
if not app.config['RATE_LIMITER_ENABLED']:
print("Rate limiter disabled in debug mode!")
limiter = Limiter(
app,
key_func=get_remote_addr,
app=app,
default_limits=["3/second;30/minute;200/hour;1000/day"],
application_limits=["10/second;200/minute;5000/hour;10000/day"],
storage_uri=environ.get("REDIS_URL", "redis://localhost"),

View file

@ -1,53 +0,0 @@
let u_username = document.getElementById('u_username')
if (u_username)
{
u_username = u_username.innerHTML
let audio = new Audio(`/@${u_username}/song`);
audio.loop=true;
function toggle() {
if (audio.paused) audio.play()
else audio.pause()
}
audio.play();
document.getElementById('userpage').addEventListener('click', () => {
if (audio.paused) audio.play();
}, {once : true});
}
else
{
let v_username = document.getElementById('v_username')
if (v_username)
{
v_username = v_username.innerHTML
const paused = localStorage.getItem("paused")
let audio = new Audio(`/@${v_username}/song`);
audio.loop=true;
function toggle() {
if (audio.paused)
{
audio.play()
localStorage.setItem("paused", "")
}
else
{
audio.pause()
localStorage.setItem("paused", "1")
}
}
if (!paused)
{
audio.play();
window.addEventListener('click', () => {
if (audio.paused) audio.play();
}, {once : true});
}
}
}

View file

@ -386,17 +386,10 @@ class Comment(Base):
def plainbody(self, v):
if self.post and self.post.club and not (v and (v.paid_dues or v.id in [self.author_id, self.post.author_id])): return f"<p>{CC} ONLY</p>"
body = self.body
if not body: return ""
return body
def print(self):
print(f'post: {self.id}, comment: {self.author_id}', flush=True)
return ''
@lazy
def collapse_for_user(self, v, path):
if v and self.author_id == v.id: return False

View file

@ -346,8 +346,8 @@ class Submission(Base):
url = self.url.replace("old.reddit.com", v.reddit)
if '/comments/' in url and "sort=" not in url:
if "?" in url: url += "&context=9"
else: url += "?context=8"
if "?" in url: url += f"&context={RENDER_DEPTH_LIMIT}"
else: url += f"?context={RENDER_DEPTH_LIMIT - 1}"
if v.controversial: url += "&sort=controversial"
return url
elif self.url:
@ -382,22 +382,13 @@ class Submission(Base):
def plainbody(self, v):
if self.club and not (v and (v.paid_dues or v.id == self.author_id)): return f"<p>{CC} ONLY</p>"
body = self.body
if not body: return ""
if v:
body = body.replace("old.reddit.com", v.reddit)
if v.nitter and '/i/' not in body and '/retweets' not in body: body = body.replace("www.twitter.com", "nitter.net").replace("twitter.com", "nitter.net")
return body
def print(self):
print(f'post: {self.id}, author: {self.author_id}', flush=True)
return ''
@lazy
def realtitle(self, v):
if self.title_html:

View file

@ -47,7 +47,6 @@ class User(Base):
theme = Column(String, default=defaulttheme, nullable=False)
themecolor = Column(String, default=DEFAULT_COLOR, nullable=False)
cardview = Column(Boolean, default=cardview, nullable=False)
song = Column(String)
highres = Column(String)
profileurl = Column(String)
bannerurl = Column(String)

View file

@ -4,6 +4,7 @@ import sqlalchemy
from werkzeug.security import generate_password_hash
from files.__main__ import app
from files.classes import User, Submission, Comment, Vote, CommentVote
from files.helpers.comments import bulk_recompute_descendant_counts
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy(app)
@ -262,5 +263,8 @@ def seed_db():
comment.realupvotes = comment.upvotes - comment.downvotes
db.session.add(comment)
print("Computing comment descendant_count")
bulk_recompute_descendant_counts(db=db.session)
db.session.commit()
db.session.flush()

14
files/helpers/captcha.py Normal file
View file

@ -0,0 +1,14 @@
from typing import Final
import requests
HCAPTCHA_URL: Final[str] = "https://hcaptcha.com/siteverify"
def validate_captcha(secret:str, sitekey: str, token: str):
if not sitekey: return True
if not token: return False
data = {"secret": secret,
"response": token,
"sitekey": sitekey
}
req = requests.post(HCAPTCHA_URL, data=data, timeout=5)
return bool(req.json()["success"])

View file

@ -1,15 +1,15 @@
from pusher_push_notifications import PushNotifications
from files.classes import Comment, Notification, Subscription
from files.classes import Comment, Notification, Subscription, User
from files.helpers.alerts import NOTIFY_USERS
from files.helpers.const import PUSHER_ID, PUSHER_KEY, SITE_ID, SITE_FULL
from files.helpers.assetcache import assetcache_path
from flask import g
from sqlalchemy import select, update
from sqlalchemy.sql.expression import func, text, alias
from sqlalchemy.orm import aliased
from sqlalchemy.orm import Query, aliased
from sys import stdout
import gevent
import typing
from typing import Optional
if PUSHER_ID != 'blahblahblah':
beams_client = PushNotifications(instance_id=PUSHER_ID, secret_key=PUSHER_KEY)
@ -217,3 +217,16 @@ def comment_on_unpublish(comment:Comment):
reflect the comments users will actually see.
"""
update_stateful_counters(comment, -1)
def comment_filter_moderated(q: Query, v: Optional[User]) -> Query:
if not (v and v.shadowbanned) and not (v and v.admin_level > 2):
q = q.join(User, User.id == Comment.author_id) \
.filter(User.shadowbanned == None)
if not v or v.admin_level < 2:
q = q.filter(
((Comment.filter_state != 'filtered')
& (Comment.filter_state != 'removed'))
| (Comment.author_id == ((v and v.id) or 0))
)
return q

View file

@ -55,6 +55,10 @@ ERROR_MESSAGES = {
}
LOGGEDIN_ACTIVE_TIME = 15 * 60
RENDER_DEPTH_LIMIT = 9
'''
The maximum depth at which a comment tree is rendered
'''
WERKZEUG_ERROR_DESCRIPTIONS = {
400: "The browser (or proxy) sent a request that this server could not understand.",

View file

@ -1,9 +1,9 @@
from collections import defaultdict
from typing import Iterable, List, Optional, Type, Union
from typing import Callable, Iterable, List, Optional, Type, Union
from flask import g
from sqlalchemy import and_, or_, func
from sqlalchemy.orm import selectinload
from sqlalchemy.orm import Query, selectinload
from files.classes import *
from files.helpers.const import AUTOJANNY_ID
@ -277,7 +277,7 @@ def get_comments(
# TODO: There is probably some way to unify this with get_comments. However, in
# the interim, it's a hot path and benefits from having tailored code.
def get_comment_trees_eager(
top_comment_ids:Iterable[int],
query_filter_callable: Callable[[Query], Query],
sort: str="old",
v: Optional[User]=None) -> List[Comment]:
@ -305,7 +305,7 @@ def get_comment_trees_eager(
else:
query = g.db.query(Comment)
query = query.filter(Comment.top_comment_id.in_(top_comment_ids))
query = query_filter_callable(query)
query = query.options(
selectinload(Comment.author).options(
selectinload(User.badges),
@ -335,13 +335,12 @@ def get_comment_trees_eager(
comments_map_parent[c.parent_comment_id].append(c)
for parent_id in comments_map_parent:
if parent_id is None: continue
comments_map_parent[parent_id] = sort_comment_results(
comments_map_parent[parent_id], sort)
if parent_id in comments_map:
comments_map[parent_id].replies2 = comments_map_parent[parent_id]
return [comments_map[tcid] for tcid in top_comment_ids]
return comments, comments_map_parent
# TODO: This function was concisely inlined into posts.py in upstream.

View file

@ -85,6 +85,7 @@ def inject_constants():
"THEMES":THEMES,
"PERMS":PERMS,
"FEATURES":FEATURES,
"RENDER_DEPTH_LIMIT":RENDER_DEPTH_LIMIT,
}

View file

@ -10,21 +10,43 @@ import re
from mistletoe import markdown
from json import loads, dump
from random import random, choice
import signal
import gevent
import time
import requests
from files.__main__ import app
TLDS = ('ac','ad','ae','aero','af','ag','ai','al','am','an','ao','aq','ar','arpa','as','asia','at','au','aw','ax','az','ba','bb','bd','be','bf','bg','bh','bi','biz','bj','bm','bn','bo','br','bs','bt','bv','bw','by','bz','ca','cafe','cat','cc','cd','cf','cg','ch','ci','ck','cl','club','cm','cn','co','com','coop','cr','cu','cv','cx','cy','cz','de','dj','dk','dm','do','dz','ec','edu','ee','eg','er','es','et','eu','fi','fj','fk','fm','fo','fr','ga','gb','gd','ge','gf','gg','gh','gi','gl','gm','gn','gov','gp','gq','gr','gs','gt','gu','gw','gy','hk','hm','hn','hr','ht','hu','id','ie','il','im','in','info','int','io','iq','ir','is','it','je','jm','jo','jobs','jp','ke','kg','kh','ki','km','kn','kp','kr','kw','ky','kz','la','lb','lc','li','lk','lr','ls','lt','lu','lv','ly','ma','mc','md','me','mg','mh','mil','mk','ml','mm','mn','mo','mobi','mp','mq','mr','ms','mt','mu','museum','mv','mw','mx','my','mz','na','name','nc','ne','net','nf','ng','ni','nl','no','np','nr','nu','nz','om','org','pa','pe','pf','pg','ph','pk','pl','pm','pn','post','pr','pro','ps','pt','pw','py','qa','re','ro','rs','ru','rw','sa','sb','sc','sd','se','sg','sh','si','sj','sk','sl','sm','sn','so','social','sr','ss','st','su','sv','sx','sy','sz','tc','td','tel','tf','tg','th','tj','tk','tl','tm','tn','to','tp','tr','travel','tt','tv','tw','tz','ua','ug','uk','us','uy','uz','va','vc','ve','vg','vi','vn','vu','wf','win','ws','xn','xxx','xyz','ye','yt','yu','za','zm','zw', 'moe')
TLDS = ('ac','ad','ae','aero','af','ag','ai','al','am','an','ao','aq','ar',
'arpa','as','asia','at','au','aw','ax','az','ba','bb','bd','be','bf','bg',
'bh','bi','biz','bj','bm','bn','bo','br','bs','bt','bv','bw','by','bz',
'ca','cafe','cat','cc','cd','cf','cg','ch','ci','ck','cl','club','cm',
'cn','co','com','coop','cr','cu','cv','cx','cy','cz','de','dj','dk','dm',
'do','dz','ec','edu','ee','eg','er','es','et','eu','fi','fj','fk','fm',
'fo','fr','ga','gb','gd','ge','gf','gg','gh','gi','gl','gm','gn','gov',
'gp','gq','gr','gs','gt','gu','gw','gy','hk','hm','hn','hr','ht','hu',
'id','ie','il','im','in','info','int','io','iq','ir','is','it','je','jm',
'jo','jobs','jp','ke','kg','kh','ki','km','kn','kp','kr','kw','ky','kz',
'la','lb','lc','li','lk','lr','ls','lt','lu','lv','ly','ma','mc','md','me',
'mg','mh','mil','mk','ml','mm','mn','mo','mobi','mp','mq','mr','ms','mt',
'mu','museum','mv','mw','mx','my','mz','na','name','nc','ne','net','nf',
'ng','ni','nl','no','np','nr','nu','nz','om','org','pa','pe','pf','pg',
'ph','pk','pl','pm','pn','post','pr','pro','ps','pt','pw','py','qa','re',
'ro','rs','ru','rw','sa','sb','sc','sd','se','sg','sh','si','sj','sk',
'sl','sm','sn','so','social','sr','ss','st','su','sv','sx','sy','sz',
'tc','td','tel','tf','tg','th','tj','tk','tl','tm','tn','to','tp','tr',
'travel','tt','tv','tw','tz','ua','ug','uk','us','uy','uz','va','vc','ve',
'vg','vi','vn','vu','wf','win','ws','xn','xxx','xyz','ye','yt','yu','za',
'zm','zw', 'moe')
allowed_tags = ('b','blockquote','br','code','del','em','h1','h2','h3','h4','h5','h6','hr','i','li','ol','p','pre','strong','sub','sup','table','tbody','th','thead','td','tr','ul','a','span','ruby','rp','rt','spoiler',)
allowed_tags = ('b','blockquote','br','code','del','em','h1','h2','h3','h4',
'h5','h6','hr','i','li','ol','p','pre','strong','sub','sup','table',
'tbody','th','thead','td','tr','ul','a','span','ruby','rp','rt',
'spoiler',)
if app.config['MULTIMEDIA_EMBEDDING_ENABLED']:
allowed_tags += ('img', 'lite-youtube', 'video', 'source',)
def allowed_attributes(tag, name, value):
if name == 'style': return True
if tag == 'a':
@ -123,31 +145,21 @@ def render_emoji(html, regexp, edit, marseys_used=set(), b=False):
return html
def with_sigalrm_timeout(timeout: int):
'Use SIGALRM to raise an exception if the function executes for longer than timeout seconds'
# while trying to test this using time.sleep I discovered that gunicorn does in fact do some
# async so if we timeout on that (or on a db op) then the process is crashed without returning
# a proper 500 error. Oh well.
def sig_handler(signum, frame):
print("Timeout!", flush=True)
raise Exception("Timeout")
def with_gevent_timeout(timeout: int):
'''
Use gevent to raise an exception if the function executes for longer than timeout seconds
Using gevent instead of a signal based approach allows for proper async and avoids some
worker crashes
'''
def inner(func):
@functools.wraps(inner)
@functools.wraps(func)
def wrapped(*args, **kwargs):
signal.signal(signal.SIGALRM, sig_handler)
signal.alarm(timeout)
try:
return func(*args, **kwargs)
finally:
signal.alarm(0)
return gevent.with_timeout(timeout, func, *args, **kwargs)
return wrapped
return inner
@with_sigalrm_timeout(2)
@with_gevent_timeout(2)
def sanitize(sanitized, alert=False, comment=False, edit=False):
# double newlines, eg. hello\nworld becomes hello\n\nworld, which later becomes <p>hello</p><p>world</p>
sanitized = linefeeds_regex.sub(r'\1\n\n\2', sanitized)
@ -190,11 +202,7 @@ def sanitize(sanitized, alert=False, comment=False, edit=False):
users = get_users(names,graceful=True)
if len(users) > app.config['MENTION_LIMIT']:
signal.alarm(0)
abort(
make_response(
jsonify(
error=f'Mentioned {len(users)} users but limit is {app.config["MENTION_LIMIT"]}'), 400))
abort(400, f'Mentioned {len(users)} users but limit is {app.config["MENTION_LIMIT"]}')
for u in users:
if not u: continue
@ -281,12 +289,8 @@ def sanitize(sanitized, alert=False, comment=False, edit=False):
sanitized = sanitized.replace('&amp;','&')
sanitized = utm_regex.sub('', sanitized)
sanitized = utm_regex2.sub('', sanitized)
sanitized = sanitized.replace('<html><body>','').replace('</body></html>','')
sanitized = bleach.Cleaner(tags=allowed_tags,
attributes=allowed_attributes,
protocols=['http', 'https'],
@ -321,17 +325,11 @@ def sanitize(sanitized, alert=False, comment=False, edit=False):
domain_list.add(new_domain)
bans = g.db.query(BannedDomain.domain).filter(BannedDomain.domain.in_(list(domain_list))).all()
if bans: abort(403, description=f"Remove the banned domains {bans} and try again!")
return sanitized
def allowed_attributes_emojis(tag, name, value):
if tag == 'img':
if name == 'loading' and value == 'lazy': return True
if name == 'data-bs-toggle' and value == 'tooltip': return True
@ -339,9 +337,8 @@ def allowed_attributes_emojis(tag, name, value):
return False
@with_sigalrm_timeout(1)
@with_gevent_timeout(1)
def filter_emojis_only(title, edit=False, graceful=False):
title = unwanted_bytes_regex.sub('', title)
title = whitespace_regex.sub(' ', title)
title = html.escape(title, quote=True)

View file

@ -1,9 +1,13 @@
from sqlalchemy.orm import Query
from files.helpers.wrappers import *
from files.helpers.get import *
from files.helpers.strings import sql_ilike_clean
from files.__main__ import app, cache, limiter
from files.classes.submission import Submission
from files.helpers.contentsorting import apply_time_filter, sort_objects
from files.helpers.comments import comment_filter_moderated
from files.helpers.contentsorting import \
apply_time_filter, sort_objects, sort_comment_results
defaulttimefilter = environ.get("DEFAULT_TIME_FILTER", "all").strip()
@ -384,27 +388,31 @@ def random_user(v):
@app.get("/comments")
@auth_required
def all_comments(v):
try: page = max(int(request.values.get("page", 1)), 1)
except: page = 1
page = max(request.values.get("page", 1, int), 1)
sort = request.values.get("sort", "new")
t=request.values.get("t", defaulttimefilter)
try: gt=int(request.values.get("after", 0))
except: gt=0
try: lt=int(request.values.get("before", 0))
except: lt=0
idlist = get_comments_idlist(v=v, page=page, sort=sort, t=t, gt=gt, lt=lt)
comments = get_comments(idlist, v=v)
time_filter = request.values.get("t", defaulttimefilter)
time_gt = request.values.get("after", 0, int)
time_lt = request.values.get("before", 0, int)
idlist = get_comments_idlist(v=v,
page=page, sort=sort, t=time_filter, gt=time_gt, lt=time_lt)
next_exists = len(idlist) > 25
idlist = idlist[:25]
if request.headers.get("Authorization"): return {"data": [x.json for x in comments]}
return render_template("home_comments.html", v=v, sort=sort, t=t, page=page, comments=comments, standalone=True, next_exists=next_exists)
def comment_tree_filter(q: Query) -> Query:
q = q.filter(Comment.id.in_(idlist))
q = comment_filter_moderated(q, v)
q = q.options(selectinload(Comment.post)) # used for post titles
return q
comments, _ = get_comment_trees_eager(comment_tree_filter, sort=sort, v=v)
comments = sort_comment_results(comments, sort=sort)
if request.headers.get("Authorization"):
return {"data": [x.json for x in comments]}
return render_template("home_comments.html", v=v,
sort=sort, t=time_filter, page=page, next_exists=next_exists,
comments=comments, standalone=True)
def get_comments_idlist(page=1, v=None, sort="new", t="all", gt=0, lt=0):

View file

@ -2,12 +2,11 @@ from urllib.parse import urlencode
from files.mail import *
from files.__main__ import app, limiter
from files.helpers.const import *
import requests
from files.helpers.captcha import validate_captcha
@app.get("/login")
@auth_desired
def login_get(v):
redir = request.values.get("redirect")
if redir:
redir = redir.replace("/logged_out", "").strip()
@ -290,20 +289,10 @@ def sign_up_post(v):
if existing_account:
return signup_error("An account with that username already exists.")
if app.config.get("HCAPTCHA_SITEKEY"):
token = request.values.get("h-captcha-response")
if not token:
return signup_error("Unable to verify captcha [1].")
data = {"secret": app.config["HCAPTCHA_SECRET"],
"response": token,
"sitekey": app.config["HCAPTCHA_SITEKEY"]}
url = "https://hcaptcha.com/siteverify"
x = requests.post(url, data=data, timeout=5)
if not x.json()["success"]:
return signup_error("Unable to verify captcha [2].")
if not validate_captcha(app.config.get("HCAPTCHA_SECRET", ""),
app.config.get("HCAPTCHA_SITEKEY", ""),
request.values.get("h-captcha-response", "")):
return signup_error("Unable to verify CAPTCHA")
session.pop("signup_token")

View file

@ -3,6 +3,7 @@ import gevent
from files.helpers.wrappers import *
from files.helpers.sanitize import *
from files.helpers.alerts import *
from files.helpers.comments import comment_filter_moderated
from files.helpers.contentsorting import sort_objects
from files.helpers.const import *
from files.helpers.strings import sql_ilike_clean
@ -17,6 +18,7 @@ from os import path
import requests
from shutil import copyfile
from sys import stdout
from sqlalchemy.orm import Query
snappyquotes = [f':#{x}:' for x in marseys_const2]
@ -139,93 +141,33 @@ def post_id(pid, anything=None, v=None, sub=None):
if post.club and not (v and (v.paid_dues or v.id == post.author_id)): abort(403)
if v:
votes = g.db.query(CommentVote).filter_by(user_id=v.id).subquery()
blocking = v.blocking.subquery()
blocked = v.blocked.subquery()
comments = g.db.query(
Comment,
votes.c.vote_type,
blocking.c.target_id,
blocked.c.target_id,
)
if not (v and v.shadowbanned) and not (v and v.admin_level > 2):
comments = comments.join(User, User.id == Comment.author_id).filter(User.shadowbanned == None)
if v.admin_level < 2:
filter_clause = ((Comment.filter_state != 'filtered') & (Comment.filter_state != 'removed')) | (Comment.author_id == v.id)
comments = comments.filter(filter_clause)
comments=comments.filter(Comment.parent_submission == post.id).join(
votes,
votes.c.comment_id == Comment.id,
isouter=True
).join(
blocking,
blocking.c.target_id == Comment.author_id,
isouter=True
).join(
blocked,
blocked.c.user_id == Comment.author_id,
isouter=True
)
output = []
for c in comments.all():
comment = c[0]
comment.voted = c[1] or 0
comment.is_blocking = c[2] or 0
comment.is_blocked = c[3] or 0
output.append(comment)
pinned = [c[0] for c in comments.filter(Comment.is_pinned != None).all()]
comments = comments.filter(Comment.level == 1, Comment.is_pinned == None)
comments = sort_objects(comments, sort, Comment)
comments = [c[0] for c in comments.all()]
else:
pinned = g.db.query(Comment).filter(Comment.parent_submission == post.id, Comment.is_pinned != None).all()
comments = g.db.query(Comment).join(User, User.id == Comment.author_id).filter(User.shadowbanned == None, Comment.parent_submission == post.id, Comment.level == 1, Comment.is_pinned == None)
comments = sort_objects(comments, sort, Comment)
filter_clause = (Comment.filter_state != 'filtered') & (Comment.filter_state != 'removed')
comments = comments.filter(filter_clause)
comments = comments.all()
offset = 0
ids = set()
limit = app.config['RESULTS_PER_PAGE_COMMENTS']
offset = 0
if post.comment_count > limit and not request.headers.get("Authorization") and not request.values.get("all"):
comments2 = []
count = 0
for comment in comments:
comments2.append(comment)
ids.add(comment.id)
count += g.db.query(Comment.id).filter_by(parent_submission=post.id, top_comment_id=comment.id).count() + 1
if count > limit: break
top_comments = g.db.query(Comment.id, Comment.descendant_count).filter(
Comment.parent_submission == post.id,
Comment.level == 1,
).order_by(Comment.is_pinned.desc().nulls_last())
top_comments = comment_filter_moderated(top_comments, v)
top_comments = sort_objects(top_comments, sort, Comment)
if len(comments) == len(comments2): offset = 0
else: offset = 1
comments = comments2
pg_top_comment_ids = []
pg_comment_qty = 0
for tc_id, tc_children_qty in top_comments.all():
if pg_comment_qty >= limit:
offset = 1
break
pg_comment_qty += tc_children_qty + 1
pg_top_comment_ids.append(tc_id)
for pin in pinned:
if pin.is_pinned_utc and int(time.time()) > pin.is_pinned_utc:
pin.is_pinned = None
pin.is_pinned_utc = None
g.db.add(pin)
pinned.remove(pin)
def comment_tree_filter(q: Query) -> Query:
q = q.filter(Comment.top_comment_id.in_(pg_top_comment_ids))
q = comment_filter_moderated(q, v)
return q
top_comments = pinned + comments
top_comment_ids = [c.id for c in top_comments]
post.replies = get_comment_trees_eager(top_comment_ids, sort, v)
comments, comment_tree = get_comment_trees_eager(comment_tree_filter, sort, v)
post.replies = comment_tree[None] # parent=None -> top-level comments
ids = {c.id for c in post.replies}
post.views += 1
g.db.expire_on_commit = False
@ -246,88 +188,52 @@ def viewmore(v, pid, sort, offset):
post = get_post(pid, v=v)
if post.club and not (v and (v.paid_dues or v.id == post.author_id)): abort(403)
offset = int(offset)
offset_prev = int(offset)
try: ids = set(int(x) for x in request.values.get("ids").split(','))
except: abort(400)
if sort == "new":
newest = g.db.query(Comment).filter(Comment.id.in_(ids)).order_by(Comment.created_utc.desc()).first()
if v:
votes = g.db.query(CommentVote).filter_by(user_id=v.id).subquery()
blocking = v.blocking.subquery()
blocked = v.blocked.subquery()
comments = g.db.query(
Comment,
votes.c.vote_type,
blocking.c.target_id,
blocked.c.target_id,
).filter(Comment.parent_submission == pid, Comment.is_pinned == None, Comment.id.notin_(ids))
if not (v and v.shadowbanned) and not (v and v.admin_level > 2):
comments = comments.join(User, User.id == Comment.author_id).filter(User.shadowbanned == None)
if not v or v.admin_level < 2:
filter_clause = (Comment.filter_state != 'filtered') & (Comment.filter_state != 'removed')
if v:
filter_clause = filter_clause | (Comment.author_id == v.id)
comments = comments.filter(filter_clause)
comments=comments.join(
votes,
votes.c.comment_id == Comment.id,
isouter=True
).join(
blocking,
blocking.c.target_id == Comment.author_id,
isouter=True
).join(
blocked,
blocked.c.user_id == Comment.author_id,
isouter=True
)
output = []
for c in comments.all():
comment = c[0]
comment.voted = c[1] or 0
comment.is_blocking = c[2] or 0
comment.is_blocked = c[3] or 0
output.append(comment)
comments = comments.filter(Comment.level == 1)
if sort == "new":
comments = comments.filter(Comment.created_utc < newest.created_utc)
comments = sort_objects(comments, sort, Comment)
comments = [c[0] for c in comments.all()]
else:
comments = g.db.query(Comment).join(User, User.id == Comment.author_id).filter(User.shadowbanned == None, Comment.parent_submission == pid, Comment.level == 1, Comment.is_pinned == None, Comment.id.notin_(ids))
if sort == "new":
comments = comments.filter(Comment.created_utc < newest.created_utc)
comments = sort_objects(comments, sort, Comment)
comments = comments.all()
comments = comments[offset:]
limit = app.config['RESULTS_PER_PAGE_COMMENTS']
comments2 = []
count = 0
offset = 0
for comment in comments:
comments2.append(comment)
ids.add(comment.id)
count += g.db.query(Comment.id).filter_by(parent_submission=post.id, top_comment_id=comment.id).count() + 1
if count > limit: break
# TODO: Unify with common post_id logic
top_comments = g.db.query(Comment.id, Comment.descendant_count).filter(
Comment.parent_submission == post.id,
Comment.level == 1,
Comment.id.notin_(ids),
Comment.is_pinned == None,
).order_by(Comment.is_pinned.desc().nulls_last())
if len(comments) == len(comments2): offset = 0
else: offset += 1
comments = comments2
if sort == "new":
newest_created_utc = g.db.query(Comment.created_utc).filter(
Comment.id.in_(ids),
Comment.is_pinned == None,
).order_by(Comment.created_utc.desc()).limit(1).scalar()
# Needs to be <=, not just <, to support seed_db data which has many identical
# created_utc values. Shouldn't cause duplication in real data because of the
# `NOT IN :ids` in top_comments.
top_comments = top_comments.filter(Comment.created_utc <= newest_created_utc)
top_comments = comment_filter_moderated(top_comments, v)
top_comments = sort_objects(top_comments, sort, Comment)
pg_top_comment_ids = []
pg_comment_qty = 0
for tc_id, tc_children_qty in top_comments.all():
if pg_comment_qty >= limit:
offset = offset_prev + 1
break
pg_comment_qty += tc_children_qty + 1
pg_top_comment_ids.append(tc_id)
def comment_tree_filter(q: Query) -> Query:
q = q.filter(Comment.top_comment_id.in_(pg_top_comment_ids))
q = comment_filter_moderated(q, v)
return q
_, comment_tree = get_comment_trees_eager(comment_tree_filter, sort, v)
comments = comment_tree[None] # parent=None -> top-level comments
ids |= {c.id for c in comments}
return render_template("comments.html", v=v, comments=comments, p=post, ids=list(ids), render_replies=True, pid=pid, sort=sort, offset=offset, ajax=True)
@ -353,7 +259,7 @@ def morecomments(v, cid):
votes.c.vote_type,
blocking.c.target_id,
blocked.c.target_id,
).filter(Comment.top_comment_id == tcid, Comment.level > 9).join(
).filter(Comment.top_comment_id == tcid, Comment.level > RENDER_DEPTH_LIMIT).join(
votes,
votes.c.comment_id == Comment.id,
isouter=True

View file

@ -1,10 +1,8 @@
from __future__ import unicode_literals
from files.helpers.alerts import *
from files.helpers.sanitize import *
from files.helpers.const import *
from files.mail import *
from files.__main__ import app, cache, limiter
import youtube_dl
from .front import frontlist
import os
from files.helpers.sanitize import filter_emojis_only
@ -634,85 +632,6 @@ def settings_name_change(v):
return redirect("/settings/profile")
@app.post("/settings/song_change")
@limiter.limit("2/second;10/day")
@auth_required
def settings_song_change(v):
song=request.values.get("song").strip()
if song == "" and v.song:
if path.isfile(f"/songs/{v.song}.mp3") and g.db.query(User.id).filter_by(song=v.song).count() == 1:
os.remove(f"/songs/{v.song}.mp3")
v.song = None
g.db.add(v)
g.db.commit()
return redirect("/settings/profile")
song = song.replace("https://music.youtube.com", "https://youtube.com")
if song.startswith(("https://www.youtube.com/watch?v=", "https://youtube.com/watch?v=", "https://m.youtube.com/watch?v=")):
id = song.split("v=")[1]
elif song.startswith("https://youtu.be/"):
id = song.split("https://youtu.be/")[1]
else:
return render_template("settings_profile.html", v=v, error="Not a youtube link.")
if "?" in id: id = id.split("?")[0]
if "&" in id: id = id.split("&")[0]
if path.isfile(f'/songs/{id}.mp3'):
v.song = id
g.db.add(v)
g.db.commit()
return redirect("/settings/profile")
req = requests.get(f"https://www.googleapis.com/youtube/v3/videos?id={id}&key={YOUTUBE_KEY}&part=contentDetails", timeout=5).json()
duration = req['items'][0]['contentDetails']['duration']
if duration == 'P0D':
return render_template("settings_profile.html", v=v, error="Can't use a live youtube video!")
if "H" in duration:
return render_template("settings_profile.html", v=v, error="Duration of the video must not exceed 15 minutes.")
if "M" in duration:
duration = int(duration.split("PT")[1].split("M")[0])
if duration > 15:
return render_template("settings_profile.html", v=v, error="Duration of the video must not exceed 15 minutes.")
if v.song and path.isfile(f"/songs/{v.song}.mp3") and g.db.query(User.id).filter_by(song=v.song).count() == 1:
os.remove(f"/songs/{v.song}.mp3")
ydl_opts = {
'outtmpl': '/songs/%(title)s.%(ext)s',
'format': 'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
try: ydl.download([f"https://youtube.com/watch?v={id}"])
except Exception as e:
print(e)
return render_template("settings_profile.html",
v=v,
error="Age-restricted videos aren't allowed.")
files = os.listdir("/songs/")
paths = [path.join("/songs/", basename) for basename in files]
songfile = max(paths, key=path.getctime)
os.rename(songfile, f"/songs/{id}.mp3")
v.song = id
g.db.add(v)
g.db.commit()
return redirect("/settings/profile")
@app.post("/settings/title_change")
@limiter.limit("1/second;30/minute;200/hour;1000/day")
@auth_required

View file

@ -2,6 +2,7 @@ from files.mail import *
from files.__main__ import app, limiter, mail
from files.helpers.alerts import *
from files.helpers.const import *
from files.helpers.captcha import validate_captcha
from files.classes.award import AWARDS
from sqlalchemy import func
from os import path
@ -280,13 +281,17 @@ def api(v):
@app.get("/media")
@auth_desired
def contact(v):
return render_template("contact.html", v=v)
return render_template("contact.html", v=v,
hcaptcha=app.config.get("HCAPTCHA_SITEKEY", ""))
@app.post("/send_admin")
@limiter.limit("1/second;2/minute;6/hour;10/day")
@auth_desired
def submit_contact(v):
def submit_contact(v: Optional[User]):
if not v and not validate_captcha(app.config.get("HCAPTCHA_SECRET", ""),
app.config.get("HCAPTCHA_SITEKEY", ""),
request.values.get("h-captcha-response", "")):
abort(403, "CAPTCHA provided was not correct. Please try it again")
body = request.values.get("message")
email = request.values.get("email")
if not body: abort(400)

View file

@ -495,20 +495,6 @@ def get_profilecss(username):
resp.headers.add("Content-Type", "text/css")
return resp
@app.get("/@<username>/song")
def usersong(username):
user = get_user(username)
if user.song: return redirect(f"/song/{user.song}.mp3")
else: abort(404)
@app.get("/song/<song>")
@app.get("/static/song/<song>")
def song(song):
resp = make_response(send_from_directory('/songs', song))
resp.headers.remove("Cache-Control")
resp.headers.add("Cache-Control", "public, max-age=3153600")
return resp
@app.post("/subscribe/<post_id>")
@limiter.limit("1/second;30/minute;200/hour;1000/day")
@auth_required

View file

@ -21,8 +21,6 @@ def admin_vote_info_get(v):
if thing.ghost and v.id != OWNER_ID: abort(403)
if not thing.author:
print(thing.id, flush=True)
if isinstance(thing, Submission):
if thing.author.shadowbanned and not (v and v.admin_level):
thing_id = g.db.query(Submission.id).filter_by(upvotes=thing.upvotes, downvotes=thing.downvotes).order_by(Submission.id).first()[0]

View file

@ -79,7 +79,7 @@
<div class="comment-body">
<div id="comment-{{c.id}}-only" class="comment-{{c.id}}-only"></div>
{% if render_replies %}
{% if level<9 %}
{% if level <= RENDER_DEPTH_LIMIT - 1 %}
<div id="replies-of-{{c.id}}" class="">
{% set standalone=False %}
{% for reply in replies %}
@ -151,7 +151,7 @@
</div>
{% endif %}
{% if c.parent_comment and c.parent_comment.sentto %}
{% if not standalone and c.parent_comment and c.parent_comment.sentto %}
{% set isreply = True %}
{% else %}
{% set isreply = False %}
@ -169,12 +169,8 @@
{% if c.ghost %}
👻
{% else %}
{% if c.author.verified %}<i class="fas fa-badge-check align-middle ml-1 {% if c.author.verified=='Glowiefied' %}glow{% endif %}" style="color:{% if c.author.verifiedcolor %}#{{c.author.verifiedcolor}}{% else %}#1DA1F2{% endif %}" data-bs-toggle="tooltip" data-bs-placement="bottom" title="{{c.author.verified}}"></i>
{% endif %}
{% if not c.author %}
{{c.print()}}
{% endif %}
{% if not should_hide_username %}
<a href="/@{{c.author_name}}" class="user-name" onclick='popclick({{c.author.json_popover(v) | tojson}}); return false' data-bs-placement="bottom" data-bs-toggle="popover" data-bs-trigger="click" data-content-id="popover" role="button" tabindex="0" style="font-size:12px; font-weight:bold;"><img loading="lazy" src="{{c.author.profile_url}}" class="profile-pic-20 mr-2"><span {% if c.author.patron and not c.distinguish_level %}class="patron" style="background-color:#{{c.author.namecolor}};"{% elif c.distinguish_level %}class="mod"{% endif %}>{{c.author_name}}</span></a>
{% endif %}
@ -184,7 +180,9 @@
data-micromodal-trigger="modal-1"
onclick='fillnote( {{c.author.json_notes(v) | tojson}}, null, {{c.id}} )'>U</span>
{% endif %}
{% if c.author.customtitle %}&nbsp;<bdi style="color: #{{c.author.titlecolor}}">&nbsp;{{c.author.customtitle | safe}}</bdi>{% endif %}
{% if c.author.customtitle and not should_hide_username -%}
&nbsp;<bdi style="color: #{{c.author.titlecolor}}">&nbsp;{{c.author.customtitle | safe}}</bdi>
{%- endif %}
{% endif %}
{% if FEATURES['AWARDS'] %}
@ -536,7 +534,7 @@
{% if render_replies %}
{% if level<9 or request.path == '/notifications' %}
{% if level <= RENDER_DEPTH_LIMIT - 1 or request.path == '/notifications' %}
<div id="replies-of-{{c.id}}">
{% for reply in replies %}
{{single_comment(reply, level=level+1)}}
@ -707,15 +705,6 @@
</div>
{% endif %}
</div>
{% endif %}

View file

@ -1,12 +1,8 @@
{% extends "default.html" %}
{% block title %}
<title>{{SITE_TITLE}} - Contact</title>
{% endblock %}
{% block content %}
{% if msg %}
<div class="alert alert-success alert-dismissible fade show my-3" role="alert">
<i class="fas fa-check-circle my-auto" aria-hidden="true"></i>
@ -18,7 +14,7 @@
</button>
</div>
{% endif %}
<section id="contact">
<h1 class="article-title">Contact {{SITE_TITLE}} Admins</h1>
<p>Use this form to contact {{SITE_TITLE}} Admins.</p>
@ -34,18 +30,16 @@
<div id="filename"><i class="far fa-image"></i></div>
<input autocomplete="off" id="file-upload" type="file" name="file" accept="image/*, video/*" {% if request.headers.get('cf-ipcountry')=="T1" %}disabled{% endif %} onchange="changename('filename','file-upload')" hidden>
</label>
{% if not v and hcaptcha %}
<div class="h-captcha" data-sitekey="{{hcaptcha}}"></div>
{% endif %}
<input type="submit" value="Submit" class="btn btn-primary mt-3">
</form>
<pre>
</pre>
</section>
<section id="canary">
<p>If you can see this line, we haven't been contacted by any law enforcement or governmental organizations in 2022 yet.</p>
<pre>
</pre>
</section>
{% if hcaptcha %}
<script src="{{ 'js/hcaptcha.js' | asset }}"></script>
{% endif %}
{% endblock %}

View file

@ -172,15 +172,9 @@
{% if v and p.filter_state == 'reported' and v.can_manage_reports() %}
<a class="btn btn-primary" id="submission-report-button" role="button" style="padding:1px 5px; font-size:10px"onclick="document.getElementById('flaggers').classList.toggle('d-none')">{{p.active_flags(v)}} Reports</a>
{% endif %}
{% if not p.author %}
{{p.print()}}
{% endif %}
{% if p.ghost %}
👻
{% else %}
{% if p.author.verified %}<i class="fas fa-badge-check align-middle ml-1 {% if p.author.verified=='Glowiefied' %}glow{% endif %}" style="color:{% if p.author.verifiedcolor %}#{{p.author.verifiedcolor}}{% else %}var(--primary){% endif %}" data-bs-toggle="tooltip" data-bs-placement="bottom" title="{{p.author.verified}}"></i>
{% endif %}

View file

@ -198,15 +198,9 @@
<span>
<span data-bs-toggle="tooltip" data-bs-placement="bottom" onmouseover="timestamp('timestamp-{{p.id}}','{{p.created_utc}}')" id="timestamp-{{p.id}}">{{p.age_string}} by </span>
{% if not p.author %}
{{p.print()}}
{% endif %}
{% if p.ghost %}
👻
{% else %}
{% if p.author.verified %}<i class="fas fa-badge-check align-middle ml-1 {% if p.author.verified=='Glowiefied' %}glow{% endif %}" style="color:{% if p.author.verifiedcolor %}#{{p.author.verifiedcolor}}{% else %}#1DA1F2{% endif %}" data-bs-toggle="tooltip" data-bs-placement="bottom" title="{{p.author.verified}}"></i>
{% endif %}
<a href="/@{{p.author_name}}"

View file

@ -190,10 +190,6 @@
<a href="/views" class="btn btn-secondary">Profile views</a>
{% endif %}
{% if u.song and v and (v.id == u.id) %}
<a class="btn btn-secondary" role="button" onclick="toggle()">Toggle anthem</a>
{% endif %}
{% if v and v.id != u.id and v.admin_level > 1 %}
<br><br>
<div class="body d-lg-flex border-bottom">
@ -416,10 +412,6 @@
<a href="/views" class="btn btn-secondary">Profile views</a>
{% endif %}
{% if u.song and v and (v.id == u.id) %}
<a class="btn btn-secondary" role="button" onclick="toggle()">Toggle anthem</a>
{% endif %}
{% if v and v.id != u.id %}
<a id="button-unsub2" class="btn btn-secondary {% if not is_following %}d-none{% endif %}" role="button" onclick="post_toast2(this,'/unfollow/{{u.username}}','button-unsub2','button-sub2')">Unfollow</a>
@ -636,22 +628,12 @@
</div>
{% if u.song %}
{% if v and v.id == u.id %}
<div id="v_username" class="d-none">{{v.username}}</div>
{% else %}
<div id="u_username" class="d-none">{{u.username}}</div>
{% endif %}
{% endif %}
{% if v %}
<div id='tax' class="d-none">{% if v.patron or u.patron or v.alts_patron or u.alts_patron %}0{% else %}0.03{% endif %}</div>
<script src="{{ 'js/userpage_v.js' | asset }}"></script>
<div id="username" class="d-none">{{u.username}}</div>
{% endif %}
<script src="{{ 'js/userpage.js' | asset }}"></script>
{% endblock %}
{% block pagenav %}

View file

@ -104,19 +104,10 @@
</div>
</div>
{% if u.song %}
{% if v and v.id == u.id %}
<div id="v_username" class="d-none">{{v.username}}</div>
{% else %}
<div id="u_username" class="d-none">{{u.username}}</div>
{% endif %}
{% endif %}
{% if v %}
<div id='tax' class="d-none">{% if v.patron or u.patron %}0{% else %}0.03{% endif %}</div>
<script src="{{ 'js/userpage_v.js' | asset }}"></script>
<div id="username" class="d-none">{{u.username}}</div>
{% endif %}
<script src="{{ 'js/userpage.js' | asset }}"></script>
{% endblock %}

View file

@ -15,20 +15,9 @@
</div>
</div>
{% if u.song %}
{% if v and v.id == u.id %}
<div id="v_username" class="d-none">{{v.username}}</div>
{% else %}
<div id="u_username" class="d-none">{{u.username}}</div>
{% endif %}
{% endif %}
{% endblock %}
{% block pagenav %}
{% if u.song %}
<div id="uid" class="d-none">{{u.id}}</div>
{% endif %}
{% if v %}
<div id='tax' class="d-none">{% if v.patron or u.patron %}0{% else %}0.03{% endif %}</div>
@ -36,5 +25,4 @@
<div id="username" class="d-none">{{u.username}}</div>
{% endif %}
<script src="{{ 'js/userpage.js' | asset }}"></script>
{% endblock %}

View file

@ -1,3 +1,4 @@
from files.helpers.const import RENDER_DEPTH_LIMIT
from . import fixture_accounts
from . import fixture_submissions
from . import fixture_comments
@ -158,10 +159,10 @@ def test_more_button_label_in_deep_threads(accounts, submissions, comments):
# only look every 5 posts to make this test not _too_ unbearably slow
view_post_response = alice_client.get(f'/post/{post.id}')
assert 200 == view_post_response.status_code
if i <= 8:
assert f'More comments ({i - 8})' not in view_post_response.text
if i <= RENDER_DEPTH_LIMIT - 1:
assert f'More comments ({i - RENDER_DEPTH_LIMIT + 1})' not in view_post_response.text
else:
assert f'More comments ({i - 8})' in view_post_response.text
assert f'More comments ({i - RENDER_DEPTH_LIMIT + 1})' in view_post_response.text
@util.no_rate_limit
def test_bulk_update_descendant_count_quick(accounts, submissions, comments):

View file

@ -0,0 +1,28 @@
"""remove users.song
Revision ID: ba8a214736eb
Revises: 1f30a37b08a0
Create Date: 2023-02-08 22:04:15.901498+00:00
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'ba8a214736eb'
down_revision = '1f30a37b08a0'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('users', 'song')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('users', sa.Column('song', sa.VARCHAR(length=50), autoincrement=False, nullable=True))
# ### end Alembic commands ###

12
poetry.lock generated
View file

@ -1006,14 +1006,6 @@ category = "main"
optional = false
python-versions = "*"
[[package]]
name = "youtube-dl"
version = "2021.12.17"
description = "YouTube video downloader"
category = "main"
optional = false
python-versions = "*"
[[package]]
name = "zope.event"
version = "4.5.0"
@ -2228,10 +2220,6 @@ wrapt = [
yattag = [
{file = "yattag-1.14.0.tar.gz", hash = "sha256:5731a31cb7452c0c6930dd1a284e0170b39eee959851a2aceb8d6af4134a5fa8"},
]
youtube-dl = [
{file = "youtube_dl-2021.12.17-py2.py3-none-any.whl", hash = "sha256:f1336d5de68647e0364a47b3c0712578e59ec76f02048ff5c50ef1c69d79cd55"},
{file = "youtube_dl-2021.12.17.tar.gz", hash = "sha256:bc59e86c5d15d887ac590454511f08ce2c47698d5a82c27bfe27b5d814bbaed2"},
]
"zope.event" = [
{file = "zope.event-4.5.0-py2.py3-none-any.whl", hash = "sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42"},
{file = "zope.event-4.5.0.tar.gz", hash = "sha256:5e76517f5b9b119acf37ca8819781db6c16ea433f7e2062c4afc2b6fbedb1330"},

View file

@ -34,7 +34,6 @@ SQLAlchemy = "^1.4.43"
user-agents = "*"
psycopg2-binary = "*"
pusher_push_notifications = "*"
youtube-dl = "*"
yattag = "*"
webptools = "*"
pytest = "*"