invisibleify completely removed trees only (fixes #431) (#535)

* invisibleify completely removed trees only (fixes #431)

* fix visibility state for shadowbanned users.
this also ends up moving some of the complexity out of the templates.

* comments: remove unused variable

* moderation state machine

* no seriously this really should check for v not being None

* fix shadowban state

* fix visibility state

* update stateful counters

* don't use bespoke function for show_descendants

* properly mock ModerationState for cron submissions

* fix approval discrepency

* remove treenukes for removed comments

* show shadowbans as removed

files/classes/comment.py

@@ -1,5 +1,4 @@
-import time
+from typing import TYPE_CHECKING, Literal, Optional
-from typing import Literal, Optional
 from urllib.parse import parse_qs, urlencode, urlparse
 
 from flask import g
@@ -9,11 +8,14 @@ from sqlalchemy.orm import relationship
 from files.classes.base import CreatedBase
 from files.helpers.config.const import *
 from files.helpers.config.environment import SCORE_HIDING_TIME_HOURS, SITE_FULL
-from files.helpers.content import (body_displayed,
+from files.helpers.content import (ModerationState, body_displayed,
                                    execute_shadowbanned_fake_votes)
 from files.helpers.lazy import lazy
 from files.helpers.time import format_age
 
+if TYPE_CHECKING:
+	from files.classes.user import User
+
 CommentRenderContext = Literal['comments', 'volunteer']
 
 class Comment(CreatedBase):
@@ -402,7 +404,6 @@ class Comment(CreatedBase):
 		if v.id == self.author_id: return 1
 		return getattr(self, 'voted', 0)
 
-
 	def sticky_api_url(self, v) -> str | None:
 		'''
 		Returns the API URL used to sticky this comment.
@@ -418,7 +419,25 @@ class Comment(CreatedBase):
 			return 'pin_comment'
 		return None
 
-
 	@lazy
 	def active_flags(self, v): 
 		return len(self.flags(v))
+
+	@lazy
+	def show_descendants(self, v:"User | None") -> bool:
+		if self.moderation_state.is_visible_to(v, getattr(self, 'is_blocking', False)):
+			return True
+		return bool(self.descendant_count)
+
+	@lazy
+	def visibility_state(self, v:"User | None") -> tuple[bool, str]:
+		'''
+		Returns a tuple of whether this content is visible and a publicly 
+		visible message to accompany it. The visibility state machine is
+		a slight mess but... this should at least unify the state checks.
+		'''
+		return self.moderation_state.visibility_state(v, getattr(self, 'is_blocking', False))
+
+	@property
+	def moderation_state(self) -> ModerationState:
+		return ModerationState.from_submittable(self)

files/classes/cron/submission.py

@@ -8,10 +8,8 @@ from sqlalchemy.sql.sqltypes import Boolean, Integer, String, Text
 from files.classes.cron.tasks import (RepeatableTask, ScheduledTaskType,
                                       TaskRunContext)
 from files.classes.submission import Submission
-from files.helpers.config.const import (RENDER_DEPTH_LIMIT,
+from files.helpers.config.const import SUBMISSION_TITLE_LENGTH_MAXIMUM
-                                        SUBMISSION_TITLE_LENGTH_MAXIMUM)
+from files.helpers.content import ModerationState, body_displayed
-from files.helpers.config.environment import SITE_FULL
-from files.helpers.content import body_displayed
 from files.helpers.lazy import lazy
 from files.helpers.sanitize import filter_emojis_only
 
@@ -172,3 +170,16 @@ class ScheduledSubmissionTask(RepeatableTask):
 	@property
 	def edit_url(self) -> str:
 		return f"/tasks/scheduled_posts/{self.id}/content"
+	
+	@property
+	def moderation_state(self) -> ModerationState:
+		return ModerationState(
+			removed=False,
+			removed_by_name=None,
+			deleted=False, # we only want to show deleted UI color if disabled
+			reports_ignored=False,
+			filtered=False,
+			op_shadowbanned=False,
+			op_id=self.author_id_submission,
+			op_name_safe=self.author_name
+		)

files/classes/submission.py

@@ -11,7 +11,7 @@ from files.helpers.assetcache import assetcache_path
 from files.helpers.config.const import *
 from files.helpers.config.environment import (SCORE_HIDING_TIME_HOURS, SITE,
                                               SITE_FULL, SITE_ID)
-from files.helpers.content import body_displayed
+from files.helpers.content import ModerationState, body_displayed
 from files.helpers.lazy import lazy
 from files.helpers.time import format_age, format_datetime
 
@@ -360,3 +360,7 @@ class Submission(CreatedBase):
 	@property
 	def edit_url(self) -> str:
 		return f"/edit_post/{self.id}"
+	
+	@property
+	def moderation_state(self) -> ModerationState:
+		return ModerationState.from_submittable(self)

files/helpers/content.py

@@ -2,7 +2,8 @@ from __future__ import annotations
 
 import random
 import urllib.parse
-from typing import TYPE_CHECKING, Optional, Union
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Optional
 
 from sqlalchemy.orm import Session
 
@@ -10,7 +11,9 @@ from files.helpers.config.const import PERMS
 
 if TYPE_CHECKING:
 	from files.classes import Comment, Submission, User
-	Submittable = Union[Submission, Comment]
+	Submittable = Comment | Submission
+else:
+	Submittable = Any
 
 
 def _replace_urls(url:str) -> str:
@@ -86,18 +89,100 @@ def canonicalize_url2(url:str, *, httpsify:bool=False) -> urllib.parse.ParseResu
 	return url_parsed
 
 
-def moderated_body(target:Submittable, v:Optional[User]) -> Optional[str]:
+@dataclass(frozen=True, kw_only=True, slots=True)
-	if v and (v.admin_level >= PERMS['POST_COMMENT_MODERATION'] \
+class ModerationState:
-			or v.id == target.author_id):
+	'''
+	The moderation state machine. This holds moderation state information,
+	including whether this was removed, deleted, filtered, whether OP was 
+	shadowbanned, etc
+	'''
+	removed: bool
+	removed_by_name: str | None
+	deleted: bool
+	reports_ignored: bool
+	filtered: bool
+	op_shadowbanned: bool
+	op_id: int
+	op_name_safe: str
+
+	@classmethod
+	def from_submittable(cls, target: Submittable) -> "ModerationState":
+		return cls(
+			removed=bool(target.is_banned or target.filter_state == 'removed'),
+			removed_by_name=target.ban_reason,  # type: ignore
+			deleted=bool(target.deleted_utc != 0),
+			reports_ignored=bool(target.filter_state == 'ignored'),
+			filtered=bool(target.filter_state == 'filtered'),
+			op_shadowbanned=bool(target.author.shadowbanned),
+			op_id=target.author_id,  # type: ignore
+			op_name_safe=target.author_name
+		)
+
+	def moderated_body(self, v: User | None) -> str | None:
+		if v and (v.admin_level >= PERMS['POST_COMMENT_MODERATION'] \
+			or v.id == self.op_id):
+			return None
+		if self.deleted: return 'Deleted'
+		if self.appear_removed(v): return 'Removed'
+		if self.filtered: return 'Filtered'
 		return None
-	if target.deleted_utc: return 'Deleted by author'
+	
-	if target.is_banned or target.filter_state == 'removed': return 'Removed'
+	def visibility_state(self, v: User | None, is_blocking: bool) -> tuple[bool, str]:
-	if target.filter_state == 'filtered': return 'Filtered'
+		'''
-	return None
+		Returns a tuple of whether this content is visible and a publicly 
+		visible message to accompany it. The visibility state machine is
+		a slight mess but... this should at least unify the state checks.
+		'''
+		def can(v: User | None, perm_level: int) -> bool:
+			return v and v.admin_level >= perm_level
+
+		can_moderate: bool = can(v, PERMS['POST_COMMENT_MODERATION'])
+		can_shadowban: bool = can(v, PERMS['USER_SHADOWBAN'])
+
+		if v and v.id == self.op_id:
+			return True, "This shouldn't be here, please report it!"
+		if (self.removed and not can_moderate) or \
+				(self.op_shadowbanned and not can_shadowban):
+			msg: str = 'Removed'
+			if self.removed_by_name:
+				msg = f'Removed by @{self.removed_by_name}'
+			return False, msg
+		if self.filtered and not can_moderate:
+			return False, 'Filtered, please go kick a mod in the ass to fix this'
+		if self.deleted and not can_moderate:
+			return False, 'Deleted by author'
+		if is_blocking:
+			return False, f'You are blocking @{self.op_name_safe}'
+		return True, "This shouldn't be here, please report it!"
+	
+	def is_visible_to(self, v: User | None, is_blocking: bool) -> bool:
+		return self.visibility_state(v, is_blocking)[0]
+	
+	def replacement_message(self, v: User | None, is_blocking: bool) -> str:
+		return self.visibility_state(v, is_blocking)[1]
+	
+	def appear_removed(self, v: User | None) -> bool:
+		if self.removed: return True
+		if not self.op_shadowbanned: return False
+		return (not v) or bool(v.admin_level < PERMS['USER_SHADOWBAN'])
+	
+	@property
+	def publicly_visible(self) -> bool:
+		return all(
+			not state for state in 
+			[self.deleted, self.removed, self.filtered, self.op_shadowbanned]
+		)
+	
+	@property
+	def explicitly_moderated(self) -> bool:
+		'''
+		Whether this was removed or filtered and not as the result of a shadowban
+		'''
+		return self.removed or self.filtered
 
 
 def body_displayed(target:Submittable, v:Optional[User], is_html:bool) -> str:
-	moderated:Optional[str] = moderated_body(target, v)
+	moderated:Optional[str] = target.moderation_state.moderated_body(v)
 	if moderated: return moderated
 
 	body = target.body_html if is_html else target.body

files/routes/admin/admin.py

@@ -276,13 +276,13 @@ def update_filter_status(v):
 		return { 'result': f'Status of {new_status} is not permitted' }
 
 	if post_id:
-		p = g.db.get(Submission, post_id)
+		target = g.db.get(Submission, post_id)
-		old_status = p.filter_state
+		old_status = target.filter_state
 		rows_updated = g.db.query(Submission).where(Submission.id == post_id) \
 							.update({Submission.filter_state: new_status})
 	elif comment_id:
-		c = g.db.get(Comment, comment_id)
+		target = g.db.get(Comment, comment_id)
-		old_status = c.filter_state
+		old_status = target.filter_state
 		rows_updated = g.db.query(Comment).where(Comment.id == comment_id) \
 							.update({Comment.filter_state: new_status})
 	else:
@@ -290,15 +290,15 @@ def update_filter_status(v):
 
 	if rows_updated == 1:
 		# If comment now visible, update state to reflect publication.
-		if (comment_id
+		if (isinstance(target, Comment)
 				and old_status in ['filtered', 'removed']
 				and new_status in ['normal', 'ignored']):
-			comment_on_publish(c)
+			comment_on_publish(target) # XXX: can cause discrepancies if removal state ≠ filter state
 
-		if (comment_id
+		if (isinstance(target, Comment)
 				and old_status in ['normal', 'ignored']
 				and new_status in ['filtered', 'removed']):
-			comment_on_unpublish(c)
+			comment_on_unpublish(target) # XXX: can cause discrepancies if removal state ≠ filter state
 
 		g.db.commit()
 		return { 'result': 'Update successful' }
@@ -1302,7 +1302,6 @@ def unsticky_comment(cid, v):
 @limiter.exempt
 @admin_level_required(2)
 def api_ban_comment(c_id, v):
-
 	comment = g.db.query(Comment).filter_by(id=c_id).one_or_none()
 	if not comment:
 		abort(404)
@@ -1310,7 +1309,7 @@ def api_ban_comment(c_id, v):
 	comment.is_banned = True
 	comment.is_approved = None
 	comment.ban_reason = v.username
-	g.db.add(comment)
+	comment_on_unpublish(comment) # XXX: can cause discrepancies if removal state ≠ filter state
 	ma=ModAction(
 		kind="ban_comment",
 		user_id=v.id,
@@ -1325,7 +1324,6 @@ def api_ban_comment(c_id, v):
 @limiter.exempt
 @admin_level_required(2)
 def api_unban_comment(c_id, v):
-
 	comment = g.db.query(Comment).filter_by(id=c_id).one_or_none()
 	if not comment: abort(404)
 	
@@ -1340,6 +1338,7 @@ def api_unban_comment(c_id, v):
 	comment.is_banned = False
 	comment.ban_reason = None
 	comment.is_approved = v.id
+	comment_on_publish(comment) # XXX: can cause discrepancies if removal state ≠ filter state
 
 	g.db.add(comment)
 

files/routes/comments.py

@@ -331,17 +331,13 @@ def edit_comment(cid, v):
 @limiter.limit("1/second;30/minute;200/hour;1000/day")
 @auth_required
 def delete_comment(cid, v):
-
 	c = get_comment(cid, v=v)
-
+	if c.deleted_utc: abort(409)
-	if not c.deleted_utc:
+	if c.author_id != v.id: abort(403)
-
+	c.deleted_utc = int(time.time())
-		if c.author_id != v.id: abort(403)
+	# TODO: update stateful counters
-
+	g.db.add(c)
-		c.deleted_utc = int(time.time())
+	g.db.commit()
-
-		g.db.add(c)
-		g.db.commit()
 
 	return {"message": "Comment deleted!"}
 
@@ -349,16 +345,13 @@ def delete_comment(cid, v):
 @limiter.limit("1/second;30/minute;200/hour;1000/day")
 @auth_required
 def undelete_comment(cid, v):
-
 	c = get_comment(cid, v=v)
-
+	if not c.deleted_utc: abort(409)
-	if c.deleted_utc:
+	if c.author_id != v.id: abort(403)
-		if c.author_id != v.id: abort(403)
+	c.deleted_utc = 0
-
+	# TODO: update stateful counters
-		c.deleted_utc = 0
+	g.db.add(c)
-
+	g.db.commit()
-		g.db.add(c)
-		g.db.commit()
 
 	return {"message": "Comment undeleted!"}
 
@@ -366,7 +359,6 @@ def undelete_comment(cid, v):
 @app.post("/pin_comment/<cid>")
 @auth_required
 def pin_comment(cid, v):
-	
 	comment = get_comment(cid, v=v)
 	
 	if not comment.is_pinned:

files/routes/posts.py

@@ -15,7 +15,6 @@ from files.__main__ import app, db_session, limiter
 from files.classes import *
 from files.helpers.alerts import *
 from files.helpers.caching import invalidate_cache
-from files.helpers.comments import comment_filter_moderated
 from files.helpers.config.const import *
 from files.helpers.content import canonicalize_url2
 from files.helpers.contentsorting import sort_objects
@@ -94,7 +93,6 @@ def post_id(pid, anything=None, v=None):
 		Comment.parent_submission == post.id,
 		Comment.level == 1,
 	).order_by(Comment.is_pinned.desc().nulls_last())
-	top_comments = comment_filter_moderated(top_comments, v)
 	top_comments = sort_objects(top_comments, sort, Comment)
 
 	pg_top_comment_ids = []
@@ -108,7 +106,6 @@ def post_id(pid, anything=None, v=None):
 
 	def comment_tree_filter(q: Query) -> Query:
 		q = q.filter(Comment.top_comment_id.in_(pg_top_comment_ids))
-		q = comment_filter_moderated(q, v)
 		return q
 
 	comments, comment_tree = get_comment_trees_eager(comment_tree_filter, sort, v)
@@ -160,7 +157,6 @@ def viewmore(v, pid, sort, offset):
 		# `NOT IN :ids` in top_comments.
 		top_comments = top_comments.filter(Comment.created_utc <= newest_created_utc)
 
-	top_comments = comment_filter_moderated(top_comments, v)
 	top_comments = sort_objects(top_comments, sort, Comment)
 
 	pg_top_comment_ids = []
@@ -174,7 +170,6 @@ def viewmore(v, pid, sort, offset):
 
 	def comment_tree_filter(q: Query) -> Query:
 		q = q.filter(Comment.top_comment_id.in_(pg_top_comment_ids))
-		q = comment_filter_moderated(q, v)
 		return q
 
 	_, comment_tree = get_comment_trees_eager(comment_tree_filter, sort, v)

files/routes/users.py

@@ -517,7 +517,7 @@ def messagereply(v):
 	g.db.add(c)
 	g.db.flush()
 
-	if user_id and user_id != v.id and user_id != 2:
+	if user_id and user_id != v.id and user_id != MODMAIL_ID:
 		notif = g.db.query(Notification).filter_by(comment_id=c.id, user_id=user_id).one_or_none()
 		if not notif:
 			notif = Notification(comment_id=c.id, user_id=user_id)

files/templates/comments.html

@@ -10,11 +10,10 @@
 	{%- set score = c.score_str(render_ctx) -%}
 	{%- set downs = c.downvotes_str(render_ctx) -%}
 
-{%- set replies = c.replies(v) -%}
+	{% set replies = c.replies(v) %}
-{%- set is_notification_page = request.path.startswith('/notifications') -%}
-
-{% if (c.is_banned or c.deleted_utc or c.is_blocking) and not (v and v.admin_level >= 2) and not (v and v.id==c.author_id) %}
 
+{% if not c.visibility_state(v)[0] %}
+{% if c.show_descendants(v) %}
 <div id="comment-{{c.id}}" class="comment">
 		<div class="comment-collapse-icon" onclick="collapse_comment('{{c.id}}', this.parentElement)"></div>
 		<div class="comment-collapse-bar-click" onclick="collapse_comment('{{c.id}}', this.parentElement)">
@@ -23,7 +22,7 @@
 
 		<div class="comment-user-info">
 			{% if standalone and c.over_18 %}<span class="badge badge-danger">+18</span>{% endif %}
-			{% if c.is_banned %}removed by @{{c.ban_reason}}{% elif c.deleted_utc %}Deleted by author{% elif c.is_blocking %}You are blocking @{{c.author_name}}{% endif %}
+			{{c.visibility_state(v)[1]}}
 		</div>
 		
 		<div class="comment-body">
@@ -54,6 +53,7 @@
 		{% endif %}
 	{% endif %}
 </div>
+{% endif %}
 </div>
 
 {% else %}