Aw shucks. No GIFs found...
Thou've reached the end of the list!
{CC} ONLY
" - - body = self.body_html or "" - - if body: - - if v: - body = body.replace("old.reddit.com", v.reddit) - - if v.nitter and not '/i/' in body and '/retweets' not in body: body = body.replace("www.twitter.com", "nitter.net").replace("twitter.com", "nitter.net") - - if v and v.controversial: - captured = [] - for i in controversial_regex.finditer(body): - if i.group(0) in captured: continue - captured.append(i.group(0)) - - url = i.group(1) - p = urlparse(url).query - p = parse_qs(p) - - if 'sort' not in p: p['sort'] = ['controversial'] - - url_noquery = url.split('?')[0] - body = body.replace(url, f"{url_noquery}?{urlencode(p, True)}") - - if v and v.shadowbanned and v.id == self.author_id and 86400 > time.time() - self.created_utc > 60: - ti = max(int((time.time() - self.created_utc)/60), 1) - maxupvotes = min(ti, 13) - rand = randint(0, maxupvotes) - if self.upvotes < rand: - amount = randint(0, 3) - if amount == 1: - self.upvotes += amount - g.db.add(self) - g.db.commit() + body = body_displayed(self, v, is_html=True) + if v and v.controversial: + captured = [] + for i in controversial_regex.finditer(body): + if i.group(0) in captured: continue + captured.append(i.group(0)) + url = i.group(1) + p = urlparse(url).query + p = parse_qs(p) + if 'sort' not in p: p['sort'] = ['controversial'] + url_noquery = url.split('?')[0] + body = body.replace(url, f"{url_noquery}?{urlencode(p, True)}") + execute_shadowbanned_fake_votes(g.db, self, v) # TODO: put in route handler? return body def plainbody(self, v): - if self.post and self.post.club and not (v and (v.paid_dues or v.id in [self.author_id, self.post.author_id])): return f"{CC} ONLY
" - body = self.body - if not body: return "" - return body + return body_displayed(self, v, is_html=False) @lazy def collapse_for_user(self, v, path): if v and self.author_id == v.id: return False - if path == '/admin/removed/comments': return False - if self.over_18 and not (v and v.over_18) and not (self.post and self.post.over_18): return True - - if self.is_banned: return True - + # we no longer collapse removed things; the mods want to see them, non-mods see a placeholder anyway if v and v.filter_words and self.body and any(x in self.body for x in v.filter_words): return True - return False @property @lazy - def is_op(self): return self.author_id==self.post.author_id + def is_op(self): + return self.author_id == self.post.author_id + @property @lazy - def active_flags(self, v): return len(self.flags(v)) + def is_comment(self) -> bool: + ''' + Returns whether this is an actual comment (i.e. not a private message) + ''' + return bool(self.parent_submission) + + @property + @lazy + def is_message(self) -> bool: + ''' + Returns whether this is a private message or modmail + ''' + return not self.is_comment + + @property + @lazy + def is_strict_message(self) -> bool: + ''' + Returns whether this is a private message or modmail + but is not a notification + ''' + return self.is_message and not self.is_notification + + @property + @lazy + def is_modmail(self) -> bool: + ''' + Returns whether this is a modmail message + ''' + if not self.is_message: return False + if self.sentto == MODMAIL_ID: return True + + top_comment: Optional["Comment"] = self.top_comment + return bool(top_comment.sentto == MODMAIL_ID) + + @property + @lazy + def is_notification(self) -> bool: + ''' + Returns whether this is a notification + ''' + return self.is_message and not self.sentto + + @lazy + def header_msg(self, v, is_notification_page: bool) -> str: + ''' + Returns a message that is in the header for a comment, usually for + display on a notification page. + ''' + if self.post: + post_html:str = f"{self.post.realtitle(v)}" + if v: + if self.level > 1 and v.id == self.parent_comment.author_id: + text = "Comment Reply" + elif self.level == 1 and v.id == self.post.author_id: + text = "Post Reply" + elif self.parent_submission in v.subscribed_idlist(): + text = "Subscribed Thread" + else: + text = "Username Mention" + if is_notification_page: + return f"{text}: {post_html}" + return post_html + elif self.author_id in {AUTOJANNY_ID, NOTIFICATIONS_ID}: + return "Notification" + elif self.sentto == MODMAIL_ID: + return "Sent to admins" + else: + return f"Sent to @{self.senttouser.username}" + + @lazy + def voted_display(self, v) -> int: + ''' + Returns data used to modify how to show the vote buttons. + + :returns: A number between `-2` and `1`. `-2` is returned if `v` is + `None`. `1` is returned if the user is the comment author. + Otherwise, a value of `-1` (downvote),` 0` (no vote or no data), or `1` + (upvote) is returned. + ''' + if not v: return -2 + if v.id == self.author_id: return 1 + return getattr(self, 'voted', 0) + + def sticky_api_url(self, v) -> str | None: + ''' + Returns the API URL used to sticky this comment. + :returns: Currently `None` always. Stickying comments was disabled + UI-side on TheMotte. + ''' + return None + if not self.is_comment: return None + if not v: return None + if v.admin_level >= 2: + return 'sticky_comment' + elif v.id == self.post.author_id: + return 'pin_comment' + return None + + @lazy + def active_flags(self, v): + return len(self.flags(v)) + + @lazy + def show_descendants(self, v:"User | None") -> bool: + if self.visibility_state.is_visible_to(v, getattr(self, 'is_blocking', False)): + return True + return bool(self.descendant_count) + + @lazy + def visibility_and_message(self, v:"User | None") -> tuple[bool, str]: + ''' + Returns a tuple of whether this content is visible and a publicly + visible message to accompany it. The visibility state machine is + a slight mess but... this should at least unify the state checks. + ''' + return self.visibility_state.visibility_and_message( + v, getattr(self, 'is_blocking', False)) + + @property + def visibility_state(self) -> VisibilityState: + return VisibilityState.from_submittable(self) + + def volunteer_janitor_is_unknown(self): + return self.volunteer_janitor_badness > 0.4 and self.volunteer_janitor_badness < 0.6 + + def volunteer_janitor_is_bad(self): + return self.volunteer_janitor_badness >= 0.6 + + def volunteer_janitor_is_notbad(self): + return self.volunteer_janitor_badness <= 0.4 + + def volunteer_janitor_confidence(self): + unitconfidence = (abs(self.volunteer_janitor_badness - 0.5) * 2) + unitanticonfidence = 1 - unitconfidence + logconfidence = -math.log(unitanticonfidence, 2) + return round(logconfidence * 10) + + def volunteer_janitor_css(self): + if self.volunteer_janitor_is_unknown(): + category = "unknown" + elif self.volunteer_janitor_is_bad(): + category = "bad" + elif self.volunteer_janitor_is_notbad(): + category = "notbad" + + strength = clamp(math.trunc(self.volunteer_janitor_confidence() / 10), 0, 3) + + return f"{category}_{strength}" diff --git a/files/classes/cron/pycallable.py b/files/classes/cron/pycallable.py new file mode 100644 index 000000000..5be567d01 --- /dev/null +++ b/files/classes/cron/pycallable.py @@ -0,0 +1,55 @@ +import importlib +from types import ModuleType +from typing import Callable + +from sqlalchemy.schema import Column, ForeignKey +from sqlalchemy.sql.sqltypes import Integer, String + +from files.classes.cron.tasks import (RepeatableTask, ScheduledTaskType, + TaskRunContext) + +__all__ = ('PythonCodeTask',) + +class PythonCodeTask(RepeatableTask): + ''' + A repeatable task that is naively calls a Python callable. It can access + all of the app state as that is provided to the callee. An example task + that sets the fictional `hams` variable on a `User` to `eggs` is shown + below. + + ```py + from files.classes.user import User + from files.classes.cron.tasks import TaskRunContext + from files.helpers.get import get_account + + def spam_task(ctx:TaskRunContext): + user:Optional[User] = get_account(1784, graceful=True, db=ctx.db) + if not user: raise Exception("User not found!") + user.hams = "eggs" + ctx.db.commit() + ``` + + The `import_path` and `callable` properties are passed to `importlib` + which then imports the module and then calls the callable with the current + task context. + ''' + + __tablename__ = "tasks_repeatable_python" + + __mapper_args__ = { + "polymorphic_identity": int(ScheduledTaskType.PYTHON_CALLABLE), + } + + id = Column(Integer, ForeignKey(RepeatableTask.id), primary_key=True) + import_path = Column(String, nullable=False) + callable = Column(String, nullable=False) + + def run_task(self, ctx:TaskRunContext): + self.get_function()(ctx) + + def get_function(self) -> Callable[[TaskRunContext], None]: + module:ModuleType = importlib.import_module(self.import_path) + fn = getattr(module, self.callable, None) + if not callable(fn): + raise TypeError("Name either does not exist or is not callable") + return fn diff --git a/files/classes/cron/submission.py b/files/classes/cron/submission.py new file mode 100644 index 000000000..89155b35c --- /dev/null +++ b/files/classes/cron/submission.py @@ -0,0 +1,185 @@ +import functools +from datetime import datetime, timezone + +from sqlalchemy.orm import relationship +from sqlalchemy.schema import Column, ForeignKey +from sqlalchemy.sql.sqltypes import Boolean, Integer, String, Text + +from files.classes.cron.tasks import (RepeatableTask, ScheduledTaskType, + TaskRunContext) +from files.classes.submission import Submission +from files.classes.visstate import StateMod, StateReport, VisibilityState +from files.helpers.config.const import SUBMISSION_TITLE_LENGTH_MAXIMUM +from files.helpers.content import body_displayed +from files.helpers.lazy import lazy +from files.helpers.sanitize import filter_emojis_only + +__all__ = ('ScheduledSubmissionTask',) + + +class ScheduledSubmissionTask(RepeatableTask): + __tablename__ = "tasks_repeatable_scheduled_submissions" + + __mapper_args__ = { + "polymorphic_identity": int(ScheduledTaskType.SCHEDULED_SUBMISSION), + } + + id = Column(Integer, ForeignKey(RepeatableTask.id), primary_key=True) + author_id_submission = Column(Integer, ForeignKey("users.id"), nullable=False) + ghost = Column(Boolean, default=False, nullable=False) + private = Column(Boolean, default=False, nullable=False) + over_18 = Column(Boolean, default=False, nullable=False) + is_bot = Column(Boolean, default=False, nullable=False) + title = Column(String(SUBMISSION_TITLE_LENGTH_MAXIMUM), nullable=False) + url = Column(String) + body = Column(Text) + body_html = Column(Text) + flair = Column(String) + embed_url = Column(String) + + author = relationship("User", foreign_keys=author_id_submission) + task = relationship(RepeatableTask) + submissions = relationship(Submission, + back_populates="task", order_by="Submission.id.desc()") + + def run_task(self, ctx:TaskRunContext) -> None: + submission:Submission = self.make_submission(ctx) + with ctx.app_context(): + # TODO: stop using app context (currently required for sanitize and + # username pings) + submission.submit(ctx.db) # TODO: thumbnails + submission.publish() + + def make_submission(self, ctx:TaskRunContext) -> Submission: + title:str = self.make_title(ctx.trigger_time) + title_html:str = filter_emojis_only(title, graceful=True) + if len(title_html) > 1500: raise ValueError("Rendered title too large") + + return Submission( + created_utc=int(ctx.trigger_time.timestamp()), + private=self.private, + author_id=self.author_id_submission, + over_18=self.over_18, + app_id=None, + is_bot =self.is_bot, + title=title, + title_html=title_html, + url=self.url, + body=self.body, + body_html=self.body_html, + flair=self.flair, + ghost=self.ghost, + state_mod=StateMod.VISIBLE, + embed_url=self.embed_url, + task_id=self.id, + ) + + def make_title(self, trigger_time:datetime) -> str: + return trigger_time.strftime(self.title) + + # properties below here are mocked in order to reuse part of the submission + # HTML template for previewing a submitted task + + @property + def state_user_deleted_utc(self) -> datetime | None: + return datetime.now(tz=timezone.utc) if not self.task.enabled else None + + @functools.cached_property + def title_html(self) -> str: + ''' + This is used as a mock property for display in submission listings that + contain scheduled posts. + + .. warning:: + This property should not be used for generating the HTML for an actual + submission as this will be missing the special formatting that may be + applies to titles. Instead call + `ScheduledSubmissionContext.make_title()` with the `datetime` that the + event was triggered at. + ''' + return filter_emojis_only(self.title) + + @property + def author_name(self) -> str: + return self.author.username + + @property + def upvotes(self) -> int: + return 1 + + @property + def score(self) -> int: + return 1 + + @property + def downvotes(self) -> int: + return 0 + + @property + def realupvotes(self) -> int: + return 1 + + @property + def comment_count(self) -> int: + return 0 + + @property + def views(self) -> int: + return 0 + + @property + def state_mod(self) -> StateMod: + return StateMod.VISIBLE + + def award_count(self, kind): + return 0 + + @lazy + def realurl(self, v): + return Submission.realurl(self, v) + + def realbody(self, v): + return body_displayed(self, v, is_html=True) + + def plainbody(self, v): + return body_displayed(self, v, is_html=False) + + @lazy + def realtitle(self, v): + return self.title_html if self.title_html else self.title + + @lazy + def plaintitle(self, v): + return self.title + + @property + def permalink(self): + return f"/tasks/scheduled_posts/{self.id}" + + @property + def shortlink(self): + return self.permalink + + @property + def is_real_submission(self) -> bool: + return False + + @property + def should_hide_score(self) -> bool: + return True + + @property + def edit_url(self) -> str: + return f"/tasks/scheduled_posts/{self.id}/content" + + @property + def visibility_state(self) -> VisibilityState: + return VisibilityState( + state_mod=StateMod.VISIBLE, + state_mod_set_by=None, + state_report=StateReport.UNREPORTED, + deleted=False, # we only want to show deleted UI color if disabled + op_shadowbanned=False, + op_id=self.author_id_submission, + op_name_safe=self.author_name + ) diff --git a/files/classes/cron/tasks.py b/files/classes/cron/tasks.py new file mode 100644 index 000000000..52a45bb6f --- /dev/null +++ b/files/classes/cron/tasks.py @@ -0,0 +1,401 @@ +from __future__ import annotations + +import contextlib +import dataclasses +from datetime import date, datetime, timedelta, timezone +from enum import IntEnum, IntFlag +from typing import TYPE_CHECKING, Final, Optional, Union + +import flask +import flask_caching +import flask_mail +import redis +from sqlalchemy.orm import relationship, Session +from sqlalchemy.schema import Column, ForeignKey +from sqlalchemy.sql.sqltypes import (Boolean, DateTime, Integer, SmallInteger, + Text, Time, String) + +from files.classes.base import CreatedBase +from files.helpers.time import format_age, format_datetime + +if TYPE_CHECKING: + from files.classes.user import User + +class ScheduledTaskType(IntEnum): + PYTHON_CALLABLE = 1 + SCHEDULED_SUBMISSION = 2 + + def __str__(self): + if not self.name: return super().__str__() + return self.name.replace('_', ' ').title() + + +class ScheduledTaskState(IntEnum): + WAITING = 1 + ''' + A task waiting to be triggered + ''' + RUNNING = 2 + ''' + A task that is currently running + ''' + + +class DayOfWeek(IntFlag): + SUNDAY = 1 << 1 + MONDAY = 1 << 2 + TUESDAY = 1 << 3 + WEDNESDAY = 1 << 4 + THURSDAY = 1 << 5 + FRIDAY = 1 << 6 + SATURDAY = 1 << 7 + + WEEKDAYS = MONDAY | TUESDAY | WEDNESDAY | THURSDAY | FRIDAY + WEEKENDS = SATURDAY | SUNDAY + + NONE = 0 << 0 + ALL = WEEKDAYS | WEEKENDS + + @classmethod + @property + def all_days(cls) -> list["DayOfWeek"]: + return [ + cls.SUNDAY, cls.MONDAY, cls.TUESDAY, cls.WEDNESDAY, + cls.THURSDAY, cls.FRIDAY, cls.SATURDAY + ] + + @property + def empty(self) -> bool: + return self not in self.ALL + + def __contains__(self, other:Union[date, "DayOfWeek"]) -> bool: + _days:dict[int, "DayOfWeek"] = { + 0: self.MONDAY, + 1: self.TUESDAY, + 2: self.WEDNESDAY, + 3: self.THURSDAY, + 4: self.FRIDAY, + 5: self.SATURDAY, + 6: self.SUNDAY + } + if not isinstance(other, date): + return super().__contains__(other) + weekday:int = other.weekday() + if not 0 <= weekday <= 6: + raise Exception( + f"Unexpected weekday value (got {weekday}, expected 0-6)") + return _days[weekday] in self + + +_UserConvertible = Union["User", str, int] + +@dataclasses.dataclass(frozen=True, kw_only=True, slots=True) +class TaskRunContext: + ''' + A full task run context, with references to all app globals embedded. + This is the entirety of the application's global state at this point. + + This is explicit state. This is useful so scheduled tasks do not have + to import from `files.__main__` and so they can use all of the features + of the application without being in a request context. + ''' + app:flask.app.Flask + ''' + The application. Many of the app functions use the app context globals and + do not have their state explicitly passed. This is a convenience get out of + jail free card so that most features (excepting those that require a + `request` context can be used.) + ''' + cache:flask_caching.Cache + ''' + A cache extension. This is useful for situations where a scheduled task + might want to interact with the cache in some way (for example invalidating + or adding something to the cache.) + ''' + db:Session + ''' + A database session. Useful for when a task needs to modify something in the + database (for example creating a submission) + ''' + mail:flask_mail.Mail + ''' + The mail extension. Needed for sending emails. + ''' + redis:redis.Redis + ''' + A direct reference to our redis connection. Normally most operations that + involve the redis datastore use flask_caching's Cache object (accessed via + the `cache` property), however this is provided as a convenience for more + granular redis operations. + ''' + task:RepeatableTask + ''' + A reference to the task that is being ran. + ''' + task_run:RepeatableTaskRun + ''' + A reference to this current run of the task. + ''' + trigger_time:datetime + ''' + The date and time (UTC) that this task was triggered + ''' + + @property + def run_time(self) -> datetime: + ''' + The date and time (UTC) that this task was actually ran + ''' + return self.task_run.created_datetime_py + + @contextlib.contextmanager + def app_context(self, *, v:Optional[_UserConvertible]=None): + ''' + Context manager that uses `self.app` to generate an app context and set + up the application with expected globals. This assigns `g.db`, `g.v`, + and `g.debug`. + + This is intended for use with legacy code that does not pass state + explicitly and instead relies on the use of `g` for state passing. If + at all possible, state should be passed explicitly to functions that + require it. + + Usage is simple: + ```py + with ctx.app_context() as app_ctx: + # code that requires g + ``` + + Any code that uses `g` can be ran here. As this is intended for + scenarios that may be outside of a request context code that uses the + request context may still raise `RuntimeException`s. + + An example + + ```py + from flask import g, request # import works ok + + def legacy_function(): + u:Optional[User] = g.db.get(User, 1784) # works ok! :) + u.admin_level = \\ + request.values.get("admin_level", default=9001, type=int) + # raises a RuntimeError :( + g.db.commit() + ``` + + This is because there is no actual request being made. Creating a + mock request context is doable but outside of the scope of this + function as this is often not needed outside of route handlers (where + this function is out of scope anyway). + + :param v: A `User`, an `int`, a `str`, or `None`. `g.v` will be set + using the following rules: + + 1. If `v` is an `int`, `files.helpers.get_account` is called and the + result of that is stored in `g.v`. + + 2. If `v` is an `str`, `files.helpers.get_user` is called and the + result of that is stored in `g.v`. + + 3. If `v` is a `User`, it is stored in `g.v`. + + It is expected that callees will provide a valid user ID or username. + If an invalid one is provided, *no* exception will be raised and `g.v` + will be set to `None`. + + This is mainly provided as an optional feature so that tasks can be + somewhat "sudo"ed as a particular user. Note that `g.v` is always + assigned (even if to `None`) in order to prevent code that depends on + its existence from raising an exception. + ''' + with self.app.app_context() as app_ctx: + app_ctx.g.db = self.db + + from files.helpers.get import get_account, get_user + + if isinstance(v, str): + v = get_user(v, graceful=True) + elif isinstance(v, int): + v = get_account(v, graceful=True, db=self.db) + + app_ctx.g.v = v + app_ctx.g.debug = self.app.debug + yield app_ctx + + @contextlib.contextmanager + def db_transaction(self): + try: + yield + self.db.commit() + except: + self.db.rollback() + + +_TABLE_NAME: Final[str] = "tasks_repeatable" + + +class RepeatableTask(CreatedBase): + __tablename__ = _TABLE_NAME + + id = Column(Integer, primary_key=True, nullable=False) + author_id = Column(Integer, ForeignKey("users.id"), nullable=False) + type_id = Column(SmallInteger, nullable=False) + enabled = Column(Boolean, default=True, nullable=False) + run_state = Column(SmallInteger, default=int(ScheduledTaskState.WAITING), nullable=False) + run_time_last = Column(DateTime, default=None) + + frequency_day = Column(SmallInteger, nullable=False) + time_of_day_utc = Column(Time, nullable=False) + + # used for the cron hardcoding system + label = Column(String, nullable=True, unique=True) + + runs = relationship("RepeatableTaskRun", back_populates="task") + + @property + def type(self) -> ScheduledTaskType: + return ScheduledTaskType(self.type_id) + + @type.setter + def type(self, value:ScheduledTaskType): + self.type_id = value + + @property + def frequency_day_flags(self) -> DayOfWeek: + return DayOfWeek(self.frequency_day) + + @frequency_day_flags.setter + def frequency_day_flags(self, value:DayOfWeek): + self.frequency_day = int(value) + + @property + def run_state_enum(self) -> ScheduledTaskState: + return ScheduledTaskState(self.run_state) + + @run_state_enum.setter + def run_state_enum(self, value:ScheduledTaskState): + self.run_state = int(value) + + @property + def run_time_last_or_created_utc(self) -> datetime: + return self.run_time_last or self.created_datetime_py + + @property + def run_time_last_str(self) -> str: + if not self.run_time_last: return 'Never' + return (f'{format_datetime(self.run_time_last)} ' + f'({format_age(self.run_time_last)})') + + @property + def trigger_time(self) -> datetime | None: + return self.next_trigger(self.run_time_last_or_created_utc) + + def can_run(self, now: datetime) -> bool: + return not ( + self.trigger_time is None + or now < self.trigger_time + or self.run_state_enum != ScheduledTaskState.WAITING) + + def next_trigger(self, anchor: datetime) -> datetime | None: + if not self.enabled: return None + if self.frequency_day_flags.empty: return None + + day:timedelta = timedelta(1.0) + target_date:datetime = anchor - day # incremented at start of for loop + + for i in range(8): + target_date = target_date + day + if i == 0 and target_date.time() > self.time_of_day_utc: continue + if target_date in self.frequency_day_flags: break + else: + raise Exception("Could not find suitable timestamp to run next task") + + return datetime.combine(target_date, self.time_of_day_utc, tzinfo=timezone.utc) # type: ignore + + def run(self, db: Session, trigger_time: datetime) -> RepeatableTaskRun: + run:RepeatableTaskRun = RepeatableTaskRun(task_id=self.id) + try: + from files.__main__ import app, cache, mail, r # i know + ctx: TaskRunContext = TaskRunContext( + app=app, + cache=cache, + db=db, + mail=mail, + redis=r, + task=self, + task_run=run, + trigger_time=trigger_time, + ) + self.run_task(ctx) + except Exception as e: + run.exception = e + run.completed_utc = datetime.now(tz=timezone.utc) + db.add(run) + return run + + def run_task(self, ctx:TaskRunContext): + raise NotImplementedError() + + def contains_day_str(self, day_str:str) -> bool: + return (bool(day_str) + and DayOfWeek[day_str.upper()] in self.frequency_day_flags) + + def __repr__(self) -> str: + return f'<{self.__class__.__name__}(id={self.id}, created_utc={self.created_date}, author_id={self.author_id})>' + + __mapper_args__ = { + "polymorphic_on": type_id, + } + + +class RepeatableTaskRun(CreatedBase): + __tablename__ = "tasks_repeatable_runs" + + id = Column(Integer, primary_key=True) + task_id = Column(Integer, ForeignKey(RepeatableTask.id), nullable=False) + manual = Column(Boolean, default=False, nullable=False) + traceback_str = Column(Text, nullable=True) + + completed_utc = Column(DateTime) + + task = relationship(RepeatableTask, back_populates="runs") + + _exception: Optional[Exception] = None # not part of the db model + + @property + def completed_datetime_py(self) -> datetime | None: + if self.completed_utc is None: + return None + return datetime.combine( + self.completed_utc.date(), + self.completed_utc.time(), + timezone.utc) + + @property + def completed_datetime_str(self) -> str: + return format_datetime(self.completed_utc) + + @property + def status_text(self) -> str: + if not self.completed_utc: return "Running" + return "Failed" if self.traceback_str else "Completed" + + @property + def time_elapsed(self) -> Optional[timedelta]: + if self.completed_datetime_py is None: return None + return self.completed_datetime_py - self.created_datetime_py + + @property + def time_elapsed_str(self) -> str: + elapsed:Optional[timedelta] = self.time_elapsed + if not elapsed: return '' + return str(elapsed) + + @property + def exception(self) -> Optional[Exception]: + return self._exception + + @exception.setter + def exception(self, value: Optional[Exception]) -> None: + self._exception = value + self.traceback_str = str(value) if value else None diff --git a/files/classes/domains.py b/files/classes/domains.py index f9acb7b32..10da5862e 100644 --- a/files/classes/domains.py +++ b/files/classes/domains.py @@ -1,8 +1,7 @@ from sqlalchemy import * -from files.__main__ import Base +from files.classes.base import Base class BannedDomain(Base): - __tablename__ = "banneddomains" domain = Column(String, primary_key=True) reason = Column(String, nullable=False) diff --git a/files/classes/exiles.py b/files/classes/exiles.py deleted file mode 100644 index f02da7133..000000000 --- a/files/classes/exiles.py +++ /dev/null @@ -1,18 +0,0 @@ -from sqlalchemy import * -from sqlalchemy.orm import relationship -from files.__main__ import Base - -class Exile(Base): - - __tablename__ = "exiles" - user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) - sub = Column(String, ForeignKey("subs.name"), primary_key=True) - exiler_id = Column(Integer, ForeignKey("users.id"), nullable=False) - - Index('fki_exile_exiler_fkey', exiler_id) - Index('fki_exile_sub_fkey', sub) - - exiler = relationship("User", primaryjoin="User.id==Exile.exiler_id", viewonly=True) - - def __repr__(self): - return f"<{self.__class__.__name__}(user_id={self.user_id}, sub={self.sub})>" diff --git a/files/classes/flags.py b/files/classes/flags.py index 12804d7a0..3f74c720a 100644 --- a/files/classes/flags.py +++ b/files/classes/flags.py @@ -1,77 +1,46 @@ from sqlalchemy import * from sqlalchemy.orm import relationship -from files.__main__ import Base +from files.classes.base import CreatedBase, CreatedDateTimeBase, Base from files.helpers.lazy import lazy -from files.helpers.const import * -import time +from files.helpers.config.const import * -class Flag(Base): + +class Flag(CreatedBase): __tablename__ = "flags" id = Column(Integer, primary_key=True) post_id = Column(Integer, ForeignKey("submissions.id"), nullable=False) user_id = Column(Integer, ForeignKey("users.id"), nullable=False) reason = Column(String) - created_utc = Column(Integer, nullable=False) Index('flag_user_idx', user_id) user = relationship("User", primaryjoin = "Flag.user_id == User.id", uselist = False, viewonly=True) - def __init__(self, *args, **kwargs): - if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time()) - super().__init__(*args, **kwargs) - def __repr__(self): return f"<{self.__class__.__name__}(id={self.id})>" - @property - @lazy - def created_date(self): - return time.strftime("%d %B %Y", time.gmtime(self.created_utc)) - - @property - @lazy - def created_datetime(self): - return time.strftime("%d/%B/%Y %H:%M:%S UTC", time.gmtime(self.created_utc)) - @lazy def realreason(self, v): return self.reason -class CommentFlag(Base): - +class CommentFlag(CreatedDateTimeBase): __tablename__ = "commentflags" id = Column(Integer, primary_key=True) comment_id = Column(Integer, ForeignKey("comments.id"), nullable=False) user_id = Column(Integer, ForeignKey("users.id"), nullable=False) reason = Column(String) - created_utc = Column(Integer, nullable=False) Index('cflag_user_idx', user_id) user = relationship("User", primaryjoin = "CommentFlag.user_id == User.id", uselist = False, viewonly=True) - def __init__(self, *args, **kwargs): - if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time()) - super().__init__(*args, **kwargs) - def __repr__(self): return f"<{self.__class__.__name__}(id={self.id})>" - @property - @lazy - def created_date(self): - return time.strftime("%d %B %Y", time.gmtime(self.created_utc)) - - @property - @lazy - def created_datetime(self): - return time.strftime("%d/%B/%Y %H:%M:%S UTC", time.gmtime(self.created_utc)) - @lazy def realreason(self, v): return self.reason diff --git a/files/classes/follows.py b/files/classes/follows.py index c372adafb..8bffa1e9d 100644 --- a/files/classes/follows.py +++ b/files/classes/follows.py @@ -1,22 +1,19 @@ from sqlalchemy import * from sqlalchemy.orm import relationship -from files.__main__ import Base -import time +from files.classes.base import CreatedBase -class Follow(Base): +class Follow(CreatedBase): __tablename__ = "follows" target_id = Column(Integer, ForeignKey("users.id"), primary_key=True) user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) - created_utc = Column(Integer, nullable=False) Index('follow_user_id_index', user_id) user = relationship("User", uselist=False, primaryjoin="User.id==Follow.user_id", viewonly=True) target = relationship("User", primaryjoin="User.id==Follow.target_id", viewonly=True) - def __init__(self, *args, **kwargs): - if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time()) - super().__init__(*args, **kwargs) - def __repr__(self): - return f"<{self.__class__.__name__}(id={self.id})>" + return ( + f"<{self.__class__.__name__}(" + f"target_id={self.target_id}, user_id={self.user_id})>" + ) diff --git a/files/classes/leaderboard.py b/files/classes/leaderboard.py index 4697d39bf..b505901b8 100644 --- a/files/classes/leaderboard.py +++ b/files/classes/leaderboard.py @@ -2,9 +2,9 @@ from dataclasses import dataclass from typing import Any, Callable, Final, Optional from sqlalchemy import Column, func -from sqlalchemy.orm import scoped_session, Query +from sqlalchemy.orm import Session, Query -from files.helpers.const import LEADERBOARD_LIMIT +from files.helpers.config.const import LEADERBOARD_LIMIT from files.classes.badges import Badge from files.classes.marsey import Marsey @@ -51,9 +51,9 @@ class Leaderboard: raise NotImplementedError() class SimpleLeaderboard(Leaderboard): - def __init__(self, v:User, meta:LeaderboardMeta, db:scoped_session, users_query:Query, column:Column): + def __init__(self, v:User, meta:LeaderboardMeta, db:Session, users_query:Query, column:Column): super().__init__(v, meta) - self.db:scoped_session = db + self.db:Session = db self.users_query:Query = users_query self.column:Column = column self._calculate() @@ -62,7 +62,7 @@ class SimpleLeaderboard(Leaderboard): self._all_users = self.users_query.order_by(self.column.desc()).limit(self.meta.limit).all() if self.v not in self._all_users: sq = self.db.query(User.id, self.column, func.rank().over(order_by=self.column.desc()).label("rank")).subquery() - sq_data = self.db.query(sq.c.id, sq.c.column, sq.c.rank).filter(sq.c.id == self.v.id).limit(1).one() + sq_data = self.db.query(sq.c.id, sq.c[self.column.name], sq.c.rank).filter(sq.c.id == self.v.id).limit(1).one() self._v_value:int = sq_data[1] self._v_position:int = sq_data[2] @@ -92,9 +92,9 @@ class _CountedAndRankedLeaderboard(Leaderboard): return func.rank().over(order_by=func.count(criteria).desc()).label("rank") class BadgeMarseyLeaderboard(_CountedAndRankedLeaderboard): - def __init__(self, v:User, meta:LeaderboardMeta, db:scoped_session, column:Column): + def __init__(self, v:User, meta:LeaderboardMeta, db:Session, column:Column): super().__init__(v, meta) - self.db:scoped_session = db + self.db:Session = db self.column = column self._calculate() @@ -132,12 +132,12 @@ class BadgeMarseyLeaderboard(_CountedAndRankedLeaderboard): @property def value_func(self) -> Callable[[User], int]: - return lambda u:self._all_users[u] + return lambda u: self._all_users[u] class UserBlockLeaderboard(_CountedAndRankedLeaderboard): - def __init__(self, v:User, meta:LeaderboardMeta, db:scoped_session, column:Column): + def __init__(self, v:User, meta:LeaderboardMeta, db:Session, column:Column): super().__init__(v, meta) - self.db:scoped_session = db + self.db:Session = db self.column = column self._calculate() @@ -168,8 +168,12 @@ class UserBlockLeaderboard(_CountedAndRankedLeaderboard): def v_value(self) -> int: return self._v_value + @property + def value_func(self) -> Callable[[User], int]: + return lambda u: self._all_users[u] + class RawSqlLeaderboard(Leaderboard): - def __init__(self, meta:LeaderboardMeta, db:scoped_session, query:str) -> None: # should be LiteralString on py3.11+ + def __init__(self, meta:LeaderboardMeta, db:Session, query:str) -> None: # should be LiteralString on py3.11+ super().__init__(None, meta) self.db = db self._calculate(query) @@ -234,7 +238,7 @@ FROM cv_for_user cvfu ORDER BY count DESC LIMIT 25 """ - def __init__(self, meta:LeaderboardMeta, db:scoped_session) -> None: + def __init__(self, meta:LeaderboardMeta, db:Session) -> None: super().__init__(meta, db, self._query) class GivenUpvotesLeaderboard(RawSqlLeaderboard): @@ -248,5 +252,5 @@ FULL OUTER JOIN (SELECT user_id, COUNT(*) FROM commentvotes WHERE vote_type = 1 ORDER BY count DESC LIMIT 25 """ - def __init__(self, meta:LeaderboardMeta, db:scoped_session) -> None: + def __init__(self, meta:LeaderboardMeta, db:Session) -> None: super().__init__(meta, db, self._query) diff --git a/files/classes/marsey.py b/files/classes/marsey.py index 4aba397f9..06edcc208 100644 --- a/files/classes/marsey.py +++ b/files/classes/marsey.py @@ -1,5 +1,5 @@ from sqlalchemy import * -from files.__main__ import Base +from files.classes.base import Base class Marsey(Base): __tablename__ = "marseys" diff --git a/files/classes/mod.py b/files/classes/mod.py deleted file mode 100644 index fb8812a95..000000000 --- a/files/classes/mod.py +++ /dev/null @@ -1,26 +0,0 @@ -from sqlalchemy import * -from sqlalchemy.orm import relationship -from files.__main__ import Base -from files.helpers.lazy import * -import time - -class Mod(Base): - - __tablename__ = "mods" - user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) - sub = Column(String, ForeignKey("subs.name"), primary_key=True) - created_utc = Column(Integer, nullable=False) - - Index('fki_mod_sub_fkey', sub) - - def __init__(self, *args, **kwargs): - if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time()) - super().__init__(*args, **kwargs) - - def __repr__(self): - return f"<{self.__class__.__name__}(user_id={self.user_id}, sub={self.sub})>" - - @property - @lazy - def created_datetime(self): - return time.strftime("%d/%B/%Y %H:%M:%S UTC", time.gmtime(self.created_utc)) diff --git a/files/classes/mod_logs.py b/files/classes/mod_logs.py index 278a1a5ed..245bb98fc 100644 --- a/files/classes/mod_logs.py +++ b/files/classes/mod_logs.py @@ -1,14 +1,15 @@ +import logging +from copy import deepcopy + from sqlalchemy import * from sqlalchemy.orm import relationship -from files.__main__ import Base -import time -from files.helpers.lazy import lazy -from os import environ -from copy import deepcopy -from files.helpers.const import * -import logging -class ModAction(Base): +from files.classes.base import CreatedBase +from files.helpers.config.const import * +from files.helpers.lazy import lazy + + +class ModAction(CreatedBase): __tablename__ = "modactions" id = Column(Integer, primary_key=True) user_id = Column(Integer, ForeignKey("users.id")) @@ -17,7 +18,6 @@ class ModAction(Base): target_submission_id = Column(Integer, ForeignKey("submissions.id")) target_comment_id = Column(Integer, ForeignKey("comments.id")) _note=Column(String) - created_utc = Column(Integer, nullable=False) Index('fki_modactions_user_fkey', target_user_id) Index('modaction_action_idx', kind) @@ -29,49 +29,12 @@ class ModAction(Base): target_user = relationship("User", primaryjoin="User.id==ModAction.target_user_id", viewonly=True) target_post = relationship("Submission", viewonly=True) - def __init__(self, *args, **kwargs): - if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time()) - super().__init__(*args, **kwargs) - def __repr__(self): return f"<{self.__class__.__name__}(id={self.id})>" - @property - @lazy - def age_string(self): - - age = int(time.time()) - self.created_utc - - if age < 60: - return "just now" - elif age < 3600: - minutes = int(age / 60) - return f"{minutes}m ago" - elif age < 86400: - hours = int(age / 3600) - return f"{hours}hr ago" - elif age < 2678400: - days = int(age / 86400) - return f"{days}d ago" - - now = time.gmtime() - ctd = time.gmtime(self.created_utc) - - months = now.tm_mon - ctd.tm_mon + 12 * (now.tm_year - ctd.tm_year) - if now.tm_mday < ctd.tm_mday: - months -= 1 - - if months < 12: - return f"{months}mo ago" - else: - years = int(months / 12) - return f"{years}yr ago" - - @property def note(self): - - if self.kind=="ban_user": + if self.kind == "ban_user": if self.target_post: return f'for post' elif self.target_comment_id: return f'for comment' else: return self._note @@ -92,11 +55,9 @@ class ModAction(Base): @property @lazy def string(self): - output = self.lookup_action_type()["str"].format(self=self, cc=CC_TITLE) - - if self.note: output += f" ({self.note})" - + if not self.note: return output + output += f" ({self.note})" return output @property @@ -124,6 +85,26 @@ class ModAction(Base): return f"/log/{self.id}" ACTIONTYPES = { + 'approve_post': { + "str": 'approved post {self.target_link}', + "icon": 'fa-feather-alt', + "color": 'bg-success' + }, + 'approve_comment': { + "str": 'approved {self.target_link}', + "icon": 'fa-comment', + "color": 'bg-success' + }, + 'remove_post': { + "str": 'removed post {self.target_link}', + "icon": 'fa-feather-alt', + "color": 'bg-danger' + }, + 'remove_comment': { + "str": 'removed {self.target_link}', + "icon": 'fa-comment', + "color": 'bg-danger' + }, 'approve_app': { "str": 'approved an application by {self.target_link}', "icon": 'fa-robot', @@ -139,21 +120,11 @@ ACTIONTYPES = { "icon": 'fa-badge', "color": 'bg-danger' }, - 'ban_comment': { - "str": 'removed {self.target_link}', - "icon": 'fa-comment', - "color": 'bg-danger' - }, 'ban_domain': { "str": 'banned a domain', "icon": 'fa-globe', "color": 'bg-danger' }, - 'ban_post': { - "str": 'removed post {self.target_link}', - "icon": 'fa-feather-alt', - "color": 'bg-danger' - }, 'ban_user': { "str": 'banned user {self.target_link}', "icon": 'fa-user-slash', @@ -279,11 +250,6 @@ ACTIONTYPES = { "icon": 'fa-sack-dollar', "color": 'bg-success' }, - 'move_hole': { - "str": 'moved {self.target_link} to /h/{self.target_post.sub}', - "icon": 'fa-manhole', - "color": 'bg-primary' - }, 'nuke_user': { "str": 'removed all content of {self.target_link}', "icon": 'fa-radiation-alt', @@ -344,21 +310,11 @@ ACTIONTYPES = { "icon": 'fa-eye-slash', "color": 'bg-danger' }, - 'unban_comment': { - "str": 'reinstated {self.target_link}', - "icon": 'fa-comment', - "color": 'bg-success' - }, 'unban_domain': { "str": 'unbanned a domain', "icon": 'fa-globe', "color": 'bg-success' }, - 'unban_post': { - "str": 'reinstated post {self.target_link}', - "icon": 'fa-feather-alt', - "color": 'bg-success' - }, 'unban_user': { "str": 'unbanned user {self.target_link}', "icon": 'fa-user', diff --git a/files/classes/notifications.py b/files/classes/notifications.py index a8a190513..b4b87510b 100644 --- a/files/classes/notifications.py +++ b/files/classes/notifications.py @@ -1,16 +1,13 @@ from sqlalchemy import * from sqlalchemy.orm import relationship -from files.__main__ import Base -import time - -class Notification(Base): +from files.classes.base import CreatedBase +class Notification(CreatedBase): __tablename__ = "notifications" user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) comment_id = Column(Integer, ForeignKey("comments.id"), primary_key=True) read = Column(Boolean, default=False, nullable=False) - created_utc = Column(Integer, nullable=False) Index('notification_read_idx', read) Index('notifications_comment_idx', comment_id) @@ -19,9 +16,5 @@ class Notification(Base): comment = relationship("Comment", viewonly=True) user = relationship("User", viewonly=True) - def __init__(self, *args, **kwargs): - if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time()) - super().__init__(*args, **kwargs) - def __repr__(self): - return f"<{self.__class__.__name__}(id={self.id})>" + return f"<{self.__class__.__name__}(user_id={self.user_id}, comment_id={self.comment_id})>" diff --git a/files/classes/saves.py b/files/classes/saves.py index 1e89898b6..3d753864d 100644 --- a/files/classes/saves.py +++ b/files/classes/saves.py @@ -1,24 +1,21 @@ from sqlalchemy import * from sqlalchemy.orm import relationship -from files.__main__ import Base +from files.classes.base import Base class SaveRelationship(Base): + __tablename__ = "save_relationship" - __tablename__="save_relationship" - - user_id=Column(Integer, ForeignKey("users.id"), primary_key=True) - submission_id=Column(Integer, ForeignKey("submissions.id"), primary_key=True) + user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) + submission_id = Column(Integer, ForeignKey("submissions.id"), primary_key=True) Index('fki_save_relationship_submission_fkey', submission_id) - class CommentSaveRelationship(Base): + __tablename__ = "comment_save_relationship" - __tablename__="comment_save_relationship" - - user_id=Column(Integer, ForeignKey("users.id"), primary_key=True) - comment_id=Column(Integer, ForeignKey("comments.id"), primary_key=True) + user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) + comment_id = Column(Integer, ForeignKey("comments.id"), primary_key=True) Index('fki_comment_save_relationship_comment_fkey', comment_id) diff --git a/files/classes/sub.py b/files/classes/sub.py deleted file mode 100644 index 04780cfda..000000000 --- a/files/classes/sub.py +++ /dev/null @@ -1,50 +0,0 @@ -from sqlalchemy import * -from sqlalchemy.orm import relationship -from files.__main__ import Base -from files.helpers.lazy import lazy -from os import environ -from .sub_block import * - -SITE = environ.get("DOMAIN", '').strip() -if SITE == "localhost": SITE_FULL = 'http://' + SITE -else: SITE_FULL = 'https://' + SITE - -class Sub(Base): - - __tablename__ = "subs" - name = Column(String, primary_key=True) - sidebar = Column(String) - sidebar_html = Column(String) - sidebarurl = Column(String) - bannerurl = Column(String) - css = Column(String) - - Index('subs_idx', name) - - blocks = relationship("SubBlock", lazy="dynamic", primaryjoin="SubBlock.sub==Sub.name", viewonly=True) - - - def __repr__(self): - return f"<{self.__class__.__name__}(name={self.name})>" - - @property - @lazy - def sidebar_url(self): - if self.sidebarurl: return SITE_FULL + self.sidebarurl - return None # Add default sidebar for subs if subs are used again - - @property - @lazy - def banner_url(self): - if self.bannerurl: return SITE_FULL + self.bannerurl - return None # Add default banner for subs if subs are used again - - @property - @lazy - def subscription_num(self): - return self.subscriptions.count() - - @property - @lazy - def block_num(self): - return self.blocks.count() diff --git a/files/classes/sub_block.py b/files/classes/sub_block.py deleted file mode 100644 index 9a41b0664..000000000 --- a/files/classes/sub_block.py +++ /dev/null @@ -1,14 +0,0 @@ -from sqlalchemy import * -from sqlalchemy.orm import relationship -from files.__main__ import Base - -class SubBlock(Base): - - __tablename__ = "sub_blocks" - user_id = Column(Integer, ForeignKey("users.id"), primary_key=True) - sub = Column(String, ForeignKey("subs.name"), primary_key=True) - - Index('fki_sub_blocks_sub_fkey', sub) - - def __repr__(self): - return f"<{self.__class__.__name__}(user_id={self.user_id}, sub={self.sub})>" diff --git a/files/classes/submission.py b/files/classes/submission.py index ca74f594f..1857d3a88 100644 --- a/files/classes/submission.py +++ b/files/classes/submission.py @@ -1,43 +1,39 @@ -from os import environ -import random -import re -import time from urllib.parse import urlparse -from flask import render_template -from sqlalchemy import * -from sqlalchemy.orm import relationship, deferred -from files.__main__ import Base, app -from files.helpers.const import * -from files.helpers.lazy import lazy -from files.helpers.assetcache import assetcache_path -from .flags import Flag -from .comment import Comment -from flask import g -from .sub import Sub -from .votes import CommentVote -class Submission(Base): +from flask import g +from sqlalchemy import * +from sqlalchemy.orm import Session, declared_attr, deferred, relationship + +from files.classes.base import CreatedBase +from files.classes.flags import Flag +from files.classes.visstate import StateMod, StateReport, VisibilityState +from files.classes.votes import Vote +from files.helpers.assetcache import assetcache_path +from files.helpers.config.const import * +from files.helpers.config.environment import (SCORE_HIDING_TIME_HOURS, SITE, + SITE_FULL, SITE_ID) +from files.helpers.content import body_displayed +from files.helpers.lazy import lazy +from files.helpers.time import format_age, format_datetime + + +class Submission(CreatedBase): __tablename__ = "submissions" id = Column(Integer, primary_key=True) author_id = Column(Integer, ForeignKey("users.id"), nullable=False) edited_utc = Column(Integer, default=0, nullable=False) - created_utc = Column(Integer, nullable=False) thumburl = Column(String) - is_banned = Column(Boolean, default=False, nullable=False) bannedfor = Column(Boolean) ghost = Column(Boolean, default=False, nullable=False) views = Column(Integer, default=0, nullable=False) - deleted_utc = Column(Integer, default=0, nullable=False) distinguish_level = Column(Integer, default=0, nullable=False) stickied = Column(String) stickied_utc = Column(Integer) - sub = Column(String, ForeignKey("subs.name")) is_pinned = Column(Boolean, default=False, nullable=False) private = Column(Boolean, default=False, nullable=False) club = Column(Boolean, default=False, nullable=False) comment_count = Column(Integer, default=0, nullable=False) - is_approved = Column(Integer, ForeignKey("users.id")) over_18 = Column(Boolean, default=False, nullable=False) is_bot = Column(Boolean, default=False, nullable=False) upvotes = Column(Integer, default=1, nullable=False) @@ -50,36 +46,83 @@ class Submission(Base): body = Column(Text) body_html = Column(Text) flair = Column(String) - ban_reason = Column(String) embed_url = Column(String) - filter_state = Column(String, nullable=False) + task_id = Column(Integer, ForeignKey("tasks_repeatable_scheduled_submissions.id")) + + # Visibility states here + state_user_deleted_utc = Column(DateTime(timezone=True), nullable=True) # null if it hasn't been deleted by the user + state_mod = Column(Enum(StateMod), default=StateMod.FILTERED, nullable=False) # default to Filtered just to partially neuter possible exploits + state_mod_set_by = Column(String, nullable=True) # This should *really* be a User.id, but I don't want to mess with the required refactoring at the moment - it's extra hard because it could potentially be a lot of extra either data or queries + state_report = Column(Enum(StateReport), default=StateReport.UNREPORTED, nullable=False) - Index('fki_submissions_approver_fkey', is_approved) Index('post_app_id_idx', app_id) - Index('subimssion_binary_group_idx', is_banned, deleted_utc, over_18) - Index('submission_isbanned_idx', is_banned) - Index('submission_isdeleted_idx', deleted_utc) - Index('submission_new_sort_idx', is_banned, deleted_utc, created_utc.desc(), over_18) + Index('subimssion_binary_group_idx', state_mod, state_user_deleted_utc, over_18) + Index('submission_state_mod_idx', state_mod) + Index('submission_isdeleted_idx', state_user_deleted_utc) + + @declared_attr + def submission_new_sort_idx(self): + return Index('submission_new_sort_idx', self.state_mod, self.state_user_deleted_utc, self.created_utc.desc(), self.over_18) + Index('submission_pinned_idx', is_pinned) Index('submissions_author_index', author_id) - Index('submissions_created_utc_asc_idx', created_utc.nullsfirst()) - Index('submissions_created_utc_desc_idx', created_utc.desc()) + + @declared_attr + def submisission_created_utc_asc_idx(self): + return Index('submissions_created_utc_asc_idx', self.created_utc.nulls_first()) + + @declared_attr + def submissions_created_utc_desc_idx(self): + return Index('submissions_created_utc_desc_idx', self.created_utc.desc()) + Index('submissions_over18_index', over_18) author = relationship("User", primaryjoin="Submission.author_id==User.id") oauth_app = relationship("OauthApp", viewonly=True) - approved_by = relationship("User", uselist=False, primaryjoin="Submission.is_approved==User.id", viewonly=True) awards = relationship("AwardRelationship", viewonly=True) reports = relationship("Flag", viewonly=True) comments = relationship("Comment", primaryjoin="Comment.parent_submission==Submission.id") - subr = relationship("Sub", primaryjoin="foreign(Submission.sub)==remote(Sub.name)", viewonly=True) notes = relationship("UserNote", back_populates="post") + task = relationship("ScheduledSubmissionTask", back_populates="submissions") bump_utc = deferred(Column(Integer, server_default=FetchedValue())) - def __init__(self, *args, **kwargs): - if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time()) - super().__init__(*args, **kwargs) + def submit(self, db: Session): + # create submission... + db.add(self) + db.flush() + + # then create vote... + vote = Vote( + user_id=self.author_id, + vote_type=1, + submission_id=self.id + ) + db.add(vote) + author = self.author + author.post_count = db.query(Submission.id).filter_by( + author_id=self.author_id, + state_mod=StateMod.VISIBLE, + state_user_deleted_utc=None).count() + db.add(author) + + def publish(self): + # this is absolutely horrifying. imports are very very tangled and the + # spider web of imports is hard to maintain. we defer loading these + # imports until as late as possible. otherwise there are import loops + # that would require much more work to untangle + from files.helpers.alerts import notify_submission_publish + from files.helpers.caching import invalidate_cache + + if self.private: return + if not self.ghost: + notify_submission_publish(self) + invalidate_cache( + frontlist=True, + userpagelisting=True, + changeloglist=("[changelog]" in self.title.lower() + or "(changelog)" in self.title.lower()), + ) def __repr__(self): return f"<{self.__class__.__name__}(id={self.id})>" @@ -87,9 +130,8 @@ class Submission(Base): @property @lazy def should_hide_score(self): - submission_age_seconds = int(time.time()) - self.created_utc - submission_age_hours = submission_age_seconds / (60*60) - return submission_age_hours < app.config['SCORE_HIDING_TIME_HOURS'] + submission_age_hours = self.age_seconds / (60*60) + return submission_age_hours < SCORE_HIDING_TIME_HOURS @property @lazy @@ -100,89 +142,19 @@ class Submission(Base): @lazy def flags(self, v): flags = g.db.query(Flag).filter_by(post_id=self.id).order_by(Flag.created_utc).all() - if not (v and (v.shadowbanned or v.admin_level > 2)): + if not (v and (v.shadowbanned or v.admin_level >= 3)): for flag in flags: if flag.user.shadowbanned: flags.remove(flag) return flags @property - @lazy - def created_datetime(self): - return time.strftime("%d/%B/%Y %H:%M:%S UTC", time.gmtime(self.created_utc)) - - @property - @lazy - def created_datetime(self): - return time.strftime("%d/%B/%Y %H:%M:%S UTC", time.gmtime(self.created_utc)) - - @property - @lazy - def age_string(self): - - age = int(time.time()) - self.created_utc - - if age < 60: - return "just now" - elif age < 3600: - minutes = int(age / 60) - return f"{minutes}m ago" - elif age < 86400: - hours = int(age / 3600) - return f"{hours}hr ago" - elif age < 2678400: - days = int(age / 86400) - return f"{days}d ago" - - now = time.gmtime() - ctd = time.gmtime(self.created_utc) - - months = now.tm_mon - ctd.tm_mon + 12 * (now.tm_year - ctd.tm_year) - if now.tm_mday < ctd.tm_mday: - months -= 1 - - if months < 12: - return f"{months}mo ago" - else: - years = int(months / 12) - return f"{years}yr ago" - - @property - @lazy def edited_string(self): - - if not self.edited_utc: return "never" - - age = int(time.time()) - self.edited_utc - - if age < 60: - return "just now" - elif age < 3600: - minutes = int(age / 60) - return f"{minutes}m ago" - elif age < 86400: - hours = int(age / 3600) - return f"{hours}hr ago" - elif age < 2678400: - days = int(age / 86400) - return f"{days}d ago" - - now = time.gmtime() - ctd = time.gmtime(self.edited_utc) - months = now.tm_mon - ctd.tm_mon + 12 * (now.tm_year - ctd.tm_year) - - if months < 12: - return f"{months}mo ago" - else: - years = now.tm_year - ctd.tm_year - return f"{years}yr ago" - + return format_age(self.edited_utc) if self.edited_utc else "never" @property - @lazy def edited_datetime(self): - return time.strftime("%d/%B/%Y %H:%M:%S UTC", time.gmtime(self.edited_utc)) - + return format_datetime(self.edited_utc) if self.edited_utc else "" @property @lazy @@ -192,15 +164,12 @@ class Submission(Base): @property @lazy def fullname(self): - return f"t2_{self.id}" - + return f"post_{self.id}" @property @lazy def shortlink(self): link = f"/post/{self.id}" - if self.sub: link = f"/h/{self.sub}{link}" - if self.club: return link + '/-' output = title_regex.sub('', self.title.lower()) @@ -261,8 +230,6 @@ class Submission(Base): data = {'author_name': self.author_name if self.author else '', 'permalink': self.permalink, 'shortlink': self.shortlink, - 'is_banned': bool(self.is_banned), - 'deleted_utc': self.deleted_utc, 'created_utc': self.created_utc, 'id': self.id, 'title': self.title, @@ -287,26 +254,22 @@ class Submission(Base): 'club': self.club, } - if self.ban_reason: - data["ban_reason"]=self.ban_reason - return data @property @lazy def json_core(self): - - if self.is_banned: - return {'is_banned': True, - 'deleted_utc': self.deleted_utc, - 'ban_reason': self.ban_reason, + if self.state_mod != StateMod.VISIBLE: + return { + 'state_user_deleted_utc': self.state_user_deleted_utc, + 'state_mod_set_by': self.state_mod_set_by, 'id': self.id, 'title': self.title, 'permalink': self.permalink, } - elif self.deleted_utc: - return {'is_banned': bool(self.is_banned), - 'deleted_utc': True, + elif self.state_user_deleted_utc: + return { + 'state_user_deleted_utc': self.state_user_deleted_utc, 'id': self.id, 'title': self.title, 'permalink': self.permalink, @@ -317,10 +280,9 @@ class Submission(Base): @property @lazy def json(self): - data=self.json_core - if self.deleted_utc or self.is_banned: + if self.state_user_deleted_utc or self.state_mod != StateMod.VISIBLE: return data data["author"]='👻' if self.ghost else self.author.json_core @@ -357,44 +319,14 @@ class Submission(Base): else: return "" def realbody(self, v): - if self.club and not (v and (v.paid_dues or v.id == self.author_id)): return f"{CC} ONLY
" - - body = self.body_html or "" - - if v: - body = body.replace("old.reddit.com", v.reddit) - - if v.nitter and '/i/' not in body and '/retweets' not in body: body = body.replace("www.twitter.com", "nitter.net").replace("twitter.com", "nitter.net") - - if v and v.shadowbanned and v.id == self.author_id and 86400 > time.time() - self.created_utc > 20: - ti = max(int((time.time() - self.created_utc)/60), 1) - maxupvotes = min(ti, 11) - rand = random.randint(0, maxupvotes) - if self.upvotes < rand: - amount = random.randint(0, 3) - if amount == 1: - self.views += amount*random.randint(3, 5) - self.upvotes += amount - g.db.add(self) - g.db.commit() - - return body + return body_displayed(self, v, is_html=True) def plainbody(self, v): - if self.club and not (v and (v.paid_dues or v.id == self.author_id)): return f"{CC} ONLY
" - body = self.body - if not body: return "" - if v: - body = body.replace("old.reddit.com", v.reddit) - if v.nitter and '/i/' not in body and '/retweets' not in body: body = body.replace("www.twitter.com", "nitter.net").replace("twitter.com", "nitter.net") - return body + return body_displayed(self, v, is_html=False) @lazy def realtitle(self, v): - if self.title_html: - return self.title_html - else: - return self.title + return self.title_html if self.title_html else self.title @lazy def plaintitle(self, v): @@ -413,4 +345,17 @@ class Submission(Base): return False @lazy - def active_flags(self, v): return len(self.flags(v)) + def active_flags(self, v): + return len(self.flags(v)) + + @property + def is_real_submission(self) -> bool: + return True + + @property + def edit_url(self) -> str: + return f"/edit_post/{self.id}" + + @property + def visibility_state(self) -> VisibilityState: + return VisibilityState.from_submittable(self) diff --git a/files/classes/subscriptions.py b/files/classes/subscriptions.py index 72a58a104..d25091a3e 100644 --- a/files/classes/subscriptions.py +++ b/files/classes/subscriptions.py @@ -1,6 +1,6 @@ from sqlalchemy import * from sqlalchemy.orm import relationship -from files.__main__ import Base +from files.classes.base import Base class Subscription(Base): __tablename__ = "subscriptions" diff --git a/files/classes/user.py b/files/classes/user.py index d7d7bf835..0050144df 100644 --- a/files/classes/user.py +++ b/files/classes/user.py @@ -1,35 +1,39 @@ -from sqlalchemy.orm import deferred, aliased -from secrets import token_hex -import pyotp -from files.helpers.media import * -from files.helpers.const import * -from .alts import Alt -from .saves import * -from .notifications import Notification -from .award import AwardRelationship -from .follows import * -from .subscriptions import * -from .userblock import * -from .badges import * -from .clients import * -from .mod_logs import * -from .mod import * -from .exiles import * -from .sub_block import * -from files.__main__ import app, Base, cache -from files.helpers.security import * -from files.helpers.assetcache import assetcache_path -from files.helpers.contentsorting import apply_time_filter, sort_objects -import random -from datetime import datetime -from os import environ, remove, path +from __future__ import annotations +import time +from typing import TYPE_CHECKING, Union + +import pyotp +from flask import g, session +from sqlalchemy.orm import aliased, declared_attr, deferred, relationship + +from files.classes.alts import Alt +from files.classes.award import AwardRelationship +from files.classes.badges import Badge +from files.classes.base import CreatedBase +from files.classes.clients import * # note: imports Comment and Submission +from files.classes.follows import Follow +from files.classes.mod_logs import ModAction +from files.classes.notifications import Notification +from files.classes.saves import CommentSaveRelationship, SaveRelationship +from files.classes.subscriptions import Subscription +from files.classes.userblock import UserBlock +from files.classes.visstate import StateMod +from files.helpers.assetcache import assetcache_path +from files.helpers.config.const import * +from files.helpers.config.environment import (CARD_VIEW, + CLUB_TRUESCORE_MINIMUM, + DEFAULT_COLOR, + DEFAULT_TIME_FILTER, SITE_FULL, + SITE_ID) +from files.helpers.security import * + +if TYPE_CHECKING: + from files.classes.cron.submission import ScheduledSubmissionTask defaulttheme = "TheMotte" -defaulttimefilter = environ.get("DEFAULT_TIME_FILTER", "all").strip() -cardview = bool(int(environ.get("CARD_VIEW", 1))) -class User(Base): +class User(CreatedBase): __tablename__ = "users" __table_args__ = ( UniqueConstraint('bannerurl', name='one_banner'), @@ -40,13 +44,12 @@ class User(Base): id = Column(Integer, primary_key=True) username = Column(String(length=255), nullable=False) namecolor = Column(String(length=6), default=DEFAULT_COLOR, nullable=False) - background = Column(String) customtitle = Column(String) customtitleplain = deferred(Column(String)) titlecolor = Column(String(length=6), default=DEFAULT_COLOR, nullable=False) theme = Column(String, default=defaulttheme, nullable=False) themecolor = Column(String, default=DEFAULT_COLOR, nullable=False) - cardview = Column(Boolean, default=cardview, nullable=False) + cardview = Column(Boolean, default=CARD_VIEW, nullable=False) highres = Column(String) profileurl = Column(String) bannerurl = Column(String) @@ -63,7 +66,6 @@ class User(Base): post_count = Column(Integer, default=0, nullable=False) comment_count = Column(Integer, default=0, nullable=False) received_award_count = Column(Integer, default=0, nullable=False) - created_utc = Column(Integer, nullable=False) admin_level = Column(Integer, default=0, nullable=False) coins_spent = Column(Integer, default=0, nullable=False) lootboxes_bought = Column(Integer, default=0, nullable=False) @@ -103,14 +105,14 @@ class User(Base): stored_subscriber_count = Column(Integer, default=0, nullable=False) defaultsortingcomments = Column(String, default="new", nullable=False) defaultsorting = Column(String, default="new", nullable=False) - defaulttime = Column(String, default=defaulttimefilter, nullable=False) + defaulttime = Column(String, default=DEFAULT_TIME_FILTER, nullable=False) is_nofollow = Column(Boolean, default=False, nullable=False) custom_filter_list = Column(String) ban_evade = Column(Integer, default=0, nullable=False) original_username = deferred(Column(String)) referred_by = Column(Integer, ForeignKey("users.id")) - subs_created = Column(Integer, default=0, nullable=False) volunteer_last_started_utc = Column(DateTime, nullable=True) + volunteer_janitor_correctness = Column(Float, default=0, nullable=False) Index( 'users_original_username_trgm_idx', @@ -128,7 +130,11 @@ class User(Base): Index('fki_user_referrer_fkey', referred_by) Index('user_banned_idx', is_banned) Index('user_private_idx', is_private) - Index('users_created_utc_index', created_utc) + + @declared_attr + def users_created_utc_index(self): + return Index('users_created_utc_index', self.created_utc) + Index('users_subs_idx', stored_subscriber_count) Index('users_unbanutc_idx', unban_utc.desc()) @@ -148,12 +154,8 @@ class User(Base): if "password" in kwargs: kwargs["passhash"] = self.hash_password(kwargs["password"]) kwargs.pop("password") - - if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time()) - super().__init__(**kwargs) - def can_manage_reports(self): return self.admin_level > 1 @@ -163,8 +165,10 @@ class User(Base): @property def should_comments_be_filtered(self): + from files.__main__ import app # avoiding import loop if self.admin_level > 0: return False + # TODO: move settings out of app.config site_settings = app.config['SETTINGS'] min_comments = site_settings.get('FilterCommentsMinComments', 0) min_karma = site_settings.get('FilterCommentsMinKarma', 0) @@ -173,58 +177,30 @@ class User(Base): or self.age_days < min_age \ or self.truescore < min_karma - @lazy - def mods(self, sub): - return self.admin_level == 3 or bool(g.db.query(Mod.user_id).filter_by(user_id=self.id, sub=sub).one_or_none()) - @lazy - def exiled_from(self, sub): - return self.admin_level < 2 and bool(g.db.query(Exile.user_id).filter_by(user_id=self.id, sub=sub).one_or_none()) + def can_change_user_privacy(self, v: "User") -> bool: + from files.__main__ import app # avoiding import loop + if v.admin_level >= PERMS['USER_SET_PROFILE_PRIVACY']: return True + if self.id != v.id: return False # non-admin changing someone else's things, hmm... - @property - @lazy - def all_blocks(self): - return [x[0] for x in g.db.query(SubBlock.sub).filter_by(user_id=self.id).all()] + # TODO: move settings out of app.config + site_settings = app.config['SETTINGS'] + min_comments: int = site_settings.get('min_comments_private_profile', 0) + min_truescore: int = site_settings.get('min_truescore_private_profile', 0) + min_age_days: int = site_settings.get('min_age_days_private_profile', 0) + user_age_days: int = self.age_timedelta.days - @lazy - def blocks(self, sub): - return g.db.query(SubBlock).filter_by(user_id=self.id, sub=sub).one_or_none() - - @lazy - def mod_date(self, sub): - if self.id == OWNER_ID: return 1 - mod = g.db.query(Mod).filter_by(user_id=self.id, sub=sub).one_or_none() - if not mod: return None - return mod.created_utc + return ( + self.comment_count >= min_comments + and self.truecoins >= min_truescore + and user_age_days >= min_age_days) + @property @lazy def csslazy(self): return self.css - @property - @lazy - def created_date(self): - - return time.strftime("%d %b %Y", time.gmtime(self.created_utc)) - - @property - @lazy - def discount(self): - if self.patron == 1: discount = 0.90 - elif self.patron == 2: discount = 0.85 - elif self.patron == 3: discount = 0.80 - elif self.patron == 4: discount = 0.75 - elif self.patron == 5: discount = 0.70 - elif self.patron == 6: discount = 0.65 - else: discount = 1 - - for badge in [69,70,71,72,73]: - if self.has_badge(badge): discount -= discounts[badge] - - return discount - - @property @lazy def user_awards(self): @@ -244,25 +220,18 @@ class User(Base): @property @lazy def paid_dues(self): - return not self.shadowbanned and not (self.is_banned and not self.unban_utc) and (self.admin_level or self.club_allowed or (self.club_allowed != False and self.truescore > dues)) + return not self.shadowbanned and not (self.is_banned and not self.unban_utc) and (self.admin_level or self.club_allowed or (self.club_allowed != False and self.truescore > CLUB_TRUESCORE_MINIMUM)) @lazy def any_block_exists(self, other): - return g.db.query(UserBlock).filter( or_(and_(UserBlock.user_id == self.id, UserBlock.target_id == other.id), and_( UserBlock.user_id == other.id, UserBlock.target_id == self.id))).first() def validate_2fa(self, token): - x = pyotp.TOTP(self.mfa_secret) return x.verify(token, valid_window=1) - @property - @lazy - def age(self): - return int(time.time()) - self.created_utc - @property @lazy def ban_reason_link(self): @@ -285,22 +254,6 @@ class User(Base): if u.patron: return True return False - @cache.memoize(timeout=86400) - def userpagelisting(self, site=None, v=None, page=1, sort="new", t="all"): - if self.shadowbanned and not (v and (v.admin_level > 1 or v.id == self.id)): return [] - - posts = g.db.query(Submission.id).filter_by(author_id=self.id, is_pinned=False) - - if not (v and (v.admin_level > 1 or v.id == self.id)): - posts = posts.filter_by(deleted_utc=0, is_banned=False, private=False, ghost=False) - - posts = apply_time_filter(posts, t, Submission) - posts = sort_objects(posts, sort, Submission) - - posts = posts.offset(25 * (page - 1)).limit(26).all() - - return [x[0] for x in posts] - @property @lazy def follow_count(self): @@ -315,7 +268,7 @@ class User(Base): @property @lazy def fullname(self): - return f"t1_{self.id}" + return f"user_{self.id}" @property @lazy @@ -336,7 +289,6 @@ class User(Base): @property @lazy def formkey(self): - msg = f"{session['session_id']}+{self.id}+{self.login_nonce}" return generate_hash(msg) @@ -409,7 +361,7 @@ class User(Base): @property @lazy def notifications_count(self): - notifs = g.db.query(Notification.user_id).join(Comment).filter(Notification.user_id == self.id, Notification.read == False, Comment.is_banned == False, Comment.deleted_utc == 0) + notifs = g.db.query(Notification.user_id).join(Comment).filter(Notification.user_id == self.id, Notification.read == False, Comment.state_mod == StateMod.VISIBLE, Comment.state_user_deleted_utc == None) if not self.shadowbanned and self.admin_level < 3: notifs = notifs.join(User, User.id == Comment.author_id).filter(User.shadowbanned == None) @@ -421,30 +373,19 @@ class User(Base): def post_notifications_count(self): return g.db.query(Notification.user_id).join(Comment).filter(Notification.user_id == self.id, Notification.read == False, Comment.author_id == AUTOJANNY_ID).count() - @property - @lazy - def reddit_notifications_count(self): - return g.db.query(Notification.user_id).join(Comment).filter(Notification.user_id == self.id, Notification.read == False, Comment.is_banned == False, Comment.deleted_utc == 0, Comment.body_html.like('%New site mention: = PERMS['POST_EDITING']
+ if target.__class__.__name__ == "ScheduledSubmissionTask":
+ # XXX: avoiding import loop. should be fixed someday.
+ return self.admin_level >= PERMS['SCHEDULER_POSTS']
+
@property
def can_see_shadowbanned(self):
return self.admin_level >= PERMS['USER_SHADOWBAN'] or self.shadowbanned
diff --git a/files/classes/userblock.py b/files/classes/userblock.py
index f9e3cb8b6..435062392 100644
--- a/files/classes/userblock.py
+++ b/files/classes/userblock.py
@@ -1,6 +1,6 @@
from sqlalchemy import *
from sqlalchemy.orm import relationship
-from files.__main__ import Base
+from files.classes.base import Base
class UserBlock(Base):
diff --git a/files/classes/usernotes.py b/files/classes/usernotes.py
index 665691d72..ee9f7937c 100644
--- a/files/classes/usernotes.py
+++ b/files/classes/usernotes.py
@@ -1,9 +1,7 @@
-import time
-from flask import *
from sqlalchemy import *
from sqlalchemy.orm import relationship
-from files.__main__ import Base
-from files.helpers.const import *
+from files.classes.base import CreatedBase
+from files.helpers.config.const import *
from enum import Enum
from sqlalchemy import Enum as EnumType
@@ -17,13 +15,11 @@ class UserTag(Enum):
Spam = 6
Bot = 7
-class UserNote(Base):
-
+class UserNote(CreatedBase):
__tablename__ = "usernotes"
id = Column(Integer, primary_key=True)
author_id = Column(Integer, ForeignKey("users.id"), nullable=False)
- created_utc = Column(Integer, nullable=False)
reference_user = Column(Integer, ForeignKey("users.id", ondelete='CASCADE'), nullable=False)
reference_comment = Column(Integer, ForeignKey("comments.id", ondelete='SET NULL'))
reference_post = Column(Integer, ForeignKey("submissions.id", ondelete='SET NULL'))
@@ -35,11 +31,6 @@ class UserNote(Base):
comment = relationship("Comment", back_populates="notes")
post = relationship("Submission", back_populates="notes")
-
- def __init__(self, *args, **kwargs):
- if "created_utc" not in kwargs:
- kwargs["created_utc"] = int(time.time())
- super().__init__(*args, **kwargs)
def __repr__(self):
return f"<{self.__class__.__name__}(id={self.id})>"
@@ -63,4 +54,4 @@ class UserNote(Base):
'tag': self.tag.value
}
- return data
\ No newline at end of file
+ return data
diff --git a/files/classes/views.py b/files/classes/views.py
index 29a206577..18ad428d7 100644
--- a/files/classes/views.py
+++ b/files/classes/views.py
@@ -1,11 +1,14 @@
-from sqlalchemy import *
-from sqlalchemy.orm import relationship
-from files.__main__ import Base
-from files.helpers.lazy import *
import time
-class ViewerRelationship(Base):
+from sqlalchemy import *
+from sqlalchemy.orm import relationship
+from files.classes.base import Base
+from files.helpers.lazy import lazy
+from files.helpers.time import format_age
+
+
+class ViewerRelationship(Base):
__tablename__ = "viewers"
user_id = Column(Integer, ForeignKey('users.id'), primary_key=True)
@@ -17,7 +20,6 @@ class ViewerRelationship(Base):
viewer = relationship("User", primaryjoin="ViewerRelationship.viewer_id == User.id", viewonly=True)
def __init__(self, **kwargs):
-
if 'last_view_utc' not in kwargs:
kwargs['last_view_utc'] = int(time.time())
@@ -26,36 +28,9 @@ class ViewerRelationship(Base):
@property
@lazy
def last_view_since(self):
-
return int(time.time()) - self.last_view_utc
@property
@lazy
def last_view_string(self):
-
- age = self.last_view_since
-
- if age < 60:
- return "just now"
- elif age < 3600:
- minutes = int(age / 60)
- return f"{minutes}m ago"
- elif age < 86400:
- hours = int(age / 3600)
- return f"{hours}hr ago"
- elif age < 2678400:
- days = int(age / 86400)
- return f"{days}d ago"
-
- now = time.gmtime()
- ctd = time.gmtime(self.created_utc)
-
- months = now.tm_mon - ctd.tm_mon + 12 * (now.tm_year - ctd.tm_year)
- if now.tm_mday < ctd.tm_mday:
- months -= 1
-
- if months < 12:
- return f"{months}mo ago"
- else:
- years = int(months / 12)
- return f"{years}yr ago"
+ return format_age(self.last_view_utc)
diff --git a/files/classes/visstate.py b/files/classes/visstate.py
new file mode 100644
index 000000000..a0bfd9fc0
--- /dev/null
+++ b/files/classes/visstate.py
@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import enum
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from files.helpers.config.const import PERMS
+
+if TYPE_CHECKING:
+ from files.classes.user import User
+ from files.helpers.content import Submittable
+
+
+class StateMod(enum.Enum):
+ VISIBLE = 0
+ FILTERED = 1
+ REMOVED = 2
+
+class StateReport(enum.Enum):
+ UNREPORTED = 0
+ RESOLVED = 1
+ REPORTED = 2
+ IGNORED = 3
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class VisibilityState:
+ '''
+ The full moderation state machine. It holds the moderation state, report
+ state, deleted information, and shadowban information. A decision to show
+ or hide a post or comment should be able to be done with information from
+ this alone.
+ '''
+ state_mod: StateMod
+ state_mod_set_by: str | None
+ state_report: StateReport
+ state_mod_set_by: str | None
+ deleted: bool
+ op_shadowbanned: bool
+ op_id: int
+ op_name_safe: str
+
+ @property
+ def removed(self) -> bool:
+ return self.state_mod == StateMod.REMOVED
+
+ @property
+ def filtered(self) -> bool:
+ return self.state_mod == StateMod.FILTERED
+
+ @property
+ def reports_ignored(self) -> bool:
+ return self.state_report == StateReport.IGNORED
+
+ @classmethod
+ def from_submittable(cls, target: Submittable) -> "VisibilityState":
+ return cls(
+ state_mod=target.state_mod,
+ state_mod_set_by=target.state_mod_set_by, # type: ignore
+ state_report=target.state_report,
+ deleted=bool(target.state_user_deleted_utc != None),
+ op_shadowbanned=bool(target.author.shadowbanned),
+ op_id=target.author_id, # type: ignore
+ op_name_safe=target.author_name
+ )
+
+ def moderated_body(self, v: User | None, is_blocking: bool=False) -> str | None:
+ if v and (v.admin_level >= PERMS['POST_COMMENT_MODERATION'] \
+ or v.id == self.op_id):
+ return None
+ if self.deleted: return 'Deleted'
+ if self.appear_removed(v): return 'Removed'
+ if self.filtered: return 'Filtered'
+ if is_blocking: return f'You are blocking @{self.op_name_safe}'
+ return None
+
+ def visibility_and_message(self, v: User | None, is_blocking: bool) -> tuple[bool, str]:
+ '''
+ Returns a tuple of whether this content is visible and a publicly
+ visible message to accompany it. The visibility state machine is
+ a slight mess but... this should at least unify the state checks.
+ '''
+ def can(v: User | None, perm_level: int) -> bool:
+ return v and v.admin_level >= perm_level
+
+ can_moderate: bool = can(v, PERMS['POST_COMMENT_MODERATION'])
+ can_shadowban: bool = can(v, PERMS['USER_SHADOWBAN'])
+
+ if v and v.id == self.op_id:
+ return True, "This shouldn't be here, please report it!"
+ if (self.removed and not can_moderate) or \
+ (self.op_shadowbanned and not can_shadowban):
+ msg: str = 'Removed'
+ if self.state_mod_set_by:
+ msg = f'Removed by @{self.state_mod_set_by}'
+ return False, msg
+ if self.filtered and not can_moderate:
+ return False, 'Filtered'
+ if self.deleted and not can_moderate:
+ return False, 'Deleted by author'
+ if is_blocking:
+ return False, f'You are blocking @{self.op_name_safe}'
+ return True, "This shouldn't be here, please report it!"
+
+ def is_visible_to(self, v: User | None, is_blocking: bool) -> bool:
+ return self.visibility_and_message(v, is_blocking)[0]
+
+ def replacement_message(self, v: User | None, is_blocking: bool) -> str:
+ return self.visibility_and_message(v, is_blocking)[1]
+
+ def appear_removed(self, v: User | None) -> bool:
+ if self.removed: return True
+ if not self.op_shadowbanned: return False
+ return (not v) or bool(v.admin_level < PERMS['USER_SHADOWBAN'])
+
+ @property
+ def publicly_visible(self) -> bool:
+ return all(
+ not state for state in
+ [self.deleted, self.removed, self.filtered, self.op_shadowbanned]
+ )
+
+ @property
+ def explicitly_moderated(self) -> bool:
+ '''
+ Whether this was removed or filtered and not as the result of a shadowban
+ '''
+ return self.removed or self.filtered
diff --git a/files/classes/volunteer_janitor.py b/files/classes/volunteer_janitor.py
index aa2e07417..19dc2df74 100644
--- a/files/classes/volunteer_janitor.py
+++ b/files/classes/volunteer_janitor.py
@@ -1,6 +1,6 @@
import enum
-from files.__main__ import Base
+from files.classes.base import Base
from sqlalchemy import *
from sqlalchemy.orm import relationship
@@ -14,7 +14,6 @@ class VolunteerJanitorResult(enum.Enum):
Ban = 6
class VolunteerJanitorRecord(Base):
-
__tablename__ = "volunteer_janitor"
id = Column(Integer, primary_key=True)
diff --git a/files/classes/votes.py b/files/classes/votes.py
index 1112d01b6..4163dc46f 100644
--- a/files/classes/votes.py
+++ b/files/classes/votes.py
@@ -1,12 +1,11 @@
-from flask import *
from sqlalchemy import *
from sqlalchemy.orm import relationship
-from files.__main__ import Base
+
+from files.classes.base import CreatedBase
from files.helpers.lazy import lazy
-import time
-class Vote(Base):
+class Vote(CreatedBase):
__tablename__ = "votes"
submission_id = Column(Integer, ForeignKey("submissions.id"), primary_key=True)
@@ -14,7 +13,6 @@ class Vote(Base):
vote_type = Column(Integer, nullable=False)
app_id = Column(Integer, ForeignKey("oauth_apps.id"))
real = Column(Boolean, default=True, nullable=False)
- created_utc = Column(Integer, nullable=False)
Index('votes_type_index', vote_type)
Index('vote_user_index', user_id)
@@ -22,10 +20,6 @@ class Vote(Base):
user = relationship("User", lazy="subquery", viewonly=True)
post = relationship("Submission", lazy="subquery", viewonly=True)
- def __init__(self, *args, **kwargs):
- if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time())
- super().__init__(*args, **kwargs)
-
def __repr__(self):
return f"<{self.__class__.__name__}(id={self.id})>"
@@ -45,12 +39,10 @@ class Vote(Base):
data=self.json_core
data["user"]=self.user.json_core
data["post"]=self.post.json_core
-
return data
-class CommentVote(Base):
-
+class CommentVote(CreatedBase):
__tablename__ = "commentvotes"
comment_id = Column(Integer, ForeignKey("comments.id"), primary_key=True)
@@ -58,7 +50,6 @@ class CommentVote(Base):
vote_type = Column(Integer, nullable=False)
app_id = Column(Integer, ForeignKey("oauth_apps.id"))
real = Column(Boolean, default=True, nullable=False)
- created_utc = Column(Integer, nullable=False)
Index('cvote_user_index', user_id)
Index('commentvotes_comments_type_index', vote_type)
@@ -66,10 +57,6 @@ class CommentVote(Base):
user = relationship("User", lazy="subquery")
comment = relationship("Comment", lazy="subquery", viewonly=True)
- def __init__(self, *args, **kwargs):
- if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time())
- super().__init__(*args, **kwargs)
-
def __repr__(self):
return f"<{self.__class__.__name__}(id={self.id})>"
@@ -89,5 +76,4 @@ class CommentVote(Base):
data=self.json_core
data["user"]=self.user.json_core
data["comment"]=self.comment.json_core
-
return data
diff --git a/files/cli.py b/files/cli.py
index a8461d700..a2f9ed6a6 100644
--- a/files/cli.py
+++ b/files/cli.py
@@ -1,7 +1,12 @@
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
-from .__main__ import app
-from .commands.seed_db import seed_db
+
+from files.__main__ import app
+from files.commands.cron import cron_app_worker
+from files.commands.seed_db import seed_db
+from files.commands.volunteer_janitor_recalc import volunteer_janitor_recalc
+from files.commands.volunteer_janitor_histogram import volunteer_janitor_histogram_cmd
+from files.commands.cron_setup import cron_setup
import files.classes
db = SQLAlchemy(app)
diff --git a/files/commands/cron.py b/files/commands/cron.py
new file mode 100644
index 000000000..4f672f332
--- /dev/null
+++ b/files/commands/cron.py
@@ -0,0 +1,139 @@
+import contextlib
+import logging
+import time
+from datetime import datetime, timezone
+from typing import Final
+
+from sqlalchemy.orm import sessionmaker, Session
+
+from files.__main__ import app, db_session_factory
+from files.classes.cron.tasks import (DayOfWeek, RepeatableTask,
+ RepeatableTaskRun, ScheduledTaskState)
+
+CRON_SLEEP_SECONDS: Final[int] = 15
+'''
+How long the app will sleep for between runs. Lower values give better
+resolution, but will hit the database more.
+
+The cost of a lower value is potentially higher lock contention. A value below
+`0` will raise a `ValueError` (on call to `time.sleep`). A value of `0` is
+possible but not recommended.
+
+The sleep time is not guaranteed to be exactly this value (notably, it may be
+slightly longer if the system is very busy)
+
+This value is passed to `time.sleep()`. For more information on that, see
+the Python documentation: https://docs.python.org/3/library/time.html
+'''
+
+_CRON_COMMAND_NAME = "cron"
+
+
+@app.cli.command(_CRON_COMMAND_NAME)
+def cron_app_worker():
+ '''
+ The "worker" process task. This actually executes tasks.
+ '''
+
+ # someday we'll clean this up further, for now I need debug info
+ logging.basicConfig(level=logging.INFO)
+
+ logging.info("Starting scheduler worker process")
+ while True:
+ try:
+ _run_tasks(db_session_factory)
+ except Exception as e:
+ logging.exception(
+ "An unhandled exception occurred while running tasks",
+ exc_info=e
+ )
+ time.sleep(CRON_SLEEP_SECONDS)
+
+
+@contextlib.contextmanager
+def _acquire_lock_exclusive(db: Session, table: str):
+ '''
+ Acquires an exclusive lock on the table provided by the `table` parameter.
+ This can be used for synchronizing the state of the specified table and
+ making sure no readers can access it while in the critical section.
+ '''
+ # TODO: make `table` the type LiteralString once we upgrade to python 3.11
+ db.begin() # we want to raise an exception if there's a txn in progress
+ db.execute(f"LOCK TABLE {table} IN ACCESS EXCLUSIVE MODE")
+ try:
+ yield
+ db.commit()
+ except Exception:
+ logging.error(
+ "An exception occurred during an operation in a critical section. "
+ "A task might not occur or might be duplicated."
+ )
+ try:
+ db.rollback()
+ except:
+ logging.warning(
+ f"Failed to rollback database. The table {table} might still "
+ "be locked.")
+ raise
+
+
+def _run_tasks(db_session_factory: sessionmaker):
+ '''
+ Runs tasks, attempting to guarantee that a task is ran once and only once.
+ This uses postgres to lock the table containing our tasks at key points in
+ in the process (reading the tasks and writing the last updated time).
+
+ The task itself is ran outside of this context; this is so that a long
+ running task does not lock the entire table for its entire run, which would
+ for example, prevent any statistics about status from being gathered.
+ '''
+ db: Session = db_session_factory()
+
+ with _acquire_lock_exclusive(db, RepeatableTask.__tablename__):
+ now: datetime = datetime.now(tz=timezone.utc)
+
+ tasks: list[RepeatableTask] = db.query(RepeatableTask).filter(
+ RepeatableTask.enabled == True,
+ RepeatableTask.frequency_day != int(DayOfWeek.NONE),
+ RepeatableTask.run_state != int(ScheduledTaskState.RUNNING),
+ (RepeatableTask.run_time_last <= now)
+ | (RepeatableTask.run_time_last == None),
+ ).all()
+
+ # SQLA needs to query again for the inherited object info anyway
+ # so it's fine that objects in the list get expired on txn end.
+ # Prefer more queries to risk of task run duplication.
+ tasks_to_run: list[RepeatableTask] = list(filter(
+ lambda task: task.can_run(now), tasks))
+
+ for task in tasks_to_run:
+ now = datetime.now(tz=timezone.utc)
+ with _acquire_lock_exclusive(db, RepeatableTask.__tablename__):
+ # We need to check for runnability again because we don't mutex
+ # the RepeatableTask.run_state until now.
+ if not task.can_run(now):
+ continue
+ task.run_time_last = now
+ task.run_state_enum = ScheduledTaskState.RUNNING
+
+ # This *must* happen before we start doing db queries, including sqlalchemy db queries
+ db.begin()
+ task_debug_identifier = f"(ID {task.id}:{task.label})"
+ logging.info(f"Running task {task_debug_identifier}")
+
+ run: RepeatableTaskRun = task.run(db, task.run_time_last_or_created_utc)
+
+ if run.exception:
+ # TODO: collect errors somewhere other than just here and in the
+ # task run object itself (see #220).
+ logging.exception(
+ f"Exception running task {task_debug_identifier}",
+ exc_info=run.exception
+ )
+ db.rollback()
+ else:
+ db.commit()
+ logging.info(f"Finished task {task_debug_identifier}")
+
+ with _acquire_lock_exclusive(db, RepeatableTask.__tablename__):
+ task.run_state_enum = ScheduledTaskState.WAITING
diff --git a/files/commands/cron_setup.py b/files/commands/cron_setup.py
new file mode 100644
index 000000000..e44edf7b2
--- /dev/null
+++ b/files/commands/cron_setup.py
@@ -0,0 +1,53 @@
+
+from typing import Optional
+import datetime
+
+import sqlalchemy
+from sqlalchemy.orm import Session
+
+from files.__main__ import app, db_session
+from files.classes.cron.pycallable import PythonCodeTask
+from files.classes.cron.tasks import DayOfWeek
+from files.helpers.config.const import AUTOJANNY_ID
+
+
+@app.cli.command('cron_setup')
+def cron_setup():
+ db: Session = db_session()
+
+ tasklist = db.query(PythonCodeTask)
+
+ # I guess in theory we should load this from a file or something, but, ehhhh
+ hardcoded_cron_jobs = {
+ 'volunteer_janitor_recalc': {
+ 'frequency_day': DayOfWeek.ALL,
+ 'time_of_day_utc': datetime.time(0, 0),
+ 'import_path': 'files.commands.volunteer_janitor_recalc',
+ 'callable': 'volunteer_janitor_recalc_cron',
+ },
+ }
+
+ print(f"{tasklist.count()} tasks")
+ for task in tasklist:
+ if task.label and task.label in hardcoded_cron_jobs:
+ print(f"Cron: Updating {task.label}")
+ ref = hardcoded_cron_jobs[task.label]
+ task.frequency_day = ref["frequency_day"]
+ task.time_of_day_utc = ref["time_of_day_utc"]
+ task.import_path = ref["import_path"]
+ task.callable = ref["callable"]
+ del hardcoded_cron_jobs[task.label]
+
+ for label, ref in hardcoded_cron_jobs.items():
+ print(f"Cron: Creating {label}")
+ task: PythonCodeTask = PythonCodeTask(
+ label = label,
+ author_id = AUTOJANNY_ID,
+ frequency_day = ref["frequency_day"],
+ time_of_day_utc = ref["time_of_day_utc"],
+ import_path = ref["import_path"],
+ callable = ref["callable"],
+ )
+ db.add(task)
+
+ db.commit()
diff --git a/files/commands/seed_db.py b/files/commands/seed_db.py
index 08329866d..23e306047 100644
--- a/files/commands/seed_db.py
+++ b/files/commands/seed_db.py
@@ -1,15 +1,17 @@
import hashlib
import math
from typing import Optional
-import sqlalchemy
-from sqlalchemy.orm import scoped_session
+import sqlalchemy
+from sqlalchemy.orm import Session
from werkzeug.security import generate_password_hash
from files.__main__ import app, db_session
-from files.classes import User, Submission, Comment, Vote, CommentVote
+from files.classes import Comment, CommentVote, Submission, User, Vote
+from files.classes.visstate import StateMod
from files.helpers.comments import bulk_recompute_descendant_counts
+
@app.cli.command('seed_db')
def seed_db():
seed_db_worker()
@@ -23,7 +25,7 @@ def seed_db_worker(num_users = 900, num_posts = 40, num_toplevel_comments = 1000
COMMENT_UPVOTE_PROB = 0.0008
COMMENT_DOWNVOTE_PROB = 0.0003
- db: scoped_session = db_session()
+ db: Session = db_session()
def detrand():
detrand.randstate = bytes(hashlib.sha256(detrand.randstate).hexdigest(), 'utf-8')
@@ -102,9 +104,8 @@ def seed_db_worker(num_users = 900, num_posts = 40, num_toplevel_comments = 1000
embed_url=None,
title=f'Clever unique post title number {i}',
title_html=f'Clever unique post title number {i}',
- sub=None,
ghost=False,
- filter_state='normal'
+ state_mod=StateMod.VISIBLE,
)
db.add(post)
posts.append(post)
@@ -127,7 +128,8 @@ def seed_db_worker(num_users = 900, num_posts = 40, num_toplevel_comments = 1000
app_id=None,
body_html=f'toplevel {i}',
body=f'toplevel {i}',
- ghost=False
+ ghost=False,
+ state_mod=StateMod.VISIBLE,
)
db.add(comment)
comments.append(comment)
@@ -156,7 +158,8 @@ def seed_db_worker(num_users = 900, num_posts = 40, num_toplevel_comments = 1000
app_id=None,
body_html=f'reply {i}',
body=f'reply {i}',
- ghost=False
+ ghost=False,
+ state_mod=StateMod.VISIBLE,
)
db.add(comment)
comments.append(comment)
diff --git a/files/commands/volunteer_janitor_histogram.py b/files/commands/volunteer_janitor_histogram.py
new file mode 100644
index 000000000..c68a57bc5
--- /dev/null
+++ b/files/commands/volunteer_janitor_histogram.py
@@ -0,0 +1,35 @@
+
+import pprint
+
+from files.classes.volunteer_janitor import VolunteerJanitorRecord
+
+from files.__main__ import app, db_session
+
+@app.cli.command('volunteer_janitor_histogram')
+def volunteer_janitor_histogram_cmd():
+ import pandas as pd
+ import matplotlib.pyplot as plt
+
+ result_set = db_session().query(VolunteerJanitorRecord.recorded_utc).all()
+
+ # convert the result into a pandas DataFrame
+ df = pd.DataFrame(result_set, columns=['recorded_utc'])
+
+ # convert the date column to datetime
+ df['recorded_utc'] = pd.to_datetime(df['recorded_utc'])
+
+ # set 'recorded_utc' as the index of the DataFrame
+ df.set_index('recorded_utc', inplace=True)
+
+ # resample the data to daily frequency
+ df_resampled = df.resample('D').size()
+
+ # plot the resampled DataFrame
+ df_resampled.plot(kind='line')
+ plt.title('Density of Dates over Time')
+ plt.xlabel('Date')
+ plt.ylabel('Count')
+
+ # save the figure in SVG format
+ plt.savefig('output.svg', format='svg')
+ print(len(result_set))
diff --git a/files/commands/volunteer_janitor_recalc.py b/files/commands/volunteer_janitor_recalc.py
new file mode 100644
index 000000000..5941c6efb
--- /dev/null
+++ b/files/commands/volunteer_janitor_recalc.py
@@ -0,0 +1,379 @@
+
+import pprint
+
+import sqlalchemy
+from sqlalchemy.orm import Session
+
+from alive_progress import alive_it
+from collections import defaultdict
+from files.classes import User, Comment, UserNote, UserTag
+from files.classes.cron.tasks import TaskRunContext
+from files.classes.volunteer_janitor import VolunteerJanitorRecord, VolunteerJanitorResult
+from files.helpers.volunteer_janitor import evaluate_badness_of, userweight_from_user_accuracy, calculate_final_comment_badness, update_comment_badness
+from files.helpers.math import saturate, remap, lerp
+
+import logging
+import random
+
+from files.__main__ import app, db_session
+
+CONFIG_modhat_weight = 4
+CONFIG_admin_volunteer_weight = 4
+CONFIG_new_user_damping = 2
+CONFIG_default_user_accuracy = 0.2
+CONFIG_user_correctness_lerp = 0.2
+
+def _compile_records(db):
+ vrecords = db.query(VolunteerJanitorRecord).order_by(VolunteerJanitorRecord.recorded_utc).all()
+
+ # get the info we need for all mentioned posts
+ reported_comment_ids = {record.comment_id for record in vrecords}
+ reported_comments = db.query(Comment).where(Comment.id.in_(reported_comment_ids)).options(sqlalchemy.orm.load_only('id', 'state_user_deleted_utc'))
+ reported_comments = {comment.id: comment for comment in reported_comments}
+
+ # get our compiled data
+ records_compiled = {}
+ for record in vrecords:
+ # we're just going to ignore deleted comments entirely
+ if reported_comments[record.comment_id].state_user_deleted_utc != None:
+ continue
+
+ # unique identifier for user/comment report pair
+ uic = (record.user_id, record.comment_id)
+
+ if record.result == VolunteerJanitorResult.Pending:
+ if uic in records_compiled:
+ # something wonky happened, we went back to pending after getting a result?
+ records_compiled[uic]["status"] = "wonky"
+ else:
+ # fill out the pending field
+ records_compiled[uic] = {"status": "pending"}
+ else:
+ if not uic in records_compiled:
+ # something wonky happened, we never asked them for the info to begin with
+ records_compiled[uic] = {"status": "wonky"}
+ elif records_compiled[uic]["status"] != "pending":
+ # received two submissions; practically we'll just use their first submission
+ records_compiled[uic]["status"] = "resubmit"
+ else:
+ # actually got a result, yay
+ records_compiled[uic]["status"] = "submit"
+ records_compiled[uic]["result"] = record.result
+
+ # todo:
+ # filter out anything submitted *after* a mod chimed in
+ # filter out anything submitted too long after the request
+
+ users_compiled = defaultdict(lambda: {
+ "pending": 0,
+ "wonky": 0,
+ "submit": 0,
+ "resubmit": 0,
+ })
+ for key, result in records_compiled.items():
+ #pprint.pprint(key)
+ userid = key[0]
+
+ users_compiled[key[0]][result["status"]] += 1
+
+ #pprint.pprint(records_compiled)
+ #pprint.pprint(users_compiled)
+
+ # strip out invalid records
+ random_removal = -1 # this is sometimes useful for testing that our algorithm is somewhat stable; removing a random half of all responses shouldn't drastically invert people's quality scores, for example
+ records_compiled = {key: value for key, value in records_compiled.items() if "result" in value and random.random() > random_removal}
+
+ return records_compiled, users_compiled
+
+def dbg_commentdump(cid, records_compiled, users, user_accuracy):
+ print(f"Dump for comment {cid}")
+
+ from tabulate import tabulate
+
+ dats = []
+ for key, value in [(key, value) for key, value in records_compiled.items() if key[1] == cid]:
+ uid = key[0]
+ dats.append({
+ "vote": value["result"],
+ "username": users[uid]["username"],
+ "accuracy": user_accuracy[uid],
+ })
+ print(tabulate(dats, headers = "keys"))
+
+def dbg_userdump(uid, records_compiled, users, comment_calculated_badness_user):
+ print(f"Dump for user {users[uid]['username']}")
+
+ from tabulate import tabulate
+
+ dats = []
+ for key, value in [(key, value) for key, value in records_compiled.items() if key[0] == uid]:
+ cid = key[1]
+ bad, weight = evaluate_badness_of(value["result"])
+ dats.append({
+ "cid": cid,
+ "vote": value["result"],
+ "calculated": evaluate_badness_of(value["result"]),
+ "badness": comment_calculated_badness_user[cid],
+ "correctness": evaluate_correctness_single(bad, weight, comment_calculated_badness_user[cid]),
+ })
+ print(tabulate(dats, headers = "keys"))
+
+# Calculates how correct a user is, based on whether they thought it was bad, how confident they were, and how bad we think it is
+# Returns (IsCorrect, Confidence)
+def evaluate_correctness_single(bad, user_weight, calculated_badness):
+ # Boolean for whether this comment is bad
+ calculated_badbool = calculated_badness > 0.5
+
+ # Boolean for whether the user was correct
+ correctness_result = (bad == calculated_badbool) and 1 or 0
+
+ # "how confident are we that this is bad/notbad", range [0, 0.5]
+ calculated_badness_confidence = abs(calculated_badness - 0.5)
+
+ # "how much do we want this to influence the user's correctness"
+ # there's a deadzone around not-confident where we just push it to 0 and don't make it relevant
+ calculated_badness_weight = saturate(remap(calculated_badness_confidence, 0.1, 0.5, 0, 1))
+
+ # see how correct we think the user is
+ user_correctness = user_weight * calculated_badness_weight
+
+ return correctness_result, user_correctness
+
+def volunteer_janitor_recalc(db: Session, diagnostics: bool = False):
+ logging.info("Starting full janitor recalculation")
+
+ # Get our full list of data
+ records_compiled, users_compiled = _compile_records(db)
+
+ reporting_user_ids = {record[0] for record in records_compiled}
+ reported_comment_ids = {record[1] for record in records_compiled}
+
+ # Get some metadata for all reported comments
+ comments = db.query(Comment) \
+ .where(Comment.id.in_(reported_comment_ids)) \
+ .options(sqlalchemy.orm.load_only('id', 'created_utc', 'author_id'))
+ comments = {comment.id: comment for comment in comments}
+
+ reported_user_ids = {comment.author_id for comment in comments.values()}
+
+ # Get mod intervention data
+ modhats_raw = db.query(Comment) \
+ .where(Comment.parent_comment_id.in_(reported_comment_ids)) \
+ .where(Comment.distinguish_level > 0) \
+ .options(sqlalchemy.orm.load_only('parent_comment_id', 'created_utc'))
+
+ modhats = {}
+ # we jump through some hoops to deduplicate this; I guess we just pick the last one in our list for now
+ for modhat in modhats_raw:
+ modhats[modhat.parent_comment_id] = modhat
+
+ usernotes_raw = db.query(UserNote) \
+ .where(UserNote.tag.in_([UserTag.Warning, UserTag.Tempban, UserTag.Permban, UserTag.Spam, UserTag.Bot])) \
+ .options(sqlalchemy.orm.load_only('reference_user', 'created_utc', 'tag'))
+
+ # Here we're trying to figure out whether modhats are actually warnings/bans
+ # We don't have a formal connection between "a comment is bad" and "the user got a warning", so we're kind of awkwardly trying to derive it from our database
+ # In addition, sometimes someone posts a lot of bad comments and only gets modhatted for one of them
+ # That doesn't mean the other comments weren't bad
+ # It just means we picked the worst one
+ # So we ignore comments near the actual modhat time
+
+ commentresults = {}
+ for uid in reported_user_ids:
+ # For each user, figure out when modhats happened
+ # this is slow but whatever
+ modhat_times = []
+ for modhat in modhats.values():
+ if comments[modhat.parent_comment_id].author_id != uid:
+ continue
+
+ modhat_times.append(modhat.created_utc)
+
+ usernote_times = []
+ for usernote in usernotes_raw:
+ if usernote.reference_user != uid:
+ continue
+
+ usernote_times.append(usernote.created_utc)
+
+ # For each comment . . .
+ for comment in comments.values():
+ if comment.author_id != uid:
+ continue
+
+ if comment.id in modhats:
+ modhat_comment = modhats[comment.id]
+ else:
+ modhat_comment = None
+
+ # if the comment was modhatted *and* resulted in a negative usernote near the modhat time, it's bad
+ if modhat_comment is not None and next((time for time in usernote_times if abs(modhat_comment.created_utc - time) < 60 * 15), None) is not None:
+ commentresults[comment.id] = "bad"
+ # otherwise, if the comment was posted less than 48 hours before a negative usernote, we ignore it for processing on the assumption that it may just have been part of a larger warning
+ elif next((time for time in usernote_times if comment.created_utc < time and comment.created_utc + 48 * 60 * 60 > time), None) is not None:
+ commentresults[comment.id] = "ignored"
+ # otherwise, we call it not-bad
+ else:
+ commentresults[comment.id] = "notbad"
+
+ # get per-user metadata
+ users = db.query(User) \
+ .where(User.id.in_(reporting_user_ids)) \
+ .options(sqlalchemy.orm.load_only('id', 'username', 'admin_level'))
+ users = {user.id: {"username": user.username, "admin": user.admin_level != 0} for user in users}
+
+ user_accuracy = defaultdict(lambda: CONFIG_default_user_accuracy)
+
+ # Do an update loop!
+ for lid in range(0, 100):
+
+ # Accumulated weight/badness, taking admin flags into account
+ # This is used for training
+ comment_weight_admin = defaultdict(lambda: 0)
+ comment_badness_admin = defaultdict(lambda: 0)
+
+ # Accumulated weight/badness, not taking admin flags into account
+ # This is used for output and display
+ comment_weight_user = defaultdict(lambda: 0)
+ comment_badness_user = defaultdict(lambda: 0)
+
+ # accumulate modhat weights
+ for cid in reported_comment_ids:
+ result = commentresults[cid]
+
+ if result == "ignored":
+ # I guess we'll just let the users decide?
+ continue
+
+ if result == "bad":
+ comment_weight_admin[cid] += CONFIG_modhat_weight
+ comment_badness_admin[cid] += CONFIG_modhat_weight
+
+ if result == "notbad":
+ comment_weight_admin[cid] += CONFIG_modhat_weight
+
+ # accumulate volunteer weights
+ for key, value in records_compiled.items():
+ uid, cid = key
+
+ # Calculate how much to weight a user; highly inaccurate users are not inverted! They just don't get contribution
+ # (losers)
+ userweight_user = userweight_from_user_accuracy(user_accuracy[uid]);
+
+ if users[uid]["admin"]:
+ userweight_admin = CONFIG_admin_volunteer_weight
+ else:
+ userweight_admin = userweight_user
+
+ bad, weight = evaluate_badness_of(value["result"])
+
+ # Accumulate these to our buffers
+ comment_weight_admin[cid] += userweight_admin * weight
+ comment_weight_user[cid] += userweight_user * weight
+
+ if bad:
+ comment_badness_admin[cid] += userweight_admin * weight
+ comment_badness_user[cid] += userweight_user * weight
+
+ # Calculated badnesses, both taking admins into account and not doing so, and "theoretical idea" versus a conversative view designed to be more skeptical of low-weighted comments
+ comment_calculated_badness_admin = {cid: calculate_final_comment_badness(comment_badness_admin[cid], comment_weight_admin[cid], False) for cid in reported_comment_ids}
+ comment_calculated_badness_admin_conservative = {cid: calculate_final_comment_badness(comment_badness_admin[cid], comment_weight_admin[cid], True) for cid in reported_comment_ids}
+ comment_calculated_badness_user = {cid: calculate_final_comment_badness(comment_badness_user[cid], comment_weight_user[cid], False) for cid in reported_comment_ids}
+ comment_calculated_badness_user_conservative = {cid: calculate_final_comment_badness(comment_badness_user[cid], comment_weight_user[cid], True) for cid in reported_comment_ids}
+
+ # go through user submissions and count up how good users seem to be at this
+ user_correctness_weight = defaultdict(lambda: CONFIG_new_user_damping)
+ user_correctness_value = defaultdict(lambda: CONFIG_default_user_accuracy * user_correctness_weight[0])
+
+ for key, value in records_compiled.items():
+ uid, cid = key
+
+ # if this is "ignored", I don't trust that we have a real answer, so we just skip it for training purposes
+ if commentresults[cid] == "ignored":
+ continue
+
+ bad, weight = evaluate_badness_of(value["result"])
+
+ correctness, weight = evaluate_correctness_single(bad, weight, comment_calculated_badness_admin[cid])
+
+ user_correctness_weight[uid] += weight
+ user_correctness_value[uid] += correctness * weight
+
+ # calculate new correctnesses
+ for uid in reporting_user_ids:
+ target_user_correctness = user_correctness_value[uid] / user_correctness_weight[uid]
+
+ # lerp slowly to the new values
+ user_accuracy[uid] = lerp(user_accuracy[uid], target_user_correctness, CONFIG_user_correctness_lerp)
+
+ if diagnostics:
+ # debug print
+
+ from tabulate import tabulate
+
+ commentscores = [{
+ "link": f"https://themotte.org/comment/{cid}",
+ "badness": comment_calculated_badness_admin[cid],
+ "badnessuser": comment_calculated_badness_user[cid],
+ "badnessusercons": comment_calculated_badness_user_conservative[cid],
+ "participation": comment_weight_user[cid],
+ "mh": commentresults[cid]} for cid in reported_comment_ids]
+ commentscores.sort(key = lambda item: item["badnessusercons"] + item["badnessuser"] / 100)
+ print(tabulate(commentscores, headers = "keys"))
+
+ results = [{
+ "user": f"https://themotte.org/@{users[uid]['username']}",
+ "accuracy": user_accuracy[uid],
+ "submit": users_compiled[uid]["submit"],
+ "nonsubmit": sum(users_compiled[uid].values()) - users_compiled[uid]["submit"],
+ "admin": users[uid]["admin"] and "Admin" or "",
+ } for uid in reporting_user_ids]
+ results.sort(key = lambda k: k["accuracy"])
+ print(tabulate(results, headers = "keys"))
+
+ dbg_commentdump(89681, records_compiled, users, user_accuracy)
+ print(calculate_final_comment_badness(comment_badness_user[89681], comment_weight_user[89681], True))
+
+ #dbg_userdump(131, records_compiled, users, comment_calculated_badness_user)
+
+ # Shove all this in the database, yaaay
+ # Conditional needed because sqlalchemy breaks if you try passing it zero data
+ if len(user_accuracy) > 0:
+ db.query(User) \
+ .where(User.id.in_([id for id in user_accuracy.keys()])) \
+ .update({
+ User.volunteer_janitor_correctness: sqlalchemy.sql.case(
+ user_accuracy,
+ value = User.id,
+ )
+ })
+ db.commit()
+
+ # We don't bother recalculating comment confidences here; it's a pain to do it and they shouldn't change much
+
+ logging.info("Finished full janitor recalculation")
+
+@app.cli.command('volunteer_janitor_recalc')
+def volunteer_janitor_recalc_cmd():
+ volunteer_janitor_recalc(db_session(), diagnostics = True)
+
+def volunteer_janitor_recalc_cron(ctx:TaskRunContext):
+ volunteer_janitor_recalc(ctx.db)
+
+def volunteer_janitor_recalc_all_comments(db: Session):
+ # may as well do this first
+ volunteer_janitor_recalc(db)
+
+ # I'm not sure of the details here, but there seems to be some session-related caching cruft left around
+ # so let's just nuke that
+ db.expire_all()
+
+ # going through all the comments piecemeal like this is hilariously efficient, but this entire system gets run exactly once ever, so, okay
+ for comment in alive_it(db.query(Comment).join(Comment.reports)):
+ update_comment_badness(db, comment.id)
+
+ db.commit()
+
+@app.cli.command('volunteer_janitor_recalc_all_comments')
+def volunteer_janitor_recalc_all_comments_cmd():
+ volunteer_janitor_recalc_all_comments(db_session())
diff --git a/files/helpers/alerts.py b/files/helpers/alerts.py
index cea6d52d9..75927e6e3 100644
--- a/files/helpers/alerts.py
+++ b/files/helpers/alerts.py
@@ -1,7 +1,9 @@
from files.classes import *
from flask import g
+
from .sanitize import *
-from .const import *
+from .config.const import *
+from files.classes.visstate import StateMod
def create_comment(text_html, autojanny=False):
if autojanny: author_id = AUTOJANNY_ID
@@ -10,7 +12,8 @@ def create_comment(text_html, autojanny=False):
new_comment = Comment(author_id=author_id,
parent_submission=None,
body_html=text_html,
- distinguish_level=6)
+ distinguish_level=6,
+ state_mod=StateMod.VISIBLE,)
g.db.add(new_comment)
g.db.flush()
@@ -19,7 +22,6 @@ def create_comment(text_html, autojanny=False):
return new_comment.id
def send_repeatable_notification(uid, text, autojanny=False):
-
if autojanny: author_id = AUTOJANNY_ID
else: author_id = NOTIFICATIONS_ID
@@ -38,13 +40,11 @@ def send_repeatable_notification(uid, text, autojanny=False):
def send_notification(uid, text, autojanny=False):
-
cid = notif_comment(text, autojanny)
add_notif(cid, uid)
def notif_comment(text, autojanny=False):
-
if autojanny:
author_id = AUTOJANNY_ID
alert = True
@@ -61,7 +61,6 @@ def notif_comment(text, autojanny=False):
def notif_comment2(p):
-
search_html = f'% has mentioned you: %'
existing = g.db.query(Comment.id).filter(Comment.author_id == NOTIFICATIONS_ID, Comment.parent_submission == None, Comment.body_html.like(search_html)).first()
@@ -69,7 +68,6 @@ def notif_comment2(p):
if existing: return existing[0]
else:
text = f"@{p.author.username} has mentioned you: [{p.title}](/post/{p.id})"
- if p.sub: text += f" in /h/{p.sub}"
text_html = sanitize(text, alert=True)
return create_comment(text_html)
@@ -81,11 +79,12 @@ def add_notif(cid, uid):
g.db.add(notif)
-def NOTIFY_USERS(text, v):
+def NOTIFY_USERS(text, v) -> set[int]:
notify_users = set()
for word, id in NOTIFIED_USERS.items():
if id == 0 or v.id == id: continue
- if word in text.lower() and id not in notify_users: notify_users.add(id)
+ if word in text.lower() and id not in notify_users:
+ notify_users.add(id)
captured = []
for i in mention_regex.finditer(text):
@@ -95,6 +94,26 @@ def NOTIFY_USERS(text, v):
captured.append(i.group(0))
user = get_user(i.group(2), graceful=True)
- if user and v.id != user.id and not v.any_block_exists(user): notify_users.add(user.id)
+ if user and v.id != user.id and not v.any_block_exists(user):
+ notify_users.add(user.id)
return notify_users
+
+def notify_submission_publish(target: Submission):
+ # Username mentions in title & body
+ text: str = f'{target.title} {target.body}'
+ notify_users = NOTIFY_USERS(text, target.author)
+ if notify_users:
+ comment_id = notif_comment2(target)
+ for user_id in notify_users:
+ add_notif(comment_id, user_id)
+
+ # Submission author followers
+ if target.author.followers:
+ message: str = (
+ f"@{target.author.username} has made a new post: "
+ f"[{target.title}]({target.shortlink})"
+ )
+ cid = notif_comment(message, autojanny=True)
+ for follow in target.author.followers:
+ add_notif(cid, follow.user_id)
diff --git a/files/helpers/caching.py b/files/helpers/caching.py
new file mode 100644
index 000000000..16094568d
--- /dev/null
+++ b/files/helpers/caching.py
@@ -0,0 +1,25 @@
+from files.__main__ import cache
+import files.helpers.listing as listing
+
+# i hate this.
+#
+# we should probably come up with a better way for this in the future.
+# flask_caching is kinda weird in that it requires you to use a function
+# reference to deleted a memoized function, which basically means fitting your
+# code to flask_caching's worldview. it's very much not ideal and ideally would
+# be less coupled in the future.
+#
+# the question is whether it's worth it.
+
+def invalidate_cache(*, frontlist=False, userpagelisting=False, changeloglist=False):
+ '''
+ Invalidates the caches for the front page listing, user page listings,
+ and optionally, the changelog listing.
+
+ :param frontlist: Whether to invalidate the `frontlist` cache.
+ :param userpagelisting: Whether to invalidate the `userpagelisting` cache.
+ :param changeloglist: Whether to invalidate the `changeloglist` cache.
+ '''
+ if frontlist: cache.delete_memoized(listing.frontlist)
+ if userpagelisting: cache.delete_memoized(listing.userpagelisting)
+ if changeloglist: cache.delete_memoized(listing.changeloglist)
diff --git a/files/helpers/comments.py b/files/helpers/comments.py
index 5e04a336e..2f409d983 100644
--- a/files/helpers/comments.py
+++ b/files/helpers/comments.py
@@ -1,16 +1,20 @@
-from pusher_push_notifications import PushNotifications
-from files.classes import Comment, Notification, Subscription, User
-from files.helpers.alerts import NOTIFY_USERS
-from files.helpers.const import PUSHER_ID, PUSHER_KEY, SITE_ID, SITE_FULL
-from files.helpers.assetcache import assetcache_path
-from flask import g
-from sqlalchemy import select, update
-from sqlalchemy.sql.expression import func, text, alias
-from sqlalchemy.orm import Query, aliased
from sys import stdout
-import gevent
from typing import Optional
+import gevent
+from flask import g, request
+from pusher_push_notifications import PushNotifications
+from sqlalchemy import select, update
+from sqlalchemy.orm import Query, aliased
+from sqlalchemy.sql.expression import alias, func, text
+
+from files.classes import Comment, Notification, Subscription, User
+from files.classes.visstate import StateMod
+from files.helpers.alerts import NOTIFY_USERS
+from files.helpers.assetcache import assetcache_path
+from files.helpers.config.environment import (PUSHER_ID, PUSHER_KEY, SITE_FULL,
+ SITE_ID)
+
if PUSHER_ID != 'blahblahblah':
beams_client = PushNotifications(instance_id=PUSHER_ID, secret_key=PUSHER_KEY)
@@ -62,8 +66,8 @@ def update_author_comment_count(comment, delta):
comment.author.comment_count = g.db.query(Comment).filter(
Comment.author_id == comment.author_id,
Comment.parent_submission != None,
- Comment.is_banned == False,
- Comment.deleted_utc == 0,
+ Comment.state_mod == StateMod.VISIBLE,
+ Comment.state_user_deleted_utc == None,
).count()
g.db.add(comment.author)
@@ -197,8 +201,11 @@ def comment_on_publish(comment:Comment):
to_notify.add(parent.author_id)
for uid in to_notify:
- notif = Notification(comment_id=comment.id, user_id=uid)
- g.db.add(notif)
+ notif = g.db.query(Notification) \
+ .filter_by(comment_id=comment.id, user_id=uid).one_or_none()
+ if not notif:
+ notif = Notification(comment_id=comment.id, user_id=uid)
+ g.db.add(notif)
update_stateful_counters(comment, +1)
@@ -212,7 +219,7 @@ def comment_on_publish(comment:Comment):
def comment_on_unpublish(comment:Comment):
"""
Run when a comment becomes invisible: when a moderator makes the comment non-visible
- by changing the filter_state to "removed", or when the user deletes the comment.
+ by changing the state_mod to "removed", or when the user deletes the comment.
Should be used to update stateful counters, notifications, etc. that
reflect the comments users will actually see.
"""
@@ -220,13 +227,12 @@ def comment_on_unpublish(comment:Comment):
def comment_filter_moderated(q: Query, v: Optional[User]) -> Query:
- if not (v and v.shadowbanned) and not (v and v.admin_level > 2):
+ if not (v and v.shadowbanned) and not (v and v.admin_level >= 3):
q = q.join(User, User.id == Comment.author_id) \
.filter(User.shadowbanned == None)
if not v or v.admin_level < 2:
q = q.filter(
- ((Comment.filter_state != 'filtered')
- & (Comment.filter_state != 'removed'))
+ (Comment.state_mod == StateMod.VISIBLE)
| (Comment.author_id == ((v and v.id) or 0))
)
return q
diff --git a/files/helpers/const.py b/files/helpers/config/const.py
similarity index 52%
rename from files/helpers/const.py
rename to files/helpers/config/const.py
index 09a38af34..1305f7b62 100644
--- a/files/helpers/const.py
+++ b/files/helpers/config/const.py
@@ -1,19 +1,47 @@
import re
+import sys
from copy import deepcopy
-from os import environ
+from enum import IntEnum
from typing import Final
from flask import request
-from files.__main__ import db_session
-from files.classes.sub import Sub
-from files.classes.marsey import Marsey
-SITE = environ.get("DOMAIN", '').strip()
-SITE_ID = environ.get("SITE_ID", '').strip()
-SITE_TITLE = environ.get("SITE_TITLE", '').strip()
-SCHEME = environ.get('SCHEME', 'http' if 'localhost' in SITE else 'https')
-SITE_FULL = SCHEME + '://' + SITE
+class Service(IntEnum):
+ '''
+ An enumeration of services provided by this application
+ '''
+ THEMOTTE = 0
+ '''
+ TheMotte web application. Handles most routes and tasks performed,
+ including all non-chat web requests.
+ '''
+ CRON = 1
+ '''
+ Runs tasks periodicially on a set schedule
+ '''
+ CHAT = 2
+ '''
+ Chat application.
+ '''
+ MIGRATION = 3
+ '''
+ Migration mode. Used for performing database migrations
+ '''
+
+ @classmethod
+ def from_argv(cls):
+ if "db" in sys.argv:
+ return cls.MIGRATION
+ if "cron" in sys.argv:
+ return cls.CRON
+ if "load_chat" in sys.argv:
+ return cls.CHAT
+ return cls.THEMOTTE
+
+ @property
+ def enable_services(self) -> bool:
+ return self not in {self.CRON, self.MIGRATION}
CC = "COUNTRY CLUB"
CC_TITLE = CC.title()
@@ -32,13 +60,12 @@ BASEDBOT_ID = 0
GIFT_NOTIF_ID = 9
OWNER_ID = 9
BUG_THREAD = 0
-WELCOME_MSG = f"Welcome to {SITE_TITLE}! Please read [the rules](/rules) first. Then [read some of our current conversations](/) and feel free to comment or post!\n\nWe encourage people to comment even if they aren't sure they fit in; as long as your comment follows [community rules](/rules), we are happy to have posters from all backgrounds, education levels, and specialties."
ROLES={}
LEADERBOARD_LIMIT: Final[int] = 25
-THEMES = {"TheMotte", "dramblr", "reddit", "transparent", "win98", "dark",
- "light", "coffee", "tron", "4chan", "midnight"}
+THEMES = ["TheMotte", "dramblr", "reddit", "win98", "dark",
+ "light", "coffee", "tron", "4chan", "midnight"]
SORTS_COMMON = {
"top": 'fa-arrow-alt-circle-up',
"bottom": 'fa-arrow-alt-circle-down',
@@ -54,11 +81,16 @@ SORTS_POSTS = {
SORTS_POSTS.update(SORTS_COMMON)
SORTS_COMMENTS = SORTS_COMMON
-PUSHER_ID = environ.get("PUSHER_ID", "").strip()
-PUSHER_KEY = environ.get("PUSHER_KEY", "").strip()
-DEFAULT_COLOR = environ.get("DEFAULT_COLOR", "fff").strip()
-COLORS = {'ff66ac','805ad5','62ca56','38a169','80ffff','2a96f3','eb4963','ff0000','f39731','30409f','3e98a7','e4432d','7b9ae4','ec72de','7f8fa6', 'f8db58','8cdbe6', DEFAULT_COLOR}
+MAX_CONTENT_LENGTH = 16 * 1024 * 1024
+SESSION_COOKIE_SAMESITE = "Lax"
+PERMANENT_SESSION_LIFETIME = 60 * 60 * 24 * 365
+DEFAULT_THEME = "TheMotte"
+FORCE_HTTPS = 1
+COLORS = {'ff66ac','805ad5','62ca56','38a169','80ffff','2a96f3','eb4963','ff0000','f39731','30409f','3e98a7','e4432d','7b9ae4','ec72de','7f8fa6', 'f8db58','8cdbe6', 'fff'}
+SUBMISSION_FLAIR_LENGTH_MAXIMUM: Final[int] = 350
+SUBMISSION_TITLE_LENGTH_MAXIMUM: Final[int] = 500
+SUBMISSION_URL_LENGTH_MAXIMUM: Final[int] = 2048
SUBMISSION_BODY_LENGTH_MAXIMUM: Final[int] = 20000
COMMENT_BODY_LENGTH_MAXIMUM: Final[int] = 10000
MESSAGE_BODY_LENGTH_MAXIMUM: Final[int] = 10000
@@ -72,6 +104,7 @@ ERROR_MESSAGES = {
405: "Something went wrong and it's probably my fault. If you can do it reliably, or it's causing problems for you, please report it!",
409: "There's a conflict between what you're trying to do and what you or someone else has done and because of that you can't do what you're trying to do.",
413: "Max file size is 8 MB",
+ 415: "That file type isn't allowed to be uploaded here",
422: "Something is wrong about your request. If you keep getting this unexpectedly, please report it!",
429: "Are you hammering the site? Stop that, yo.",
500: "Something went wrong and it's probably my fault. If you can do it reliably, or it's causing problems for you, please report it!",
@@ -98,6 +131,7 @@ WERKZEUG_ERROR_DESCRIPTIONS = {
}
IMAGE_FORMATS = ['png','gif','jpg','jpeg','webp']
+IMAGE_URL_ENDINGS = IMAGE_FORMATS + ['.webp', '.jpg', '.png', '.gif', '.jpeg', '?maxwidth=9999', '&fidelity=high']
VIDEO_FORMATS = ['mp4','webm','mov','avi','mkv','flv','m4v','3gp']
AUDIO_FORMATS = ['mp3','wav','ogg','aac','m4a','flac']
NO_TITLE_EXTENSIONS = IMAGE_FORMATS + VIDEO_FORMATS + AUDIO_FORMATS
@@ -108,7 +142,17 @@ FEATURES = {
PERMS = {
"DEBUG_LOGIN_TO_OTHERS": 3,
+ "PERFORMANCE_KILL_PROCESS": 3,
+ "PERFORMANCE_SCALE_UP_DOWN": 3,
+ "PERFORMANCE_RELOAD": 3,
+ "PERFORMANCE_STATS": 3,
+ "POST_COMMENT_MODERATION": 2,
+ "POST_EDITING": 3,
+ "SCHEDULER": 2,
+ "SCHEDULER_POSTS": 2,
+ "SCHEDULER_TASK_TRACEBACK": 3,
"USER_SHADOWBAN": 2,
+ 'USER_SET_PROFILE_PRIVACY': 2,
}
AWARDS = {}
@@ -132,82 +176,6 @@ NOTIFIED_USERS = {
patron = 'Patron'
-discounts = {
- 69: 0.02,
- 70: 0.04,
- 71: 0.06,
- 72: 0.08,
- 73: 0.10,
-}
-
-CF_KEY = environ.get("CF_KEY", "").strip()
-CF_ZONE = environ.get("CF_ZONE", "").strip()
-CF_HEADERS = {"Authorization": f"Bearer {CF_KEY}", "Content-Type": "application/json"}
-
-dues = int(environ.get("DUES").strip())
-
-christian_emojis = (':#marseyjesus:',':#marseyimmaculate:',':#marseymothermary:',':#marseyfatherjoseph:',':#gigachadorthodox:',':#marseyorthodox:',':#marseyorthodoxpat:')
-
-db = db_session()
-marseys_const = [x[0] for x in db.query(Marsey.name).filter(Marsey.name!='chudsey').all()]
-marseys_const2 = marseys_const + ['chudsey','a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','0','1','2','3','4','5','6','7','8','9','exclamationpoint','period','questionmark']
-db.close()
-
-valid_username_chars = 'a-zA-Z0-9_\\-'
-valid_username_regex = re.compile("^[a-zA-Z0-9_\\-]{3,25}$", flags=re.A)
-mention_regex = re.compile('(^|\\s| )@(([a-zA-Z0-9_\\-]){1,25})', flags=re.A)
-mention_regex2 = re.compile(' @(([a-zA-Z0-9_\\-]){1,25})', flags=re.A)
-
-valid_password_regex = re.compile("^.{8,100}$", flags=re.A)
-
-marsey_regex = re.compile("[a-z0-9]{1,30}", flags=re.A)
-
-tags_regex = re.compile("[a-z0-9: ]{1,200}", flags=re.A)
-
-valid_sub_regex = re.compile("^[a-zA-Z0-9_\\-]{3,20}$", flags=re.A)
-
-query_regex = re.compile("(\\w+):(\\S+)", flags=re.A)
-
-title_regex = re.compile("[^\\w ]", flags=re.A)
-
-based_regex = re.compile("based and (.{1,20}?)(-| )pilled", flags=re.I|re.A)
-
-controversial_regex = re.compile('["> ](https:\\/\\/old\\.reddit\\.com/r/[a-zA-Z0-9_]{3,20}\\/comments\\/[\\w\\-.#&/=\\?@%+]{5,250})["< ]', flags=re.A)
-
-fishylinks_regex = re.compile("https?://\\S+", flags=re.A)
-
-spoiler_regex = re.compile('''\\|\\|(.+)\\|\\|''', flags=re.A)
-reddit_regex = re.compile('(^|\\s| )\\/?((r|u)\\/(\\w|-){3,25})(?![^<]*<\\/(code|pre|a)>)', flags=re.A)
-sub_regex = re.compile('(^|\\s| )\\/?(h\\/(\\w|-){3,25})', flags=re.A)
-
-# Bytes that shouldn't be allowed in user-submitted text
-# U+200E is LTR toggle, U+200F is RTL toggle, U+200B and U+FEFF are Zero-Width Spaces,
-# and U+1242A is a massive and terrifying cuneiform numeral
-unwanted_bytes_regex = re.compile("\u200e|\u200f|\u200b|\ufeff|\U0001242a")
-
-whitespace_regex = re.compile('\\s+')
-
-strikethrough_regex = re.compile('''~{1,2}([^~]+)~{1,2}''', flags=re.A)
-
-mute_regex = re.compile("/mute @([a-z0-9_\\-]{3,25}) ([0-9])+", flags=re.A)
-
-emoji_regex = re.compile(f"[^a]>\\s*(:[!#@]{{0,3}}[{valid_username_chars}]+:\\s*)+<\\/", flags=re.A)
-emoji_regex2 = re.compile(f"(?([\\w:~,()\\-.#&\\/=?@%;+]{5,250})<\\/a>', flags=re.A)
-
-# Technically this allows stuff that is not a valid email address, but realistically
-# we care "does this email go to the correct person" rather than "is this email
-# address syntactically valid", so if we care we should be sending a confirmation
-# link, and otherwise should be pretty liberal in what we accept here.
-email_regex = re.compile('[^@]+@[^@]+\\.[^@]+', flags=re.A)
-
-utm_regex = re.compile('utm_[a-z]+=[a-z0-9_]+&', flags=re.A)
-utm_regex2 = re.compile('[?&]utm_[a-z]+=[a-z0-9_]+', flags=re.A)
-
-YOUTUBE_KEY = environ.get("YOUTUBE_KEY", "").strip()
-
proxies = {}
approved_embed_hosts = [
@@ -265,23 +233,12 @@ approved_embed_hosts = [
]
hosts = "|".join(approved_embed_hosts).replace('.','\\.')
-
image_check_regex = re.compile(f'!\\[\\]\\(((?!(https:\\/\\/([a-z0-9-]+\\.)*({hosts})\\/|\\/)).*?)\\)', flags=re.A)
-
embed_fullmatch_regex = re.compile(f'https:\\/\\/([a-z0-9-]+\\.)*({hosts})\\/[\\w:~,()\\-.#&\\/=?@%;+]*', flags=re.A)
-
video_sub_regex = re.compile(f'( [^<]*)(https:\\/\\/([a-z0-9-]+\\.)*({hosts})\\/[\\w:~,()\\-.#&\\/=?@%;+]*?\\.(mp4|webm|mov))', flags=re.A)
-youtube_regex = re.compile('( [^<]*)(https:\\/\\/youtube\\.com\\/watch\\?v\\=([a-z0-9-_]{5,20})[\\w\\-.#&/=\\?@%+]*)', flags=re.I|re.A)
-
-yt_id_regex = re.compile('[a-z0-9-_]{5,20}', flags=re.I|re.A)
-
-image_regex = re.compile("(^|\\s)(https:\\/\\/[\\w\\-.#&/=\\?@%;+]{5,250}(\\.png|\\.jpg|\\.jpeg|\\.gif|\\.webp|maxwidth=9999|fidelity=high))($|\\s)", flags=re.I|re.A)
-
procoins_li = (0,2500,5000,10000,25000,50000,125000,250000)
-linefeeds_regex = re.compile("([^\\n])\\n([^\\n])", flags=re.A)
-
-html_title_regex = re.compile(" )@(([a-zA-Z0-9_\\-]){1,25})', flags=re.A)
+mention_regex2 = re.compile(' @(([a-zA-Z0-9_\\-]){1,25})', flags=re.A)
+
+valid_password_regex = re.compile("^.{8,100}$", flags=re.A)
+
+marsey_regex = re.compile("[a-z0-9]{1,30}", flags=re.A)
+
+tags_regex = re.compile("[a-z0-9: ]{1,200}", flags=re.A)
+
+valid_sub_regex = re.compile("^[a-zA-Z0-9_\\-]{3,20}$", flags=re.A)
+
+query_regex = re.compile("(\\w+):(\\S+)", flags=re.A)
+
+title_regex = re.compile("[^\\w ]", flags=re.A)
+
+based_regex = re.compile("based and (.{1,20}?)(-| )pilled", flags=re.I|re.A)
+
+controversial_regex = re.compile('["> ](https:\\/\\/old\\.reddit\\.com/r/[a-zA-Z0-9_]{3,20}\\/comments\\/[\\w\\-.#&/=\\?@%+]{5,250})["< ]', flags=re.A)
+
+fishylinks_regex = re.compile("https?://\\S+", flags=re.A)
+
+spoiler_regex = re.compile('''\\|\\|(.+)\\|\\|''', flags=re.A)
+reddit_regex = re.compile('(^|\\s| )\\/?((r|u)\\/(\\w|-){3,25})(?![^<]*<\\/(code|pre|a)>)', flags=re.A)
+sub_regex = re.compile('(^|\\s| )\\/?(h\\/(\\w|-){3,25})', flags=re.A)
+
+
+unwanted_bytes_regex = re.compile("\u200e|\u200f|\u200b|\ufeff|\U0001242a")
+'''
+Bytes that shouldn't be allowed in user-submitted text
+U+200E is LTR toggle, U+200F is RTL toggle, U+200B and U+FEFF are Zero-Width
+Spaces, and U+1242A is a massive and terrifying cuneiform numeral
+'''
+
+whitespace_regex = re.compile('\\s+')
+
+strikethrough_regex = re.compile('''~{1,2}([^~]+)~{1,2}''', flags=re.A)
+
+mute_regex = re.compile("/mute @([a-z0-9_\\-]{3,25}) ([0-9])+", flags=re.A)
+
+emoji_regex = re.compile(f"[^a]>\\s*(:[!#@]{{0,3}}[{valid_username_chars}]+:\\s*)+<\\/", flags=re.A)
+emoji_regex2 = re.compile(f"(?([\\w:~,()\\-.#&\\/=?@%;+]{5,250})<\\/a>', flags=re.A)
+
+email_regex = re.compile('[^@]+@[^@]+\\.[^@]+', flags=re.A)
+'''
+Regex to use for email addresses.
+
+.. note::
+ Technically this allows stuff that is not a valid email address, but
+ realistically we care "does this email go to the correct person" rather
+ than "is this email address syntactically valid", so if we care we should
+ be sending a confirmation link, and otherwise should be pretty liberal in
+ what we accept here.
+'''
+
+utm_regex = re.compile('utm_[a-z]+=[a-z0-9_]+&', flags=re.A)
+utm_regex2 = re.compile('[?&]utm_[a-z]+=[a-z0-9_]+', flags=re.A)
+
+
+# urls
+
+youtube_regex = re.compile('( [^<]*)(https:\\/\\/youtube\\.com\\/watch\\?v\\=([a-z0-9-_]{5,20})[\\w\\-.#&/=\\?@%+]*)', flags=re.I|re.A)
+
+yt_id_regex = re.compile('[a-z0-9-_]{5,20}', flags=re.I|re.A)
+
+image_regex = re.compile("(^|\\s)(https:\\/\\/[\\w\\-.#&/=\\?@%;+]{5,250}(\\.png|\\.jpg|\\.jpeg|\\.gif|\\.webp|maxwidth=9999|fidelity=high))($|\\s)", flags=re.I|re.A)
+
+linefeeds_regex = re.compile("([^\\n])\\n([^\\n])", flags=re.A)
+
+html_title_regex = re.compile(r" hello world New site mention: 2)):
- comments = comments.join(User, User.id == Comment.author_id).filter(User.shadowbanned == None)
+ if not v.shadowbanned and v.admin_level < 3:
+ comments = comments.join(Comment.author).filter(User.shadowbanned == None)
- comments = comments.offset(25 * (page - 1)).limit(26).all()
+ comments = comments.offset(25 * (page - 1)).limit(26).all()
- next_exists = (len(comments) > 25)
- comments = comments[:25]
+ next_exists = (len(comments) > 25)
+ comments = comments[:25]
- cids = [x[0].id for x in comments]
+ for c, n in comments:
+ c.notif_utc = n.created_utc
+ c.unread = not n.read
+ n.read = True
- comms = get_comments(cids, v=v)
+ listing: list[Comment] = [c for c, _ in comments]
- listing = []
- for c, n in comments:
- if n.created_utc > 1620391248: c.notif_utc = n.created_utc
- if not n.read:
- n.read = True
- c.unread = True
- g.db.add(n)
+ # TODO: commit after request rendered, then default session expiry is fine
+ g.db.expire_on_commit = False
+ g.db.commit()
+ g.db.expire_on_commit = True
- if c.parent_submission:
- if c.replies2 == None:
- c.replies2 = c.child_comments.filter(or_(Comment.author_id == v.id, Comment.id.in_(cids))).all()
- for x in c.replies2:
- if x.replies2 == None: x.replies2 = []
- count = 0
- while count < 50 and c.parent_comment and (c.parent_comment.author_id == v.id or c.parent_comment.id in cids):
- count += 1
- c = c.parent_comment
- if c.replies2 == None:
- c.replies2 = c.child_comments.filter(or_(Comment.author_id == v.id, Comment.id.in_(cids))).all()
- for x in c.replies2:
- if x.replies2 == None:
- x.replies2 = x.child_comments.filter(or_(Comment.author_id == v.id, Comment.id.in_(cids))).all()
- else:
- while c.parent_comment:
- c = c.parent_comment
- c.replies2 = g.db.query(Comment).filter_by(parent_comment_id=c.id).order_by(Comment.id).all()
+ if request.headers.get("Authorization"):
+ return {"data": [x.json for x in listing]}
- if c not in listing: listing.append(c)
+ return render_template("notifications.html",
+ v=v,
+ notifications=listing,
+ next_exists=next_exists,
+ page=page,
+ standalone=True,
+ render_replies=False,
+ is_notification_page=True,
+ )
+
+
+@app.get("/notifications/posts")
+@auth_required
+def notifications_posts(v: User):
+ page: int = max(request.values.get("page", 1, int) or 1, 1)
+
+ notifications = (g.db.query(Notification, Comment)
+ .join(Comment, Notification.comment_id == Comment.id)
+ .filter(Notification.user_id == v.id, Comment.author_id == AUTOJANNY_ID)
+ .order_by(Notification.created_utc.desc()).offset(25 * (page - 1)).limit(101).all())
+
+ listing = []
+
+ for index, x in enumerate(notifications[:100]):
+ n, c = x
+ if n.read and index > 24: break
+ elif not n.read:
+ n.read = True
+ c.unread = True
+ g.db.add(n)
+ if n.created_utc > 1620391248: c.notif_utc = n.created_utc
+ listing.append(c)
+
+ next_exists = (len(notifications) > len(listing))
g.db.commit()
- if request.headers.get("Authorization"): return {"data":[x.json for x in listing]}
+ if request.headers.get("Authorization"):
+ return {"data": [x.json for x in listing]}
return render_template("notifications.html",
- v=v,
- notifications=listing,
- next_exists=next_exists,
- page=page,
- standalone=True,
- render_replies=True
- )
+ v=v,
+ notifications=listing,
+ next_exists=next_exists,
+ page=page,
+ standalone=True,
+ render_replies=True,
+ is_notification_page=True,
+ )
+
+
+@app.get("/notifications/modmail")
+@admin_level_required(2)
+def notifications_modmail(v: User):
+ page: int = max(request.values.get("page", 1, int) or 1, 1)
+
+ comments = (g.db.query(Comment)
+ .filter(Comment.sentto == MODMAIL_ID)
+ .order_by(Comment.id.desc()).offset(25 * (page - 1)).limit(26).all())
+ next_exists = (len(comments) > 25)
+ listing = comments[:25]
+
+ if request.headers.get("Authorization"):
+ return {"data": [x.json for x in listing]}
+
+ return render_template("notifications.html",
+ v=v,
+ notifications=listing,
+ next_exists=next_exists,
+ page=page,
+ standalone=True,
+ render_replies=True,
+ is_notification_page=True,
+ )
+
+
+@app.get("/notifications/messages")
+@auth_required
+def notifications_messages(v: User):
+ page: int = max(request.values.get("page", 1, int) or 1, 1)
+
+ comments = g.db.query(Comment).filter(
+ Comment.sentto != None,
+ or_(Comment.author_id==v.id, Comment.sentto==v.id),
+ Comment.parent_submission == None,
+ Comment.level == 1,
+ )
+
+ if not v.shadowbanned and v.admin_level < 3:
+ comments = comments.join(Comment.author).filter(User.shadowbanned == None)
+
+ comments = comments.order_by(Comment.id.desc()).offset(25 * (page - 1)).limit(26).all()
+
+ next_exists = (len(comments) > 25)
+ listing = comments[:25]
+
+ if request.headers.get("Authorization"):
+ return {"data": [x.json for x in listing]}
+
+ return render_template("notifications.html",
+ v=v,
+ notifications=listing,
+ next_exists=next_exists,
+ page=page,
+ standalone=True,
+ render_replies=True,
+ is_notification_page=True,
+ )
@app.get("/")
@app.get("/catalog")
-# @app.get("/h/")
-# @app.get("/s/")
@limiter.limit("3/second;30/minute;1000/hour;5000/day")
@auth_desired
-def front_all(v, sub=None, subdomain=None):
- if sub: sub = g.db.query(Sub).filter_by(name=sub.strip().lower()).one_or_none()
-
- if (request.path.startswith('/h/') or request.path.startswith('/s/')) and not sub: abort(404)
-
+def front_all(v, subdomain=None):
if g.webview and not session.get("session_id"):
session["session_id"] = secrets.token_hex(49)
@@ -198,7 +217,7 @@ def front_all(v, sub=None, subdomain=None):
try: lt=int(request.values.get("before", 0))
except: lt=0
- ids, next_exists = frontlist(sort=sort,
+ ids, next_exists = listing.frontlist(sort=sort,
page=page,
t=t,
v=v,
@@ -206,8 +225,6 @@ def front_all(v, sub=None, subdomain=None):
filter_words=v.filter_words if v else [],
gt=gt,
lt=lt,
- sub=sub,
- site=SITE
)
posts = get_posts(ids, v=v, eager=True)
@@ -230,91 +247,7 @@ def front_all(v, sub=None, subdomain=None):
g.db.commit()
if request.headers.get("Authorization"): return {"data": [x.json for x in posts], "next_exists": next_exists}
- return render_template("home.html", v=v, listing=posts, next_exists=next_exists, sort=sort, t=t, page=page, ccmode=ccmode, sub=sub, home=True)
-
-
-
-@cache.memoize(timeout=86400)
-def frontlist(v=None, sort='new', page=1, t="all", ids_only=True, ccmode="false", filter_words='', gt=0, lt=0, sub=None, site=None):
-
- posts = g.db.query(Submission)
-
- if v and v.hidevotedon:
- voted = [x[0] for x in g.db.query(Vote.submission_id).filter_by(user_id=v.id).all()]
- posts = posts.filter(Submission.id.notin_(voted))
-
- if not v or v.admin_level < 2:
- filter_clause = (Submission.filter_state != 'filtered') & (Submission.filter_state != 'removed')
- if v:
- filter_clause = filter_clause | (Submission.author_id == v.id)
- posts = posts.filter(filter_clause)
-
- if sub: posts = posts.filter_by(sub=sub.name)
- elif v: posts = posts.filter(or_(Submission.sub == None, Submission.sub.notin_(v.all_blocks)))
-
- if gt: posts = posts.filter(Submission.created_utc > gt)
- if lt: posts = posts.filter(Submission.created_utc < lt)
-
- if not gt and not lt:
- posts = apply_time_filter(posts, t, Submission)
-
- if (ccmode == "true"):
- posts = posts.filter(Submission.club == True)
-
- posts = posts.filter_by(is_banned=False, private=False, deleted_utc = 0)
-
- if ccmode == "false" and not gt and not lt:
- posts = posts.filter_by(stickied=None)
-
- if v and v.admin_level < 2:
- posts = posts.filter(Submission.author_id.notin_(v.userblocks))
-
- if not (v and v.changelogsub):
- posts=posts.filter(not_(Submission.title.ilike('[changelog]%')))
-
- if v and filter_words:
- for word in filter_words:
- word = sql_ilike_clean(word).strip()
- posts=posts.filter(not_(Submission.title.ilike(f'%{word}%')))
-
- if not (v and v.shadowbanned):
- posts = posts.join(User, User.id == Submission.author_id).filter(User.shadowbanned == None)
-
- posts = sort_objects(posts, sort, Submission)
-
- if v: size = v.frontsize or 0
- else: size = 25
-
- posts = posts.offset(size * (page - 1)).limit(size+1).all()
-
- next_exists = (len(posts) > size)
-
- posts = posts[:size]
-
- if page == 1 and ccmode == "false" and not gt and not lt:
- pins = g.db.query(Submission).filter(Submission.stickied != None, Submission.is_banned == False)
- if sub: pins = pins.filter_by(sub=sub.name)
- elif v:
- pins = pins.filter(or_(Submission.sub == None, Submission.sub.notin_(v.all_blocks)))
- if v.admin_level < 2:
- pins = pins.filter(Submission.author_id.notin_(v.userblocks))
-
- pins = pins.all()
-
- for pin in pins:
- if pin.stickied_utc and int(time.time()) > pin.stickied_utc:
- pin.stickied = None
- pin.stickied_utc = None
- g.db.add(pin)
- pins.remove(pin)
-
- posts = pins + posts
-
- if ids_only: posts = [x.id for x in posts]
-
- g.db.commit()
-
- return posts, next_exists
+ return render_template("home.html", v=v, listing=posts, next_exists=next_exists, sort=sort, t=t, page=page, ccmode=ccmode, home=True)
@app.get("/changelog")
@@ -326,11 +259,10 @@ def changelog(v):
sort=request.values.get("sort", "new")
t=request.values.get('t', "all")
- ids = changeloglist(sort=sort,
+ ids = listing.changeloglist(sort=sort,
page=page,
t=t,
v=v,
- site=SITE
)
next_exists = (len(ids) > 25)
@@ -342,29 +274,9 @@ def changelog(v):
return render_template("changelog.html", v=v, listing=posts, next_exists=next_exists, sort=sort, t=t, page=page)
-@cache.memoize(timeout=86400)
-def changeloglist(v=None, sort="new", page=1, t="all", site=None):
-
- posts = g.db.query(Submission.id).filter_by(is_banned=False, private=False,).filter(Submission.deleted_utc == 0)
-
- if v.admin_level < 2:
- posts = posts.filter(Submission.author_id.notin_(v.userblocks))
-
- admins = [x[0] for x in g.db.query(User.id).filter(User.admin_level > 0).all()]
- posts = posts.filter(Submission.title.ilike('_changelog%'), Submission.author_id.in_(admins))
-
- if t != 'all':
- posts = apply_time_filter(posts, t, Submission)
- posts = sort_objects(posts, sort, Submission)
-
- posts = posts.offset(25 * (page - 1)).limit(26).all()
-
- return [x[0] for x in posts]
-
-
@app.get("/random_post")
def random_post():
- p = g.db.query(Submission.id).filter(Submission.deleted_utc == 0, Submission.is_banned == False, Submission.private == False).order_by(func.random()).first()
+ p = g.db.query(Submission.id).filter(Submission.state_user_deleted_utc == None, Submission.state_mod == StateMod.VISIBLE, Submission.private == False).order_by(func.random()).first()
if p: p = p[0]
else: abort(404)
@@ -387,7 +299,7 @@ def random_user():
def all_comments(v):
page = max(request.values.get("page", 1, int), 1)
sort = request.values.get("sort", "new")
- time_filter = request.values.get("t", defaulttimefilter)
+ time_filter = request.values.get("t", DEFAULT_TIME_FILTER)
time_gt = request.values.get("after", 0, int)
time_lt = request.values.get("before", 0, int)
@@ -421,11 +333,10 @@ def get_comments_idlist(page=1, v=None, sort="new", t="all", gt=0, lt=0):
if v.admin_level < 2:
comments = comments.filter(
Comment.author_id.notin_(v.userblocks),
- Comment.is_banned == False,
- Comment.deleted_utc == 0,
+ Comment.state_mod == StateMod.VISIBLE,
+ Comment.state_user_deleted_utc == None,
Submission.private == False, # comment parent post not private
User.shadowbanned == None, # comment author not shadowbanned
- Comment.filter_state.notin_(('filtered', 'removed')),
)
if gt: comments = comments.filter(Comment.created_utc > gt)
diff --git a/files/routes/importstar.py b/files/routes/importstar.py
new file mode 100644
index 000000000..b232bc286
--- /dev/null
+++ b/files/routes/importstar.py
@@ -0,0 +1,17 @@
+'''
+Module that can be safely imported with the syntax
+`from files.routes.importstar import *`. This essentially
+contains flask stuff for routes that are used by pretty much
+all routes.
+
+This should only be used from the route handlers. Flask imports
+are used in pretty much every place, but they shouldn't be used
+from the models if at all possible.
+
+Ideally we'd import only what we need but this is just for ease
+of development. Feel free to remove.
+'''
+
+from flask import (Response, abort, g, jsonify, make_response, redirect,
+ render_template, request, send_file, send_from_directory,
+ session)
diff --git a/files/routes/login.py b/files/routes/login.py
index 81f9a32b4..d8111f739 100644
--- a/files/routes/login.py
+++ b/files/routes/login.py
@@ -1,7 +1,8 @@
from urllib.parse import urlencode
+from files.helpers.config.environment import HCAPTCHA_SECRET, HCAPTCHA_SITEKEY, WELCOME_MSG
from files.mail import *
from files.__main__ import app, limiter
-from files.helpers.const import *
+from files.helpers.config.const import *
from files.helpers.captcha import validate_captcha
@app.get("/login")
@@ -16,7 +17,7 @@ def login_get(v):
if redir.startswith(f'{SITE_FULL}/'): return redirect(redir)
elif redir.startswith('/'): return redirect(f'{SITE_FULL}{redir}')
- return render_template("login.html", failed=False, redirect=redir)
+ return render_template("login/login.html", failed=False, redirect=redir)
def check_for_alts(current_id):
@@ -94,18 +95,18 @@ def login_post():
if not account:
time.sleep(random.uniform(0, 2))
- return render_template("login.html", failed=True)
+ return render_template("login/login.html", failed=True)
if request.values.get("password"):
if not account.verifyPass(request.values.get("password")):
time.sleep(random.uniform(0, 2))
- return render_template("login.html", failed=True)
+ return render_template("login/login.html", failed=True)
if account.mfa_secret:
now = int(time.time())
hash = generate_hash(f"{account.id}+{now}+2fachallenge")
- return render_template("login_2fa.html",
+ return render_template("login/login_2fa.html",
v=account,
time=now,
hash=hash,
@@ -123,7 +124,7 @@ def login_post():
if not account.validate_2fa(request.values.get("2fa_token", "").strip()):
hash = generate_hash(f"{account.id}+{time}+2fachallenge")
- return render_template("login_2fa.html",
+ return render_template("login/login_2fa.html",
v=account,
time=now,
hash=hash,
@@ -166,7 +167,6 @@ def me(v):
@limiter.limit("1/second;30/minute;200/hour;1000/day")
@auth_required
def logout(v):
-
session.pop("session_id", None)
session.pop("lo_user", None)
@@ -193,7 +193,7 @@ def sign_up_get(v):
ref_user = None
if ref_user and (ref_user.id in session.get("history", [])):
- return render_template("sign_up_failed_ref.html")
+ return render_template("login/sign_up_failed_ref.html")
now = int(time.time())
token = token_hex(16)
@@ -201,20 +201,21 @@ def sign_up_get(v):
formkey_hashstr = str(now) + token + agent
- formkey = hmac.new(key=bytes(environ.get("MASTER_KEY"), "utf-16"),
+ formkey = hmac.new(key=bytes(SECRET_KEY, "utf-16"),
msg=bytes(formkey_hashstr, "utf-16"),
digestmod='md5'
).hexdigest()
error = request.values.get("error")
- return render_template("sign_up.html",
- formkey=formkey,
- now=now,
- ref_user=ref_user,
- hcaptcha=app.config["HCAPTCHA_SITEKEY"],
- error=error
- )
+ return render_template(
+ "login/sign_up.html",
+ formkey=formkey,
+ now=now,
+ ref_user=ref_user,
+ hcaptcha=HCAPTCHA_SITEKEY,
+ error=error
+ )
@app.post("/signup")
@@ -237,7 +238,7 @@ def sign_up_post(v):
correct_formkey_hashstr = form_timestamp + submitted_token + agent
- correct_formkey = hmac.new(key=bytes(environ.get("MASTER_KEY"), "utf-16"),
+ correct_formkey = hmac.new(key=bytes(SECRET_KEY, "utf-16"),
msg=bytes(correct_formkey_hashstr, "utf-16"),
digestmod='md5'
).hexdigest()
@@ -289,8 +290,7 @@ def sign_up_post(v):
if existing_account:
return signup_error("An account with that username already exists.")
- if not validate_captcha(app.config.get("HCAPTCHA_SECRET", ""),
- app.config.get("HCAPTCHA_SITEKEY", ""),
+ if not validate_captcha(HCAPTCHA_SECRET, HCAPTCHA_SITEKEY,
request.values.get("h-captcha-response", "")):
return signup_error("Unable to verify CAPTCHA")
@@ -358,20 +358,19 @@ def sign_up_post(v):
@app.get("/forgot")
def get_forgot():
- return render_template("forgot_password.html")
+ return render_template("login/forgot_password.html")
@app.post("/forgot")
@limiter.limit("1/second;30/minute;200/hour;1000/day")
def post_forgot():
-
username = request.values.get("username")
if not username: abort(400)
email = request.values.get("email",'').strip().lower()
if not email_regex.fullmatch(email):
- return render_template("forgot_password.html", error="Invalid email.")
+ return render_template("login/forgot_password.html", error="Invalid email.")
username = username.lstrip('@')
@@ -391,13 +390,12 @@ def post_forgot():
v=user)
)
- return render_template("forgot_password.html",
+ return render_template("login/forgot_password.html",
msg="If the username and email matches an account, you will be sent a password reset email. You have ten minutes to complete the password reset process.")
@app.get("/reset")
def get_reset():
-
user_id = request.values.get("id")
timestamp = int(request.values.get("time",0))
@@ -422,7 +420,7 @@ def get_reset():
reset_token = generate_hash(f"{user.id}+{timestamp}+reset+{user.login_nonce}")
- return render_template("reset_password.html",
+ return render_template("login/reset_password.html",
v=user,
token=reset_token,
time=timestamp,
@@ -458,7 +456,7 @@ def post_reset(v):
abort(404)
if password != confirm_password:
- return render_template("reset_password.html",
+ return render_template("login/reset_password.html",
v=user,
token=token,
time=timestamp,
@@ -476,16 +474,14 @@ def post_reset(v):
@app.get("/lost_2fa")
@auth_desired
def lost_2fa(v):
-
return render_template(
- "lost_2fa.html",
+ "login/lost_2fa.html",
v=v
)
@app.post("/request_2fa_disable")
@limiter.limit("1/second;6/minute;200/hour;1000/day")
def request_2fa_disable():
-
username=request.values.get("username")
user=get_user(username, graceful=True)
if not user or not user.email or not user.mfa_secret:
@@ -523,7 +519,6 @@ def request_2fa_disable():
@app.get("/reset_2fa")
def reset_2fa():
-
now=int(time.time())
t = request.values.get("t")
if not t: abort(400)
diff --git a/files/routes/oauth.py b/files/routes/oauth.py
index e684bc0cf..b8cc73a90 100644
--- a/files/routes/oauth.py
+++ b/files/routes/oauth.py
@@ -1,12 +1,14 @@
-from files.helpers.wrappers import *
-from files.helpers.alerts import *
-from files.helpers.get import *
-from files.helpers.const import *
-from files.classes import *
-from flask import *
-from files.__main__ import app, limiter
import sqlalchemy.exc
+from files.__main__ import app, limiter
+from files.classes import *
+from files.helpers.alerts import *
+from files.helpers.config.const import *
+from files.helpers.get import *
+from files.helpers.wrappers import *
+from files.routes.importstar import *
+
+
@app.get("/authorize")
@auth_required
def authorize_prompt(v):
@@ -20,7 +22,6 @@ def authorize_prompt(v):
@limiter.limit("1/second;30/minute;200/hour;1000/day")
@auth_required
def authorize(v):
-
client_id = request.values.get("client_id")
application = g.db.query(OauthApp).filter_by(client_id=client_id).one_or_none()
if not application: return {"oauth_error": "Invalid `client_id`"}, 401
@@ -42,7 +43,6 @@ def authorize(v):
@limiter.limit("1/second;30/minute;200/hour;1000/day")
@is_not_permabanned
def request_api_keys(v):
-
new_app = OauthApp(
app_name=request.values.get('name').replace('<','').replace('>',''),
redirect_uri=request.values.get('redirect_uri'),
@@ -62,7 +62,8 @@ def request_api_keys(v):
level=1,
body_html=body_html,
sentto=MODMAIL_ID,
- distinguish_level=6
+ distinguish_level=6,
+ state_mod=StateMod.VISIBLE,
)
g.db.add(new_comment)
g.db.flush()
@@ -83,7 +84,6 @@ def request_api_keys(v):
@limiter.limit("1/second;30/minute;200/hour;1000/day")
@auth_required
def delete_oauth_app(v, aid):
-
aid = int(aid)
app = g.db.query(OauthApp).filter_by(id=aid).one_or_none()
@@ -103,7 +103,6 @@ def delete_oauth_app(v, aid):
@limiter.limit("1/second;30/minute;200/hour;1000/day")
@is_not_permabanned
def edit_oauth_app(v, aid):
-
aid = int(aid)
app = g.db.query(OauthApp).filter_by(id=aid).one_or_none()
@@ -124,7 +123,6 @@ def edit_oauth_app(v, aid):
@limiter.limit("1/second;30/minute;200/hour;1000/day")
@admin_level_required(3)
def admin_app_approve(v, aid):
-
app = g.db.query(OauthApp).filter_by(id=aid).one_or_none()
user = app.author
@@ -158,7 +156,6 @@ def admin_app_approve(v, aid):
@limiter.limit("1/second;30/minute;200/hour;1000/day")
@admin_level_required(2)
def admin_app_revoke(v, aid):
-
app = g.db.query(OauthApp).filter_by(id=aid).one_or_none()
if app:
for auth in g.db.query(ClientAuth).filter_by(oauth_client=app.id).all(): g.db.delete(auth)
@@ -183,7 +180,6 @@ def admin_app_revoke(v, aid):
@limiter.limit("1/second;30/minute;200/hour;1000/day")
@admin_level_required(2)
def admin_app_reject(v, aid):
-
app = g.db.query(OauthApp).filter_by(id=aid).one_or_none()
if app:
@@ -208,9 +204,6 @@ def admin_app_reject(v, aid):
@app.get("/admin/app/'
emoji_partial = '
'
@@ -178,10 +184,7 @@ def sanitize_raw(sanitized:Optional[str], allow_newlines:bool, length_limit:Opti
@with_gevent_timeout(2)
def sanitize(sanitized, alert=False, comment=False, edit=False):
- # double newlines, eg. hello\nworld becomes hello\n\nworld, which later becomes
@{u.username}''', 1)
soup = BeautifulSoup(sanitized, 'lxml')
- if app.config['MULTIMEDIA_EMBEDDING_ENABLED']:
+ if MULTIMEDIA_EMBEDDING_ENABLED:
for tag in soup.find_all("img"):
if tag.get("src") and not tag["src"].startswith('/pp/'):
tag["loading"] = "lazy"
@@ -280,13 +283,13 @@ def sanitize(sanitized, alert=False, comment=False, edit=False):
if "https://youtube.com/watch?v=" in sanitized: sanitized = sanitized.replace("?t=", "&t=")
- if app.config['MULTIMEDIA_EMBEDDING_ENABLED']:
+ if MULTIMEDIA_EMBEDDING_ENABLED:
captured = []
for i in youtube_regex.finditer(sanitized):
if i.group(0) in captured: continue
captured.append(i.group(0))
- params = parse_qs(urlparse(i.group(2).replace('&','&')).query)
+ params = urllib.parse.parse_qs(urllib.parse.urlparse(i.group(2).replace('&','&')).query)
t = params.get('t', params.get('start', [0]))[0]
if isinstance(t, str): t = t.replace('s','')
@@ -296,7 +299,7 @@ def sanitize(sanitized, alert=False, comment=False, edit=False):
sanitized = sanitized.replace(i.group(0), htmlsource)
- if app.config['MULTIMEDIA_EMBEDDING_ENABLED']:
+ if MULTIMEDIA_EMBEDDING_ENABLED:
sanitized = video_sub_regex.sub(r'\1', sanitized)
if comment:
@@ -317,8 +320,6 @@ def sanitize(sanitized, alert=False, comment=False, edit=False):
strip=True,
).clean(sanitized)
-
-
soup = BeautifulSoup(sanitized, 'lxml')
links = soup.find_all("a")
@@ -330,7 +331,7 @@ def sanitize(sanitized, alert=False, comment=False, edit=False):
href = link.get("href")
if not href: continue
- url = urlparse(href)
+ url = urllib.parse.urlparse(href)
domain = url.netloc
url_path = url.path
domain_list.add(domain+url_path)
@@ -377,4 +378,6 @@ def validate_css(css:str) -> tuple[bool, str]:
practical concern) or causing styling issues with the rest of the page.
'''
if ' bool:
+def bool_from_string(input: typing.Union[str, int, bool]) -> bool:
if isinstance(input, bool):
return input
+ elif isinstance(input, int):
+ return bool(input)
if input.lower() in ("yes", "true", "t", "on", "1"):
return True
if input.lower() in ("no", "false", "f", "off", "0"):
diff --git a/files/helpers/time.py b/files/helpers/time.py
new file mode 100644
index 000000000..3257fd1e2
--- /dev/null
+++ b/files/helpers/time.py
@@ -0,0 +1,71 @@
+import calendar
+import time
+from datetime import datetime, timedelta
+from typing import Final, Union
+
+DATE_FORMAT: Final[str] = '%Y %B %d'
+DATETIME_FORMAT: Final[str] = '%Y %B %d %H:%M:%S UTC'
+
+AgeFormattable = Union[int, timedelta]
+TimestampFormattable = Union[int, float, datetime, time.struct_time]
+
+def format_datetime(timestamp: TimestampFormattable | None) -> str:
+ return _format_timestamp(timestamp, DATETIME_FORMAT)
+
+
+def format_date(timestamp: TimestampFormattable | None) -> str:
+ return _format_timestamp(timestamp, DATE_FORMAT)
+
+
+def format_age(timestamp: TimestampFormattable | None) -> str:
+ if timestamp is None:
+ return ""
+
+ timestamp = _make_timestamp(timestamp)
+ age:int = int(time.time()) - timestamp
+
+ if age < 60: return "just now"
+ if age < 3600:
+ minutes = int(age / 60)
+ return f"{minutes}m ago"
+ if age < 86400:
+ hours = int(age / 3600)
+ return f"{hours}hr ago"
+ if age < 2678400:
+ days = int(age / 86400)
+ return f"{days}d ago"
+
+ now = time.gmtime()
+ ctd = time.gmtime(timestamp)
+
+ months = now.tm_mon - ctd.tm_mon + 12 * (now.tm_year - ctd.tm_year)
+ if now.tm_mday < ctd.tm_mday:
+ months -= 1
+
+ if months < 12:
+ return f"{months}mo ago"
+ else:
+ years = int(months / 12)
+ return f"{years}yr ago"
+
+
+def _format_timestamp(timestamp: TimestampFormattable | None, format: str) -> str:
+ if timestamp is None:
+ return ""
+ elif isinstance(timestamp, datetime):
+ return timestamp.strftime(format)
+ elif isinstance(timestamp, (int, float)):
+ timestamp = time.gmtime(timestamp)
+ elif not isinstance(timestamp, time.struct_time):
+ raise TypeError("Invalid argument type (must be one of int, float, "
+ "datettime, or struct_time)")
+ return time.strftime(format, timestamp)
+
+
+def _make_timestamp(timestamp: TimestampFormattable) -> int:
+ if isinstance(timestamp, (int, float)):
+ return int(timestamp)
+ if isinstance(timestamp, datetime):
+ return int(timestamp.timestamp())
+ if isinstance(timestamp, time.struct_time):
+ return calendar.timegm(timestamp)
diff --git a/files/helpers/validators.py b/files/helpers/validators.py
new file mode 100644
index 000000000..d5bf8fbf5
--- /dev/null
+++ b/files/helpers/validators.py
@@ -0,0 +1,192 @@
+import shutil
+import time
+import urllib.parse
+from dataclasses import dataclass
+from typing import Optional
+
+from flask import Request, abort, request
+from werkzeug.datastructures import FileStorage
+
+import files.helpers.embeds as embeds
+import files.helpers.sanitize as sanitize
+from files.helpers.config.environment import SITE_FULL, YOUTUBE_KEY
+from files.helpers.config.const import (SUBMISSION_BODY_LENGTH_MAXIMUM,
+ SUBMISSION_TITLE_LENGTH_MAXIMUM,
+ SUBMISSION_URL_LENGTH_MAXIMUM)
+from files.helpers.content import canonicalize_url2
+from files.helpers.media import process_image
+
+
+def guarded_value(val:str, min_len:int, max_len:int) -> str:
+ '''
+ Get request value `val` and ensure it is within length constraints
+ Requires a request context and either aborts early or returns a good value
+ '''
+ raw = request.values.get(val, '').strip()
+ raw = raw.replace('\u200e', '')
+
+ if len(raw) < min_len: abort(400, f"Minimum length for {val} is {min_len}")
+ if len(raw) > max_len: abort(400, f"Maximum length for {val} is {max_len}")
+ # TODO: it may make sense to do more sanitisation here
+ return raw
+
+
+def int_ranged(val:str, min:int, max:int) -> int:
+ raw:Optional[int] = request.values.get(val, default=None, type=int)
+ if raw is None or raw < min or raw > max:
+ abort(400,
+ f"Invalid input ('{val}' must be an integer and be between {min} and {max})")
+ return raw
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class ValidatedSubmissionLike:
+ title: str
+ title_html: str
+ body: str
+ body_raw: Optional[str]
+ body_html: str
+ url: Optional[str]
+ thumburl: Optional[str]
+
+ @property
+ def embed_slow(self) -> Optional[str]:
+ url:Optional[str] = self.url
+ url_canonical: Optional[urllib.parse.ParseResult] = self.url_canonical
+ if not url or not url_canonical: return None
+
+ embed:Optional[str] = None
+ domain:str = url_canonical.netloc
+
+ if domain == "twitter.com":
+ embed = embeds.twitter(url)
+
+ if url.startswith('https://youtube.com/watch?v=') and YOUTUBE_KEY:
+ embed = embeds.youtube(url)
+
+ if SITE_FULL in domain and "/post/" in url and "context" not in url:
+ id = url.split("/post/")[1]
+ if "/" in id: id = id.split("/")[0]
+ embed = str(int(id))
+
+ return embed if embed and len(embed) <= 1500 else None
+
+ @property
+ def repost_search_url(self) -> Optional[str]:
+ search_url = self.url_canonical_str
+ if not search_url: return None
+
+ if search_url.endswith('/'):
+ search_url = search_url[:-1]
+ return search_url
+
+ @property
+ def url_canonical(self) -> Optional[urllib.parse.ParseResult]:
+ if not self.url: return None
+ return canonicalize_url2(self.url, httpsify=True)
+
+ @property
+ def url_canonical_str(self) -> Optional[str]:
+ url_canonical:Optional[urllib.parse.ParseResult] = self.url_canonical
+ if not url_canonical: return None
+ return url_canonical.geturl()
+
+ @classmethod
+ def from_flask_request(cls,
+ request:Request,
+ *,
+ allow_embedding:bool,
+ allow_media_url_upload:bool=True,
+ embed_url_file_key:str="file2",
+ edit:bool=False) -> "ValidatedSubmissionLike":
+ '''
+ Creates the basic structure for a submission and validating it. The
+ normal submission API has a lot of duplicate code and while this is not
+ a pretty solution, this essentially forces all submission-likes through
+ a central interface.
+
+ :param request: The Flask Request object.
+ :param allow_embedding: Whether to allow embedding. This should usually
+ be the value from the environment.
+ :param allow_media_url_upload: Whether to allow media URL upload. This
+ should generally be `True` for submission submitting if file uploads
+ are allowed and `False` in other contexts (such as editing)
+ :param embed_url_file_key: The key to use for inline file uploads.
+ :param edit: The value of `edit` to pass to `sanitize`
+ '''
+
+ def _process_media(file:Optional[FileStorage]) -> tuple[bool, Optional[str], Optional[str]]:
+ if request.headers.get("cf-ipcountry") == "T1": # forbid Tor uploads
+ return False, None, None
+ elif not file:
+ # We actually care about falseyness, not just `is not None` because
+ # no attachment is
'
else: abort(400, "Image files only")
- new_comment = Comment(author_id=v.id if v else NOTIFICATIONS_ID,
- parent_submission=None,
- level=1,
- body_html=html,
- sentto=MODMAIL_ID,
- )
+ new_comment = Comment(
+ author_id=v.id if v else NOTIFICATIONS_ID,
+ parent_submission=None,
+ level=1,
+ body_html=html,
+ sentto=MODMAIL_ID,
+ state_mod=StateMod.VISIBLE,
+ )
g.db.add(new_comment)
g.db.flush()
new_comment.top_comment_id = new_comment.id
@@ -407,14 +402,12 @@ def blocks(v):
@app.get("/banned")
@auth_desired
def banned(v):
-
users = [x for x in g.db.query(User).filter(User.is_banned > 0, User.unban_utc == 0).all()]
return render_template("banned.html", v=v, users=users)
@app.get("/formatting")
@auth_desired
def formatting(v):
-
return render_template("formatting.html", v=v)
@app.get("/service-worker.js")
@@ -424,7 +417,6 @@ def serviceworker():
@app.get("/settings/security")
@auth_required
def settings_security(v):
-
return render_template("settings_security.html",
v=v,
mfa_secret=pyotp.random_base32() if not v.mfa_secret else None
diff --git a/files/routes/subs.py b/files/routes/subs.py
deleted file mode 100644
index d366ea8cb..000000000
--- a/files/routes/subs.py
+++ /dev/null
@@ -1,387 +0,0 @@
-from files.__main__ import app, limiter, mail
-from files.helpers.alerts import *
-from files.helpers.wrappers import *
-from files.classes import *
-from .front import frontlist
-
-
-
-@app.post("/exile/post/
Admin Tools
+Admin Tools
Users Here Now: {{g.loggedin_counter + g.loggedout_counter}} —
@@ -16,92 +14,99 @@
{{g.loggedout_counter}} Logged-Out
-Content
+Content
-Filtering
+Filtering
-Users
+Users
-Safety
+Safety
-Grant
+Grant
-API Access Control
+API Access Control
-Statistics
+Statistics
+Scheduler
+
+ {%- if v.admin_level >= PERMS['SCHEDULER'] -%}
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+Performance
+
+
+Site Settings
+ {%- macro site_setting_bool(name, label) -%}
+
+
+
+
+ {%- endmacro -%}
+ {%- macro site_setting_int(name, label) -%}
+
+
+
+
+ {%- endmacro -%}
+ {{site_setting_bool('signups', 'Signups')}}
+ {{site_setting_bool('bots', 'Bots')}}
+ {{site_setting_bool('FilterNewPosts', 'Filter New Posts')}}
+ {{site_setting_bool('Read-only mode', 'Read-Only Mode')}}
-Comment Filtering
-
-
-
-
-
-
-
-
-
-
-
-
+ Comment Filtering
+ {{site_setting_int('FilterCommentsMinComments', 'Minimum Comments')}}
+ {{site_setting_int('FilterCommentsMinKarma', 'Minimum Karma')}}
+ {{site_setting_int('FilterCommentsMinAgeDays', 'Minimum Account Age (Days)')}}
+
+ Private Mode Requirements
+ {{site_setting_int('min_comments_private_profile', 'Minimum Comments')}}
+ {{site_setting_int('min_truescore_private_profile', 'Minimum Karma')}}
+ {{site_setting_int('min_age_days_private_profile', 'Minimum Account Age (Days)')}}
diff --git a/files/templates/admin/alt_votes.html b/files/templates/admin/alt_votes.html
index e9d5841a9..ee9c4aaab 100644
--- a/files/templates/admin/alt_votes.html
+++ b/files/templates/admin/alt_votes.html
@@ -74,7 +74,7 @@
Link Accounts
