diff --git a/files/helpers/const.py b/files/helpers/const.py index ada9c440a..c070f80a6 100644 --- a/files/helpers/const.py +++ b/files/helpers/const.py @@ -108,6 +108,10 @@ FEATURES = { PERMS = { "DEBUG_LOGIN_TO_OTHERS": 3, + 'PERFORMANCE_KILL_PROCESS': 3, + 'PERFORMANCE_SCALE_UP_DOWN': 3, + 'PERFORMANCE_RELOAD': 3, + 'PERFORMANCE_STATS': 3, "POST_COMMENT_MODERATION": 2, "USER_SHADOWBAN": 2, } diff --git a/files/helpers/jinja2.py b/files/helpers/jinja2.py index ce427f2d0..f1899787d 100644 --- a/files/helpers/jinja2.py +++ b/files/helpers/jinja2.py @@ -9,6 +9,9 @@ from .get import * from .const import * from files.helpers.assetcache import assetcache_path +@app.template_filter("computer_size") +def computer_size(size_bytes:int) -> str: + return f'{size_bytes // 1024 // 1024} MiB' @app.template_filter("shuffle") @pass_context diff --git a/files/helpers/time.py b/files/helpers/time.py new file mode 100644 index 000000000..244654663 --- /dev/null +++ b/files/helpers/time.py @@ -0,0 +1,24 @@ +import time +from datetime import datetime +from typing import Final, Union + +DATE_FORMAT: Final[str] = '%Y %B %d' +DATETIME_FORMAT: Final[str] = '%Y %B %d %H:%M:%S UTC' + +TimestampFormattable = Union[int, float, datetime, time.struct_time] + +def format_datetime(timestamp:TimestampFormattable) -> str: + return _format_timestamp(timestamp, DATETIME_FORMAT) + +def format_date(timestamp:TimestampFormattable) -> str: + return _format_timestamp(timestamp, DATE_FORMAT) + +def _format_timestamp(timestamp:TimestampFormattable, format:str) -> str: + if isinstance(timestamp, datetime): + return timestamp.strftime(format) + elif isinstance(timestamp, (int, float)): + timestamp = time.gmtime(timestamp) + elif not isinstance(timestamp, time.struct_time): + raise TypeError("Invalid argument type (must be one of int, float, " + "datettime, or struct_time)") + return time.strftime(format, timestamp) diff --git a/files/routes/admin/__init__.py b/files/routes/admin/__init__.py new file mode 100644 index 000000000..dfd2c169d --- /dev/null +++ b/files/routes/admin/__init__.py @@ -0,0 +1,2 @@ +from .admin import * +from .performance import * diff --git a/files/routes/admin.py b/files/routes/admin/admin.py similarity index 99% rename from files/routes/admin.py rename to files/routes/admin/admin.py index 22c58c52a..712a14adf 100644 --- a/files/routes/admin.py +++ b/files/routes/admin/admin.py @@ -10,7 +10,7 @@ from files.helpers.const import * from files.classes import * from flask import * from files.__main__ import app, cache, limiter -from .front import frontlist +from ..front import frontlist from files.helpers.comments import comment_on_publish, comment_on_unpublish from datetime import datetime import requests diff --git a/files/routes/admin/performance.py b/files/routes/admin/performance.py new file mode 100644 index 000000000..b06463829 --- /dev/null +++ b/files/routes/admin/performance.py @@ -0,0 +1,131 @@ +import os +from dataclasses import dataclass +from signal import Signals +from typing import Final + +import psutil +from flask import abort, render_template, request + +from files.helpers.const import PERMS +from files.helpers.time import format_datetime +from files.helpers.wrappers import admin_level_required +from files.__main__ import app + +PROCESS_NAME: Final[str] = "gunicorn" +''' +The name of the master and worker processes +''' + +INIT_PID: Final[int] = 1 +''' +The PID of the init process. Used to check an edge case for orphaned workers. +''' + +MEMORY_RSS_WARN_LEVELS_MASTER: dict[int, str] = { + 0: '', + 50 * 1024 * 1024: 'text-warn', + 75 * 1024 * 1024: 'text-danger', +} +''' +Levels to warn for in RAM memory usage for the master process. The master +process shouldn't be using much RAM at all since all it basically does is +orchestrate workers. +''' + +MEMORY_RSS_WARN_LEVELS_WORKER: dict[int, str] = { + 0: '', + 200 * 1024 * 1024: 'text-warn', + 300 * 1024 * 1024: 'text-danger', +} +''' +Levels to warn for in RAM memory usage. There are no warning levels for VM +usage because Python seems to intentionally overallocate (presumably to make +the interpreter faster) and doesn't tend to touch many of its owned pages. +''' + +@dataclass(frozen=True, slots=True) +class RenderedPerfInfo: + pid:int + started_at_utc:float + memory_rss:int + memory_vms:int + + @classmethod + def from_process(cls, p:psutil.Process) -> "RenderedPerfInfo": + with p.oneshot(): + mem = p.memory_info() + return cls(pid=p.pid, started_at_utc=p.create_time(), + memory_rss=mem.rss, memory_vms=mem.vms) + + @property + def is_master(self) -> bool: + return self.pid == os.getppid() and self.pid != INIT_PID + + @property + def is_current(self) -> bool: + return self.pid == os.getpid() + + @property + def memory_rss_css_class(self) -> str: + last = '' + levels: dict[int, str] = MEMORY_RSS_WARN_LEVELS_MASTER \ + if self.is_master else MEMORY_RSS_WARN_LEVELS_WORKER + for mem, css in levels.items(): + if self.memory_rss < mem: return last + last = css + return last + + @property + def started_at_utc_str(self) -> str: + return format_datetime(self.started_at_utc) + +@app.get('/performance/') +@admin_level_required(PERMS['PERFORMANCE_STATS']) +def performance_get_stats(v): + system_vm = psutil.virtual_memory() + processes = {p.pid:RenderedPerfInfo.from_process(p) + for p in psutil.process_iter() + if p.name() == PROCESS_NAME} + return render_template('admin/performance/memory.html', v=v, processes=processes, system_vm=system_vm) + +def _signal_master_process(signal:int) -> None: + ppid:int = os.getppid() + if ppid == INIT_PID: # shouldn't happen but handle the orphaned worker case just in case + abort(500, "This worker is an orphan!") + os.kill(ppid, signal) + +def _signal_worker_process(pid:int, signal:int) -> None: + workers:set[int] = {p.pid + for p in psutil.process_iter() + if p.name() == PROCESS_NAME} + workers.discard(os.getppid()) # don't allow killing the master process + + if not pid in workers: + abort(404, "Worker process not found") + os.kill(pid, signal) + +@app.post('/performance/workers/reload') +@admin_level_required(PERMS['PERFORMANCE_RELOAD']) +def performance_reload_workers(v): + _signal_master_process(Signals.SIGHUP) + return {'message': 'Sent reload signal successfully'} + +@app.post('/performance/workers//terminate') +@admin_level_required(PERMS['PERFORMANCE_KILL_PROCESS']) +def performance_terminate_worker_process(v, pid:int): + _signal_worker_process(pid, Signals.SIGTERM) + return {"message": f"Gracefully shut down worker PID {pid} successfully"} + +@app.post('/performance/workers//kill') +@admin_level_required(PERMS['PERFORMANCE_KILL_PROCESS']) +def performance_kill_worker_process(v, pid:int): + _signal_worker_process(pid, Signals.SIGKILL) + return {"message": f"Killed worker with PID {pid} successfully"} + +@app.post('/performance/workers/+1') +@app.post('/performance/workers/-1') +@admin_level_required(PERMS['PERFORMANCE_SCALE_UP_DOWN']) +def performance_scale_up_down(v): + scale_up:bool = '+1' in request.url + _signal_master_process(Signals.SIGTTIN if scale_up else Signals.SIGTTOU) + return {"message": "Sent signal to master to scale " + ("up" if scale_up else "down")} diff --git a/files/templates/admin/admin_home.html b/files/templates/admin/admin_home.html index 46957bcd0..c1e6c2ec5 100644 --- a/files/templates/admin/admin_home.html +++ b/files/templates/admin/admin_home.html @@ -59,6 +59,15 @@
  • Daily Stat Chart
  • +{% if v.admin_level >= 3 %} +
    +

    Performance

    + +
    +{% endif %} + {% if v.admin_level >= 3 %}
    
     	
    diff --git a/files/templates/admin/performance/memory.html b/files/templates/admin/performance/memory.html new file mode 100644 index 000000000..70937cf69 --- /dev/null +++ b/files/templates/admin/performance/memory.html @@ -0,0 +1,60 @@ +{%- extends 'default.html' -%} +{% block content %} +{%- macro post_toast_button(text, url, css_class) -%} + +{%- endmacro -%} +
    System Info
    +
    +
    + + + + + + + + + + + +
    Total Physical RAM{{system_vm.total | computer_size}}
    Available{{system_vm.available | computer_size}}
    +
    +
    +
    Worker Info
    +
    +
    + + + + + + + + + + + + + {% for pid, process in processes.items() %} + + + + + + + + + {% endfor %} + +
    PIDTypeStartedMemory Usage (Virtual)Memory Usage (Physical)Actions
    {{process.pid}}{{'Master' if process.is_master else 'Worker'}}{{process.started_at_utc_str}}{{process.memory_vms | computer_size}}{{process.memory_rss | computer_size}}{%- if process.is_master -%} + {{post_toast_button('Scale Up', '/performance/workers/+1', 'btn-secondary')}} + {{post_toast_button('Scale Down', '/performance/workers/-1', 'btn-secondary')}} + {{post_toast_button('Reload', '/performance/workers/reload', 'btn-danger')}} + {%- else -%} + {{post_toast_button('Shutdown', '/performance/workers/' ~ process.pid ~ '/terminate', 'btn-danger')}} + {{post_toast_button('Kill', '/performance/workers/' ~ process.pid ~ '/kill', 'btn-danger')}} + {%- endif -%} +
    +
    +
    +{% endblock %}