performance: add performance monitor

files/helpers/const.py

@@ -108,6 +108,10 @@ FEATURES = {
 
 PERMS = {
 	"DEBUG_LOGIN_TO_OTHERS": 3,
+	'PERFORMANCE_KILL_PROCESS': 3,
+	'PERFORMANCE_SCALE_UP_DOWN': 3,
+	'PERFORMANCE_RELOAD': 3,
+	'PERFORMANCE_STATS': 3,
 	"POST_COMMENT_MODERATION": 2,
 	"USER_SHADOWBAN": 2,
 }

files/helpers/jinja2.py

@@ -9,6 +9,9 @@ from .get import *
 from .const import * 
 from files.helpers.assetcache import assetcache_path
 
+@app.template_filter("computer_size")
+def computer_size(size_bytes:int) -> str:
+	return f'{size_bytes // 1024 // 1024} MiB'
 
 @app.template_filter("shuffle")
 @pass_context

files/helpers/time.py

@@ -0,0 +1,24 @@
+import time
+from datetime import datetime
+from typing import Final, Union
+
+DATE_FORMAT: Final[str] = '%Y %B %d'
+DATETIME_FORMAT: Final[str] = '%Y %B %d %H:%M:%S UTC'
+
+TimestampFormattable = Union[int, float, datetime, time.struct_time]
+
+def format_datetime(timestamp:TimestampFormattable) -> str:
+	return _format_timestamp(timestamp, DATETIME_FORMAT)
+
+def format_date(timestamp:TimestampFormattable) -> str:
+	return _format_timestamp(timestamp, DATE_FORMAT)
+
+def _format_timestamp(timestamp:TimestampFormattable, format:str) -> str:
+	if isinstance(timestamp, datetime):
+		return timestamp.strftime(format)
+	elif isinstance(timestamp, (int, float)):
+		timestamp = time.gmtime(timestamp)
+	elif not isinstance(timestamp, time.struct_time):
+		raise TypeError("Invalid argument type (must be one of int, float, "
+						"datettime, or struct_time)")
+	return time.strftime(format, timestamp)

files/routes/admin/__init__.py

@@ -0,0 +1,2 @@
+from .admin import *
+from .performance import *

files/routes/admin/admin.py

@@ -10,7 +10,7 @@ from files.helpers.const import *
 from files.classes import *
 from flask import *
 from files.__main__ import app, cache, limiter
-from .front import frontlist
+from ..front import frontlist
 from files.helpers.comments import comment_on_publish, comment_on_unpublish
 from datetime import datetime
 import requests

files/routes/admin/performance.py

@@ -0,0 +1,131 @@
+import os
+from dataclasses import dataclass
+from signal import Signals
+from typing import Final
+
+import psutil
+from flask import abort, render_template, request
+
+from files.helpers.const import PERMS
+from files.helpers.time import format_datetime
+from files.helpers.wrappers import admin_level_required
+from files.__main__ import app
+
+PROCESS_NAME: Final[str] = "gunicorn"
+'''
+The name of the master and worker processes
+'''
+
+INIT_PID: Final[int] = 1
+'''
+The PID of the init process. Used to check an edge case for orphaned workers.
+'''
+
+MEMORY_RSS_WARN_LEVELS_MASTER: dict[int, str] = {
+	0: '',
+	50 * 1024 * 1024: 'text-warn',
+	75 * 1024 * 1024: 'text-danger',
+}
+'''
+Levels to warn for in RAM memory usage for the master process. The master 
+process shouldn't be using much RAM at all since all it basically does is
+orchestrate workers.
+'''
+
+MEMORY_RSS_WARN_LEVELS_WORKER: dict[int, str] = {
+	0: '',
+	200 * 1024 * 1024: 'text-warn',
+	300 * 1024 * 1024: 'text-danger',
+}
+'''
+Levels to warn for in RAM memory usage. There are no warning levels for VM 
+usage because Python seems to intentionally overallocate (presumably to make 
+the interpreter faster) and doesn't tend to touch many of its owned pages.
+'''
+
+@dataclass(frozen=True, slots=True)
+class RenderedPerfInfo:
+	pid:int
+	started_at_utc:float
+	memory_rss:int
+	memory_vms:int
+
+	@classmethod
+	def from_process(cls, p:psutil.Process) -> "RenderedPerfInfo":
+		with p.oneshot():
+			mem = p.memory_info()
+			return cls(pid=p.pid, started_at_utc=p.create_time(),
+	                   memory_rss=mem.rss, memory_vms=mem.vms)
+	
+	@property
+	def is_master(self) -> bool:
+		return self.pid == os.getppid() and self.pid != INIT_PID
+
+	@property
+	def is_current(self) -> bool:
+		return self.pid == os.getpid()
+	
+	@property
+	def memory_rss_css_class(self) -> str:
+		last = ''
+		levels: dict[int, str] = MEMORY_RSS_WARN_LEVELS_MASTER \
+			if self.is_master else MEMORY_RSS_WARN_LEVELS_WORKER
+		for mem, css in levels.items():
+			if self.memory_rss < mem: return last
+			last = css
+		return last
+
+	@property
+	def started_at_utc_str(self) -> str:
+		return format_datetime(self.started_at_utc)
+
+@app.get('/performance/')
+@admin_level_required(PERMS['PERFORMANCE_STATS'])
+def performance_get_stats(v):
+	system_vm = psutil.virtual_memory()
+	processes = {p.pid:RenderedPerfInfo.from_process(p) 
+	      for p in psutil.process_iter() 
+		  if p.name() == PROCESS_NAME}
+	return render_template('admin/performance/memory.html', v=v, processes=processes, system_vm=system_vm)
+
+def _signal_master_process(signal:int) -> None:
+	ppid:int = os.getppid()
+	if ppid == INIT_PID: # shouldn't happen but handle the orphaned worker case just in case
+		abort(500, "This worker is an orphan!")
+	os.kill(ppid, signal)
+
+def _signal_worker_process(pid:int, signal:int) -> None:
+	workers:set[int] = {p.pid 
+		     for p in psutil.process_iter() 
+			 if p.name() == PROCESS_NAME}
+	workers.discard(os.getppid()) # don't allow killing the master process
+
+	if not pid in workers:
+		abort(404, "Worker process not found")
+	os.kill(pid, signal)
+
+@app.post('/performance/workers/reload')
+@admin_level_required(PERMS['PERFORMANCE_RELOAD'])
+def performance_reload_workers(v):
+	_signal_master_process(Signals.SIGHUP)
+	return {'message': 'Sent reload signal successfully'}
+
+@app.post('/performance/workers/<int:pid>/terminate')
+@admin_level_required(PERMS['PERFORMANCE_KILL_PROCESS'])
+def performance_terminate_worker_process(v, pid:int):
+	_signal_worker_process(pid, Signals.SIGTERM)
+	return {"message": f"Gracefully shut down worker PID {pid} successfully"}
+
+@app.post('/performance/workers/<int:pid>/kill')
+@admin_level_required(PERMS['PERFORMANCE_KILL_PROCESS'])
+def performance_kill_worker_process(v, pid:int):
+	_signal_worker_process(pid, Signals.SIGKILL)
+	return {"message": f"Killed worker with PID {pid} successfully"}
+
+@app.post('/performance/workers/+1')
+@app.post('/performance/workers/-1')
+@admin_level_required(PERMS['PERFORMANCE_SCALE_UP_DOWN'])
+def performance_scale_up_down(v):
+	scale_up:bool = '+1' in request.url
+	_signal_master_process(Signals.SIGTTIN if scale_up else Signals.SIGTTOU)
+	return {"message": "Sent signal to master to scale " + ("up" if scale_up else "down")}

files/templates/admin/admin_home.html

@@ -59,6 +59,15 @@
 	<li><a href="/daily_chart">Daily Stat Chart</a></li>
 </ul>
 
+{% if v.admin_level >= 3 %}
+<section id="admin-section-performance" class="admin-section mt-3">
+	<h4>Performance</h4>
+	<ul>
+		<li><a href="/performance/">Performance</a></li>
+	</ul>
+</section>
+{% endif %}
+
 {% if v.admin_level >= 3 %}
 	<pre></pre>
 	<div class="custom-control custom-switch">

files/templates/admin/performance/memory.html

@@ -0,0 +1,60 @@
+{%- extends 'default.html' -%}
+{% block content %}
+{%- macro post_toast_button(text, url, css_class) -%}
+<button class="btn{% if css_class %} {{css_class}}{% endif %}" onclick="post_toast(this, '{{url}}')">{{text}}</button>
+{%- endmacro -%}
+<h5 class="mt-3">System Info</h5>
+<section class="system-resource-usage">
+	<div class="overflow-x-auto">
+		<table class="table table-striped mb-5">
+			<tbody>
+				<tr>
+					<td>Total Physical RAM</td>
+					<td>{{system_vm.total | computer_size}}</td>
+				</tr>
+				<tr>
+					<td>Available</td>
+					<td>{{system_vm.available | computer_size}}</td>
+				</tr>
+			</tbody>
+		</table>
+	</div>
+</section>
+<h5 class="mt-3">Worker Info</h5>
+<section class="worker-resource-usage">
+	<div class="overflow-x-auto">
+		<table class="table table-striped mb-5">
+			<thead class="bg-primary text-white">
+				<tr>
+					<th>PID</th>
+					<th>Type</th>
+					<th>Started</th>
+					<th>Memory Usage (Virtual)</th>
+					<th>Memory Usage (Physical)</th>
+					<th>Actions</th>
+				</tr>
+			</thead>
+			<tbody>
+			{% for pid, process in processes.items() %}
+				<tr{% if process.is_current %} class='self'{% endif %}>
+					<td>{{process.pid}}</td>
+					<td>{{'Master' if process.is_master else 'Worker'}}</td>
+					<td>{{process.started_at_utc_str}}</td>
+					<td>{{process.memory_vms | computer_size}}</td>
+					<td class="{{process.memory_rss_css_class}}">{{process.memory_rss | computer_size}}</td>
+					<td>{%- if process.is_master -%}
+							{{post_toast_button('Scale Up', '/performance/workers/+1', 'btn-secondary')}}
+							{{post_toast_button('Scale Down', '/performance/workers/-1', 'btn-secondary')}}
+							{{post_toast_button('Reload', '/performance/workers/reload', 'btn-danger')}}
+						{%- else -%}
+							{{post_toast_button('Shutdown', '/performance/workers/' ~ process.pid ~ '/terminate', 'btn-danger')}}
+							{{post_toast_button('Kill', '/performance/workers/' ~ process.pid ~ '/kill', 'btn-danger')}}
+						{%- endif -%}
+					</td>
+				</tr>
+			{% endfor %}
+			</tbody>
+		</table>
+	</div>
+</section>
+{% endblock %}