Save chat history in database.

chat/global.d.ts

@@ -5,7 +5,6 @@ declare const process: {
 declare interface IChatMessage {
   id: string;
   username: string;
-  user_id?: string;
   avatar: string;
   text: string;
   text_html: string;

chat/src/features/chat/ChatMessage.tsx

@@ -39,7 +39,6 @@ export function ChatMessage({
 }: ChatMessageProps) {
   const {
     id,
-    user_id,
     avatar,
     username,
     text,
@@ -69,7 +68,7 @@ export function ChatMessage({
       username !== userUsername);
   const isDirect = id === DIRECT_MESSAGE_ID;
   const isOptimistic = id === OPTIMISTIC_MESSAGE_ID;
-  
+
   const timestamp = useMemo(
     () => formatTimeAgo(time),
     [time, timestampUpdates]
@@ -77,11 +76,11 @@ export function ChatMessage({
 
   const handleDeleteMessage = useCallback(() => {
     if (confirmedDelete) {
-      deleteMessage(text);
+      deleteMessage(id);
     } else {
       setConfirmedDelete(true);
     }
-  }, [text, confirmedDelete]);
+  }, [id, confirmedDelete]);
 
   const handleQuoteMessageAction = useCallback(() => {
     quoteMessage(message);

chat/src/hooks/useChat.tsx

@@ -32,7 +32,7 @@ interface ChatProviderContext {
   updateDraft: React.Dispatch<React.SetStateAction<string>>;
   sendMessage(): void;
   quoteMessage(message: null | IChatMessage): void;
-  deleteMessage(withText: string): void;
+  deleteMessage(withId: string): void;
 }
 
 const ChatContext = createContext<ChatProviderContext>({
@@ -119,16 +119,16 @@ export function ChatProvider({ children }: PropsWithChildren) {
     setDraft("");
   }, [draft, quote]);
 
-  const requestDeleteMessage = useCallback((withText: string) => {
-    socket.current?.emit(ChatHandlers.DELETE, withText);
+  const requestDeleteMessage = useCallback((withId: string) => {
+    socket.current?.emit(ChatHandlers.DELETE, withId);
   }, []);
 
-  const deleteMessage = useCallback((withText: string) => {
+  const deleteMessage = useCallback((withId: string) => {
     setMessages((prev) =>
-      prev.filter((prevMessage) => prevMessage.text !== withText)
+      prev.filter((prevMessage) => prevMessage.id !== withId)
     );
 
-    if (quote?.text === withText) {
+    if (quote?.id === withId) {
       setQuote(null);
     }
   }, []);

files/classes/__init__.py

@@ -56,6 +56,7 @@ from urllib.parse import urlencode, urlparse, parse_qs
 from .alts import Alt
 from .award import AwardRelationship
 from .badges import BadgeDef, Badge
+from .chat_message import ChatMessage
 from .clients import OauthApp, ClientAuth
 from .comment import Comment
 from .domains import BannedDomain

files/classes/chat_message.py

@@ -0,0 +1,36 @@
+
+from files.classes.base import CreatedDateTimeBase
+from files.helpers.lazy import lazy
+from sqlalchemy import *
+from sqlalchemy.orm import declared_attr, relationship
+
+class ChatMessage(CreatedDateTimeBase):
+    __tablename__ = "chat_message"
+
+    id = Column(Integer, primary_key=True)
+    author_id = Column(Integer, ForeignKey("users.id"), nullable=False)
+    quote_id = Column(Integer, ForeignKey("chat_message.id"), nullable=True)
+    text = Column(String, nullable=False)
+    text_html = Column(String, nullable=False)
+    
+    author = relationship("User", primaryjoin="User.id==ChatMessage.author_id")
+
+    @declared_attr
+    def created_datetimez_index(self):
+        return Index('created_datetimez_index', self.created_datetimez)
+    
+    Index('quote_index', quote_id)
+    
+    @lazy
+    def json_speak(self):
+        data = {
+               'id': str(self.id),
+               'quotes': None if self.quote_id is None else str(self.quote_id),
+               'avatar': self.author.profile_url,
+               'username': self.author.username,
+               'text': self.text,
+               'text_html': self.text_html,
+               'time': int(self.created_datetimez.timestamp()),
+        }
+
+        return data

files/routes/chat.py

@@ -62,25 +62,35 @@ CHAT_SCROLLBACK_ITEMS: Final[int] = 500
 
 typing: list[str] = []
 online: list[str] = []	# right now we maintain this but don't actually use it anywhere
-muted: dict[str, int] = cache.get(f'{SITE}_muted') or {}
-messages: list[dict[str, Any]] = cache.get(f'{SITE}_chat') or []
-total: int = cache.get(f'{SITE}_total') or 0
 socket_ids_to_user_ids = {}
 user_ids_to_socket_ids = {}
 
 def send_system_reply(text):
 	data = {
 		"id": str(uuid.uuid4()),
-		"quotes": [],
 		"avatar": g.db.query(User).filter(User.id == NOTIFICATIONS_ID).one().profile_url,
 		"user_id": NOTIFICATIONS_ID,
 		"username": "System",
 		"text": text,
 		"text_html": sanitize(text),
-		"time": int(time.time()),
+		'time': int(self.created_datetimez.timestamp()),
 	}
 	emit('speak', data)
 
+def get_chat_messages():
+	# Query for the last visible chat messages
+	result = (g.db.query(ChatMessage)
+          .join(User, User.id == ChatMessage.author_id)  # Join with the User table to fetch related user data
+          .order_by(ChatMessage.created_datetimez.desc())
+          .limit(CHAT_SCROLLBACK_ITEMS)
+          .all())
+	
+	# Convert the list of ChatMessage objects into a list of dictionaries
+	# Also, most recent at the bottom, not the top.
+	messages = [item.json_speak() for item in result[::-1]]
+	
+	return messages
+
 def get_chat_userlist():
 	# Query for the User.username column for users with chat_authorized == True
     result = g.db.query(User.username).filter(User.chat_authorized == True).all()
@@ -94,7 +104,7 @@ def get_chat_userlist():
 @is_not_permabanned
 @chat_is_allowed()
 def chat(v):
-	return render_template("chat.html", v=v, messages=messages)
+	return render_template("chat.html", v=v, messages=get_chat_messages())
 
 
 @socketio.on('speak')
@@ -104,13 +114,6 @@ def speak(data, v):
 	limiter.check()
 	if v.is_banned: return '', 403
 
-	vname = v.username.lower()
-	if vname in muted and not v.admin_level >= PERMS['CHAT_BYPASS_MUTE']:
-		if time.time() < muted[vname]: return '', 403
-		else: del muted[vname]
-
-	global messages, total
-
 	text = sanitize_raw(
 		data['message'],
 		allow_newlines=True,
@@ -132,27 +135,16 @@ def speak(data, v):
 
 	text_html = sanitize(text)
 	quotes = data['quotes']
-	data = {
-		"id": str(uuid.uuid4()),
-		"quotes": quotes,
-		"avatar": v.profile_url,
-		"user_id": v.id,
-		"username": v.username,
-		"text": text,
-		"text_html": text_html,
-		"time": int(time.time()),
-	}
 	
-	if v.shadowbanned:
-		emit('speak', data)
-	else:
-		emit('speak', data, broadcast=True)
-		messages.append(data)
-		messages = messages[-CHAT_SCROLLBACK_ITEMS:]
+	chat_message = ChatMessage()
+	chat_message.author_id = v.id
+	chat_message.quote_id = quotes
+	chat_message.text = text
+	chat_message.text_html = text_html
+	g.db.add(chat_message)
+	g.db.commit()
 
-	total += 1
-
-	chat_save()
+	emit('speak', chat_message.json_speak(), broadcast=True)
 
 
 @socketio.on('connect')
@@ -166,7 +158,7 @@ def connect(v):
 		user_ids_to_socket_ids[v.id] = request.sid
 
 	emit('online', get_chat_userlist())
-	emit('catchup', messages)
+	emit('catchup', get_chat_messages())
 	emit('typing', typing)
 
 
@@ -198,17 +190,19 @@ def typing_indicator(data, v):
 
 @socketio.on('delete')
 @chat_is_allowed(PERMS['CHAT_MODERATION'])
-def delete(text, v):
-	for message in messages:
-		if message['text'] == text:
-			messages.remove(message)
+def delete(id, v):
+    chat_message = g.db.query(ChatMessage).filter(ChatMessage.id == id).one_or_none()
+    if chat_message:
+        # Zero out all the quote_id references to this message
+        messages_quoting_this = g.db.query(ChatMessage).filter(ChatMessage.quote_id == id).all()
+        for message in messages_quoting_this:
+            message.quote_id = None 
 
-	emit('delete', text, broadcast=True)
+        # Now, delete the chat_message
+        g.db.delete(chat_message)
+        g.db.commit()
 
-def chat_save():
-	cache.set(f'{SITE}_chat', messages)
-	cache.set(f'{SITE}_total', total)
-	cache.set(f'{SITE}_muted', muted)
+        emit('delete', id, broadcast=True)
 
 @register_command('add', PERMS['CHAT_FULL_CONTROL'])
 def add(user):
@@ -228,6 +222,7 @@ def add(user):
 	else:
 		send_system_reply(f"Could not find user {user}.")
 
+
 @register_command('remove', PERMS['CHAT_FULL_CONTROL'])
 def remove(user):
 	print("Removing user", user)

migrations/versions/2023_08_27_13_34_34_850c47d647ba_add_chat_message_table.py

@@ -0,0 +1,40 @@
+"""add chat_message table
+
+Revision ID: 850c47d647ba
+Revises: c41b790058ad
+Create Date: 2023-08-27 13:34:34.601539+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '850c47d647ba'
+down_revision = 'c41b790058ad'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+	# ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('chat_message',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('quote_id', sa.Integer(), nullable=True),
+    sa.Column('text', sa.String(), nullable=False),
+    sa.Column('text_html', sa.String(), nullable=False),
+    sa.Column('created_datetimez', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.ForeignKeyConstraint(['author_id'], ['users.id'], ),
+    sa.ForeignKeyConstraint(['quote_id'], ['chat_message.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index('created_datetimez_index', 'chat_message', ['created_datetimez'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+	# ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('created_datetimez_index', table_name='chat_message')
+    op.drop_table('chat_message')
+    # ### end Alembic commands ###