sanitize: sanitize raw content (fixes #214)

files/helpers/sanitize.py

@@ -158,6 +158,24 @@ def with_gevent_timeout(timeout: int):
 		return wrapped
 	return inner
 
+REMOVED_CHARACTERS = ['\u200e', '\u200b', '\ufeff']
+"""
+Characters which are removed from content
+"""
+
+def sanitize_raw(sanitized:Optional[str], allow_newlines:bool, length_limit:Optional[int]) -> str:
+	if not sanitized: return ""
+	for char in REMOVED_CHARACTERS:
+		sanitized = sanitized.replace(char, '')
+	if allow_newlines:
+		sanitized = sanitized.replace("\r\n", "\n")
+	else:
+		sanitized = sanitized.replace("\r","").replace("\n", "")
+	sanitized = sanitized.strip()
+	if length_limit is not None:
+		sanitized = sanitized[:length_limit]
+	return sanitized
+
 @with_gevent_timeout(2)
 def sanitize(sanitized, alert=False, comment=False, edit=False):
 	# double newlines, eg. hello\nworld becomes hello\n\nworld, which later becomes <p>hello</p><p>world</p>

files/routes/comments.py

@@ -112,7 +112,7 @@ def post_pid_comment_cid(cid, pid=None, anything=None, v=None, sub=None):
 def api_comment(v):
 	if v.is_suspended: abort(403, "You can't perform this action while banned.")
 
-	parent_fullname = request.values.get("parent_fullname").strip()
+	parent_fullname = request.values.get("parent_fullname", "").strip()
 
 	if len(parent_fullname) < 4: abort(400)
 	id = parent_fullname[3:]
@@ -134,9 +134,9 @@ def api_comment(v):
 	sub = parent_post.sub
 	if sub and v.exiled_from(sub): abort(403, f"You're exiled from /h/{sub}")
 
-	body = request.values.get("body", "").strip()[:10000]
+	body = sanitize_raw(request.values.get("body"), allow_newlines=True, length_limit=10000)
-
+	if not body and not request.files.get('file'): 
-	if not body and not request.files.get('file'): abort(400, "You need to actually write something!")
+		abort(400, "You need to actually write something!")
 
 	if request.files.get("file") and request.headers.get("cf-ipcountry") != "T1":
 		files = request.files.getlist('file')[:4]
@@ -259,12 +259,9 @@ def api_comment(v):
 @limiter.limit("1/second;30/minute;200/hour;1000/day")
 @auth_required
 def edit_comment(cid, v):
-
 	c = get_comment(cid, v=v)
-
 	if c.author_id != v.id: abort(403)
-
+	body = sanitize_raw(request.values.get("body"), allow_newlines=True, length_limit=10000)
-	body = request.values.get("body", "").strip()[:10000]
 
 	if len(body) < 1 and not (request.files.get("file") and request.headers.get("cf-ipcountry") != "T1"):
 		abort(400, "You have to actually type something!")

files/routes/posts.py

@@ -303,7 +303,10 @@ def edit_post(pid, v):
 	if p.author_id != v.id and not (v.admin_level > 1 and v.admin_level > 2): abort(403)
 
 	title = guarded_value("title", 1, MAX_TITLE_LENGTH)
+	title = sanitize_raw(title, allow_newlines=False, length_limit=MAX_TITLE_LENGTH)
+
 	body = guarded_value("body", 0, MAX_BODY_LENGTH)
+	body = sanitize_raw(body, allow_newlines=True, length_limit=MAX_BODY_LENGTH)
 
 	if title != p.title:
 		p.title = title
@@ -558,9 +561,15 @@ def submit_post(v, sub=None):
 		SUBS = [x[0] for x in g.db.query(Sub.name).order_by(Sub.name).all()]
 		return render_template("submit.html", SUBS=SUBS, v=v, error=error, title=title, url=url, body=body), 400
 
+	if v.is_suspended: return error("You can't perform this action while banned.")
+
 	title = guarded_value("title", 1, MAX_TITLE_LENGTH)
+	title = sanitize_raw(title, allow_newlines=False, length_limit=MAX_TITLE_LENGTH)
+
 	url = guarded_value("url", 0, MAX_URL_LENGTH)
+	
 	body = guarded_value("body", 0, MAX_BODY_LENGTH)
+	body = sanitize_raw(body, allow_newlines=True, length_limit=MAX_BODY_LENGTH)
 
 	sub = request.values.get("sub")
 	if sub: sub = sub.replace('/h/','').replace('s/','')
@@ -572,8 +581,6 @@ def submit_post(v, sub=None):
 		sub = sub[0]
 		if v.exiled_from(sub): return error(f"You're exiled from /h/{sub}")
 	else: sub = None
-
-	if v.is_suspended: return error("You can't perform this action while banned.")
 	
 	title_html = filter_emojis_only(title, graceful=True)
 

files/tests/test_basic.py

@@ -1,4 +1,3 @@
-
 from . import fixture_accounts
 from . import util
 

files/tests/test_no_content.py

@@ -0,0 +1,74 @@
+from . import fixture_accounts
+from . import util
+
+@util.no_rate_limit
+def test_no_content_submissions(accounts):
+	client = accounts.client_for_account()
+
+	# get our formkey
+	submit_get_response = client.get("/submit")
+	assert submit_get_response.status_code == 200
+
+	title = '\u200e\u200e\u200e\u200e\u200e\u200e'
+	body = util.generate_text()
+	formkey = util.formkey_from(submit_get_response.text)
+
+	# test bad title against good content
+	submit_post_response = client.post("/submit", data={
+		"title": title,
+		"body": body,
+		"formkey": formkey,
+	})
+
+	assert submit_post_response.status_code == 400
+
+	title, body = body, title
+	# test good title against bad content
+	submit_post_response = client.post("/submit", data={
+		"title": title,
+		"body": body,
+		"formkey": formkey,
+	})
+
+	assert submit_post_response.status_code == 400
+
+@util.no_rate_limit
+def test_no_content_comments(accounts):
+	client = accounts.client_for_account()
+
+	# get our formkey
+	submit_get_response = client.get("/submit")
+	assert submit_get_response.status_code == 200
+
+	# make the post
+	post_title = util.generate_text()
+	post_body = util.generate_text()
+	submit_post_response = client.post("/submit", data={
+		"title": post_title,
+		"body": post_body,
+		"formkey": util.formkey_from(submit_get_response.text),
+	})
+
+	assert submit_post_response.status_code == 200
+	assert post_title in submit_post_response.text
+	assert post_body in submit_post_response.text
+
+	# verify it actually got posted
+	root_response = client.get("/")
+	assert root_response.status_code == 200
+	assert post_title in root_response.text
+	assert post_body in root_response.text
+
+	# yank the ID out
+	post = util.ItemData.from_html(submit_post_response.text)
+	
+	# post a comment child
+	comment_body = '\ufeff\ufeff\ufeff\ufeff\ufeff'
+	submit_comment_response = client.post("/comment", data={
+		"parent_fullname": post.id_full,
+		"parent_level": 1,
+		"submission": post.id,
+		"body": comment_body,
+		"formkey": util.formkey_from(submit_post_response.text),
+	})
+	assert submit_comment_response.status_code == 400

files/tests/test_seed_db.py

@@ -1,4 +1,3 @@
-
 from files.commands.seed_db import seed_db_worker
 
 def test_seed_db():