Simplify migrations and redesign utility scripts.

.github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: "run_tests.py"
+name: "E2ETests"
 
 on: [push, pull_request]
 
@@ -9,6 +9,6 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
-    - name: run_tests.py
+    - name: Run tests
       run: |
-        ./run_tests.py
+        ./util/test.py

migrations/versions/2022_05_22_21_59_30_dd9a3c418274_add_filter_state_and_update_flags_schema.py

@@ -27,4 +27,4 @@ def upgrade():
 def downgrade():
 	op.drop_column('submissions', 'filter_state')
 	op.drop_column('flags', 'id')
-	op.create_primary_key('flags_id_pk', 'flags', ['post_id', 'user_id'])
+	op.create_primary_key('flags_pkey', 'flags', ['post_id', 'user_id'])

readme.md

@@ -1,5 +1,5 @@
 
-[![Build status](https://img.shields.io/github/workflow/status/TheMotte/rDrama/run_tests.py/frost)](https://github.com/TheMotte/rDrama/actions?query=workflow%3Arun_tests.py+branch%3Afrost)
+[![Build status](https://img.shields.io/github/workflow/status/TheMotte/rDrama/E2ETests/frost)](https://github.com/TheMotte/rDrama/actions?query=workflow%3AE2ETests+branch%3Afrost)
 
 This code runs https://www.themotte.org .
 
@@ -27,7 +27,7 @@ docker-compose up
 
 # Run the E2E tests:
 
-`./run_tests.py`
+`./util/test.py`
 
 # Database Stuff
 
@@ -92,12 +92,23 @@ which, if you're using the docker-compose, looks like
 docker-compose exec -T files bash -c 'cd /service/; FLASK_APP="files/cli:app" flask "$@"' . db upgrade
 ```
 
+Or with the util scripts:
+```sh
+./util/migrate.py upgrade
+```
+
 ## Running migrations someone else checked in
 
-If another dev made schema changes, and you just merged them in, you can get your local database up to date with the changes by running
+If you've just merged schema changes that another dev made, you can get your local database up to date by:
+
+* Open two terminals in the root of the project
+* Run `docker-compose up` in one terminal
+* Run this command in the other:
 	```sh
-docker-compose exec -T files bash -c 'cd /service/; FLASK\_APP="files/cli:app" flask db upgrade
+	./util/test.py upgrade
 	```
+   (or see above section for manual upgrade command)
+
 You should not have to reboot your container, though it might be a good idea to do so anyway if the changes you are merging in are nontrivial (particularly if there have been changes to `docker-compose.yml` or `Dockerfile`).
 
 ## So what's up with schema.sql, can I just change that?

run_tests.py

@@ -1,57 +0,0 @@
-#!/usr/bin/python3
-
-import subprocess
-import sys
-
-# we want to leave the container in whatever state it currently is, so check to see if it's running
-docker_inspect = subprocess.run([
-            "docker",
-            "container",
-            "inspect",
-            "-f", "{{.State.Status}}",
-            "themotte",
-        ],
-        capture_output = True,
-    ).stdout.decode("utf-8").strip()
-
-was_running = docker_inspect == "running"
-
-# update containers, just in case they're out of date
-if was_running:
-    print("Updating containers . . .")
-else:
-    print("Starting containers . . .")
-subprocess.run([
-            "docker-compose",
-            "up",
-            "--build",
-            "-d",
-        ],
-        check = True,
-    )
-
-# run the test
-print("Running test . . .")
-result = subprocess.run([
-        "docker-compose",
-        "exec",
-        '-T',
-        "files",
-        "bash", "-c", ' && '.join([
-            "cd service",
-            "FLASK_APP=files/cli:app python3 -m flask db upgrade",
-            "python3 -m pytest -s",
-        ])
-    ])
-
-if not was_running:
-    # shut down, if we weren't running in the first place
-    print("Shutting down containers . . .")
-    subprocess.run([
-            "docker-compose",
-            "stop",
-        ],
-        check = True,
-    )
-
-sys.exit(result.returncode)

util/common/__init__.py

@@ -0,0 +1,65 @@
+import sys, subprocess, time
+
+def _execute(command,**kwargs):
+    check = kwargs.get('check',False)
+    return subprocess.run(
+        command,
+        check = check,
+        universal_newlines=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+
+def _docker(command):
+    return _execute([
+        "docker-compose",
+        "exec", '-T',
+        "files",
+        "bash", "-c", 
+        ' && '.join(command)
+    ])
+
+def _running():
+    command = ['docker','container','inspect','-f','{{.State.Status}}','themotte']
+    result = _execute(command,check=False).stdout.strip()
+    return result == "running"
+
+def _start():
+    command = ['docker-compose','up','--build','-d']
+    result = _execute(command,check=True)
+    time.sleep(1)
+    return result
+
+def _stop():
+    command = ['docker-compose','down']
+    result = _execute(command,check=True)
+    time.sleep(1)
+    return result
+
+def _operation(name, command):
+    # check if running and start if not
+    running = _running()
+
+    if not running:
+        print("Starting containers...")
+        _start()
+
+    # run operation in docker container
+    print(f"Running {name} . . .")
+    result = _docker(command)
+    print(result.stdout)
+
+    if not running:
+        print("Stopping containers...")
+        _stop()
+
+    return result
+
+def run_help():
+    print("Available commands: (test|migrate|help)")
+    print("Usage: './manage.py <command> [options]'")
+    exit(0)
+
+def error(message,code=1):
+    print(message,file=sys.stderr)
+    exit(code)

util/manage.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+import sys
+from common import error, run_help
+from migrate import run_migrate
+from test import run_test
+
+if __name__=='__main__':
+
+    if len(sys.argv) < 2:
+        error("Usage: './manage.py <command> [options]'")
+
+    name = sys.argv[1]
+    args = sys.argv[1:]
+
+    if name == "test":
+        run_test(args)
+    elif name == "migrate":
+        run_migrate(args)
+    elif name == "help":
+        run_help()
+    else:
+        error("Not a command")

util/migrate.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import sys
+from common import _operation
+
+def run_migrate(args):
+    command = 'upgrade'
+
+    if len(args) > 1:
+        command = args[1]
+
+    result = _operation(command,[
+        "cd service",
+        "export FLASK_APP=files/cli:app",
+        f"python3 -m flask db {command}",
+    ])
+
+    sys.exit(result.returncode)
+
+if __name__=='__main__':
+    run_migrate(sys.argv)

util/test.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+
+import sys
+from common import _operation
+
+def run_test(args):
+    result = _operation("tests",[
+        "cd service",
+        "FLASK_APP=files/cli:app python3 -m flask db upgrade",
+        "python3 -m pytest -s",
+    ])
+
+    sys.exit(result.returncode)
+
+if __name__=='__main__':
+    run_test(sys.argv)