From 17217d56c2581f341429f06b65ed542646f12c09 Mon Sep 17 00:00:00 2001 From: Michael House Date: Wed, 29 Jun 2022 00:18:14 -0500 Subject: [PATCH] Simplify migrations and redesign utility scripts. --- .github/workflows/test.yml | 6 +- ...dd_filter_state_and_update_flags_schema.py | 2 +- readme.md | 23 +++++-- run_tests.py | 57 ---------------- util/common/__init__.py | 65 +++++++++++++++++++ util/manage.py | 23 +++++++ util/migrate.py | 21 ++++++ util/test.py | 16 +++++ 8 files changed, 146 insertions(+), 67 deletions(-) delete mode 100755 run_tests.py create mode 100644 util/common/__init__.py create mode 100755 util/manage.py create mode 100755 util/migrate.py create mode 100755 util/test.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52a3e83aa..e7bae96aa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: "run_tests.py" +name: "E2ETests" on: [push, pull_request] @@ -9,6 +9,6 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - - name: run_tests.py + - name: Run tests run: | - ./run_tests.py + ./util/test.py diff --git a/migrations/versions/2022_05_22_21_59_30_dd9a3c418274_add_filter_state_and_update_flags_schema.py b/migrations/versions/2022_05_22_21_59_30_dd9a3c418274_add_filter_state_and_update_flags_schema.py index 0c6bd22e2..10528822b 100644 --- a/migrations/versions/2022_05_22_21_59_30_dd9a3c418274_add_filter_state_and_update_flags_schema.py +++ b/migrations/versions/2022_05_22_21_59_30_dd9a3c418274_add_filter_state_and_update_flags_schema.py @@ -27,4 +27,4 @@ def upgrade(): def downgrade(): op.drop_column('submissions', 'filter_state') op.drop_column('flags', 'id') - op.create_primary_key('flags_id_pk', 'flags', ['post_id', 'user_id']) + op.create_primary_key('flags_pkey', 'flags', ['post_id', 'user_id']) diff --git a/readme.md b/readme.md index 63bf1c16c..1c7a353cc 100644 --- a/readme.md +++ b/readme.md @@ -1,5 +1,5 @@ -[![Build status](https://img.shields.io/github/workflow/status/TheMotte/rDrama/run_tests.py/frost)](https://github.com/TheMotte/rDrama/actions?query=workflow%3Arun_tests.py+branch%3Afrost) +[![Build status](https://img.shields.io/github/workflow/status/TheMotte/rDrama/E2ETests/frost)](https://github.com/TheMotte/rDrama/actions?query=workflow%3AE2ETests+branch%3Afrost) This code runs https://www.themotte.org . @@ -27,7 +27,7 @@ docker-compose up # Run the E2E tests: -`./run_tests.py` +`./util/test.py` # Database Stuff @@ -92,12 +92,23 @@ which, if you're using the docker-compose, looks like docker-compose exec -T files bash -c 'cd /service/; FLASK_APP="files/cli:app" flask "$@"' . db upgrade ``` +Or with the util scripts: +```sh +./util/migrate.py upgrade +``` + ## Running migrations someone else checked in -If another dev made schema changes, and you just merged them in, you can get your local database up to date with the changes by running -```sh -docker-compose exec -T files bash -c 'cd /service/; FLASK\_APP="files/cli:app" flask db upgrade -``` +If you've just merged schema changes that another dev made, you can get your local database up to date by: + +* Open two terminals in the root of the project +* Run `docker-compose up` in one terminal +* Run this command in the other: + ```sh + ./util/test.py upgrade + ``` + (or see above section for manual upgrade command) + You should not have to reboot your container, though it might be a good idea to do so anyway if the changes you are merging in are nontrivial (particularly if there have been changes to `docker-compose.yml` or `Dockerfile`). ## So what's up with schema.sql, can I just change that? diff --git a/run_tests.py b/run_tests.py deleted file mode 100755 index e6259434a..000000000 --- a/run_tests.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/python3 - -import subprocess -import sys - -# we want to leave the container in whatever state it currently is, so check to see if it's running -docker_inspect = subprocess.run([ - "docker", - "container", - "inspect", - "-f", "{{.State.Status}}", - "themotte", - ], - capture_output = True, - ).stdout.decode("utf-8").strip() - -was_running = docker_inspect == "running" - -# update containers, just in case they're out of date -if was_running: - print("Updating containers . . .") -else: - print("Starting containers . . .") -subprocess.run([ - "docker-compose", - "up", - "--build", - "-d", - ], - check = True, - ) - -# run the test -print("Running test . . .") -result = subprocess.run([ - "docker-compose", - "exec", - '-T', - "files", - "bash", "-c", ' && '.join([ - "cd service", - "FLASK_APP=files/cli:app python3 -m flask db upgrade", - "python3 -m pytest -s", - ]) - ]) - -if not was_running: - # shut down, if we weren't running in the first place - print("Shutting down containers . . .") - subprocess.run([ - "docker-compose", - "stop", - ], - check = True, - ) - -sys.exit(result.returncode) diff --git a/util/common/__init__.py b/util/common/__init__.py new file mode 100644 index 000000000..55f82a527 --- /dev/null +++ b/util/common/__init__.py @@ -0,0 +1,65 @@ +import sys, subprocess, time + +def _execute(command,**kwargs): + check = kwargs.get('check',False) + return subprocess.run( + command, + check = check, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + +def _docker(command): + return _execute([ + "docker-compose", + "exec", '-T', + "files", + "bash", "-c", + ' && '.join(command) + ]) + +def _running(): + command = ['docker','container','inspect','-f','{{.State.Status}}','themotte'] + result = _execute(command,check=False).stdout.strip() + return result == "running" + +def _start(): + command = ['docker-compose','up','--build','-d'] + result = _execute(command,check=True) + time.sleep(1) + return result + +def _stop(): + command = ['docker-compose','down'] + result = _execute(command,check=True) + time.sleep(1) + return result + +def _operation(name, command): + # check if running and start if not + running = _running() + + if not running: + print("Starting containers...") + _start() + + # run operation in docker container + print(f"Running {name} . . .") + result = _docker(command) + print(result.stdout) + + if not running: + print("Stopping containers...") + _stop() + + return result + +def run_help(): + print("Available commands: (test|migrate|help)") + print("Usage: './manage.py [options]'") + exit(0) + +def error(message,code=1): + print(message,file=sys.stderr) + exit(code) diff --git a/util/manage.py b/util/manage.py new file mode 100755 index 000000000..60343df06 --- /dev/null +++ b/util/manage.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +import sys +from common import error, run_help +from migrate import run_migrate +from test import run_test + +if __name__=='__main__': + + if len(sys.argv) < 2: + error("Usage: './manage.py [options]'") + + name = sys.argv[1] + args = sys.argv[1:] + + if name == "test": + run_test(args) + elif name == "migrate": + run_migrate(args) + elif name == "help": + run_help() + else: + error("Not a command") \ No newline at end of file diff --git a/util/migrate.py b/util/migrate.py new file mode 100755 index 000000000..42d854f8c --- /dev/null +++ b/util/migrate.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +import sys +from common import _operation + +def run_migrate(args): + command = 'upgrade' + + if len(args) > 1: + command = args[1] + + result = _operation(command,[ + "cd service", + "export FLASK_APP=files/cli:app", + f"python3 -m flask db {command}", + ]) + + sys.exit(result.returncode) + +if __name__=='__main__': + run_migrate(sys.argv) \ No newline at end of file diff --git a/util/test.py b/util/test.py new file mode 100755 index 000000000..e8bf68b04 --- /dev/null +++ b/util/test.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +import sys +from common import _operation + +def run_test(args): + result = _operation("tests",[ + "cd service", + "FLASK_APP=files/cli:app python3 -m flask db upgrade", + "python3 -m pytest -s", + ]) + + sys.exit(result.returncode) + +if __name__=='__main__': + run_test(sys.argv) \ No newline at end of file