From 937d36de31c62a38d5d02163feaee342d8968b8b Mon Sep 17 00:00:00 2001 From: Ben Rog-Wilhelm Date: Thu, 10 Nov 2022 16:43:26 -0600 Subject: [PATCH] A large number of small tooling and README updates. --- Dockerfile | 1 + readme.md | 48 +++++++++++------------------------ supervisord.conf | 2 +- util/command.py | 17 +++++++++++++ util/common/__init__.py | 55 +++++++++++++++++++++++++---------------- util/manage.py | 7 +++--- util/migrate.py | 20 --------------- util/test.py | 7 ++++-- 8 files changed, 76 insertions(+), 81 deletions(-) create mode 100755 util/command.py delete mode 100755 util/migrate.py diff --git a/Dockerfile b/Dockerfile index ce418e6b3..7878d2263 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ RUN mkdir /images && mkdir /songs EXPOSE 80/tcp +ENV FLASK_APP=files/cli:app CMD [ "/usr/bin/supervisord", "-c", "/etc/supervisord.conf" ] diff --git a/readme.md b/readme.md index 1c7a353cc..508f8ae43 100644 --- a/readme.md +++ b/readme.md @@ -5,11 +5,11 @@ This code runs https://www.themotte.org . # Installation (Windows/Linux/MacOS) -1 - Install Docker on your machine. +1 - Install [a container runtime and the Docker commandline tools](https://docs.docker.com/get-docker/) on your machine. -[Docker installation](https://docs.docker.com/get-docker/) +On Windows, Docker will pester you to pay them money for licensing. If you want something less proprietary, consider [Container Desktop](https://container-desktop.io) or [Stevedore](http://github.com/slonopotamus/stevedore) instead. -2 - If hosting on localhost and/or without HTTPS, change```"SESSION_COOKIE_SECURE"``` in ```__main__.py``` to "False" +2 - Install [Git](https://git-scm.com/). If you're on Windows and want a GUI, [Github Desktop](https://desktop.github.com/) is quite nice. 3 - Run the following commands in the terminal: @@ -18,12 +18,14 @@ git clone https://github.com/themotte/rDrama/ cd rDrama -docker-compose up +docker-compose up --build ``` +The first time you do this, it will take a while. It'll be (much) faster next time. + 4 - That's it! Visit `localhost` in your browser. -5 - Optional: to change the domain from "localhost" to something else and configure the site settings, as well as integrate it with the external services the website uses, please edit the variables in the `env` file and then restart the docker container. +Code edits will be reflected (almost) immediately. If you make any setup changes or database changes, you'll need to ctrl-C the docker-compose status log and run `docker-compose up --build` again. # Run the E2E tests: @@ -54,14 +56,13 @@ As an example, let's say we want to add a column `is\_flagged` to the `comments` ... + is_flagged = Column(Boolean, default=False, nullable=False) ``` + 2. Autogenerate a migration with a descriptive message. To do this, run ```sh -flask db revision --autogenerate --message="add is_flagged field to comments" +./util/command.py db revision --autogenerate --message="add is_flagged field to comments" ``` from the flask server within the directory the flask app is being served from, with an env var of `FLASK\_APP="files/cli:app"`. If you are running flask using `docker-compose` as described above, this looks like -```sh -docker-compose exec -T files bash -c 'cd /service/; FLASK_APP="files/cli:app" flask "$@"' . db revision --autogenerate --message="add is_flagged field to comments" -``` + This will create a migration in the `migrations/versions` directory with a name like `migrations/versions/2022\_05\_23\_05\_38\_40\_9c27db0b3918\_add\_is\_flagged\_field\_to\_comments.py` and content like ```python """add is_flagged field to comments @@ -83,33 +84,12 @@ def downgrade(): ``` 3. Examine the autogenerated migration to make sure that everything looks right (it adds the column you expected it to add and nothing else, all constraints are named, etc. If you see a `None` in one of the alembic operations, e.g. `op.create\_foreign\_key\_something(None, 'usernotes', 'users', ['author\_id'])`, please replace it with a descriptive string before you commit the migration). -4. Run the migration to make sure it works. You can run a migration with the command + +4. Restart the Docker container to make sure it works. + ```sh -flask db upgrade +docker-compose up --build ``` -which, if you're using the docker-compose, looks like -```sh -docker-compose exec -T files bash -c 'cd /service/; FLASK_APP="files/cli:app" flask "$@"' . db upgrade -``` - -Or with the util scripts: -```sh -./util/migrate.py upgrade -``` - -## Running migrations someone else checked in - -If you've just merged schema changes that another dev made, you can get your local database up to date by: - -* Open two terminals in the root of the project -* Run `docker-compose up` in one terminal -* Run this command in the other: - ```sh - ./util/test.py upgrade - ``` - (or see above section for manual upgrade command) - -You should not have to reboot your container, though it might be a good idea to do so anyway if the changes you are merging in are nontrivial (particularly if there have been changes to `docker-compose.yml` or `Dockerfile`). ## So what's up with schema.sql, can I just change that? diff --git a/supervisord.conf b/supervisord.conf index e07910100..261f3f50e 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -5,7 +5,7 @@ logfile=/tmp/supervisord.log [program:service] directory=/service -command=sh -c 'FLASK_APP=files/cli:app python3 -m flask db upgrade && gunicorn files.__main__:app -k gevent -w 1 --reload -b 0.0.0.0:80 --max-requests 1000 --max-requests-jitter 500' +command=sh -c 'python3 -m flask db upgrade && gunicorn files.__main__:app -k gevent -w 1 --reload -b 0.0.0.0:80 --max-requests 1000 --max-requests-jitter 500' stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr diff --git a/util/command.py b/util/command.py new file mode 100755 index 000000000..1b2a781a6 --- /dev/null +++ b/util/command.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +import sys +from common import _operation + +def run_command(argv): + result = _operation(f"command",[ + [ + "python3", + "-m", "flask", + ] + argv[1:] + ]) + + sys.exit(result.returncode) + +if __name__=='__main__': + run_command(sys.argv) diff --git a/util/common/__init__.py b/util/common/__init__.py index 0bf97f41d..e4607c41a 100644 --- a/util/common/__init__.py +++ b/util/common/__init__.py @@ -1,6 +1,13 @@ -import sys, subprocess, time + +import pprint +import subprocess +import sys +import time def _execute(command,**kwargs): + #print("Running:") + #pprint.pprint(command) + check = kwargs.get('check', True) on_stdout_line = kwargs.get('on_stdout_line', None) on_stderr_line = kwargs.get('on_stderr_line', None) @@ -47,14 +54,12 @@ def _docker(command, **kwargs): "docker-compose", "exec", '-T', "files", - "bash", "-c", - ' && '.join(command) - ], **kwargs) + ] + command, + **kwargs) def _status_single(server): command = ['docker', 'container', 'inspect', '-f', '{{.State.Status}}', server] result = _execute(command, check=False).stdout.strip() - print(f"{server}: {result}") return result # this should really be yanked out of the docker-compose somehow @@ -93,30 +98,38 @@ def _start(): return result def _stop(): - command = ['docker-compose','down'] + # use "stop" instead of "down" to avoid killing all stored data + command = ['docker-compose','stop'] result = _execute(command) time.sleep(1) return result -def _operation(name, command): - # check if running and start if not - running = _all_running() +def _operation(name, commands): + # restart to make sure they're in the right mode + print("Stopping containers . . .") + _stop() - if not running: - print("Starting containers . . .") - _start() + print("Starting containers in operation mode . . .") + _start() - # run operation in docker container + # prepend our upgrade, since right now we're always using it + commands = [[ + "python3", + "-m", "flask", + "db", "upgrade" + ]] + commands + + # run operations in docker container print(f"Running {name} . . .") - result = _docker( - command, - on_stdout_line=lambda line: print(line, end=''), - on_stderr=lambda line: print(line, end=''), - ) + for command in commands: + result = _docker( + command, + on_stdout_line=lambda line: print(line, end=''), + on_stderr=lambda line: print(line, end=''), + ) - if not running: - print("Stopping containers . . .") - _stop() + print("Stopping containers . . .") + _stop() return result diff --git a/util/manage.py b/util/manage.py index 60343df06..efb23a0a6 100755 --- a/util/manage.py +++ b/util/manage.py @@ -2,7 +2,8 @@ import sys from common import error, run_help -from migrate import run_migrate + +from command import run_command from test import run_test if __name__=='__main__': @@ -15,8 +16,8 @@ if __name__=='__main__': if name == "test": run_test(args) - elif name == "migrate": - run_migrate(args) + elif name == "command": + run_command(args) elif name == "help": run_help() else: diff --git a/util/migrate.py b/util/migrate.py deleted file mode 100755 index 7d5e9a8d8..000000000 --- a/util/migrate.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python3 - -import sys -from common import _operation - -def run_migrate(args): - command = 'upgrade' - - if len(args) > 1: - command = args[1] - - result = _operation(command,[ - "export FLASK_APP=files/cli:app", - f"python3 -m flask db {command}", - ]) - - sys.exit(result.returncode) - -if __name__=='__main__': - run_migrate(sys.argv) \ No newline at end of file diff --git a/util/test.py b/util/test.py index cc2f199e0..4fed548bf 100755 --- a/util/test.py +++ b/util/test.py @@ -5,8 +5,11 @@ from common import _operation def run_test(args): result = _operation("tests", [ - "FLASK_APP=files/cli:app python3 -m flask db upgrade", - "python3 -m pytest -s", + [ + "python3", + "-m", "pytest", + "-s", + ] ]) sys.exit(result.returncode)