diff --git a/thirdparty/sqlalchemy-easy-profile/.github/ISSUE_TEMPLATE/bug_report.md b/thirdparty/sqlalchemy-easy-profile/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..feccbd2b7 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,20 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: dmvass + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior or test case. + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Additional context** +Add any other context about the problem here. diff --git a/thirdparty/sqlalchemy-easy-profile/.github/ISSUE_TEMPLATE/feature_request.md b/thirdparty/sqlalchemy-easy-profile/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..f67d9bb44 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[feature]" +labels: enhancement +assignees: dmvass + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/thirdparty/sqlalchemy-easy-profile/.github/dependabot.yml b/thirdparty/sqlalchemy-easy-profile/.github/dependabot.yml new file mode 100644 index 000000000..b38df29f4 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" diff --git a/thirdparty/sqlalchemy-easy-profile/.github/workflows/codeql-analysis.yml b/thirdparty/sqlalchemy-easy-profile/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..75c78110f --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/.github/workflows/codeql-analysis.yml @@ -0,0 +1,68 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# ******** NOTE ******** + +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '23 21 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/thirdparty/sqlalchemy-easy-profile/.github/workflows/python-publish.yml b/thirdparty/sqlalchemy-easy-profile/.github/workflows/python-publish.yml new file mode 100644 index 000000000..4e1ef42d2 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/.github/workflows/python-publish.yml @@ -0,0 +1,31 @@ +# This workflows will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/thirdparty/sqlalchemy-easy-profile/.gitignore b/thirdparty/sqlalchemy-easy-profile/.gitignore new file mode 100644 index 000000000..5f13ccd9e --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/.gitignore @@ -0,0 +1,18 @@ +*.pyc + +# Packages +*.egg-info +.eggs +build +dist + +# IDE files +.idea +.vscode + +# Environment +.venv* + +# Testing +.tox +.coverage diff --git a/thirdparty/sqlalchemy-easy-profile/.travis.yml b/thirdparty/sqlalchemy-easy-profile/.travis.yml new file mode 100644 index 000000000..38f26b540 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/.travis.yml @@ -0,0 +1,21 @@ +sudo: false +language: python +dist: xenial +python: + - "3.7" + - "3.8" + - "3.9" +matrix: + include: + - name: "PEP8" + python: 3.7 + env: TOXENV=pep8 +install: + - pip install tox-travis codecov +script: + - tox +after_success: + - codecov +branches: + only: + - master diff --git a/thirdparty/sqlalchemy-easy-profile/CHANGELOG.md b/thirdparty/sqlalchemy-easy-profile/CHANGELOG.md new file mode 100644 index 000000000..c5dec895a --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/CHANGELOG.md @@ -0,0 +1,100 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [1.2.1] - 2021-05-14 +- Fixed install requires SQLAlchemy version + +## [1.2.0] - 2021-03-31 +- Added support of python 3.9 +- Added support of SQLAlchemy 1.4 +- Removed support of python 3.6 +- Removed support of SQLAlchemy 1.1, 1.2 + +## [1.1.2] - 2020-10-21 +- Fixed queries for UNIX platforms [issue-18] + +## [1.1.1] - 2020-07-26 +- Fixed deprecated time.clock [issue-16] + +## [1.1.0] - 2020-06-29 +- Removed support of python 2.7, 3.5 +- Updated documentation +- Added code of conduct + +## [1.0.3] - 2019-11-04 +- Fixed an issue where concurrent calls to an API would cause "Profiling session has already begun" exception. + +## [1.0.2] - 2019-04-06 +### Changed +- Profiler stats type from `dict` on `OrderedDict` +### Fixed +- Readme examples imports from [@dbourdeveloper](https://github.com/dbourdeveloper) +- Profiler duplicates counter (now it's begin countig from `0`) + +## [1.0.1] - 2019-04-04 +### Added +- Human readable sql output to the console [@Tomasz-Kluczkowski](https://github.com/Tomasz-Kluczkowski) +- Py2 unicode support +- Docstring improvements + +## [1.0.0] - 2019-03-25 +### Added +- Supports for SQLAlchemy 1.3 +- Makefile +- setup.cfg +- pep8 tox env +### Changed +- Set new GitHub username in the README +- Update setup requirements +### Fixed +- flake8 issues +### Removed +- Supports for SQLAlchemy 1.0 +- .bumpversion (moved to setup.cfg) + +## [0.5.0] - 2018-11-12 +### Added +- Supports for SQLAlchemy 1.0, 1.1 and 1.2 versions + +## [0.4.1] - 2018-11-08 +### Fixed +- Report example image link in the README + +## [0.4.0] - 2018-11-08 +### Added +- README + +## [0.3.4] - 2018-11-08 +### Fixed +- Travis CI pipy provider secure password + +## [0.3.2] - 2018-11-07 +### Added +- Bump version config +- Travis CI deploy to pypi + +## [0.3.1] - 2018-11-07 +### Changed +- Travis CI python3.7 on python3.7-dev + +## [0.3.0] - 2018-11-07 +### Added +- TOX +- Setuptools configuration +- Support py2/py3 +- Travic CI + +## [0.2.0] - 2018-11-07 +### Added +- Middleware tests and fixes +- Profiler tests and fixes +- Reporters tests and fixes +- Termcolors tests and fixes + +## [0.1.0] - 2018-11-04 +- Initial commit. diff --git a/thirdparty/sqlalchemy-easy-profile/CODE_OF_CONDUCT.md b/thirdparty/sqlalchemy-easy-profile/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..7a5f27edf --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at vasilishin.d.o@gmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/thirdparty/sqlalchemy-easy-profile/LICENSE b/thirdparty/sqlalchemy-easy-profile/LICENSE new file mode 100644 index 000000000..268ec0422 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Dmitri Vasilishin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/thirdparty/sqlalchemy-easy-profile/README.md b/thirdparty/sqlalchemy-easy-profile/README.md new file mode 100644 index 000000000..b6dd0a502 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/README.md @@ -0,0 +1,141 @@ +# SQLAlchemy Easy Profile +[![Build Status](https://travis-ci.com/dmvass/sqlalchemy-easy-profile.svg?branch=master)](https://travis-ci.com/dmvass/sqlalchemy-easy-profile) +[![image](https://img.shields.io/pypi/v/sqlalchemy-easy-profile.svg)](https://pypi.python.org/pypi/sqlalchemy-easy-profile) +[![codecov](https://codecov.io/gh/dmvass/sqlalchemy-easy-profile/branch/master/graph/badge.svg)](https://codecov.io/gh/dmvass/sqlalchemy-easy-profile) +[![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/dmvass/sqlalchemy-easy-profile/blob/master/LICENSE) + +Inspired by [django-querycount](https://github.com/bradmontgomery/django-querycount), +is a library that hooks into SQLAlchemy to collect metrics, streaming statistics into +console output and help you understand where in application you have slow or redundant +queries. + +![report example](https://raw.githubusercontent.com/dmvass/sqlalchemy-easy-profile/master/images/report-example.png?raw=true) + +## Installation +Install the package with pip: +``` +pip install sqlalchemy-easy-profile +``` + +## Session profiler +The profiling session hooks into SQLAlchemy and captures query statements, duration information, +and query parameters. You also may have multiple profiling sessions active at the same +time on the same or different Engines. If multiple profiling sessions are active on the +same engine, queries on that engine will be collected by both sessions and reported on +different reporters. + +You may begin and commit a profiling session as much as you like. Calling begin on an already +started session or commit on an already committed session will raise an `AssertionError`. +You also can use a contextmanager interface for session profiling or used it like a decorator. +This has the effect of only profiling queries occurred within the decorated function or inside +a manager context. + +How to use `begin` and `commit`: +```python +from easy_profile import SessionProfiler + +profiler = SessionProfiler() + +profiler.begin() +session.query(User).filter(User.name == "Arthur Dent").first() +profiler.commit() + +print(profiler.stats) +``` + +How to use as a context manager interface: +```python +profiler = SessionProfiler() +with profiler: + session.query(User).filter(User.name == "Arthur Dent").first() + +print(profiler.stats) +``` + +How to use profiler as a decorator: +```python +profiler = SessionProfiler() + +class UsersResource: + @profiler() + def on_get(self, req, resp, **args, **kwargs): + return session.query(User).all() +``` + +Keep in mind that profiler decorator interface accepts a special reporter and +If it was not defined by default will be used a base streaming reporter. Decorator +also accept `name` and `name_callback` optional parameters. + +## WSGI integration +Easy Profiler provides a specified middleware which can prints the number of database +queries for each HTTP request and can be applied as a WSGI server middleware. So you +can easily integrate Easy Profiler into any WSGI application. + +How to integrate with a Flask application: +```python +from flask import Flask +from easy_profile import EasyProfileMiddleware + +app = Flask(__name__) +app.wsgi_app = EasyProfileMiddleware(app.wsgi_app) +``` + +How to integrate with a Falcon application: +```python +import falcon +from easy_profile import EasyProfileMiddleware + +api = application = falcon.API() +application = EasyProfileMiddleware(application) +``` + +## How to customize output + +The `StreamReporter` accepts medium-high thresholds, output file destination (stdout by default), a special +flag for disabling color formatting and number of displayed duplicated queries: + +```python +from flask import Flask +from easy_profile import EasyProfileMiddleware, StreamReporter + +app = Flask(__name__) +app.wsgi_app = EasyProfileMiddleware(app.wsgi_app, reporter=StreamReporter(display_duplicates=100)) +``` + +Any custom reporter can be created as: + +```python +from easy_profile.reporters import Reporter + +class CustomReporter(Reporter): + + def report(self, path, stats): + """Do something with path and stats. + + :param str path: where profiling occurred + :param dict stats: profiling statistics + + """ + ... + +``` + +## Testing +To run the tests: +``` +python setup.py test +``` + +Or use `tox` for running in all tests environments. + +## License +This code is distributed under the terms of the MIT license. + +## Changes +A full changelog is maintained in the [CHANGELOG](https://github.com/dmvass/sqlalchemy-easy-profile/blob/master/CHANGELOG.md) file. + +## Contributing +**sqlalchemy-easy-profile** is an open source project and contributions are +welcome! Check out the [Issues](https://github.com/dmvass/sqlalchemy-easy-profile/issues) +page to see if your idea for a contribution has already been mentioned, and feel +free to raise an issue or submit a pull request. diff --git a/thirdparty/sqlalchemy-easy-profile/easy_profile/__init__.py b/thirdparty/sqlalchemy-easy-profile/easy_profile/__init__.py new file mode 100644 index 000000000..db9d906e7 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/easy_profile/__init__.py @@ -0,0 +1,12 @@ +# The following names are available as part of the public API for +# ``sqlalchemy-easy-profile``. End users of this package can import +# these names by doing ``from easy_profile import SessionProfiler``, +# for example. + +from .middleware import EasyProfileMiddleware +from .profiler import SessionProfiler +from .reporters import StreamReporter + +__all__ = ["EasyProfileMiddleware", "SessionProfiler", "StreamReporter"] +__author__ = "Dmitry Vasilishin" +__version__ = "1.2.1" diff --git a/thirdparty/sqlalchemy-easy-profile/easy_profile/middleware.py b/thirdparty/sqlalchemy-easy-profile/easy_profile/middleware.py new file mode 100644 index 000000000..9d6d5af89 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/easy_profile/middleware.py @@ -0,0 +1,63 @@ +import re + +from .profiler import SessionProfiler +from .reporters import Reporter, StreamReporter + + +class EasyProfileMiddleware(object): + """This middleware prints the number of database queries for each HTTP + request and can be applied as a WSGI server middleware. + + :param app: WSGI application server + :param sqlalchemy.engine.base.Engine engine: sqlalchemy database engine + :param Reporter reporter: reporter instance + :param list exclude_path: a list of regex patterns for excluding requests + :param int min_time: minimal queries duration to logging + :param int min_query_count: minimal queries count to logging + + """ + + def __init__(self, + app, + engine=None, + reporter=None, + exclude_path=None, + min_time=0, + min_query_count=1): + + if reporter: + if not isinstance(reporter, Reporter): + raise TypeError("reporter must be inherited from 'Reporter'") + self.reporter = reporter + else: + self.reporter = StreamReporter() + + self.app = app + self.engine = engine + self.exclude_path = exclude_path or [] + self.min_time = min_time + self.min_query_count = min_query_count + + def __call__(self, environ, start_response): + profiler = SessionProfiler(self.engine) + path = environ.get("PATH_INFO", "") + if not self._ignore_request(path): + method = environ.get("REQUEST_METHOD") + if method: + path = "{0} {1}".format(method, path) + try: + with profiler: + response = self.app(environ, start_response) + finally: + self._report_stats(path, profiler.stats) + return response + return self.app(environ, start_response) + + def _ignore_request(self, path): + """Check to see if we should ignore the request.""" + return any(re.match(pattern, path) for pattern in self.exclude_path) + + def _report_stats(self, path, stats): + if (stats["total"] >= self.min_query_count and + stats["duration"] >= self.min_time): + self.reporter.report(path, stats) diff --git a/thirdparty/sqlalchemy-easy-profile/easy_profile/profiler.py b/thirdparty/sqlalchemy-easy-profile/easy_profile/profiler.py new file mode 100644 index 000000000..ce5793a5c --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/easy_profile/profiler.py @@ -0,0 +1,186 @@ +from collections import Counter, namedtuple, OrderedDict +import functools +import inspect +from queue import Queue +import re +import sys +import time + +from sqlalchemy import event +from sqlalchemy.engine.base import Engine + +from .reporters import StreamReporter + +# Optimize timer function for the platform +if sys.platform == "win32": # pragma: no cover + _timer = time.perf_counter +else: + _timer = time.time + + +SQL_OPERATORS = ["select", "insert", "update", "delete"] +OPERATOR_REGEX = re.compile("(%s) *." % "|".join(SQL_OPERATORS), re.IGNORECASE) + + +def _get_object_name(obj): + module = getattr(obj, "__module__", inspect.getmodule(obj).__name__) + if hasattr(obj, "__qualname__"): + name = obj.__qualname__ + else: + name = obj.__name__ + return module + "." + name + + +_DebugQuery = namedtuple( + "_DebugQuery", "statement,parameters,start_time,end_time" +) + + +class DebugQuery(_DebugQuery): + """Public implementation of the debug query class""" + + @property + def duration(self): + return self.end_time - self.start_time + + +class SessionProfiler: + """A session profiler for sqlalchemy queries. + + :param Engine engine: sqlalchemy database engine + + :attr bool alive: is True if profiling in progress + :attr Queue queries: sqlalchemy queries queue + + """ + + _before = "before_cursor_execute" + _after = "after_cursor_execute" + + def __init__(self, engine=None): + if engine is None: + self.engine = Engine + self.db_name = "default" + else: + self.engine = engine + self.db_name = engine.url.database or "undefined" + + self.alive = False + self.queries = None + + self._stats = None + + def __enter__(self): + self.begin() + + def __exit__(self, exc_type, exc_val, exc_tb): + self.commit() + + def __call__(self, path=None, path_callback=None, reporter=None): + """Decorate callable object and profile sqlalchemy queries. + + If reporter was not defined by default will be used a base + streaming reporter. + + :param easy_profile.reporters.Reporter reporter: profiling reporter + :param collections.abc.Callable path_callback: callback for getting + more complex path + + """ + if reporter is None: + reporter = StreamReporter() + + def decorator(func): + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if path_callback is not None: + _path = path_callback(func, *args, **kwargs) + else: + _path = path or _get_object_name(func) + + self.begin() + try: + result = func(*args, **kwargs) + finally: + self.commit() + reporter.report(_path, self.stats) + return result + + return wrapper + + return decorator + + @property + def stats(self): + if self._stats is None: + self._reset_stats() + return self._stats + + def begin(self): + """Begin profiling session. + + :raises AssertionError: When the session is already alive. + + """ + if self.alive: + raise AssertionError("Profiling session has already begun") + + self.alive = True + self.queries = Queue() + self._reset_stats() + + event.listen(self.engine, self._before, self._before_cursor_execute) + event.listen(self.engine, self._after, self._after_cursor_execute) + + def commit(self): + """Commit profiling session. + + :raises AssertionError: When the session is not alive. + + """ + if not self.alive: + raise AssertionError("Profiling session is already committed") + + self.alive = False + self._get_stats() + + event.remove(self.engine, self._before, self._before_cursor_execute) + event.remove(self.engine, self._after, self._after_cursor_execute) + + def _get_stats(self): + """Calculate and returns session statistics.""" + while not self.queries.empty(): + query = self.queries.get() + self._stats["call_stack"].append(query) + match = OPERATOR_REGEX.match(query.statement) + if match: + self._stats[match.group(1).lower()] += 1 + self._stats["total"] += 1 + self._stats["duration"] += query.duration + duplicates = self._stats["duplicates"].get(query.statement, -1) + self._stats["duplicates"][query.statement] = duplicates + 1 + + return self._stats + + def _reset_stats(self): + self._stats = OrderedDict() + self._stats["db"] = self.db_name + + for operator in SQL_OPERATORS: + self._stats[operator] = 0 + + self._stats["total"] = 0 + self._stats["duration"] = 0 + self._stats["call_stack"] = [] + self._stats["duplicates"] = Counter() + + def _before_cursor_execute(self, conn, cursor, statement, parameters, + context, executemany): + context._query_start_time = _timer() + + def _after_cursor_execute(self, conn, cursor, statement, parameters, + context, executemany): + self.queries.put(DebugQuery( + statement, parameters, context._query_start_time, _timer() + )) diff --git a/thirdparty/sqlalchemy-easy-profile/easy_profile/reporters.py b/thirdparty/sqlalchemy-easy-profile/easy_profile/reporters.py new file mode 100644 index 000000000..d1c77a7bc --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/easy_profile/reporters.py @@ -0,0 +1,161 @@ +from abc import ABC, abstractmethod +from collections import OrderedDict +import sys + +import sqlparse + +from .termcolors import colorize + + +def shorten(text, length, placeholder="..."): + """Truncate the given text to fit in the given length. + + :param str text: string for truncate + :param int length: max length of string + :param str placeholder: append to the end of truncated text + + :return: truncated string + + """ + if len(text) > length: + return text[:length - len(placeholder)] + placeholder + return text + + +class Reporter(ABC): + """Abstract class for profiler reporters.""" + + @abstractmethod + def report(self, path, stats): + """Reports profiling statistic to a stream. + + :param str path: where profiling occurred + :param dict stats: profiling statistics + + """ + + +class StreamReporter(Reporter): + """A base reporter for streaming to a file. By default reports + will be written to ``sys.stdout``. + + :param int medium: a medium threshold count + :param int high: a high threshold count + :param file: output destination (stdout by default) + :param bool colorized: set True if output should be colorized + :param int display_duplicates: how much sql duplicates will be displayed + + """ + + _display_names = OrderedDict([ + ("Database", "db"), + ("SELECT", "select"), + ("INSERT", "insert"), + ("UPDATE", "update"), + ("DELETE", "delete"), + ("Totals", "total"), + ("Duplicates", "duplicates_count"), + ]) + + def __init__(self, + medium=50, + high=100, + file=sys.stdout, + colorized=True, + display_duplicates=5): + + if medium >= high: + raise ValueError("Medium must be less than high") + + self._medium = medium + self._high = high + self._file = file + self._colorized = colorized + self._display_duplicates = display_duplicates or 0 + + def report(self, path, stats): + duplicates = stats["duplicates"] + stats["duplicates_count"] = sum(duplicates.values()) + stats["db"] = shorten(stats["db"], 10) + + output = self._colorize("\n{0}\n".format(path), ["bold"], fg="blue") + output += self.stats_table(stats) + + total = stats["total"] + duration = float(stats["duration"]) + summary = "Total queries: {0} in {1:.3}s".format(total, duration) + output += self._info_line("\n{0}\n".format(summary), total) + + # Display duplicated sql statements. + # + # Get top counters were value greater than 1 and write to + # a stream. It will be skipped if `display_duplicates` was + # set to `0` or `None`. + most_common = duplicates.most_common(self._display_duplicates) + for statement, count in most_common: + if count < 1: + continue + # Wrap SQL statement and returning a list of wrapped lines + statement = sqlparse.format( + statement, reindent=True, keyword_case="upper" + ) + text = "\nRepeated {0} times:\n{1}\n".format(count + 1, statement) + output += self._info_line(text, count) + + self._file.write(output) + + def stats_table(self, stats, sep="|"): + """Formats profiling statistics as table. + + :param dict stats: profiling statistics + :param str sep: columns separator character + + :return: formatted table + :rtype: str + + """ + line = sep + "{}" + sep + "\n" + h_names = [n.center(len(n) + 2) for n in self._display_names] + breakline = line.format(sep.join("-" * len(n) for n in h_names)) + + # Creates table and writes a header + output = "" + output += breakline + output += line.format(sep.join(h_names)) + output += breakline + + # Formats and writes row values in order by display_names. + # + # Row with values can be colorized for better perception. It's + # can be activated/deactivated through `colorized` parameter. + values = [] + for name, key in self._display_names.items(): + value = stats[key] + size = len(name) + 2 + values.append(str(value).center(size)) + + row = line.format(sep.join(values)) + output += self._info_line(row, stats["total"]) + output += breakline + + return output + + def _info_line(self, line, total): + """Returns colorized text according threshold. + + :param str line: text which should be colorized + :param int total: threshold count + + :return: colorized text + + """ + if total > self._high: + return self._colorize(line, ["bold"], fg="red") + elif total > self._medium: + return self._colorize(line, ["bold"], fg="yellow") + return self._colorize(line, ["bold"], fg="green") + + def _colorize(self, text, opts=(), fg=None, bg=None): + if not self._colorized: + return text + return colorize(text, opts, fg=fg, bg=bg) diff --git a/thirdparty/sqlalchemy-easy-profile/easy_profile/termcolors.py b/thirdparty/sqlalchemy-easy-profile/easy_profile/termcolors.py new file mode 100644 index 000000000..85160ce5f --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/easy_profile/termcolors.py @@ -0,0 +1,68 @@ +ansi_colors = { + "black": 30, + "red": 31, + "green": 32, + "yellow": 33, + "blue": 34, + "magenta": 35, + "cyan": 36, + "white": 37, + "bright_black": 90, + "bright_red": 91, + "bright_green": 92, + "bright_yellow": 93, + "bright_blue": 94, + "bright_magenta": 95, + "bright_cyan": 96, + "bright_white": 97, +} + +ansi_reset = "\033[0m" + +ansi_options = { + "bold": 1, + "underscore": 4, + "blink": 5, + "reverse": 7, + "conceal": 8, +} + + +def colorize(text, opts=(), fg=None, bg=None): + """Colorize text enclosed in ANSI graphics codes. + + Depends on the keyword arguments 'fg' and 'bg', and the contents of + the opts tuple/list. + + Valid colors: + 'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white' + + Valid options: + 'bold', 'underscore', 'blink', 'reverse', 'conceal' + 'noreset' - string will not be terminated with the reset code + + :param str text: your text + :param tuple opts: text options + :param str fg: foreground color name + :param str bg: background color name + + :return: colorized text + + """ + codes = [] + if len(opts) == 1 and opts[0] == "reset": + return ansi_reset + + if fg and fg in ansi_colors: + codes.append("\033[{0}m".format(ansi_colors[fg])) + elif bg and bg in ansi_colors: + codes.append("\033[{0}m".format(ansi_colors[bg] + 10)) + + for opt in opts: + if opt in ansi_options: + codes.append("\033[{0}m".format(ansi_options[opt])) + + if "noreset" not in opts: + text += ansi_reset + + return "".join(codes) + text diff --git a/thirdparty/sqlalchemy-easy-profile/images/report-example.png b/thirdparty/sqlalchemy-easy-profile/images/report-example.png new file mode 100644 index 000000000..44d2363bb Binary files /dev/null and b/thirdparty/sqlalchemy-easy-profile/images/report-example.png differ diff --git a/thirdparty/sqlalchemy-easy-profile/setup.cfg b/thirdparty/sqlalchemy-easy-profile/setup.cfg new file mode 100644 index 000000000..37fbdba27 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/setup.cfg @@ -0,0 +1,13 @@ +[flake8] +max-complexity = 10 +exclude = .eggs,.tox,.venv*,build,dist +max-line-length = 79 +import-order-style = google +inline-quotes = double +docstring-quotes = double +application-import-names = easy_profile + +[coverage:run] +branch = True +source = easy_profile +omit = setup.py diff --git a/thirdparty/sqlalchemy-easy-profile/setup.py b/thirdparty/sqlalchemy-easy-profile/setup.py new file mode 100644 index 000000000..200dcc5e1 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/setup.py @@ -0,0 +1,54 @@ +from os.path import dirname, join +import re + +import setuptools + + +def find_version(fname): + """Attempts to find the version number in the file names fname. + Raises RuntimeError if not found. + + """ + version = "" + with open(fname, "r") as fp: + regex = re.compile(r'__version__ = [\'"]([^\'"]*)[\'"]') + for line in fp: + m = regex.match(line) + if m: + version = m.group(1) + break + if not version: + raise RuntimeError("Cannot find version information") + return version + + +def read(fname): + with open(join(dirname(__file__), fname), "r") as fh: + return fh.read() + + +setuptools.setup( + name="sqlalchemy-easy-profile", + version=find_version("easy_profile/__init__.py"), + author="Dmitri Vasilishin", + author_email="vasilishin.d.o@gmail.com", + description="An easy profiler for SQLAlchemy queries", + long_description=read("README.md"), + long_description_content_type="text/markdown", + url="https://github.com/dmvass/sqlalchemy-easy-profile", + packages=setuptools.find_packages(exclude=("test*",)), + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + keywords=["sqlalchemy", "easy", "profile", "profiler", "profiling"], + install_requires=["sqlalchemy<1.5", "sqlparse>=0.3.0"], + tests_require=["coverage"], + extras_require={"dev": ["tox"]} +) diff --git a/thirdparty/sqlalchemy-easy-profile/tests/__init__.py b/thirdparty/sqlalchemy-easy-profile/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/thirdparty/sqlalchemy-easy-profile/tests/test_middleware.py b/thirdparty/sqlalchemy-easy-profile/tests/test_middleware.py new file mode 100644 index 000000000..3e0ae3427 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/tests/test_middleware.py @@ -0,0 +1,151 @@ +from queue import Queue +from threading import Thread +from time import sleep +import unittest +from unittest import mock + +from easy_profile.middleware import EasyProfileMiddleware +from easy_profile.reporters import Reporter, StreamReporter + + +class TestEasyProfileMiddleware(unittest.TestCase): + + def test_initialization_default(self): + mocked_app = mock.Mock() + mw = EasyProfileMiddleware(mocked_app) + self.assertEqual(mw.app, mocked_app) + self.assertIs(mw.engine, None) + self.assertIsInstance(mw.reporter, StreamReporter) + self.assertEqual(mw.exclude_path, []) + self.assertEqual(mw.min_time, 0) + self.assertEqual(mw.min_query_count, 1) + + def test_initialize_custom(self): + mocked_app = mock.Mock() + mocked_reporter = mock.Mock(spec=Reporter) + expected_exclude_path = ["/api/users"] + mw = EasyProfileMiddleware( + mocked_app, + reporter=mocked_reporter, + exclude_path=expected_exclude_path, + min_time=42, + min_query_count=42, + ) + self.assertEqual(mw.app, mocked_app) + self.assertEqual(mw.reporter, mocked_reporter) + self.assertEqual(mw.exclude_path, expected_exclude_path) + self.assertEqual(mw.min_time, 42) + self.assertEqual(mw.min_query_count, 42) + + def test_initialize_reporter_type_error(self): + with self.assertRaises(TypeError) as exec_info: + EasyProfileMiddleware(mock.Mock(), reporter=mock.Mock()) + self.assertEqual( + str(exec_info.exception), + "reporter must be inherited from 'Reporter'" + ) + + def test__report_stats(self): + mocked_reporter = mock.Mock(spec=Reporter) + mw = EasyProfileMiddleware( + mock.Mock(), + reporter=mocked_reporter, + min_time=42, + min_query_count=42, + ) + + # Test that report called + mw._report_stats("path", dict(total=43, duration=43)) + mw.reporter.report.assert_called() + + # Test that report not called + mw.reporter = mock.Mock(spec=Reporter) + mw._report_stats("path", dict(total=41, duration=41)) + mw.reporter.report.assert_not_called() + + def test__ignore_request(self): + patterns = [r"^/api/users", r"^/api/roles", r"^/api/permissions"] + mw = EasyProfileMiddleware(mock.Mock(), exclude_path=patterns) + # Test unavailable path + for path in ["/api/users", "/api/roles", "/api/permissions"]: + self.assertTrue(mw._ignore_request(path)) + # Test available path + for path in ["/faq", "/about", "/search"]: + self.assertFalse(mw._ignore_request(path)) + + def test__call__for_available_path(self): + mw = EasyProfileMiddleware( + mock.Mock(), + reporter=mock.Mock(spec=Reporter), + exclude_path=[r"^/api/users"] + ) + with mock.patch.object(mw, "_report_stats") as mocked_report_stats: + environ = dict(PATH_INFO="/api/roles", REQUEST_METHOD="GET") + mw(environ, None) + mocked_report_stats.assert_called() + expected = environ["REQUEST_METHOD"] + " " + environ["PATH_INFO"] + self.assertEqual(mocked_report_stats.call_args[0][0], expected) + + def test__call__for_unavailable_path(self): + mw = EasyProfileMiddleware( + mock.Mock(), + reporter=mock.Mock(spec=Reporter), + exclude_path=[r"^/api/users"] + ) + with mock.patch.object(mw, "_report_stats") as mocked_report_stats: + environ = dict(PATH_INFO="/api/users", REQUEST_METHOD="GET") + mw(environ, None) + mocked_report_stats.assert_not_called() + + def test__call__with_exception_triggered_when_getting_response(self): + app = mock.Mock() + app.side_effect = Exception("boom") + mw = EasyProfileMiddleware( + app=app, + reporter=mock.Mock(spec=Reporter), + exclude_path=[r"^/api/users"] + ) + with mock.patch.object(mw, "_report_stats") as mocked_report_stats: + environ = dict(PATH_INFO="/api/roles", REQUEST_METHOD="GET") + with self.assertRaises(Exception): + mw(environ, None) + mocked_report_stats.assert_called() + expected = environ["REQUEST_METHOD"] + " " + environ["PATH_INFO"] + self.assertEqual(mocked_report_stats.call_args[0][0], expected) + + def test__call__with_multiple_concurrent_calls(self): + fake_response = "fake response" + + def fake_call(*args, **kwargs): + sleep(1) + return fake_response + + mock_app = mock.Mock() + mock_app.side_effect = fake_call + mw = EasyProfileMiddleware( + mock_app, + reporter=mock.Mock(spec=Reporter), + exclude_path=[r"^/api/users"] + ) + + thread_queue = Queue() + threads = [] + repeats = 5 + + with mock.patch.object(mw, "_report_stats"): + environ = dict(PATH_INFO="/api/roles", REQUEST_METHOD="GET") + for i in range(repeats): + thread = Thread( + target=lambda queue, fn, env: queue.put(fn(env, None)), + args=(thread_queue, mw, environ) + ) + thread.start() + threads.append(thread) + [thread.join() for thread in threads] + + results = [] + while not thread_queue.empty(): + results.append(thread_queue.get()) + + assert len(results) == repeats + assert set(results) == {fake_response} diff --git a/thirdparty/sqlalchemy-easy-profile/tests/test_profiler.py b/thirdparty/sqlalchemy-easy-profile/tests/test_profiler.py new file mode 100644 index 000000000..12ecca181 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/tests/test_profiler.py @@ -0,0 +1,258 @@ +from collections import Counter +from queue import Queue +import time +import unittest +from unittest import mock + +from sqlalchemy import create_engine, event +from sqlalchemy.engine.base import Engine + +from easy_profile.profiler import DebugQuery, SessionProfiler, SQL_OPERATORS +from easy_profile.reporters import Reporter + + +debug_queries = [ + DebugQuery("SELECT id FROM users", {}, 1541489542, 1541489543), + DebugQuery("SELECT id FROM users", {}, 1541489542, 1541489543), + DebugQuery("SELECT name FROM users", {}, 1541489543, 1541489544), + DebugQuery("SELECT gender FROM users", {}, 1541489544, 1541489545), + DebugQuery( + "INSERT INTO users (name) VALUES (%(param_1)s)", + {"param_1": "Arthur Dent"}, + 1541489545, + 1541489546 + ), + DebugQuery( + "INSERT INTO users (name) VALUES (%(param_1)s)", + {"param_1": "Ford Prefect"}, + 1541489546, + 1541489547 + ), + DebugQuery( + "UPDATE users SET name=(%(param_1)s)", + {"param_1": "Prefect Ford"}, + 1541489547, + 1541489548 + ), + DebugQuery("DELETE FROM users", {}, 1541489548, 1541489549), +] + + +class TestSessionProfiler(unittest.TestCase): + + def test_initialization_default(self): + profiler = SessionProfiler() + self.assertIs(profiler.engine, Engine) + self.assertEqual(profiler.db_name, "default") + self.assertFalse(profiler.alive) + self.assertIsNone(profiler.queries) + + def test_initialization_custom(self): + engine = create_engine("sqlite:///test") + profiler = SessionProfiler(engine) + self.assertIs(profiler.engine, engine) + self.assertEqual(profiler.db_name, "test") + + def test_begin(self): + profiler = SessionProfiler() + with mock.patch.object(profiler, "_reset_stats") as mocked: + profiler.begin() + mocked.assert_called() + self.assertTrue(profiler.alive) + self.assertIsInstance(profiler.queries, Queue) + self.assertTrue(profiler.queries.empty()) + self.assertTrue(event.contains( + profiler.engine, + profiler._before, + profiler._before_cursor_execute + )) + self.assertTrue(event.contains( + profiler.engine, + profiler._after, + profiler._after_cursor_execute + )) + + def test_begin_alive(self): + profiler = SessionProfiler() + profiler.alive = True + with self.assertRaises(AssertionError) as exec_info: + profiler.begin() + + error = exec_info.exception + self.assertEqual(str(error), "Profiling session has already begun") + + def test_commit(self): + profiler = SessionProfiler() + profiler.begin() + with mock.patch.object(profiler, "_get_stats") as mocked: + profiler.commit() + mocked.assert_called() + self.assertFalse(profiler.alive) + self.assertFalse(event.contains( + profiler.engine, + profiler._before, + profiler._before_cursor_execute + )) + self.assertFalse(event.contains( + profiler.engine, + profiler._after, + profiler._after_cursor_execute + )) + + def test_commit_alive(self): + profiler = SessionProfiler() + profiler.alive = False + with self.assertRaises(AssertionError) as exec_info: + profiler.commit() + + error = exec_info.exception + self.assertEqual(str(error), "Profiling session is already committed") + + def test__reset_stats(self): + profiler = SessionProfiler() + profiler._reset_stats() + self.assertEqual(profiler._stats["total"], 0) + self.assertEqual(profiler._stats["duration"], 0) + self.assertEqual(profiler._stats["select"], 0) + self.assertEqual(profiler._stats["insert"], 0) + self.assertEqual(profiler._stats["update"], 0) + self.assertEqual(profiler._stats["call_stack"], []) + self.assertEqual(profiler._stats["duplicates"], Counter()) + self.assertEqual(profiler._stats["db"], profiler.db_name) + + def test__get_stats(self): + profiler = SessionProfiler() + profiler.queries = Queue() + profiler._reset_stats() + duplicates = Counter() + for query in debug_queries: + profiler.queries.put(query) + duplicates_count = duplicates.get(query.statement, -1) + duplicates[query.statement] = duplicates_count + 1 + + stats = profiler._get_stats() + + for op in SQL_OPERATORS: + res = filter(lambda x: op.upper() in x.statement, debug_queries) + self.assertEqual(stats[op], len(list(res))) + + self.assertEqual(stats["db"], profiler.db_name) + self.assertEqual(stats["total"], len(debug_queries)) + self.assertListEqual(debug_queries, stats["call_stack"]) + self.assertDictEqual(stats["duplicates"], duplicates) + + @mock.patch("easy_profile.profiler._timer") + def test__before_cursor_execute(self, mocked): + expected_time = time.time() + mocked.return_value = expected_time + profiler = SessionProfiler() + context = mock.Mock() + profiler._before_cursor_execute( + conn=None, + cursor=None, + statement=None, + parameters={}, + context=context, + executemany=None + ) + self.assertEqual(context._query_start_time, expected_time) + + @mock.patch("easy_profile.profiler._timer") + def test__after_cursor_execute(self, mocked): + expected_query = debug_queries[0] + mocked.return_value = expected_query.end_time + profiler = SessionProfiler() + context = mock.Mock() + context._query_start_time = expected_query.start_time + with profiler: + profiler._after_cursor_execute( + conn=None, + cursor=None, + statement=expected_query.statement, + parameters=expected_query.parameters, + context=context, + executemany=None + ) + actual_query = profiler.queries.get() + self.assertEqual(actual_query, expected_query) + + def test_stats(self): + profiler = SessionProfiler() + self.assertIsNotNone(profiler.stats) + + @mock.patch("easy_profile.profiler.SessionProfiler.begin") + @mock.patch("easy_profile.profiler.SessionProfiler.commit") + def test_contextmanager_interface(self, mocked_commit, mocked_begin): + profiler = SessionProfiler() + with profiler: + mocked_begin.assert_called() + mocked_commit.assert_called() + + def test_decorator(self): + engine = self._create_engine() + profiler = SessionProfiler(engine) + wrapper = profiler() + wrapper(self._decorated_func)(engine) + # Test profile statistics + self.assertEqual(profiler.stats["db"], "undefined") + self.assertEqual(profiler.stats["total"], 4) + self.assertEqual(profiler.stats["select"], 3) + self.assertEqual(profiler.stats["delete"], 1) + self.assertEqual(profiler.stats["duplicates_count"], 1) + + def test_decorator_path(self): + expected_path = "test_path" + engine = self._create_engine() + profiler = SessionProfiler(engine) + reporter = mock.Mock(spec=Reporter) + # Get profiler decorator with specified path + wrapper = profiler(path=expected_path, reporter=reporter) + wrapper(self._decorated_func)(engine) + # Test that reporter method report was called with expected path + reporter.report.assert_called_with(expected_path, profiler.stats) + + def test_decorator_path_callback(self): + expected_path = "path_callback" + + def _callback(func, *args, **kwargs): + return expected_path + + engine = self._create_engine() + profiler = SessionProfiler(engine) + reporter = mock.Mock(spec=Reporter) + # Get profiler decorator with specified path_callback + wrapper = profiler(path_callback=_callback, reporter=reporter) + wrapper(self._decorated_func)(engine) + # Test that reporter method report was called with expected path + reporter.report.assert_called_with(expected_path, profiler.stats) + + def test_decorator_path_and_path_callback(self): + expected_path = "path_callback" + + def _callback(func, *args, **kwargs): + return expected_path + + engine = self._create_engine() + profiler = SessionProfiler(engine) + reporter = mock.Mock(spec=Reporter) + # Get profiler decorator with specified path_callback + wrapper = profiler( + path="fail", + path_callback=_callback, + reporter=reporter + ) + wrapper(self._decorated_func)(engine) + # Test that reporter method report was called with expected path + reporter.report.assert_called_with(expected_path, profiler.stats) + + def _create_engine(self): + """Creates and returns sqlalchemy engine.""" + return create_engine("sqlite://") + + def _decorated_func(self, engine): + """Function for testing profiler as decorator.""" + engine.execute("CREATE TABLE users (id int, name varchar(8))") + engine.execute("SELECT id FROM users") + engine.execute("SELECT id FROM users") + engine.execute("SELECT name FROM users") + engine.execute("DELETE FROM users") diff --git a/thirdparty/sqlalchemy-easy-profile/tests/test_reporters.py b/thirdparty/sqlalchemy-easy-profile/tests/test_reporters.py new file mode 100644 index 000000000..92848c085 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/tests/test_reporters.py @@ -0,0 +1,141 @@ +from collections import Counter +import unittest +from unittest import mock + +import sqlparse + +from easy_profile.reporters import shorten, StreamReporter + + +expected_table = """ +|----------|--------|--------|--------|--------|--------|------------| +| Database | SELECT | INSERT | UPDATE | DELETE | Totals | Duplicates | +|----------|--------|--------|--------|--------|--------|------------| +| default | 8 | 2 | 3 | 0 | 13 | 3 | +|----------|--------|--------|--------|--------|--------|------------| +""" + +expected_table_stats = { + "db": "default", + "select": 8, + "insert": 2, + "update": 3, + "delete": 0, + "total": 13, + "duration": 0.0345683, + "duplicates": Counter({ + "SELECT id FROM users": 2, + "SELECT id, name FROM users": 1, + }), + "duplicates_count": 3, +} + + +class TestShorten(unittest.TestCase): + + def test_shorten(self): + # Test not longer string + expected = "test" + self.assertEqual(shorten(expected, len(expected)), expected) + + # Test longer string + expected = "test..." + self.assertEqual(shorten("test test", 7), expected) + + # Test with placeholder + expected = "test!!!" + self.assertEqual(shorten("test test", 7, placeholder="!!!"), expected) + + +class TestStreamReporter(unittest.TestCase): + + def test_initialization(self): + mocked_file = mock.Mock() + reporter = StreamReporter( + medium=1, + high=2, + file=mocked_file, + colorized=False, + display_duplicates=0 + ) + self.assertEqual(reporter._medium, 1) + self.assertEqual(reporter._high, 2) + self.assertEqual(reporter._file, mocked_file) + self.assertFalse(reporter._colorized) + self.assertEqual(reporter._display_duplicates, 0) + + def test_initialization_default(self): + reporter = StreamReporter() + self.assertEqual(reporter._medium, 50) + self.assertEqual(reporter._high, 100) + self.assertTrue(reporter._colorized) + self.assertEqual(reporter._display_duplicates, 5) + + def test_initialization_error(self): + with self.assertRaises(ValueError): + StreamReporter(medium=100, high=50) + + def test__colorize_on_deactivated(self): + with mock.patch("easy_profile.reporters.colorize") as mocked: + reporter = StreamReporter(colorized=False) + reporter._colorize("test") + mocked.assert_not_called() + + def test__colorize_on_activated(self): + with mock.patch("easy_profile.reporters.colorize") as mocked: + reporter = StreamReporter(colorized=True) + reporter._colorize("test") + mocked.assert_called() + + def test__info_line_on_high(self): + with mock.patch.object(StreamReporter, "_colorize") as mocked: + reporter = StreamReporter() + reporter._info_line("test", reporter._high + 1) + mocked.assert_called_with("test", ["bold"], fg="red") + + def test__info_line_on_medium(self): + with mock.patch.object(StreamReporter, "_colorize") as mocked: + reporter = StreamReporter() + reporter._info_line("test", reporter._medium + 1) + mocked.assert_called_with("test", ["bold"], fg="yellow") + + def test__info_line_on_low(self): + with mock.patch.object(StreamReporter, "_colorize") as mocked: + reporter = StreamReporter() + reporter._info_line("test", reporter._medium - 1) + mocked.assert_called_with("test", ["bold"], fg="green") + + def test_stats_table(self): + reporter = StreamReporter(colorized=False) + actual_table = reporter.stats_table(expected_table_stats) + self.assertEqual(actual_table.strip(), expected_table.strip()) + + def test_stats_table_change_sep(self): + sep = "+" + reporter = StreamReporter(colorized=False) + actual_table = reporter.stats_table(expected_table_stats, sep=sep) + expected = expected_table.replace("|", sep) + self.assertEqual(actual_table.strip(), expected.strip()) + + def test_report(self): + dest = mock.Mock() + reporter = StreamReporter(colorized=False, file=dest) + reporter.report("test", expected_table_stats) + + expected_output = "\ntest" + expected_output += expected_table + + total = expected_table_stats["total"] + duration = expected_table_stats["duration"] + summary = "\nTotal queries: {0} in {1:.3}s\n".format(total, duration) + expected_output += summary + + actual_output = dest.write.call_args[0][0] + self.assertRegexpMatches(actual_output, expected_output) + + for statement, count in expected_table_stats["duplicates"].items(): + statement = sqlparse.format( + statement, reindent=True, keyword_case="upper" + ) + text = "\nRepeated {0} times:\n{1}\n".format(count + 1, statement) + self.assertRegexpMatches(actual_output, text) diff --git a/thirdparty/sqlalchemy-easy-profile/tests/test_termcolors.py b/thirdparty/sqlalchemy-easy-profile/tests/test_termcolors.py new file mode 100644 index 000000000..a94406482 --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/tests/test_termcolors.py @@ -0,0 +1,47 @@ +import unittest + +from easy_profile.termcolors import ansi_options, ansi_reset, colorize + + +class TestColorize(unittest.TestCase): + + def test_fg(self): + colors = [ + "black", "red", "green", "yellow", "blue", + "magenta", "cyan", "white" + ] + for color, code in dict(zip(colors, range(30, 38))).items(): + expected = "\033[{0}m".format(code) + "test" + ansi_reset + self.assertEqual(colorize("test", fg=color), expected) + + def test_bg(self): + colors = [ + "bright_black", "bright_red", "bright_green", "bright_yellow", + "bright_blue", "bright_magenta", "bright_cyan", "bright_white" + ] + for color, code in dict(zip(colors, range(90, 98))).items(): + expected = "\033[{0}m".format(code + 10) + "test" + ansi_reset + self.assertEqual(colorize("test", bg=color), expected) + + def test_options(self): + for opt, code in ansi_options.items(): + expected = "\033[{0}m".format(code) + "test" + ansi_reset + self.assertEqual(colorize("test", [opt]), expected) + + def test_noreset(self): + self.assertEqual(colorize("test", ["noreset"]), "test") + + def test_reset(self): + self.assertEqual(colorize("test", ["reset"]), ansi_reset) + + def test_complex(self): + text = "test" + expected = [ + "\033[30m", # fg=black, + "\033[1m", # bold + "\033[4m", # underscore + text, + ansi_reset, + ] + actual = colorize(text, ["bold", "underscore"], fg="black") + self.assertEqual(actual, "".join(expected)) diff --git a/thirdparty/sqlalchemy-easy-profile/tox.ini b/thirdparty/sqlalchemy-easy-profile/tox.ini new file mode 100644 index 000000000..2d220e18d --- /dev/null +++ b/thirdparty/sqlalchemy-easy-profile/tox.ini @@ -0,0 +1,22 @@ +[tox] +envlist = + sa13-py{37,38,39} + sa14-py{37,38,39} + pep8 + +[testenv] +passenv = CI TRAVIS TRAVIS_* +deps = + codecov + sa13: SQLAlchemy>=1.3,<1.4 + sa14: SQLAlchemy>=1.4,<1.5 + +commands = coverage run setup.py test + +[testenv:pep8] +basepython = python3.7 +deps = flake8 + flake8-import-order + flake8-quotes + +commands = flake8