diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..94988fb --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,33 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: [push, pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[test] + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint + run: | + ./lint.sh + - name: Test with pytest + run: | + #pytest + ./test_pytracing.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..91680cd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,26 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.5.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: detect-aws-credentials + args: [--allow-missing-credentials] + +- repo: https://github.com/humitos/mirrors-autoflake.git + rev: v1.3 + hooks: + - id: autoflake + args: ['--in-place', '--expand-star-imports', '--ignore-init-module-imports', '--remove-all-unused-imports'] + +- repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + args: [--line-length=120] + + #- repo: https://github.com/pre-commit/mirrors-isort + # rev: v4.3.21 + # hooks: + # - id: isort diff --git a/LICENSE b/LICENSE index 5cc10a6..b83c351 100644 --- a/LICENSE +++ b/LICENSE @@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/lint.sh b/lint.sh new file mode 100755 index 0000000..c71099a --- /dev/null +++ b/lint.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +pre-commit run -v -a +flake8 . --config=setup.cfg diff --git a/pytracing/__init__.py b/pytracing/__init__.py index aa0ec66..dc7dc34 100644 --- a/pytracing/__init__.py +++ b/pytracing/__init__.py @@ -6,4 +6,4 @@ from .pytracing import TraceProfiler -__all__ = ['TraceProfiler'] +__all__ = ["TraceProfiler"] diff --git a/pytracing/pytracing.py b/pytracing/pytracing.py index 607f532..57f551a 100644 --- a/pytracing/pytracing.py +++ b/pytracing/pytracing.py @@ -9,120 +9,135 @@ import json import time import threading +import logging from contextlib import contextmanager +from typing import Dict, Any + +logger = logging.getLogger(__name__) try: - from queue import Queue + from queue import Queue except ImportError: - from Queue import Queue + from Queue import Queue def to_microseconds(s): - return 1000000 * float(s) + return 1000000 * float(s) class TraceWriter(threading.Thread): - - def __init__(self, terminator, input_queue, output_stream): - threading.Thread.__init__(self) - self.daemon = True - self.terminator = terminator - self.input = input_queue - self.output = output_stream - - def _open_collection(self): - """Write the opening of a JSON array to the output.""" - self.output.write(b'[') - - def _close_collection(self): - """Write the closing of a JSON array to the output.""" - self.output.write(b'{}]') # empty {} so the final entry doesn't end with a comma - - def run(self): - self._open_collection() - while not self.terminator.is_set() or not self.input.empty(): - item = self.input.get() - self.output.write((json.dumps(item) + ',\n').encode('ascii')) - self._close_collection() + def __init__(self, terminator, input_queue, output_stream): + threading.Thread.__init__(self) + self.daemon = True + self.terminator = terminator + self.input = input_queue + self.output = output_stream + + def _open_collection(self): + """Write the opening of a JSON array to the output.""" + logger.debug("_open_collection") + self.output.write("[") + + def _close_collection(self): + """Write the closing of a JSON array to the output.""" + logger.debug("_close_collection") + self.output.write("{}]") # empty {} so the final entry doesn't end with a comma + + def run(self): + self._open_collection() + while not self.terminator.is_set() or not self.input.empty(): + item = self.input.get() + self.output.write(json.dumps(item) + ",\n") + self._close_collection() class TraceProfiler(object): - """A python trace profiler that outputs Chrome Trace-Viewer format (about://tracing). - - Usage: - - from pytracing import TraceProfiler - tp = TraceProfiler(output=open('/tmp/trace.out', 'wb')) - with tp.traced(): - ... - - """ - TYPES = {'call': 'B', 'return': 'E'} - - def __init__(self, output, clock=None): - self.output = output - self.clock = clock or time.time - self.pid = os.getpid() - self.queue = Queue() - self.terminator = threading.Event() - self.writer = TraceWriter(self.terminator, self.queue, self.output) - - @property - def thread_id(self): - return threading.current_thread().name - - @contextmanager - def traced(self): - """Context manager for install/shutdown in a with block.""" - self.install() - try: - yield - finally: - self.shutdown() - - def install(self): - """Install the trace function and open the JSON output stream.""" - self.writer.start() # Start the writer thread. - sys.setprofile(self.tracer) # Set the trace/profile function. - threading.setprofile(self.tracer) # Set the trace/profile function for threads. - - def shutdown(self): - sys.setprofile(None) # Clear the trace/profile function. - threading.setprofile(None) # Clear the trace/profile function for threads. - self.terminator.set() # Stop the writer thread. - self.writer.join() # Join the writer thread. - - def fire_event(self, event_type, func_name, func_filename, func_line_no, - caller_filename, caller_line_no): - """Write a trace event to the output stream.""" - timestamp = to_microseconds(self.clock()) - # https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview - - event = dict( - name=func_name, # Event Name. - cat=func_filename, # Event Category. - tid=self.thread_id, # Thread ID. - ph=self.TYPES[event_type], # Event Type. - pid=self.pid, # Process ID. - ts=timestamp, # Timestamp. - args=dict( - function=':'.join([str(x) for x in (func_filename, func_line_no, func_name)]), - caller=':'.join([str(x) for x in (caller_filename, caller_line_no)]), - ) - ) - self.queue.put(event) - - def tracer(self, frame, event_type, arg): - """Bound tracer function for sys.settrace().""" - try: - if event_type in self.TYPES.keys() and frame.f_code.co_name != 'write': - self.fire_event( - event_type=event_type, - func_name=frame.f_code.co_name, - func_filename=frame.f_code.co_filename, - func_line_no=frame.f_lineno, - caller_filename=frame.f_back.f_code.co_filename, - caller_line_no=frame.f_back.f_lineno, + """A python trace profiler that outputs Chrome Trace-Viewer format (about://tracing). + + Usage: + + from pytracing import TraceProfiler + tp = TraceProfiler(output=open('/tmp/trace.out', 'wb')) + with tp.traced(): + ... + + """ + + TYPES = {"call": "B", "return": "E"} + + def __init__(self, output, clock=None): + self.output = output + self.clock = clock or time.time + self.pid = os.getpid() + self.queue = Queue() + self.terminator = threading.Event() + self.writer = TraceWriter(self.terminator, self.queue, self.output) + self.event_callbacks = [self._chrome_tracing_event] + + @property + def thread_id(self): + return threading.current_thread().name + + @contextmanager + def traced(self): + """Context manager for install/shutdown in a with block.""" + self.install() + try: + yield + finally: + self.shutdown() + + def install(self): + """Install the trace function and open the JSON output stream.""" + self.writer.start() # Start the writer thread. + sys.setprofile(self.tracer) # Set the trace/profile function. + threading.setprofile(self.tracer) # Set the trace/profile function for threads. + + def shutdown(self): + sys.setprofile(None) # Clear the trace/profile function. + threading.setprofile(None) # Clear the trace/profile function for threads. + self.terminator.set() # Stop the writer thread. + self.writer.join() # Join the writer thread. + + def _chrome_tracing_event( + self, event_type, func_name, func_filename, func_line_no, caller_filename, caller_line_no + ) -> Dict[str, Any]: + """ + Format a Chrome tracing event that can be encoded to JSON + https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview + """ + timestamp = to_microseconds(self.clock()) + event = dict( + name=func_name, # Event Name. + cat=func_filename, # Event Category. + tid=self.thread_id, # Thread ID. + ph=self.TYPES[event_type], # Event Type. + pid=self.pid, # Process ID. + ts=timestamp, # Timestamp. + args=dict( + function=":".join([str(x) for x in (func_filename, func_line_no, func_name)]), + caller=":".join([str(x) for x in (caller_filename, caller_line_no)]), + ), ) - except Exception: - pass # Don't disturb execution if we can't log the trace. + return event + + def fire_event(self, event_type, func_name, func_filename, func_line_no, caller_filename, caller_line_no): + """Trigger event callbacks.""" + for cb in self.event_callbacks: + event = cb(event_type, func_name, func_filename, func_line_no, caller_filename, caller_line_no) + self.queue.put(event) + + def tracer(self, frame, event_type, arg): + """Bound tracer function for sys.settrace().""" + try: + if event_type in self.TYPES.keys() and frame.f_code.co_name != "write": + self.fire_event( + event_type=event_type, + func_name=frame.f_code.co_name, + func_filename=frame.f_code.co_filename, + func_line_no=frame.f_lineno, + caller_filename=frame.f_back.f_code.co_filename, + caller_line_no=frame.f_back.f_lineno, + ) + except Exception: + pass # Don't disturb execution if we can't log the trace. diff --git a/setup.cfg b/setup.cfg index b88034e..15bb1fe 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,17 @@ [metadata] description-file = README.md + +[flake8] +exclude = + venv*, + .pybuilder, + build, + dist +max-line-length = 120 +select = E9,F63,F7,F82 +max-complexity = 10 +verbose = false +jobs = auto +count = true +show-source = true +statistics = true diff --git a/setup.py b/setup.py index 250b8c2..0c0516e 100644 --- a/setup.py +++ b/setup.py @@ -3,11 +3,12 @@ from setuptools import setup setup( - name='pytracing', - version='0.4', - description='a python trace profiler that outputs to chrome trace-viewer format (about://tracing).', - author='Kris Wilson', - author_email='kwilson@twitter.com', - url='https://www.github.com/kwlzn/pytracing', - packages=['pytracing'] + name="pytracing", + version="0.4", + description="a python trace profiler that outputs to chrome trace-viewer format (about://tracing).", + author="Kris Wilson", + author_email="kwilson@twitter.com", + extras_require={"test": ["flake8==3.9.2", "pre-commit"]}, + url="https://www.github.com/kwlzn/pytracing", + packages=["pytracing"], ) diff --git a/test_pytracing.py b/test_pytracing.py index aa114d3..255676d 100755 --- a/test_pytracing.py +++ b/test_pytracing.py @@ -13,29 +13,28 @@ def function_a(x): - print('sleeping {}'.format(x)) - time.sleep(x) - return + print("sleeping {}".format(x)) + time.sleep(x) + return def function_b(x): - function_a(x) + function_a(x) def main(): - function_a(1) - function_b(2) + function_a(1) + function_b(2) -if __name__ == '__main__': - with io.open('./trace.out', mode='w', encoding='utf-8') as fh: - tp = TraceProfiler(output=fh) - tp.install() - main() - tp.shutdown() - print('wrote trace.out') - - # ensure the output is at least valid JSON - with io.open('./trace.out', encoding='utf-8') as fh: - json.load(fh) +if __name__ == "__main__": + with io.open("./trace.out", mode="w", encoding="utf-8") as fh: + tp = TraceProfiler(output=fh) + tp.install() + main() + tp.shutdown() + print("wrote trace.out") + # ensure the output is at least valid JSON + with io.open("./trace.out", encoding="utf-8") as fh: + json.load(fh)