From ad5ea7d37876daf3c5c54c85cc623041453d9125 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Tue, 29 Dec 2020 18:41:14 +0100 Subject: [PATCH 01/37] dtrace_ctypes_test.py: Add missing return 0 in callback This avoids a ctypes warning since the expected return type is int and not "None". --- tests/dtrace_ctypes_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/dtrace_ctypes_test.py b/tests/dtrace_ctypes_test.py index 4cdc56a..ebc2101 100644 --- a/tests/dtrace_ctypes_test.py +++ b/tests/dtrace_ctypes_test.py @@ -32,3 +32,4 @@ def test_run_for_sanity(self): def _get_output(self, data, arg): tmp = c_char_p(data.contents.dtbda_buffered).value.strip() self.out = tmp + return 0 From 7241861fef21bc0c7b80af8ae1b168932a564c43 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Wed, 30 Dec 2020 21:12:09 +0100 Subject: [PATCH 02/37] More code style & cleanup stuff. --- examples/cli.py | 14 +++++++++----- examples/hello_world.py | 5 +++-- examples/read_bytes.py | 6 ++++-- examples/read_distribution_quantize.py | 8 +++++--- examples/syscall_by_zone.py | 11 +++++++---- examples/syscall_count.py | 5 +++-- examples/syscall_count_continuous.py | 6 ++++-- examples/syscall_count_own_walk.py | 8 +++++--- examples/syscall_lquantize.py | 8 +++++--- tests/dtrace_ctypes_test.py | 2 +- 10 files changed, 46 insertions(+), 27 deletions(-) diff --git a/examples/cli.py b/examples/cli.py index 44e7afa..6adfd43 100755 --- a/examples/cli.py +++ b/examples/cli.py @@ -8,10 +8,13 @@ @author: tmetsch """ + from __future__ import print_function -from dtrace import DTraceConsumerThread + import sys +import dtrace + def print_lquantize(values): """ @@ -26,14 +29,14 @@ def print_lquantize(values): for item in values: if item[0][0] > 0: print('%10s ' % item[0][0], end=' ') - for i in range(0, ((40 * int(item[1])) / maxi)): + for _ in range(0, ((40 * int(item[1])) / maxi)): sys.stdout.write('*') - for i in range(((40 * int(item[1])) / maxi), 40): + for _ in range(((40 * int(item[1])) / maxi), 40): sys.stdout.write(' ') print(' %5s' % item[1]) -def pretty_print(iden, action, keys, values): +def pretty_print(_iden, action, keys, values): """ Pretty print aggregation walker. """ @@ -60,7 +63,7 @@ def run_dtrace(script): """ Run DTrace till Ctrl+C is pressed... """ - thr = DTraceConsumerThread(script, False, walk_func=pretty_print) + thr = dtrace.DTraceConsumerThread(script, False, walk_func=pretty_print) thr.start() brendan() try: @@ -70,6 +73,7 @@ def run_dtrace(script): thr.stop() thr.join() + if __name__ == '__main__': # run_dtrace('dtrace:::BEGIN {trace("Hello World"); exit(0);}') # run_dtrace('syscall:::entry { @num[pid,execname] = count(); }') diff --git a/examples/hello_world.py b/examples/hello_world.py index 3484f45..c345bee 100755 --- a/examples/hello_world.py +++ b/examples/hello_world.py @@ -8,7 +8,7 @@ @author: tmetsch """ -from dtrace import DTraceConsumer +import dtrace SCRIPT = 'dtrace:::BEGIN {trace("Hello World"); exit(0);}' @@ -17,10 +17,11 @@ def main(): """ Run DTrace... """ - consumer = DTraceConsumer() + consumer = dtrace.DTraceConsumer() # Even when the runtime is set to run 10sec this will terminate immediately # because of the exit statement in the D script. consumer.run(SCRIPT, runtime=10) + if __name__ == '__main__': main() diff --git a/examples/read_bytes.py b/examples/read_bytes.py index c542459..fa1d06d 100755 --- a/examples/read_bytes.py +++ b/examples/read_bytes.py @@ -8,7 +8,8 @@ @author: tmetsch """ from __future__ import print_function -from dtrace import DTraceConsumer + +import dtrace SCRIPT = 'sysinfo:::readch { @bytes[execname] = sum(arg0); }' @@ -19,8 +20,9 @@ def main(): """ print('Hint: if you don\'t get any output try running it with pfexec...') - consumer = DTraceConsumer() + consumer = dtrace.DTraceConsumer() consumer.run(SCRIPT, 4) + if __name__ == '__main__': main() diff --git a/examples/read_distribution_quantize.py b/examples/read_distribution_quantize.py index cb2c7e0..1ce625b 100755 --- a/examples/read_distribution_quantize.py +++ b/examples/read_distribution_quantize.py @@ -8,13 +8,14 @@ @author: tmetsch """ from __future__ import print_function -from dtrace import DTraceConsumer + +import dtrace # SCRIPT = 'io:::start { @bytes = quantize(args[0]->b_bcount); }' SCRIPT = 'sysinfo:::readch { @dist[execname] = quantize(arg0); }' -def my_walk(action, identifier, key, values): +def my_walk(_action, _identifier, key, values): """ Walk the aggregrate. """ @@ -28,8 +29,9 @@ def main(): """ Run DTrace... """ - consumer = DTraceConsumer(walk_func=my_walk) + consumer = dtrace.DTraceConsumer(walk_func=my_walk) consumer.run(SCRIPT, 10) + if __name__ == '__main__': main() diff --git a/examples/syscall_by_zone.py b/examples/syscall_by_zone.py index cfa3ebb..ca73ae1 100755 --- a/examples/syscall_by_zone.py +++ b/examples/syscall_by_zone.py @@ -8,24 +8,26 @@ @author: tmetsch """ from __future__ import print_function -from dtrace import DTraceConsumerThread + import time +import dtrace + SCRIPT = 'syscall:::entry { @num[zonename] = count(); }' -def my_walk(action, identifier, key, value): +def my_walk(_action, _identifier, key, value): """ A simple aggregate walker. """ - print('Zone "{0:s}" made {1:d} syscalls.'.format(key[0], value)) + print('Zone "{0:s}" made {1:d} syscalls.'.format(key[0].decode(), value)) def main(): """ Run DTrace... """ - thr = DTraceConsumerThread(SCRIPT, walk_func=my_walk, sleep=1) + thr = dtrace.DTraceConsumerThread(SCRIPT, walk_func=my_walk, sleep=1) thr.start() # we will stop the thread after some time... @@ -35,5 +37,6 @@ def main(): thr.stop() thr.join() + if __name__ == '__main__': main() diff --git a/examples/syscall_count.py b/examples/syscall_count.py index b597622..f4fe339 100755 --- a/examples/syscall_count.py +++ b/examples/syscall_count.py @@ -8,7 +8,7 @@ @author: tmetsch """ -from dtrace import DTraceConsumer +import dtrace SCRIPT = 'syscall:::entry { @num[pid,execname] = count(); }' @@ -17,8 +17,9 @@ def main(): """ Run DTrace... """ - consumer = DTraceConsumer() + consumer = dtrace.DTraceConsumer() consumer.run(SCRIPT, 2) + if __name__ == '__main__': main() diff --git a/examples/syscall_count_continuous.py b/examples/syscall_count_continuous.py index 34e4113..08604a8 100755 --- a/examples/syscall_count_continuous.py +++ b/examples/syscall_count_continuous.py @@ -9,9 +9,10 @@ @author: tmetsch """ -from dtrace import DTraceConsumerThread import time +import dtrace + SCRIPT = 'syscall:::entry { @num[execname] = count(); }' @@ -19,7 +20,7 @@ def main(): """ Run DTrace... """ - thr = DTraceConsumerThread(SCRIPT) + thr = dtrace.DTraceConsumerThread(SCRIPT) thr.start() # we will stop the thread after some time... @@ -29,5 +30,6 @@ def main(): thr.stop() thr.join() + if __name__ == '__main__': main() diff --git a/examples/syscall_count_own_walk.py b/examples/syscall_count_own_walk.py index 87ca7a2..6a9c47f 100755 --- a/examples/syscall_count_own_walk.py +++ b/examples/syscall_count_own_walk.py @@ -9,12 +9,13 @@ @author: tmetsch """ from __future__ import print_function -from dtrace import DTraceConsumer + +import dtrace SCRIPT = 'syscall:::entry { @num[execname] = count(); }' -def my_walk(action, identifier, key, value): +def my_walk(_action, identifier, key, value): """ Aggregate walker. """ @@ -25,8 +26,9 @@ def main(): """ Run DTrace... """ - consumer = DTraceConsumer(walk_func=my_walk) + consumer = dtrace.DTraceConsumer(walk_func=my_walk) consumer.run(SCRIPT, 4) + if __name__ == '__main__': main() diff --git a/examples/syscall_lquantize.py b/examples/syscall_lquantize.py index 8656027..3d429c9 100755 --- a/examples/syscall_lquantize.py +++ b/examples/syscall_lquantize.py @@ -8,12 +8,13 @@ @author: tmetsch """ from __future__ import print_function -from dtrace import DTraceConsumer + +import dtrace SCRIPT = 'syscall::read:entry { @dist[execname] = lquantize(arg0, 0, 12, 2); }' -def my_walk(action, identifier, key, values): +def my_walk(_action, _identifier, key, values): """ Walk the aggregate. """ @@ -27,8 +28,9 @@ def main(): """ Run DTrace... """ - consumer = DTraceConsumer(walk_func=my_walk) + consumer = dtrace.DTraceConsumer(walk_func=my_walk) consumer.run(SCRIPT, 5) + if __name__ == '__main__': main() diff --git a/tests/dtrace_ctypes_test.py b/tests/dtrace_ctypes_test.py index 576b45c..314f688 100644 --- a/tests/dtrace_ctypes_test.py +++ b/tests/dtrace_ctypes_test.py @@ -29,6 +29,6 @@ def test_run_for_sanity(self): self.consumer.run(SCRIPT) self.assertEqual(self.out, b'Hello World') - def _get_output(self, data, arg): + def _get_output(self, data, _arg): tmp = c_char_p(data.contents.dtbda_buffered).value.strip() self.out = tmp From 58019c9a12022203a9ffda286dd8b41f1a5ace42 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Tue, 29 Dec 2020 14:56:50 +0100 Subject: [PATCH 03/37] Add support for building on FreeBSD On FreeBSD the DTrace headers are not installed by default so we need to include them from the source directory. --- setup.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 968aa29..3fbcffc 100644 --- a/setup.py +++ b/setup.py @@ -10,14 +10,28 @@ from distutils.core import setup from distutils.extension import Extension +import os +import platform import sys try: from Cython.Build import build_ext, cythonize - + extra_args = {} + if platform.system().lower().startswith("freebsd"): + # On FreeBSD the dtrace headers are not installed by default. + src_dir = os.getenv("FREEBSD_SRC_DIR", "/usr/src") + if not os.path.exists(os.path.join(src_dir, "sys/cddl")): + raise ImportError("Cannot find FreeBSD DTrace headers") + extra_args["include_dirs"] = [ + os.path.join(src_dir, "sys/cddl/compat/opensolaris"), + os.path.join(src_dir, "sys/cddl/contrib/opensolaris/uts/common"), + os.path.join(src_dir, "cddl/contrib/opensolaris/lib/libdtrace/common"), + ] BUILD_EXTENSION = {'build_ext': build_ext} - EXT_MODULES = cythonize([Extension("dtrace", ["dtrace_cython/dtrace_h.pxd", - "dtrace_cython/consumer.pyx"], - libraries=["dtrace"])], + EXT_MODULES = cythonize([ + Extension("dtrace", ["dtrace_cython/dtrace_h.pxd", + "dtrace_cython/consumer.pyx"], + libraries=["dtrace"], + **extra_args)], language_level=sys.version_info.major) except ImportError: From f54e6b7e8d1be8e4025f1082aa5210534a5c8697 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Wed, 30 Dec 2020 21:16:46 +0100 Subject: [PATCH 04/37] code style changes + merge requests. --- tests/__init__.py | 3 +++ tests/dtrace_ctypes_test.py | 11 +++++++++-- tests/dtrace_cython_test.py | 21 ++++++++++++++++++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 49a2fb9..6aa8002 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1,4 @@ +""" +Unit tests. +""" __author__ = 'tmetsch' diff --git a/tests/dtrace_ctypes_test.py b/tests/dtrace_ctypes_test.py index debfa87..c232846 100644 --- a/tests/dtrace_ctypes_test.py +++ b/tests/dtrace_ctypes_test.py @@ -4,11 +4,12 @@ __author__ = 'tmetsch' -from dtrace_ctypes import consumer + +import unittest from ctypes import c_char_p -import unittest +from dtrace_ctypes import consumer SCRIPT = 'dtrace:::BEGIN {trace("Hello World");}' @@ -23,9 +24,15 @@ def setUp(self): self.consumer = consumer.DTraceConsumer(out_func=self._get_output) def test_run_for_success(self): + """ + Test for success. + """ self.consumer.run(SCRIPT) def test_run_for_sanity(self): + """ + Test for sanity. + """ self.consumer.run(SCRIPT) self.assertEqual(self.out, b'Hello World') diff --git a/tests/dtrace_cython_test.py b/tests/dtrace_cython_test.py index 75dcdc6..cc47cc3 100644 --- a/tests/dtrace_cython_test.py +++ b/tests/dtrace_cython_test.py @@ -4,9 +4,10 @@ __author__ = 'tmetsch' -import dtrace import unittest +import dtrace + SCRIPT = 'dtrace:::BEGIN {trace("Hello World");}' @@ -22,24 +23,42 @@ def setUp(self): # Test fo success. def test_compile_for_success(self): + """ + Test for success. + """ self.consumer.compile(SCRIPT) def test_run_for_success(self): + """ + Test for success. + """ self.consumer.run(SCRIPT) # Test fo failure. def test_compile_for_failure(self): + """ + Test for failure. + """ self.assertRaises(Exception, self.consumer.compile, 'foo') def test_run_for_failure(self): + """ + Test for failure. + """ self.assertRaises(Exception, self.consumer.run, 'sadf') # Test fo sanity. def test_run_for_sanity(self): + """ + Test for sanity. + """ self.consumer.run(SCRIPT) self.assertEqual(b'Hello World', self.out) def _get_output(self, tmp): + """ + Test for sanity. + """ self.out = tmp From e8528682d3a228c9a40f070cadb8f3f137a283a4 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Wed, 30 Dec 2020 21:20:02 +0100 Subject: [PATCH 05/37] cython: use str instead of unicode This allows the cython wrapper to work with both py2 and py3 strings. --- dtrace_cython/consumer.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index 7ffd9b4..c4a5430 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -265,7 +265,7 @@ cdef class DTraceConsumer: if hasattr(self, 'handle') and self.handle != NULL: dtrace_close(self.handle) - cpdef compile(self, unicode script): + cpdef compile(self, str script): """ Compile a DTrace script and return errors if any. @@ -279,7 +279,7 @@ cdef class DTraceConsumer: dtrace_errmsg(self.handle, dtrace_errno(self.handle))) - cpdef run(self, unicode script, runtime=1): + cpdef run(self, str script, runtime=1): """ Run a DTrace script for a number of seconds defined by the runtime. @@ -355,7 +355,7 @@ cdef class DTraceContinuousConsumer: cdef object chewrec_func cdef object script - def __init__(self, unicode script, chew_func=None, chewrec_func=None, + def __init__(self, str script, chew_func=None, chewrec_func=None, out_func=None, walk_func=None): """ Constructor. will get the DTrace handle From b76c73145d36f34440824d53183e0c5de3b7be72 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Wed, 30 Dec 2020 21:33:31 +0100 Subject: [PATCH 06/37] deleted old pydev project files. --- .project | 17 ----------------- .pydevproject | 10 ---------- 2 files changed, 27 deletions(-) delete mode 100644 .project delete mode 100644 .pydevproject diff --git a/.project b/.project deleted file mode 100644 index f62d591..0000000 --- a/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - python-dtrace - - - - - - org.python.pydev.PyDevBuilder - - - - - - org.python.pydev.pythonNature - - diff --git a/.pydevproject b/.pydevproject deleted file mode 100644 index 3573661..0000000 --- a/.pydevproject +++ /dev/null @@ -1,10 +0,0 @@ - - - - -Default -python 2.7 - -/python-dtrace - - From c0104447e78cab597981f9bb3294ee7b4d91b703 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Wed, 30 Dec 2020 21:36:35 +0100 Subject: [PATCH 07/37] code cleanup. --- setup.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 3fbcffc..b11e422 100644 --- a/setup.py +++ b/setup.py @@ -22,17 +22,23 @@ if not os.path.exists(os.path.join(src_dir, "sys/cddl")): raise ImportError("Cannot find FreeBSD DTrace headers") extra_args["include_dirs"] = [ - os.path.join(src_dir, "sys/cddl/compat/opensolaris"), - os.path.join(src_dir, "sys/cddl/contrib/opensolaris/uts/common"), - os.path.join(src_dir, "cddl/contrib/opensolaris/lib/libdtrace/common"), + os.path.join(src_dir, + "sys/cddl/compat/opensolaris"), + os.path.join(src_dir, + "sys/cddl/contrib/opensolaris/uts/common"), + os.path.join(src_dir, + "cddl/contrib/opensolaris/lib/libdtrace/common"), ] BUILD_EXTENSION = {'build_ext': build_ext} - EXT_MODULES = cythonize([ - Extension("dtrace", ["dtrace_cython/dtrace_h.pxd", - "dtrace_cython/consumer.pyx"], - libraries=["dtrace"], - **extra_args)], - language_level=sys.version_info.major) + EXT_MODULES = cythonize( + [ + Extension("dtrace", ["dtrace_cython/dtrace_h.pxd", + "dtrace_cython/consumer.pyx"], + libraries=["dtrace"], + **extra_args) + ], + language_level=sys.version_info.major + ) except ImportError: BUILD_EXTENSION = {} @@ -45,8 +51,8 @@ setup(name='python-dtrace', version='0.0.11', description='DTrace consumer for Python based on libdtrace. Use Python' - + ' as DTrace Consumer and Provider! See the homepage for' - + ' more information.', + ' as DTrace Consumer and Provider! See the homepage for' + ' more information.', license='MIT', keywords='DTrace', url='http://tmetsch.github.com/python-dtrace/', From 0debbc23158987a4f1859285707195b9523f7e7c Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Tue, 29 Dec 2020 18:40:02 +0100 Subject: [PATCH 08/37] Avoid crashing on macOS Buffered output handling appears to be broken on macOS. The -B flag is disabled in dtrace.c (https://opensource.apple.com/source/dtrace/dtrace-338.40.5/cmd/dtrace/dtrace.c.auto.html) and compiling dtrace.c with that flag enabled results in the same crash that I see with python-dtrace: ``` * thread 1, queue = 'com.apple.main-thread', stop reason = EXC_BAD_ACCESS (code=1, address=0x7fb100000000) * frame 0: 0x00007fff6f83de52 libsystem_platform.dylib`_platform_strlen + 18 frame 1: 0x00007fff6f6d9891 libsystem_c.dylib`__vfprintf + 5379 frame 2: 0x00007fff6f6ffad3 libsystem_c.dylib`__v2printf + 475 frame 3: 0x00007fff6f6e5ee7 libsystem_c.dylib`_vsnprintf + 417 frame 4: 0x00007fff6f6e5f90 libsystem_c.dylib`vsnprintf + 68 frame 5: 0x00007fff6d3650e3 libdtrace.dylib`dt_printf + 524 frame 6: 0x00007fff6d33da7f libdtrace.dylib`dt_consume_cpu + 2536 frame 7: 0x00007fff6d33c9e0 libdtrace.dylib`dtrace_consume + 1090 frame 8: 0x00007fff6d366534 libdtrace.dylib`dtrace_work + 116 frame 9: 0x000000010d6be9ff dtrace2`main(argc=4, argv=0x00007ffee2547250) at dtrace2.c:1834:17 frame 10: 0x00007fff6f647cc9 libdyld.dylib`start + 1 ``` To work around this crash, output to stderr for now. --- dtrace_ctypes/consumer.py | 18 +++++++++++++----- dtrace_cython/consumer.pyx | 15 +++++++++++++-- dtrace_cython/dtrace_h.pxd | 3 ++- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/dtrace_ctypes/consumer.py b/dtrace_ctypes/consumer.py index 4ac7291..9e13d13 100644 --- a/dtrace_ctypes/consumer.py +++ b/dtrace_ctypes/consumer.py @@ -10,7 +10,7 @@ import threading import time from ctypes import (byref, c_char_p, c_int, c_uint, c_void_p, cast, cdll, - CFUNCTYPE, POINTER) + CDLL, CFUNCTYPE, POINTER) from threading import Thread from dtrace_ctypes.dtrace_structs import (dtrace_aggdata, dtrace_bufdata, @@ -186,6 +186,14 @@ def _dtrace_open(): return handle +def _get_dtrace_work_fp(): + if platform.system().startswith("Darwin"): + # Note: macOS crashes if we pass NULL for fp (FreeBSD works fine) + # TODO: use a pipe to get output + return c_void_p.in_dll(CDLL('libc.dylib'), '__stderrp') + return c_void_p(None) + + class DTraceConsumer: """ A Pyton based DTrace consumer. @@ -264,8 +272,8 @@ def run(self, script, runtime=1): i = 0 while i < runtime: LIBRARY.dtrace_sleep(self.handle) - status = LIBRARY.dtrace_work(self.handle, None, self.chew, - self.chew_rec, None) + status = LIBRARY.dtrace_work(self.handle, _get_dtrace_work_fp(), + self.chew, self.chew_rec, None) if status == DTRACE_WORKSTATUS_ERROR: raise Exception('dtrace_work failed: ', get_error_msg(self.handle)) @@ -361,8 +369,8 @@ def run(self): # aggregate data for a few sec... while not self.stopped(): LIBRARY.dtrace_sleep(self.handle) - status = LIBRARY.dtrace_work(self.handle, None, self.chew, - self.chew_rec, None) + status = LIBRARY.dtrace_work(self.handle, _get_dtrace_work_fp(), + self.chew, self.chew_rec, None) if status == DTRACE_WORKSTATUS_ERROR: raise Exception('dtrace_work failed: ', get_error_msg(self.handle)) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index c4a5430..bce9ce2 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -4,6 +4,7 @@ import threading from threading import Thread from dtrace_cython.dtrace_h cimport * from libc.stdint cimport INT64_MAX, INT64_MIN +from libc.stdio cimport stderr # ---------------------------------------------------------------------------- # The DTrace callbacks @@ -316,10 +317,15 @@ cdef class DTraceConsumer: i = 0 args = (self.chew_func, self.chewrec_func) + cdef FILE * fp = NULL + IF UNAME_SYSNAME == "Darwin": + # Note: macOS crashes if we pass NULL for fp (FreeBSD works fine) + # TODO: use a pipe to get output + fp = stderr while i < runtime: dtrace_sleep(self.handle) - status = dtrace_work(self.handle, NULL, & chew, & chewrec, + status = dtrace_work(self.handle, fp, & chew, & chewrec, args) if status == DTRACE_WORKSTATUS_DONE: break @@ -431,7 +437,12 @@ cdef class DTraceContinuousConsumer: Snapshot the data and walk the aggregate. """ args = (self.chew_func, self.chewrec_func) - status = dtrace_work(self.handle, NULL, & chew, & chewrec, + cdef FILE * fp = NULL + IF UNAME_SYSNAME == "Darwin": + # Note: macOS crashes if we pass NULL for fp (FreeBSD works fine) + # TODO: use a pipe to get output + fp = stderr + status = dtrace_work(self.handle, fp, & chew, & chewrec, args) if status == DTRACE_WORKSTATUS_ERROR: raise Exception('dtrace_work failed: ', diff --git a/dtrace_cython/dtrace_h.pxd b/dtrace_cython/dtrace_h.pxd index a841a80..4ee5c48 100644 --- a/dtrace_cython/dtrace_h.pxd +++ b/dtrace_cython/dtrace_h.pxd @@ -1,5 +1,6 @@ from libc.stdint cimport uint16_t, int32_t, uint32_t, int64_t, uint64_t +from libc.stdio cimport FILE cdef extern from "libelf_workaround.h": @@ -118,7 +119,7 @@ cdef extern from "dtrace.h": int dtrace_go(dtrace_hdl_t *) int dtrace_stop(dtrace_hdl_t *) void dtrace_sleep(dtrace_hdl_t *) - dtrace_workstatus_t dtrace_work(dtrace_hdl_t * , char * , dtrace_consume_probe_f * , dtrace_consume_rec_f * , void *) + dtrace_workstatus_t dtrace_work(dtrace_hdl_t * , FILE * , dtrace_consume_probe_f * , dtrace_consume_rec_f * , void *) # walking aggregate int dtrace_aggregate_walk_valsorted(dtrace_hdl_t * , dtrace_aggregate_f * , void *) From a4127b104beba6646c6d53ff79629de5a0537dfd Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Tue, 29 Dec 2020 18:40:35 +0100 Subject: [PATCH 09/37] ctypes: use ctypes.util.find_library() to find libdtrace --- dtrace_ctypes/consumer.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dtrace_ctypes/consumer.py b/dtrace_ctypes/consumer.py index 9e13d13..41e5446 100644 --- a/dtrace_ctypes/consumer.py +++ b/dtrace_ctypes/consumer.py @@ -11,6 +11,7 @@ import time from ctypes import (byref, c_char_p, c_int, c_uint, c_void_p, cast, cdll, CDLL, CFUNCTYPE, POINTER) +from ctypes.util import find_library from threading import Thread from dtrace_ctypes.dtrace_structs import (dtrace_aggdata, dtrace_bufdata, @@ -20,11 +21,7 @@ DTRACE_WORKSTATUS_DONE, DTRACE_WORKSTATUS_ERROR) -if platform.system().startswith("Darwin"): - _LIBNAME = "libdtrace.dylib" -else: - _LIBNAME = "libdtrace.so" -_LIBRARY = cdll.LoadLibrary(_LIBNAME) +_LIBRARY = cdll.LoadLibrary(find_library("dtrace")) # ============================================================================= # chewing and output walkers From af5dde1db7e04a56b4252fa2b81f1da16a5cc4ee Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Tue, 29 Dec 2020 19:46:19 +0100 Subject: [PATCH 10/37] Use open_memstream(3) to work around broken macOS buffered output Passing NULL as the fp argument of dtrace_work results in segfaults on macOS. Work around this by passing a FILE * obtained from open_memstream and reading it after each call to dtrace_work(). This should generally result in the same output (the hello world test now passes on macOS), but there could be some corner cases that are different. --- dtrace_ctypes/consumer.py | 54 ++++++++++++++++++++++++++++++------- dtrace_cython/consumer.pyx | 27 +++++++++++++++---- tests/dtrace_ctypes_test.py | 4 +-- 3 files changed, 68 insertions(+), 17 deletions(-) diff --git a/dtrace_ctypes/consumer.py b/dtrace_ctypes/consumer.py index 41e5446..41bf961 100644 --- a/dtrace_ctypes/consumer.py +++ b/dtrace_ctypes/consumer.py @@ -9,8 +9,8 @@ import platform import threading import time -from ctypes import (byref, c_char_p, c_int, c_uint, c_void_p, cast, cdll, - CDLL, CFUNCTYPE, POINTER) +from ctypes import (byref, c_char_p, c_int, c_size_t, c_uint, c_void_p, cast, cdll, + CDLL, CFUNCTYPE, POINTER, Structure) from ctypes.util import find_library from threading import Thread @@ -22,6 +22,7 @@ DTRACE_WORKSTATUS_ERROR) _LIBRARY = cdll.LoadLibrary(find_library("dtrace")) +_IS_MACOS = platform.system().startswith("Darwin") # ============================================================================= # chewing and output walkers @@ -183,12 +184,31 @@ def _dtrace_open(): return handle +class FILE(Structure): + pass + + +if _IS_MACOS: + # These are needed to work around a broken libdtrace on macOS. + libc_open_memstream = CDLL('libc.dylib').open_memstream + libc_open_memstream.restype = POINTER(FILE) + libc_open_memstream.argtypes = [POINTER(c_void_p), POINTER(c_size_t)] + libc_fclose = CDLL('libc.dylib').fclose + libc_fclose.argtypes = [POINTER(FILE)] + libc_free = CDLL('libc.dylib').free + libc_free.argtypes = [c_void_p] + + def _get_dtrace_work_fp(): - if platform.system().startswith("Darwin"): + if _IS_MACOS: # Note: macOS crashes if we pass NULL for fp (FreeBSD works fine) - # TODO: use a pipe to get output - return c_void_p.in_dll(CDLL('libc.dylib'), '__stderrp') - return c_void_p(None) + # Use open_memstream() as a workaround for this bug. + memstream = c_void_p(None) + size = c_size_t(0) + fp = libc_open_memstream(byref(memstream), byref(size)) + assert cast(fp, c_void_p).value != c_void_p(None).value + return fp, memstream + return None, None class DTraceConsumer: @@ -269,8 +289,15 @@ def run(self, script, runtime=1): i = 0 while i < runtime: LIBRARY.dtrace_sleep(self.handle) - status = LIBRARY.dtrace_work(self.handle, _get_dtrace_work_fp(), - self.chew, self.chew_rec, None) + fp, memstream = _get_dtrace_work_fp() + status = LIBRARY.dtrace_work(self.handle, fp, self.chew, + self.chew_rec, None) + if memstream is not None: + assert memstream.value != 0, memstream + libc_fclose(fp) # buffer is valid after fclose(). + tmp = dtrace_bufdata() + tmp.dtbda_buffered = cast(memstream, c_char_p) + self.buf_out(byref(tmp), None) if status == DTRACE_WORKSTATUS_ERROR: raise Exception('dtrace_work failed: ', get_error_msg(self.handle)) @@ -366,8 +393,15 @@ def run(self): # aggregate data for a few sec... while not self.stopped(): LIBRARY.dtrace_sleep(self.handle) - status = LIBRARY.dtrace_work(self.handle, _get_dtrace_work_fp(), - self.chew, self.chew_rec, None) + fp, memstream = _get_dtrace_work_fp() + status = LIBRARY.dtrace_work(self.handle, fp, self.chew, + self.chew_rec, None) + if memstream is not None: + assert memstream.value != 0, memstream + libc_fclose(fp) # buffer is valid after fclose(). + tmp = dtrace_bufdata() + tmp.dtbda_buffered = cast(memstream, c_char_p) + self.buf_out(byref(tmp), None) if status == DTRACE_WORKSTATUS_ERROR: raise Exception('dtrace_work failed: ', get_error_msg(self.handle)) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index bce9ce2..7ac8b8e 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -2,9 +2,12 @@ from __future__ import print_function, division import time import threading from threading import Thread + from dtrace_cython.dtrace_h cimport * from libc.stdint cimport INT64_MAX, INT64_MIN -from libc.stdio cimport stderr +from libc.stdio cimport fclose +from libc.stdlib cimport free +from posix.stdio cimport open_memstream # ---------------------------------------------------------------------------- # The DTrace callbacks @@ -319,14 +322,21 @@ cdef class DTraceConsumer: args = (self.chew_func, self.chewrec_func) cdef FILE * fp = NULL IF UNAME_SYSNAME == "Darwin": + cdef char * memstream + cdef size_t size # Note: macOS crashes if we pass NULL for fp (FreeBSD works fine) - # TODO: use a pipe to get output - fp = stderr + # As a workaround we use open_memstream to write to memory. + fp = open_memstream(&memstream, &size) + assert fp != NULL while i < runtime: dtrace_sleep(self.handle) status = dtrace_work(self.handle, fp, & chew, & chewrec, args) + IF UNAME_SYSNAME == "Darwin": + fclose(fp) + self.out_func((memstream).strip()) + free(memstream) if status == DTRACE_WORKSTATUS_DONE: break elif status == DTRACE_WORKSTATUS_ERROR: @@ -439,11 +449,18 @@ cdef class DTraceContinuousConsumer: args = (self.chew_func, self.chewrec_func) cdef FILE * fp = NULL IF UNAME_SYSNAME == "Darwin": + cdef char * memstream + cdef size_t size # Note: macOS crashes if we pass NULL for fp (FreeBSD works fine) - # TODO: use a pipe to get output - fp = stderr + # As a workaround we use open_memstream to write to memory. + fp = open_memstream(&memstream, &size) + assert fp != NULL status = dtrace_work(self.handle, fp, & chew, & chewrec, args) + IF UNAME_SYSNAME == "Darwin": + fclose(fp) + self.out_func((memstream).strip()) + free(memstream) if status == DTRACE_WORKSTATUS_ERROR: raise Exception('dtrace_work failed: ', dtrace_errmsg(self.handle, diff --git a/tests/dtrace_ctypes_test.py b/tests/dtrace_ctypes_test.py index c232846..8cf729f 100644 --- a/tests/dtrace_ctypes_test.py +++ b/tests/dtrace_ctypes_test.py @@ -20,7 +20,7 @@ class TestDTraceConsumer(unittest.TestCase): """ def setUp(self): - self.out = '' + self.out = b'' self.consumer = consumer.DTraceConsumer(out_func=self._get_output) def test_run_for_success(self): @@ -38,5 +38,5 @@ def test_run_for_sanity(self): def _get_output(self, data, _arg): tmp = c_char_p(data.contents.dtbda_buffered).value.strip() - self.out = tmp + self.out += tmp return 0 From 045fcc821518781c3fc71727d20adfc5aeba35f9 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Wed, 30 Dec 2020 22:50:06 +0100 Subject: [PATCH 11/37] Reduce amount of duplicate code Factor out the code calling dtrace_sleep()+dtrace_work() into a helper function for the cython and ctypes wrappers. --- dtrace_ctypes/consumer.py | 45 +++++++++++++----------------- dtrace_cython/consumer.pyx | 56 ++++++++++++++++---------------------- 2 files changed, 42 insertions(+), 59 deletions(-) diff --git a/dtrace_ctypes/consumer.py b/dtrace_ctypes/consumer.py index 41bf961..bf0945a 100644 --- a/dtrace_ctypes/consumer.py +++ b/dtrace_ctypes/consumer.py @@ -211,6 +211,23 @@ def _get_dtrace_work_fp(): return None, None +def _dtrace_sleep_and_work(consumer): + LIBRARY.dtrace_sleep(consumer.handle) + fp, memstream = _get_dtrace_work_fp() + status = LIBRARY.dtrace_work(consumer.handle, fp, consumer.chew, + consumer.chew_rec, None) + if fp is not None: + assert memstream.value != 0, memstream + libc_fclose(fp) # buffer is valid after fclose(). + tmp = dtrace_bufdata() + tmp.dtbda_buffered = cast(memstream, c_char_p) + consumer.buf_out(byref(tmp), None) + if status == DTRACE_WORKSTATUS_ERROR: + raise Exception('dtrace_work failed: ', + get_error_msg(consumer.handle)) + return status + + class DTraceConsumer: """ A Pyton based DTrace consumer. @@ -288,19 +305,7 @@ def run(self, script, runtime=1): # aggregate data for a few sec... i = 0 while i < runtime: - LIBRARY.dtrace_sleep(self.handle) - fp, memstream = _get_dtrace_work_fp() - status = LIBRARY.dtrace_work(self.handle, fp, self.chew, - self.chew_rec, None) - if memstream is not None: - assert memstream.value != 0, memstream - libc_fclose(fp) # buffer is valid after fclose(). - tmp = dtrace_bufdata() - tmp.dtbda_buffered = cast(memstream, c_char_p) - self.buf_out(byref(tmp), None) - if status == DTRACE_WORKSTATUS_ERROR: - raise Exception('dtrace_work failed: ', - get_error_msg(self.handle)) + status = _dtrace_sleep_and_work(self) if status == DTRACE_WORKSTATUS_DONE: break # No more work assert status == DTRACE_WORKSTATUS_OKAY, status @@ -392,19 +397,7 @@ def run(self): # aggregate data for a few sec... while not self.stopped(): - LIBRARY.dtrace_sleep(self.handle) - fp, memstream = _get_dtrace_work_fp() - status = LIBRARY.dtrace_work(self.handle, fp, self.chew, - self.chew_rec, None) - if memstream is not None: - assert memstream.value != 0, memstream - libc_fclose(fp) # buffer is valid after fclose(). - tmp = dtrace_bufdata() - tmp.dtbda_buffered = cast(memstream, c_char_p) - self.buf_out(byref(tmp), None) - if status == DTRACE_WORKSTATUS_ERROR: - raise Exception('dtrace_work failed: ', - get_error_msg(self.handle)) + status = _dtrace_sleep_and_work(self) if status == DTRACE_WORKSTATUS_DONE: break # No more work assert status == DTRACE_WORKSTATUS_OKAY, status diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index 7ac8b8e..e66e0cc 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -224,6 +224,25 @@ def simple_walk(action, identifier, keys, value): # The consumers # ---------------------------------------------------------------------------- +cdef int _dtrace_sleep_and_work(dtrace_hdl_t * handle, void * args, out_func): + cdef FILE *fp = NULL + IF UNAME_SYSNAME == "Darwin": + cdef char *memstream + cdef size_t size + # Note: macOS crashes if we pass NULL for fp (FreeBSD works fine) + # As a workaround we use open_memstream to write to memory. + fp = open_memstream(&memstream, &size) + assert fp != NULL + status = dtrace_work(handle, fp, &chew, &chewrec, args) + IF UNAME_SYSNAME == "Darwin": + fclose(fp) + out_func(( memstream).strip()) + free(memstream) + if status == DTRACE_WORKSTATUS_ERROR: + raise Exception('dtrace_work failed: ', + dtrace_errmsg(handle, dtrace_errno(handle))) + return status + cdef class DTraceConsumer: """ @@ -320,23 +339,10 @@ cdef class DTraceConsumer: i = 0 args = (self.chew_func, self.chewrec_func) - cdef FILE * fp = NULL - IF UNAME_SYSNAME == "Darwin": - cdef char * memstream - cdef size_t size - # Note: macOS crashes if we pass NULL for fp (FreeBSD works fine) - # As a workaround we use open_memstream to write to memory. - fp = open_memstream(&memstream, &size) - assert fp != NULL - while i < runtime: dtrace_sleep(self.handle) - status = dtrace_work(self.handle, fp, & chew, & chewrec, - args) - IF UNAME_SYSNAME == "Darwin": - fclose(fp) - self.out_func((memstream).strip()) - free(memstream) + status = _dtrace_sleep_and_work(self.handle, args, + self.out_func) if status == DTRACE_WORKSTATUS_DONE: break elif status == DTRACE_WORKSTATUS_ERROR: @@ -447,24 +453,8 @@ cdef class DTraceContinuousConsumer: Snapshot the data and walk the aggregate. """ args = (self.chew_func, self.chewrec_func) - cdef FILE * fp = NULL - IF UNAME_SYSNAME == "Darwin": - cdef char * memstream - cdef size_t size - # Note: macOS crashes if we pass NULL for fp (FreeBSD works fine) - # As a workaround we use open_memstream to write to memory. - fp = open_memstream(&memstream, &size) - assert fp != NULL - status = dtrace_work(self.handle, fp, & chew, & chewrec, - args) - IF UNAME_SYSNAME == "Darwin": - fclose(fp) - self.out_func((memstream).strip()) - free(memstream) - if status == DTRACE_WORKSTATUS_ERROR: - raise Exception('dtrace_work failed: ', - dtrace_errmsg(self.handle, - dtrace_errno(self.handle))) + status = _dtrace_sleep_and_work(self.handle, args, + self.out_func) if dtrace_aggregate_snap(self.handle) != 0: raise Exception('Failed to get the aggregate: ', dtrace_errmsg(self.handle, From 312844a84f3fab94936f0356b38bd7931048ae1f Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Fri, 1 Jan 2021 18:40:36 +0100 Subject: [PATCH 12/37] Fix variable name conflict with Python3 Thread implementation The Thread class in Python 3.9 (and possible also older versions) already has a _stop member function. This function is called by thread .join(), but we have a variable with the same name so it will raise a "not callable" exception. Fix this by renaming the DTraceConsumerThread variable. --- dtrace_cython/consumer.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index c4a5430..3cbcd7d 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -461,7 +461,7 @@ class DTraceConsumerThread(Thread): Initilizes the Thread. """ Thread.__init__(self) - self._stop = threading.Event() + self._should_stop = threading.Event() self.sleep_time = sleep self.consume = consume self.consumer = DTraceContinuousConsumer(script, chew_func, @@ -498,10 +498,10 @@ class DTraceConsumerThread(Thread): """ Stop DTrace. """ - self._stop.set() + self._should_stop.set() def stopped(self): """ Used to check the status. """ - return self._stop.isSet() + return self._should_stop.isSet() From 085d3bbf42a6ca37686e7f4a46e48c0a6c379f14 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Fri, 1 Jan 2021 18:41:39 +0100 Subject: [PATCH 13/37] cython DTraceConsumerThread: Avoid error in __del__ if __init__ failed If __init__ raised an exception __del__ will still be called and raise an AttributeError when trying to access self.consumer. --- dtrace_cython/consumer.pyx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index c4a5430..7bbc034 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -472,9 +472,12 @@ class DTraceConsumerThread(Thread): """ Make sure DTrace stops. """ - if not self.consume: - self.consumer.snapshot() - del self.consumer + # This is called even if __init__ raises, so we have to check whether + # consumer was initialized. + if hasattr(self, "consumer"): + if not self.consume: + self.consumer.snapshot() + del self.consumer def run(self): """ From 15f561f75a96a038c8ce1861588aab884c534097 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Fri, 1 Jan 2021 18:42:23 +0100 Subject: [PATCH 14/37] Use DTRACE_WORKSTATUS_DONE instead of integer 1 --- dtrace_cython/consumer.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index c4a5430..c47d2e6 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -491,7 +491,7 @@ class DTraceConsumerThread(Thread): if self.consume: status = self.consumer.snapshot() - if status == 1: + if status == DTRACE_WORKSTATUS_DONE: self.stop() def stop(self): From 1eb62defb5e7717dc24880fb22043f467975ec0b Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Tue, 29 Dec 2020 18:07:17 +0100 Subject: [PATCH 15/37] Add support for building with AddressSanitizer I used this to debug crashes on macOS (which is caused by broken buffered output handling in the macOS libdtrace version, see #15). --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index b11e422..0978356 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,9 @@ os.path.join(src_dir, "cddl/contrib/opensolaris/lib/libdtrace/common"), ] + if os.getenv("ENABLE_ASAN", None) is not None: + extra_args["extra_compile_args"] = ["-fsanitize=address"] + extra_args["extra_link_args"] = ["-fsanitize=address"] BUILD_EXTENSION = {'build_ext': build_ext} EXT_MODULES = cythonize( [ From daf7a547aef54b82205f2878a9e36df8a7e461f3 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Tue, 29 Dec 2020 18:43:08 +0100 Subject: [PATCH 16/37] Add a requirements.txt with Cython This avoids warnings when editing the project in PyCharm. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..25e9025 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +Cython~=3.0a6 From 009b862c2b1d561f3a12cbb7b945733672226e63 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Tue, 29 Dec 2020 19:46:44 +0100 Subject: [PATCH 17/37] Install no-op callbacks when explicitly passing None This commit retains the previous behaviour of installing simple_* callbacks if no explicit override is passed, and allows passing None to install no-op callbacks. I generally don't want the CPU: N prints so end up having to use the API as follows: ```python def simple_walk(action, identifier, keys, value): """ action -- type of action (sum, avg, ...) identifier -- the id. keys -- list of keys. value -- the value. """ values[keys[0]] += value dtrace_thread = DTraceConsumerThread(syscall_script, walk_func=simple_walk, out_func=lambda v: None, chew_func=lambda v: None, chewrec_func=lambda v: None, sleep=1) ``` Allowing None instead of `lambda v: None` would make this a bit simpler. --- dtrace_ctypes/consumer.py | 47 +++++++++++++++++++++++++------------- dtrace_cython/consumer.pyx | 46 +++++++++++++++++++++++++------------ 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/dtrace_ctypes/consumer.py b/dtrace_ctypes/consumer.py index 4ac7291..6fc6d61 100644 --- a/dtrace_ctypes/consumer.py +++ b/dtrace_ctypes/consumer.py @@ -54,6 +54,10 @@ def simple_chew_func(data, _arg): return 0 +def noop_chew_func(_data, _arg): + return 0 + + def simple_chewrec_func(_data, rec, _arg): """ Callback for record chewing. @@ -63,6 +67,9 @@ def simple_chewrec_func(_data, rec, _arg): return 0 +noop_chewrec_func = simple_chewrec_func + + def simple_buffered_out_writer(bufdata, _arg): """ In case dtrace_work is given None as filename - this one is called. @@ -72,6 +79,10 @@ def simple_buffered_out_writer(bufdata, _arg): return 0 +def noop_buffered_out_writer(bufdata, _arg): + return 0 + + def simple_walk(data, _arg): """ Aggregate walker capable of reading a name and one value. @@ -86,6 +97,10 @@ def simple_walk(data, _arg): return 0 + +def noop_walk(_data, _arg): + return 0 + # ============================================================================= # LibDTrace function wrapper class # ============================================================================= @@ -191,32 +206,32 @@ class DTraceConsumer: A Pyton based DTrace consumer. """ def __init__(self, - chew_func=None, - chew_rec_func=None, - walk_func=None, - out_func=None): + chew_func=simple_chew_func, + chew_rec_func=simple_chewrec_func, + walk_func=simple_walk, + out_func=simple_buffered_out_writer): """ Constructor. will get the DTrace handle """ if chew_func is not None: self.chew = CHEW_FUNC(chew_func) else: - self.chew = CHEW_FUNC(simple_chew_func) + self.chew = CHEW_FUNC(noop_chew_func) if chew_rec_func is not None: self.chew_rec = CHEWREC_FUNC(chew_rec_func) else: - self.chew_rec = CHEWREC_FUNC(simple_chewrec_func) + self.chew_rec = CHEWREC_FUNC(noop_chewrec_func) if walk_func is not None: self.walk = WALK_FUNC(walk_func) else: - self.walk = WALK_FUNC(simple_walk) + self.walk = WALK_FUNC(noop_walk) if out_func is not None: self.buf_out = BUFFERED_FUNC(out_func) else: - self.buf_out = BUFFERED_FUNC(simple_buffered_out_writer) + self.buf_out = BUFFERED_FUNC(noop_buffered_out_writer) # get dtrace handle self.handle = _dtrace_open() @@ -293,10 +308,10 @@ class DTraceConsumerThread(Thread): def __init__(self, script, - chew_func=None, - chew_rec_func=None, - walk_func=None, - out_func=None): + chew_func=simple_chew_func, + chew_rec_func=simple_chewrec_func, + walk_func=simple_walk, + out_func=simple_buffered_out_writer): """ Constructor. will get the DTrace handle """ @@ -307,22 +322,22 @@ def __init__(self, if chew_func is not None: self.chew = CHEW_FUNC(chew_func) else: - self.chew = CHEW_FUNC(simple_chew_func) + self.chew = CHEW_FUNC(noop_chew_func) if chew_rec_func is not None: self.chew_rec = CHEWREC_FUNC(chew_rec_func) else: - self.chew_rec = CHEWREC_FUNC(simple_chewrec_func) + self.chew_rec = CHEWREC_FUNC(noop_chewrec_func) if walk_func is not None: self.walk = WALK_FUNC(walk_func) else: - self.walk = WALK_FUNC(simple_walk) + self.walk = WALK_FUNC(noop_walk) if out_func is not None: self.buf_out = BUFFERED_FUNC(out_func) else: - self.buf_out = BUFFERED_FUNC(simple_buffered_out_writer) + self.buf_out = BUFFERED_FUNC(noop_buffered_out_writer) # get dtrace handle self.handle = _dtrace_open() diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index c4a5430..afc692d 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -186,6 +186,10 @@ def simple_chew(cpu): print('Running on CPU:', cpu) +def noop_chew(_cpu): + pass + + def simple_chewrec(action): """ Simple chewrec callback. @@ -195,6 +199,10 @@ def simple_chewrec(action): print('Called action was:', action) +def noop_chewrec(_action): + pass + + def simple_out(value): """ A buffered output handler for all those prints. @@ -204,6 +212,10 @@ def simple_out(value): print('Value is:', value) +def noop_out(_value): + pass + + def simple_walk(action, identifier, keys, value): """ Simple aggregation walker. @@ -216,6 +228,10 @@ def simple_walk(action, identifier, keys, value): """ print(action, identifier, keys, value) + +def noop_walk(_action, _identifier, _keys, _value): + pass + # ---------------------------------------------------------------------------- # The consumers # ---------------------------------------------------------------------------- @@ -232,15 +248,16 @@ cdef class DTraceConsumer: cdef object chew_func cdef object chewrec_func - def __init__(self, chew_func=None, chewrec_func=None, out_func=None, - walk_func=None): + def __init__(self, chew_func=simple_chew, chewrec_func=simple_chewrec, + out_func=simple_out, walk_func=simple_walk): """ Constructor. Gets a DTrace handle and sets some options. + Passing None as one of the callback arguments installs a no-op callback. """ - self.chew_func = chew_func or simple_chew - self.chewrec_func = chewrec_func or simple_chewrec - self.out_func = out_func or simple_out - self.walk_func = walk_func or simple_walk + self.chew_func = noop_chew if chew_func is None else chew_func + self.chewrec_func = noop_chewrec if chewrec_func is None else simple_chewrec + self.out_func = noop_out if out_func is None else out_func + self.walk_func = noop_walk if walk_func is None else walk_func cdef int err if not hasattr(self, 'handle'): @@ -355,15 +372,15 @@ cdef class DTraceContinuousConsumer: cdef object chewrec_func cdef object script - def __init__(self, str script, chew_func=None, chewrec_func=None, - out_func=None, walk_func=None): + def __init__(self, str script, chew_func=simple_chew, chewrec_func=simple_chewrec, + out_func=simple_out, walk_func=simple_walk): """ Constructor. will get the DTrace handle """ - self.chew_func = chew_func or simple_chew - self.chewrec_func = chewrec_func or simple_chewrec - self.out_func = out_func or simple_out - self.walk_func = walk_func or simple_walk + self.chew_func = noop_chew if chew_func is None else chew_func + self.chewrec_func = noop_chewrec if chewrec_func is None else simple_chewrec + self.out_func = noop_out if out_func is None else out_func + self.walk_func = noop_walk if walk_func is None else walk_func self.script = script.encode("utf-8") cdef int err @@ -455,8 +472,9 @@ class DTraceConsumerThread(Thread): Helper Thread which can be used to continuously aggregate. """ - def __init__(self, script, consume=True, chew_func=None, chewrec_func=None, - out_func=None, walk_func=None, sleep=0): + def __init__(self, script, consume=True, chew_func=simple_chew, + chewrec_func=simple_chewrec, out_func=simple_out, + walk_func=simple_walk, sleep=0): """ Initilizes the Thread. """ From db15aef25be870e90d393fd062283211936cea64 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Fri, 1 Jan 2021 20:41:46 +0100 Subject: [PATCH 18/37] renamed to constraints.txt, as Cython is not necesarrily a hard reuquirement atm. Hope that works :-) --- requirements.txt => constraints.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename requirements.txt => constraints.txt (100%) diff --git a/requirements.txt b/constraints.txt similarity index 100% rename from requirements.txt rename to constraints.txt From fd4cd0838209131a295aab64a07e2e41097d3f29 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Fri, 1 Jan 2021 20:54:30 +0100 Subject: [PATCH 19/37] code cleanup. Got contributed name wrong in the past - no clue what went wrong there - Excuses! --- ACKNOWLEDGEMENTS | 4 ++-- dtrace_ctypes/consumer.py | 33 ++++++++++++++++++++++----------- dtrace_cython/consumer.pyx | 28 ++++++++++++++++++++++------ 3 files changed, 46 insertions(+), 19 deletions(-) diff --git a/ACKNOWLEDGEMENTS b/ACKNOWLEDGEMENTS index 74808c2..fca1df1 100644 --- a/ACKNOWLEDGEMENTS +++ b/ACKNOWLEDGEMENTS @@ -5,8 +5,8 @@ Acknowledgements Thanks to Marc Abramowitz (https://github.com/msabramo) for testing and porting this to Mac OS X. -Thanks to James Clarke (https://github.com/jrtc27) for looking into leaking of -the DTrace handles. +Thanks to Jessica Clarke (https://github.com/jrtc27) for looking into leaking +of the DTrace handles. Thanks to Alexander Richardson (https://github.com/arichardson) for doing a huge refactoring and looking into properly supporting python3. diff --git a/dtrace_ctypes/consumer.py b/dtrace_ctypes/consumer.py index 0a4383c..d879436 100644 --- a/dtrace_ctypes/consumer.py +++ b/dtrace_ctypes/consumer.py @@ -9,8 +9,8 @@ import platform import threading import time -from ctypes import (byref, c_char_p, c_int, c_size_t, c_uint, c_void_p, cast, cdll, - CDLL, CFUNCTYPE, POINTER, Structure) +from ctypes import (byref, c_char_p, c_int, c_size_t, c_uint, c_void_p, cast, + cdll, CDLL, CFUNCTYPE, POINTER, Structure) from ctypes.util import find_library from threading import Thread @@ -53,6 +53,9 @@ def simple_chew_func(data, _arg): def noop_chew_func(_data, _arg): + """ + No-op chew function. + """ return 0 @@ -77,7 +80,10 @@ def simple_buffered_out_writer(bufdata, _arg): return 0 -def noop_buffered_out_writer(bufdata, _arg): +def noop_buffered_out_writer(_bufdata, _arg): + """ + No-op buffered out writer. + """ return 0 @@ -97,6 +103,9 @@ def simple_walk(data, _arg): def noop_walk(_data, _arg): + """ + No-op walker. + """ return 0 # ============================================================================= @@ -200,7 +209,9 @@ def _dtrace_open(): class FILE(Structure): - pass + """ + Basic dummy structure. + """ if _IS_MACOS: @@ -220,20 +231,20 @@ def _get_dtrace_work_fp(): # Use open_memstream() as a workaround for this bug. memstream = c_void_p(None) size = c_size_t(0) - fp = libc_open_memstream(byref(memstream), byref(size)) - assert cast(fp, c_void_p).value != c_void_p(None).value - return fp, memstream + f_p = libc_open_memstream(byref(memstream), byref(size)) + assert cast(f_p, c_void_p).value != c_void_p(None).value + return f_p, memstream return None, None def _dtrace_sleep_and_work(consumer): LIBRARY.dtrace_sleep(consumer.handle) - fp, memstream = _get_dtrace_work_fp() - status = LIBRARY.dtrace_work(consumer.handle, fp, consumer.chew, + f_p, memstream = _get_dtrace_work_fp() + status = LIBRARY.dtrace_work(consumer.handle, f_p, consumer.chew, consumer.chew_rec, None) - if fp is not None: + if f_p is not None: assert memstream.value != 0, memstream - libc_fclose(fp) # buffer is valid after fclose(). + libc_fclose(f_p) # buffer is valid after fclose(). tmp = dtrace_bufdata() tmp.dtbda_buffered = cast(memstream, c_char_p) consumer.buf_out(byref(tmp), None) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index 98c2dcb..f9a0cb7 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -191,6 +191,9 @@ def simple_chew(cpu): def noop_chew(_cpu): + """ + No-op chew function. + """ pass @@ -204,6 +207,9 @@ def simple_chewrec(action): def noop_chewrec(_action): + """ + No-op chewrec function. + """ pass @@ -217,6 +223,9 @@ def simple_out(value): def noop_out(_value): + """ + No-op out function. + """ pass @@ -234,6 +243,9 @@ def simple_walk(action, identifier, keys, value): def noop_walk(_action, _identifier, _keys, _value): + """ + No-op walker. + """ pass # ---------------------------------------------------------------------------- @@ -262,7 +274,7 @@ cdef int _dtrace_sleep_and_work(dtrace_hdl_t * handle, void * args, out_func): cdef class DTraceConsumer: """ - A Pyton based DTrace consumer. + A Python based DTrace consumer. """ cdef dtrace_hdl_t * handle @@ -275,10 +287,12 @@ cdef class DTraceConsumer: out_func=simple_out, walk_func=simple_walk): """ Constructor. Gets a DTrace handle and sets some options. - Passing None as one of the callback arguments installs a no-op callback. + Passing None as one of the callback arguments installs a no-op + callback. """ self.chew_func = noop_chew if chew_func is None else chew_func - self.chewrec_func = noop_chewrec if chewrec_func is None else simple_chewrec + self.chewrec_func = noop_chewrec if chewrec_func is None \ + else simple_chewrec self.out_func = noop_out if out_func is None else out_func self.walk_func = noop_walk if walk_func is None else walk_func @@ -394,13 +408,15 @@ cdef class DTraceContinuousConsumer: cdef object chewrec_func cdef object script - def __init__(self, str script, chew_func=simple_chew, chewrec_func=simple_chewrec, - out_func=simple_out, walk_func=simple_walk): + def __init__(self, str script, chew_func=simple_chew, + chewrec_func=simple_chewrec, out_func=simple_out, + walk_func=simple_walk): """ Constructor. will get the DTrace handle """ self.chew_func = noop_chew if chew_func is None else chew_func - self.chewrec_func = noop_chewrec if chewrec_func is None else simple_chewrec + self.chewrec_func = noop_chewrec if chewrec_func is None \ + else simple_chewrec self.out_func = noop_out if out_func is None else out_func self.walk_func = noop_walk if walk_func is None else walk_func self.script = script.encode("utf-8") From 8afbbfa1a5c4849449c14e7c94b2ee00c6a8be21 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Fri, 1 Jan 2021 21:05:04 +0100 Subject: [PATCH 20/37] minor cleanups. --- README.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index c532121..47c5da4 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,14 @@ Python as a DTrace consumer This is a first shot into looking at getting something up and running like https://github.com/bcantrill/node-libdtrace for Python. -Note: This is far from ready - documentation on writing own DTrace consumers +**_Note_**: This is far from ready - documentation on writing own DTrace consumers is very rare :-/ The codes in the **examples** folder should give an overview of how to use Python as a DTrace consumer. -Currently this package provides two modules: one wraps libdtrace using ctypes. -The other one uses cython. Should you not have cython installed the ctypes +Currently, this package provides two modules: one wraps libdtrace using ctypes; +The other one uses cython. Should you not have cython installed, the ctypes wrapper can still be used. The Python DTrace consumer can be installed via source here on GitHub, or using @@ -23,8 +23,11 @@ Cython based wrapper The Cython wrapper is more sophisticated and generally easier to use. Initializing the Python based DTrace consumer is as simple as: - from dtrace import DTraceConsumer - consumer = DTraceConsumer(walk_func=my_walk [...]) + import dtrace + + SCRIPT = 'dtrace:::BEGIN {trace("Hello World");}' + + consumer = dtrace.DTraceConsumer(walk_func=my_walk [...]) consumer.run(SCRIPT, runtime=3) The simple DTraceConsumer can be initialized with self written callbacks which @@ -54,22 +57,22 @@ The DTraceConsumer has a *run_script* function which will run a provided DTrace script for some time or till it is finished. The time on how long it is run can be provided as parameter just like the script. During the DTrace chew, chewrec and the out callbacks are called. When the run is finished the aggregation will -be walked - Thus you can aggregate for 3 seconds and than see the results. +be walked - Thus you can aggregate for 3 seconds and then see the results. There also exists a DTraceConsumerThread which can be used to continuously run DTrace in the background. This might come in handy when you want to continuously aggregate data for e.g. an GUI. The chew, chewrec, out and walk callbacks are now called as the snapshot function of DTrace is called. -The DTraceConsumerThread has an parameter sleep which defaults to 0. This means +The DTraceConsumerThread has a parameter sleep which defaults to 0. This means that the Thread will wait for DTrace for new aggregation data to arrive. This -has a major drawback since during the wait the wait the Python GIL is acquired -and your program will stop if it needs to wait for DTrace to get new data. -Setting the parameter to 1 (or higher) will let the Thread snapshot the DTrace -aggregate every second instead of waiting for new data. Both usages might make -sense - Set the sleep parameter if you know data will arrive sporadically or -simple let it default to 0 if you know data comes in all the time - so nothing -will be blocked. +has a major drawback since during the wait the Python GIL is acquired and your +program will stop if it needs to wait for DTrace to get new data. Setting the +parameter to 1 (or higher) will let the Thread snapshot the DTrace aggregate +every second instead of waiting for new data. Both usages might make sense - +Set the sleep parameter if you know data will arrive sporadically or simple +let it default to 0 if you know data comes in all the time - so nothing will +be blocked. thr = DTraceConsumerThread(SCRIPT) thr.start() From 30cc14da4b1ec013ee4377f6c6ae0565d40bb2f6 Mon Sep 17 00:00:00 2001 From: Mateusz Piotrowski <0mp@FreeBSD.org> Date: Thu, 7 Jan 2021 17:35:31 +0100 Subject: [PATCH 21/37] Fix some typos --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 47c5da4..80d0ff7 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ has a major drawback since during the wait the Python GIL is acquired and your program will stop if it needs to wait for DTrace to get new data. Setting the parameter to 1 (or higher) will let the Thread snapshot the DTrace aggregate every second instead of waiting for new data. Both usages might make sense - -Set the sleep parameter if you know data will arrive sporadically or simple +set the sleep parameter if you know data will arrive sporadically or simply let it default to 0 if you know data comes in all the time - so nothing will be blocked. From 115cc00d9e7464262749d2786927982b9163aff9 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Mon, 8 Feb 2021 00:12:39 +0100 Subject: [PATCH 22/37] added initial support for stack(). --- dtrace_cython/consumer.pyx | 13 +++++++++++++ dtrace_cython/dtrace_h.pxd | 12 ++++++++++++ 2 files changed, 25 insertions(+) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index f9a0cb7..3b01e3d 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -9,6 +9,8 @@ from libc.stdio cimport fclose from libc.stdlib cimport free from posix.stdio cimport open_memstream +from cython.operator cimport dereference + # ---------------------------------------------------------------------------- # The DTrace callbacks # ---------------------------------------------------------------------------- @@ -77,6 +79,9 @@ cdef int walk(const dtrace_aggdata_t * data, void * arg) with gil: cdef dtrace_recdesc_t *rec cdef int64_t *tmp + cdef dtrace_syminfo_t foo + cdef GElf_Sym sym + aggrec = &desc.dtagd_rec[desc.dtagd_nrecs - 1] action = aggrec.dtrd_action @@ -87,6 +92,14 @@ cdef int walk(const dtrace_aggdata_t * data, void * arg) with gil: # TODO: need to extend this. if rec.dtrd_size == sizeof(uint32_t): keys.append((address)[0]) + if rec.dtrd_size == 160: + for j in range(rec.dtrd_arg): + # pc = *((uint64_t *)addr); + pc = dereference(address) + ret = dtrace_lookup_by_addr(data.dtada_handle, pc, &sym, &foo) + if ret == 0: + keys.append((foo.dts_object, foo.dts_name)) + address += 8 else: keys.append(address) diff --git a/dtrace_cython/dtrace_h.pxd b/dtrace_cython/dtrace_h.pxd index 4ee5c48..72460ce 100644 --- a/dtrace_cython/dtrace_h.pxd +++ b/dtrace_cython/dtrace_h.pxd @@ -2,6 +2,10 @@ from libc.stdint cimport uint16_t, int32_t, uint32_t, int64_t, uint64_t from libc.stdio cimport FILE +cdef extern from "gelf.h": + ctypedef struct GElf_Sym: + pass + cdef extern from "libelf_workaround.h": # lib elf workaround :-/ @@ -27,6 +31,7 @@ cdef extern from "sys/dtrace.h": uint16_t dtrd_action uint32_t dtrd_offset uint32_t dtrd_size + uint64_t dtrd_arg; ctypedef struct dtrace_aggdesc_t: # Taken from sys/dtrace.h:950 @@ -89,9 +94,15 @@ cdef extern from "dtrace.h": ctypedef struct dtrace_aggdata_t: # Taken from dtrace.h:351 + dtrace_hdl_t * dtada_handle dtrace_aggdesc_t * dtada_desc char * dtada_data + ctypedef struct dtrace_syminfo_t: + const char * dts_object + const char * dts_name + long bla + # from dtrace.h ctypedef int dtrace_handle_buffered_f(const dtrace_bufdata_t * buf_data, void * arg) ctypedef int dtrace_consume_probe_f(const dtrace_probedata_t *, void *) @@ -125,6 +136,7 @@ cdef extern from "dtrace.h": int dtrace_aggregate_walk_valsorted(dtrace_hdl_t * , dtrace_aggregate_f * , void *) int dtrace_aggregate_snap(dtrace_hdl_t *) int dtrace_aggregate_walk(dtrace_hdl_t *, dtrace_aggregate_f *, void *) + int dtrace_lookup_by_addr(dtrace_hdl_t *, uint64_t addr, GElf_Sym *, dtrace_syminfo_t *) # error handling... int dtrace_errno(dtrace_hdl_t * handle) From 59d239749d555868d551b1a476127c02bd242f91 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Tue, 9 Feb 2021 09:42:24 +0000 Subject: [PATCH 23/37] Stop requiring the FreeBSD source code for FreeBSD13+ Since https://github.com/freebsd/freebsd-src/commit/1f6612b444e3fc54697a4ef3167efdddff6944ea the dtrace.h header is installed to /usr/include so we no longer need to have the full source code installed just to find the headers that we need to build the cython extension. --- setup.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index 0978356..2b817d1 100644 --- a/setup.py +++ b/setup.py @@ -17,18 +17,20 @@ from Cython.Build import build_ext, cythonize extra_args = {} if platform.system().lower().startswith("freebsd"): - # On FreeBSD the dtrace headers are not installed by default. - src_dir = os.getenv("FREEBSD_SRC_DIR", "/usr/src") - if not os.path.exists(os.path.join(src_dir, "sys/cddl")): - raise ImportError("Cannot find FreeBSD DTrace headers") - extra_args["include_dirs"] = [ - os.path.join(src_dir, - "sys/cddl/compat/opensolaris"), - os.path.join(src_dir, - "sys/cddl/contrib/opensolaris/uts/common"), - os.path.join(src_dir, - "cddl/contrib/opensolaris/lib/libdtrace/common"), - ] + # On older FreeBSD versions the dtrace headers are not + # installed by default, so we need to find the full sources. + if not os.path.exists("/usr/include/dtrace.h"): + src_dir = os.getenv("FREEBSD_SRC_DIR", "/usr/src") + if not os.path.exists(os.path.join(src_dir, "sys/cddl")): + raise ImportError("Cannot find FreeBSD DTrace headers") + extra_args["include_dirs"] = [ + os.path.join(src_dir, + "sys/cddl/compat/opensolaris"), + os.path.join(src_dir, + "sys/cddl/contrib/opensolaris/uts/common"), + os.path.join(src_dir, + "cddl/contrib/opensolaris/lib/libdtrace/common"), + ] if os.getenv("ENABLE_ASAN", None) is not None: extra_args["extra_compile_args"] = ["-fsanitize=address"] extra_args["extra_link_args"] = ["-fsanitize=address"] From 4945c033f2db75e0d5736c200ae0bdb23303f1e3 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Sat, 20 Feb 2021 16:51:24 +0100 Subject: [PATCH 24/37] minor code cleanups & added example for stack(). --- dtrace_cython/consumer.pyx | 8 ++++---- dtrace_cython/dtrace_h.pxd | 8 ++++++-- examples/stack_example.py | 23 +++++++++++++++++++++++ 3 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 examples/stack_example.py diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index 3b01e3d..fe10476 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -92,14 +92,14 @@ cdef int walk(const dtrace_aggdata_t * data, void * arg) with gil: # TODO: need to extend this. if rec.dtrd_size == sizeof(uint32_t): keys.append((address)[0]) - if rec.dtrd_size == 160: + if rec.dtrd_size == 20 * sizeof(uint64_t): + # case stack() has been used --> need to lookup symbols. for j in range(rec.dtrd_arg): - # pc = *((uint64_t *)addr); pc = dereference(address) ret = dtrace_lookup_by_addr(data.dtada_handle, pc, &sym, &foo) if ret == 0: - keys.append((foo.dts_object, foo.dts_name)) - address += 8 + keys.append((foo.dts_object,foo.dts_name)) + address += sizeof(uint64_t) else: keys.append(address) diff --git a/dtrace_cython/dtrace_h.pxd b/dtrace_cython/dtrace_h.pxd index 72460ce..df4c6a2 100644 --- a/dtrace_cython/dtrace_h.pxd +++ b/dtrace_cython/dtrace_h.pxd @@ -3,6 +3,8 @@ from libc.stdint cimport uint16_t, int32_t, uint32_t, int64_t, uint64_t from libc.stdio cimport FILE cdef extern from "gelf.h": + # used for symbol lookup while using stack() + ctypedef struct GElf_Sym: pass @@ -22,7 +24,7 @@ cdef extern from "sys/dtrace.h": int DTRACEAGG_MAX int DTRACEAGG_AVG int DTRACEAGG_SUM - int DTRACEAGG_STDDEV # (unsupported) + int DTRACEAGG_STDDEV # (unsupported) int DTRACEAGG_QUANTIZE int DTRACEAGG_LQUANTIZE @@ -31,7 +33,7 @@ cdef extern from "sys/dtrace.h": uint16_t dtrd_action uint32_t dtrd_offset uint32_t dtrd_size - uint64_t dtrd_arg; + uint64_t dtrd_arg ctypedef struct dtrace_aggdesc_t: # Taken from sys/dtrace.h:950 @@ -136,6 +138,8 @@ cdef extern from "dtrace.h": int dtrace_aggregate_walk_valsorted(dtrace_hdl_t * , dtrace_aggregate_f * , void *) int dtrace_aggregate_snap(dtrace_hdl_t *) int dtrace_aggregate_walk(dtrace_hdl_t *, dtrace_aggregate_f *, void *) + + # Dealing with stack() int dtrace_lookup_by_addr(dtrace_hdl_t *, uint64_t addr, GElf_Sym *, dtrace_syminfo_t *) # error handling... diff --git a/examples/stack_example.py b/examples/stack_example.py new file mode 100644 index 0000000..2d4cd9a --- /dev/null +++ b/examples/stack_example.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +""" +Use the Python DTrace consumer and use stack(). +""" + +import dtrace + +SCRIPT = "profile-99 /arg0/ { @foo[stack()] = count();}" + + +def main(): + """ + Run DTrace... + """ + print('Hint: if you don\'t get any output try running it with pfexec...') + + consumer = dtrace.DTraceConsumer() + consumer.run(SCRIPT, 4) + + +if __name__ == '__main__': + main() From 54036c0f5162ba67d60077d29f2b602d1be1c4a0 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Sat, 20 Feb 2021 16:55:24 +0100 Subject: [PATCH 25/37] new release 0.0.12 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2b817d1..08fae54 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup(name='python-dtrace', - version='0.0.11', + version='0.0.12', description='DTrace consumer for Python based on libdtrace. Use Python' ' as DTrace Consumer and Provider! See the homepage for' ' more information.', From 41096b129d04ded2e29053d017173abfb2201b18 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Sun, 21 Feb 2021 12:32:54 +0100 Subject: [PATCH 26/37] let's properly name the var decl. --- dtrace_cython/dtrace_h.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dtrace_cython/dtrace_h.pxd b/dtrace_cython/dtrace_h.pxd index df4c6a2..cad75e1 100644 --- a/dtrace_cython/dtrace_h.pxd +++ b/dtrace_cython/dtrace_h.pxd @@ -103,7 +103,7 @@ cdef extern from "dtrace.h": ctypedef struct dtrace_syminfo_t: const char * dts_object const char * dts_name - long bla + long dts_id # from dtrace.h ctypedef int dtrace_handle_buffered_f(const dtrace_bufdata_t * buf_data, void * arg) From fed64428b6cfb987e74935699b2b5b60f414a9ee Mon Sep 17 00:00:00 2001 From: tmetsch Date: Sun, 21 Feb 2021 12:34:15 +0100 Subject: [PATCH 27/37] let's use the proper names :-) --- dtrace_cython/dtrace_h.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dtrace_cython/dtrace_h.pxd b/dtrace_cython/dtrace_h.pxd index df4c6a2..cad75e1 100644 --- a/dtrace_cython/dtrace_h.pxd +++ b/dtrace_cython/dtrace_h.pxd @@ -103,7 +103,7 @@ cdef extern from "dtrace.h": ctypedef struct dtrace_syminfo_t: const char * dts_object const char * dts_name - long bla + long dts_id # from dtrace.h ctypedef int dtrace_handle_buffered_f(const dtrace_bufdata_t * buf_data, void * arg) From 3608b6e08a4ea85ee240fc3414b9453c58699558 Mon Sep 17 00:00:00 2001 From: tmetsch Date: Wed, 24 Feb 2021 10:08:39 +0100 Subject: [PATCH 28/37] github will stop redirecting sometime soon. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 08fae54..5cfdb70 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ ' more information.', license='MIT', keywords='DTrace', - url='http://tmetsch.github.com/python-dtrace/', + url='http://tmetsch.github.io/python-dtrace/', packages=['dtrace_ctypes'], cmdclass=BUILD_EXTENSION, ext_modules=EXT_MODULES, From f06de369e1c679abb34f68265f27573556ca531a Mon Sep 17 00:00:00 2001 From: tmetsch Date: Sun, 28 Feb 2021 18:25:47 +0100 Subject: [PATCH 29/37] Update consumer.pyx switch to 64 bit uint. --- dtrace_cython/consumer.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index fe10476..a11ab7e 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -90,7 +90,7 @@ cdef int walk(const dtrace_aggdata_t * data, void * arg) with gil: address = data.dtada_data + rec.dtrd_offset # TODO: need to extend this. - if rec.dtrd_size == sizeof(uint32_t): + if rec.dtrd_size == sizeof(uint64_t): keys.append((address)[0]) if rec.dtrd_size == 20 * sizeof(uint64_t): # case stack() has been used --> need to lookup symbols. From 2e2222888ca94bd5e6fb9809bea96d1ccfbf8e22 Mon Sep 17 00:00:00 2001 From: tmetsch Date: Sun, 28 Feb 2021 18:26:57 +0100 Subject: [PATCH 30/37] Update consumer.pyx --- dtrace_cython/consumer.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index a11ab7e..9751b20 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -91,7 +91,7 @@ cdef int walk(const dtrace_aggdata_t * data, void * arg) with gil: # TODO: need to extend this. if rec.dtrd_size == sizeof(uint64_t): - keys.append((address)[0]) + keys.append((address)[0]) if rec.dtrd_size == 20 * sizeof(uint64_t): # case stack() has been used --> need to lookup symbols. for j in range(rec.dtrd_arg): From 93afc72a47054f442ea9c5dcc2f507b1076fcd77 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Sun, 28 Feb 2021 19:28:38 +0100 Subject: [PATCH 31/37] handling 32 and 64 bit ints. --- dtrace_cython/consumer.pyx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index 9751b20..8eb9658 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -90,9 +90,11 @@ cdef int walk(const dtrace_aggdata_t * data, void * arg) with gil: address = data.dtada_data + rec.dtrd_offset # TODO: need to extend this. - if rec.dtrd_size == sizeof(uint64_t): - keys.append((address)[0]) - if rec.dtrd_size == 20 * sizeof(uint64_t): + if rec.dtrd_size == sizeof(uint32_t): + keys.append((address)[0]) + elif rec.dtrd_size == sizeof(uint64_t): + keys.append(( address)[0]) + elif rec.dtrd_size == 20 * sizeof(uint64_t): # case stack() has been used --> need to lookup symbols. for j in range(rec.dtrd_arg): pc = dereference(address) From a2c394a84ae812d5c06c07d72ab950d115a78e34 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Sat, 6 Mar 2021 18:12:09 +0100 Subject: [PATCH 32/37] release fixing #25 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5cfdb70..d9d2159 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup(name='python-dtrace', - version='0.0.12', + version='0.0.13', description='DTrace consumer for Python based on libdtrace. Use Python' ' as DTrace Consumer and Provider! See the homepage for' ' more information.', From 0a889ceed8cbcc96df35d11760f62f9fb644752a Mon Sep 17 00:00:00 2001 From: Konrad Witaszczyk Date: Wed, 9 Feb 2022 13:13:58 +0000 Subject: [PATCH 33/37] Correctly free DTrace resources. Cython's cdef attributes are not regular class attributes as in Python. hasattr(object, name) always returns False for a cdef attribute corresponding to name. Instead, we should simply check a pointer's value. I couldn't confirm it in any documentation but, based on examples I found and my experiments, it seems that a cdef pointer is always initially set to NULL, at least when declared as a class attribute. --- dtrace_cython/consumer.pyx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index 8eb9658..c5a6e42 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -312,7 +312,7 @@ cdef class DTraceConsumer: self.walk_func = noop_walk if walk_func is None else walk_func cdef int err - if not hasattr(self, 'handle'): + if self.handle == NULL: # ensure we only grab 1 - cython might call init twice, of more. self.handle = dtrace_open(3, 0, &err) if self.handle == NULL: @@ -331,8 +331,9 @@ cdef class DTraceConsumer: """ Release DTrace handle. """ - if hasattr(self, 'handle') and self.handle != NULL: + if self.handle != NULL: dtrace_close(self.handle) + self.handle = NULL cpdef compile(self, str script): """ @@ -437,7 +438,7 @@ cdef class DTraceContinuousConsumer: self.script = script.encode("utf-8") cdef int err - if not hasattr(self, 'handle'): + if self.handle == NULL: # ensure we only grab 1 - cython might call init twice, of more. self.handle = dtrace_open(3, 0, &err) if self.handle == NULL: @@ -456,9 +457,10 @@ cdef class DTraceContinuousConsumer: """ Release DTrace handle. """ - if hasattr(self, 'handle') and self.handle != NULL: + if self.handle != NULL: dtrace_stop(self.handle) dtrace_close(self.handle) + self.handle = NULL cpdef go(self): """ From 56101d6d07d736c7041b0838bb138c3bfed25a81 Mon Sep 17 00:00:00 2001 From: tmetsch Date: Tue, 15 Feb 2022 14:41:33 +0100 Subject: [PATCH 34/37] Update setup.py new version prep. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d9d2159..d066c6f 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup(name='python-dtrace', - version='0.0.13', + version='0.0.14', description='DTrace consumer for Python based on libdtrace. Use Python' ' as DTrace Consumer and Provider! See the homepage for' ' more information.', From 8707df56d5d0bd434562b4e0e39404ada64bde80 Mon Sep 17 00:00:00 2001 From: tmetsch Date: Tue, 15 Feb 2022 14:44:50 +0100 Subject: [PATCH 35/37] Update ACKNOWLEDGEMENTS new version prep. --- ACKNOWLEDGEMENTS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ACKNOWLEDGEMENTS b/ACKNOWLEDGEMENTS index fca1df1..86f4514 100644 --- a/ACKNOWLEDGEMENTS +++ b/ACKNOWLEDGEMENTS @@ -10,3 +10,6 @@ of the DTrace handles. Thanks to Alexander Richardson (https://github.com/arichardson) for doing a huge refactoring and looking into properly supporting python3. + +Thanks to Konrad Witaszczyk (https://github.com/kwitaszczyk) for looking into +properly freeing up DTrace resources. From d938182122aa84a80ef1e9525f64fe4fc5a063e9 Mon Sep 17 00:00:00 2001 From: tmetsch Date: Wed, 4 May 2022 18:45:52 +0200 Subject: [PATCH 36/37] Potential fix for issue #28. --- dtrace_cython/consumer.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dtrace_cython/consumer.pyx b/dtrace_cython/consumer.pyx index c5a6e42..0401000 100644 --- a/dtrace_cython/consumer.pyx +++ b/dtrace_cython/consumer.pyx @@ -307,7 +307,7 @@ cdef class DTraceConsumer: """ self.chew_func = noop_chew if chew_func is None else chew_func self.chewrec_func = noop_chewrec if chewrec_func is None \ - else simple_chewrec + else chewrec_func self.out_func = noop_out if out_func is None else out_func self.walk_func = noop_walk if walk_func is None else walk_func @@ -432,7 +432,7 @@ cdef class DTraceContinuousConsumer: """ self.chew_func = noop_chew if chew_func is None else chew_func self.chewrec_func = noop_chewrec if chewrec_func is None \ - else simple_chewrec + else chewrec_func self.out_func = noop_out if out_func is None else out_func self.walk_func = noop_walk if walk_func is None else walk_func self.script = script.encode("utf-8") From 1cc6897eded5af05ec210d5fd08489d0ab5b9a26 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Fri, 13 May 2022 21:26:10 +0200 Subject: [PATCH 37/37] new bugfix release --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d066c6f..6f573ea 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup(name='python-dtrace', - version='0.0.14', + version='0.0.15', description='DTrace consumer for Python based on libdtrace. Use Python' ' as DTrace Consumer and Provider! See the homepage for' ' more information.',