implement and use resolve_many_addr

it uses _vmprof.resolve_many_addr on pypy, which is *much* more efficient

Author: cfbolz

vmprof/__init__.py

@@ -137,3 +137,19 @@ def get_profile_path():
     if hasattr(_vmprof, 'get_profile_path'):
         return _vmprof.get_profile_path()
     raise NotImplementedError("get_profile_path not implemented on this platform")
+
+def resolve_many_addr(addrs):
+    """ Try to symbolicate the function addresses in addrs to triples of
+    (function_name, line_number or 0, sourcefile or shared library)
+    returns a dictionary mapping the addresses where that worked to said triples.
+    """
+    if hasattr(_vmprof, 'resolve_many_addr'):
+        return _vmprof.resolve_many_addr(addrs)
+    if hasattr(_vmprof, 'resolve_addr'):
+        res = {}
+        for addr in addrs:
+            info = _vmprof.resolve_addr(addr)
+            if info is not None:
+                res[addr] = info
+        return res
+    return {} # valid result to always know nothing

vmprof/reader.py

@@ -310,24 +310,24 @@ def setup(self):
         self.dedup = set()
 
     def finished_reading_profile(self):
-        import _vmprof
+        import _vmprof, vmprof
         if not hasattr(_vmprof, 'resolve_addr'):
             # windows does not implement that!
             return
 
-        resolve_addr = _vmprof.resolve_addr
-        from _vmprof import resolve_addr
         LogReader.finished_reading_profile(self)
         if len(self.dedup) == 0:
             return
+        all_addresses = vmprof.resolve_many_addr(self.dedup)
+
         self.fileobj.seek(0, os.SEEK_END)
         # must match '<lang>:<name>:<line>:<file>'
         # 'n' has been chosen as lang here, because the symbol
         # can be generated from several languages (e.g. C, C++, ...)
 
         for addr in self.dedup:
-            bytelist = [b"\x08"]
-            result = resolve_addr(addr)
+            bytelist = [MARKER_NATIVE_SYMBOLS]
+            result = all_addresses.get(addr)
             if result is None:
                 name, lineno, srcfile = None, 0, None
             else:
@@ -348,6 +348,8 @@ def add_virtual_ip(self, marker, unique_id, name):
 
     def add_trace(self, trace, trace_count, thread_id, mem_in_kb):
         for addr in trace:
+            if not isinstance(addr, NativeCode):
+                continue
             if addr not in self.dedup:
                 self.dedup.add(addr)
 

vmprof/test/test_c_symboltable.py

@@ -11,6 +11,9 @@
     //void dump_all_known_symbols(int fd);
     int test_extract(char ** name, int * lineno, char ** src);
     int test_extract_sofile(char ** name, int * lineno, char ** src);
+
+    int somefunc();
+    void* get_somefunc(int);
     """)
     with open("src/symboltable.c", "rb") as fd:
         source = fd.read().decode()
@@ -36,7 +39,16 @@
         return vmp_resolve_addr(&abs, gname, 64,
                                 lineno, gsrc, 128);
     }
+
+    int somefunc() {return 1;}
+    void* get_somefunc(int which) {
+        if (which == 0) return &somefunc;
+        if (which == 1) return &abs;
+        return NULL;
+    }
     """
+    # replace the name, otherwise we'll get the one built into pypy
+    source = source.replace('vmp_resolve_addr', 'vmp_resolve_addr_2')
     libs = [] #['unwind', 'unwind-x86_64']
     includes = ['src']
     if sys.platform.startswith('linux'):
@@ -80,7 +92,7 @@ def test_resolve_addr(self):
         _lineno = ffi.new("int*")
         lib.test_extract(name, _lineno, src)
 
-        assert ffi.string(name[0]) == b"vmp_resolve_addr"
+        assert ffi.string(name[0]) == b"vmp_resolve_addr_2"
         srcfile = ffi.string(src[0])
         assert b"_test_symboltable" in srcfile
         if not srcfile.endswith(b"vmprof/test/_test_symboltable.c"):
@@ -90,7 +102,7 @@ def test_resolve_addr(self):
             with open("vmprof/test/_test_symboltable.c", "rb") as fd:
                 lineno = 1
                 for line in fd.readlines():
-                    if "int vmp_resolve_addr(void * addr," in line.decode():
+                    if "int vmp_resolve_addr_2(void * addr," in line.decode():
                         if _lineno[0] == lineno:
                             break
                     lineno += 1
@@ -116,3 +128,18 @@ def test_sofile_in_srcfile(self):
         elif sys.platform == "darwin":
             # osx
             assert b"libsystem_c.dylib" in ffi.string(src[0])
+
+    def test_vmprof_resolve_addr(self):
+        import _vmprof
+        res = _vmprof.resolve_addr(int(self.ffi.cast('intptr_t', self.lib.get_somefunc(0))))
+        assert res[0] == b'somefunc'
+
+    def test_vmprof_resolve_many_addr(self):
+        import vmprof
+        addrs = [int(self.ffi.cast('intptr_t', self.lib.get_somefunc(which))) for which in [0, 1, 2]]
+        res = vmprof.resolve_many_addr(addrs)
+        assert len(res) <= 2
+        if addrs[0] in res:
+            assert res[addrs[0]][0] == b'somefunc'
+        if addrs[1] in res:
+            assert res[addrs[1]][0] == b'abs'

vmprof/test/test_run.py

@@ -58,7 +58,7 @@ def read(self, count):
     PY3K = True
 else:
     PY3K = False
-if hasattr(os, 'uname') and os.uname().machine == 'ppc64le':
+if hasattr(os, 'uname') and os.uname()[4] == 'ppc64le':
     PPC64LE = True
 else:
     PPC64LE = False