From 75d64481eec6f558148a0c0137440187d4ce1cb1 Mon Sep 17 00:00:00 2001 From: Anatoly Myachev Date: Fri, 18 Jan 2019 14:36:55 +0000 Subject: [PATCH 1/3] more compatible python code --- runtool/collectors/android.py | 2 +- runtool/collectors/osx.py | 12 +- runtool/collectors/win.py | 8 +- runtool/decoders/Adreno.py | 7 +- runtool/decoders/MSNT_SystemTrace.py | 10 +- runtool/exporters/ChromeTracing.py | 37 +++-- runtool/exporters/QtCreatorProfiler.py | 33 ++-- runtool/exporters/Stat.py | 10 +- runtool/importers/etw.py | 105 +++++++------ runtool/importers/mac_log.py | 4 +- runtool/importers/osx.py | 31 ++-- runtool/importers/pprof_importer.py | 7 +- runtool/importers/qnx.py | 10 +- runtool/python_compatibility_layer.py | 22 +++ runtool/sea_runtool.py | 210 +++++++++++++------------ 15 files changed, 283 insertions(+), 225 deletions(-) create mode 100644 runtool/python_compatibility_layer.py diff --git a/runtool/collectors/android.py b/runtool/collectors/android.py index bdec485..87f2c04 100644 --- a/runtool/collectors/android.py +++ b/runtool/collectors/android.py @@ -33,7 +33,7 @@ def detect(cls): version = parts[parts.index('version') + 1] systraces.append((version, adb)) if systraces: - sorted_by_version = sorted(systraces, key=lambda(ver, _): [int(item) for item in ver.split('.')], reverse=True) + sorted_by_version = sorted(systraces, key=lambda ver__: [int(item) for item in ver__[0].split('.')], reverse=True) return '"%s"' % sorted_by_version[0][1] else: return None diff --git a/runtool/collectors/osx.py b/runtool/collectors/osx.py index 562fddf..6af1276 100644 --- a/runtool/collectors/osx.py +++ b/runtool/collectors/osx.py @@ -1,9 +1,11 @@ +from __future__ import print_function import os import sys import shutil import subprocess sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) from sea_runtool import Collector, get_decoders +from python_compatibility_layer import itervalues dtrace_context_switch = r""" @@ -205,7 +207,7 @@ def __init__(self, args): self.subcollectors = set() decoders = get_decoders() - for decoder_group in decoders.itervalues(): + for decoder_group in itervalues(decoders): for decoder_class in decoder_group: if any('Subcollector' in str(name) for name in decoder_class.__bases__): self.subcollectors.add(decoder_class) @@ -224,7 +226,7 @@ def create_ask_pass(): return path with open(path, 'w') as file: file.write(osxaskpass) - os.chmod(path, 0700) + os.chmod(path, 0o700) return path def gen_gpu_hooks(self, text): # TODO: check /System/Library/Extensions/AppleIntelSKLGraphics.kext/Contents/Info.plist @@ -286,7 +288,7 @@ def find_by_content(el, content): else: sibling = sibling.nextSibling if value != '0': - print "Warning: To enable Graphics profiling, set GraphicsFirmwareSelect to 0 in: %s\n\tThen: sudo kextcache -i / & reboot" % file_name + print('Warning: To enable Graphics profiling, set GraphicsFirmwareSelect to 0 in: %s\n\tThen: sudo kextcache -i / & reboot' % file_name) @staticmethod def gen_options(options): @@ -351,7 +353,7 @@ def start(self): def get_pid_children(parent): (out, err) = DTraceCollector.execute('ps -o pid,ppid -ax', log=False) if err: - print err + print(err) return for line in out.split('\n'): if not line: @@ -389,7 +391,7 @@ def available(cls): return False (out, err) = cls.execute('csrutil status') if 'disabled' not in out: - print 'Please do: "csrutil disable" from Recovery OS terminal to be able using dtrace...' + print('Please do: "csrutil disable" from Recovery OS terminal to be able using dtrace...') return False return True diff --git a/runtool/collectors/win.py b/runtool/collectors/win.py index 2e522b6..ef543ac 100644 --- a/runtool/collectors/win.py +++ b/runtool/collectors/win.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import sys import time @@ -11,7 +12,6 @@ from sea_runtool import Collector, is_domain_enabled import sea - def relog_etl(frm, to): sea.ITT('win').relog(frm, to) @@ -48,7 +48,7 @@ def detect(cls, statics={}): break if not res: return None - statics['res'] = sorted(res, key=lambda(_, ver): [int(item) for item in ver.split('.')], reverse=True)[0][0] + statics['res'] = sorted(res, key=lambda __ver: [int(item) for item in __ver[1].split('.')], reverse=True)[0][0] return statics['res'] @staticmethod @@ -171,7 +171,7 @@ def stop(self, wait=True, complete=True): (out, err) = self.execute('logman stop GPA_GPUVIEW -ets') if err and complete: - print err + print(err) environ = os.environ.copy() environ['TLOG'] = 'NORMAL' @@ -193,7 +193,7 @@ def merge(cls, gpuview, file, started, wait, args=None): cmd = '"%s" -merge Merged.etl IntelSEAPI.etl "%s"' % (xperf, os.path.basename(file)) (out, err) = Collector.execute(cmd, cwd=started) if err and (os.path.basename(file) not in err): - print err + print(err) relog_etl(os.path.join(started, os.path.basename(file)), file) shutil.rmtree(started) else: diff --git a/runtool/decoders/Adreno.py b/runtool/decoders/Adreno.py index 7b34af8..ca5402b 100644 --- a/runtool/decoders/Adreno.py +++ b/runtool/decoders/Adreno.py @@ -1,3 +1,8 @@ +from __future__ import print_function +import os +import sys +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) +from python_compatibility_layer import xrange TRACK_INDEX, TRACK_NAME = -1, 'GPU' @@ -173,7 +178,7 @@ def handle_record(self, proc, pid, tid, cpu, flags, timestamp, name, args): elif name in ['kgsl_mem_timestamp_queue', 'kgsl_mem_timestamp_free']: pass else: - print name + print(name) def finalize(self): pass diff --git a/runtool/decoders/MSNT_SystemTrace.py b/runtool/decoders/MSNT_SystemTrace.py index 1781195..1b272b9 100644 --- a/runtool/decoders/MSNT_SystemTrace.py +++ b/runtool/decoders/MSNT_SystemTrace.py @@ -1,5 +1,9 @@ +from __future__ import print_function +import os import sys import socket +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) +from python_compatibility_layer import basestring, iteritems def resolve_host(ip): try: @@ -9,7 +13,7 @@ def resolve_host(ip): def convert_numbers(obj): if isinstance(obj, dict): - for k, v in obj.iteritems(): + for (k, v) in iteritems(obj): obj[k] = convert_numbers(v) elif isinstance(obj, list): new = [convert_numbers(v) for v in obj] @@ -61,7 +65,7 @@ def handle_record(self, system, data, info): if info['Opcode'] in ['Video', 'NIC', 'PhyDisk', 'LogDisk', 'CPU', 'Platform']: self.callbacks.add_metadata(info['Opcode'], convert_numbers(data)) elif sys.gettrace(): - print 'EventName:', info['EventName'], 'Opcode:', info['Opcode'] + print('EventName:', info['EventName'], 'Opcode:', info['Opcode']) def on_receive(self, size, now, pid, source, target): receiver = self.tcpip.setdefault(source, {'packets': [], 'started': None}) @@ -104,7 +108,7 @@ def end_receiver(receiver): receiver['started'] = None def finalize(self): - for target, receiver in self.tcpip.iteritems(): + for (target, receiver) in iteritems(self.tcpip): self.end_receiver(receiver) packets = receiver['packets'] if packets: diff --git a/runtool/exporters/ChromeTracing.py b/runtool/exporters/ChromeTracing.py index 3d92d1b..3b8807d 100644 --- a/runtool/exporters/ChromeTracing.py +++ b/runtool/exporters/ChromeTracing.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import sys #sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) # uncomment for debugging single file @@ -14,6 +15,8 @@ MAX_GT_SIZE = 50 * 1024 * 1024 GT_FLOAT_TIME = True +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) +from python_compatibility_layer import unicode, iteritems, itervalues class GoogleTrace(TaskCombiner): @@ -64,7 +67,7 @@ def __init__(self, args, tree): handled = [] for trace in self.args.trace: if not os.path.exists(trace): - print "Error: File not found:", trace + print("Error: File not found:", trace) continue if trace.endswith(".etl"): self.handle_etw_trace(trace) @@ -79,7 +82,7 @@ def __init__(self, args, tree): self.handle_perf(trace) handled.append(trace) else: - print "Error: unsupported extension:", trace + print("Error: unsupported extension:", trace) args.trace = [trace for trace in args.trace if trace not in handled] self.start_new_trace() @@ -89,7 +92,7 @@ def start_new_trace(self): self.file = codecs.open(self.targets[-1], "wb+", 'utf-8') self.file.write('{\n"traceEvents": [\n\n') # second \n is for the rare case when there are no events, and finish cuts last two symbols - for key, value in self.tree["threads"].iteritems(): + for (key, value) in iteritems(self.tree["threads"]): pid_tid = key.split(',') self.file.write( '{"name": "thread_name", "ph":"M", "pid":%s, "tid":%s, "args": {"name":"%s(%s)"}},\n' % (pid_tid[0], pid_tid[1], value, pid_tid[1]) @@ -216,7 +219,7 @@ def calc_time_sync(time_sync, series=True): source_scale_start = time_sync[index - 1][Source] + int(diff[Source] * 0.75) # to keep the precision target_scale_start = (time_sync[index - 1][Target] + (diff[Target] * 0.25)) * 1000000. # multiplying by 1000000. to have time is microseconds (ftrace/target time was in seconds) - print "Timelines correlation precision is +- %f us" % (diff[Target] / 2. * 1000000.) + print("Timelines correlation precision is +- %f us" % (diff[Target] / 2. * 1000000.)) else: source_scale_start = time_sync[0][Source] target_scale_start = time_sync[0][Target] * 1000000. # multiplying by 1000000. to have time in microseconds (ftrace/target time was in seconds) @@ -240,7 +243,7 @@ def global_metadata(self, data): self.file.write( '{"name": "process_labels", "ph":"M", "pid":%d, "tid":%s, "args": {"labels":"%s"}},\n' % (data['pid'], data['tid'], ','.join(data['labels'])) ) - if data['tid'] >= 0 and not self.tree['threads'].has_key('%d,%d' % (data['pid'], data['tid'])): # marking the main thread + if data['tid'] >= 0 and '%d,%d' % (data['pid'], data['tid']) not in self.tree['threads']: # marking the main thread self.file.write( '{"name": "thread_name", "ph":"M", "pid":%d, "tid":%s, "args": {"name":"%s"}},\n' % (data['pid'], data['tid'], "
") ) @@ -268,14 +271,14 @@ def relation(self, data, head, tail): template = '{"ph":"%s", "name": "relation", "pid":%d, "tid":%s, "ts":%.3f, "id":%s, "args":{"name": "%s"}, "cat":"%s"},\n' else: template = '{"ph":"%s", "name": "relation", "pid":%d, "tid":%s, "ts":%d, "id":%s, "args":{"name": "%s"}, "cat":"%s"},\n' - if not data.has_key('str'): + if 'str' not in data: data['str'] = "unknown" self.file.write(template % ("s", items[0]['pid'], items[0]['tid'], self.convert_time(items[0]['time']), relation_id, data['str'], data['domain'])) self.file.write(template % ("f", items[1]['pid'], items[1]['tid'], self.convert_time(items[1]['time'] - 1), relation_id, data['str'], data['domain'])) def format_value(self, arg): # this function must add quotes if value is string, and not number/float, do this recursively for dictionary if type(arg) == type({}): - return "{" + ", ".join(['"%s":%s' % (key, self.format_value(value)) for key, value in arg.iteritems()]) + "}" + return "{" + ", ".join(['"%s":%s' % (key, self.format_value(value)) for (key, value) in iteritems(arg)]) + "}" try: val = float(arg) if float('inf') != val: @@ -289,7 +292,7 @@ def format_value(self, arg): # this function must add quotes if value is string def format_args(self, arg): # this function must add quotes if value is string, and not number/float, do this recursively for dictionary if type(arg) == type({}): - return dict([(key, self.format_args(value)) for key, value in arg.iteritems()]) + return dict([(key, self.format_args(value)) for (key, value) in iteritems(arg)]) try: val = float(arg) if float('inf') != val: @@ -309,7 +312,7 @@ def complete_task(self, type, begin, end): if self.last_task == (type, begin, end): return self.last_task = (type, begin, end) - assert (GoogleTrace.Phase.has_key(type)) + assert (type in GoogleTrace.Phase) if begin['type'] == 7: # frame_begin if 'id' not in begin: begin['id'] = id(begin) # Async events are groupped by cat & id @@ -334,7 +337,7 @@ def complete_task(self, type, begin, end): json.loads(res) except Exception as exc: import traceback - print "\n" + exc.message + ":\n" + res + "\n" + print("\n" + exc.message + ":\n" + res + "\n") traceback.print_stack() self.format_task(GoogleTrace.Phase[type], type, begin, end) res += ',\n' @@ -383,7 +386,7 @@ def format_task(self, phase, type, begin, end): res = [] res.append('{"ph":"%s"' % phase) res.append(', "pid":%(pid)d' % begin) - if begin.has_key('tid'): + if 'tid' in begin: res.append(', "tid":%(tid)d' % begin) if GT_FLOAT_TIME: res.append(', "ts":%.3f' % (self.convert_time(begin['time']))) @@ -448,7 +451,7 @@ def format_task(self, phase, type, begin, end): total = 0 breakdown = {} children = 0 - for size, values in begin['memory'].iteritems(): + for (size, values) in iteritems(begin['memory']): if size is None: # special case for children attribution children = values else: @@ -467,8 +470,8 @@ def format_task(self, phase, type, begin, end): def handle_leftovers(self): TaskCombiner.handle_leftovers(self) - for counters in self.counters.itervalues(): # workaround: google trace forgets counter last value - for counter in counters.itervalues(): + for counters in itervalues(self.counters): # workaround: google trace forgets counter last value + for counter in itervalues(counters): counter['time'] += 1 # so, we repeat it on the end of the trace self.complete_task("counter", counter, counter) @@ -481,7 +484,7 @@ def finish(self, intermediate=False): if not intermediate: if self.samples: self.file.write('], "stackFrames": {\n') - for id, frame in self.frames.iteritems(): + for (id, frame) in iteritems(self.frames): self.file.write('"%s": %s,\n' % (id, json.dumps(frame))) if self.frames: # deleting last two symbols from the file as we can't leave comma at the end due to json restrictions self.remove_last(2) @@ -494,7 +497,7 @@ def finish(self, intermediate=False): self.frames = {} if self.metadata: self.file.write('\n],\n') - for key, value in self.metadata.iteritems(): + for (key, value) in iteritems(self.metadata): self.file.write('"%s": %s,\n' % (key, json.dumps(value[0] if len(value) == 1 else value))) self.remove_last(2) # remove trailing ,\n self.file.write('\n}') @@ -516,7 +519,7 @@ def get_catapult_path(args): return path zip_path = os.path.join(args.bindir, 'catapult.zip') if os.path.exists(zip_path): - print "Extracting catapult..." + print("Extracting catapult...") from zipfile import PyZipFile pzf = PyZipFile(zip_path) pzf.extractall(args.bindir) diff --git a/runtool/exporters/QtCreatorProfiler.py b/runtool/exporters/QtCreatorProfiler.py index a951d55..260f9e7 100644 --- a/runtool/exporters/QtCreatorProfiler.py +++ b/runtool/exporters/QtCreatorProfiler.py @@ -1,6 +1,11 @@ +from __future__ import print_function import cgi from sea_runtool import TaskCombiner, get_name, to_hex +import os +import sys +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) +from python_compatibility_layer import iteritems #https://github.com/danimo/qt-creator/blob/master/src/plugins/qmlprofiler/qmlprofilertracefile.cpp #https://github.com/danimo/qt-creator/blob/master/src/plugins/qmlprofiler/qv8profilerdatamodel.cpp @@ -39,7 +44,7 @@ def complete_task(self, type, begin, end): name = get_name(begin) details = (type + ":") if type != 'task' else "" - if begin.has_key('parent'): + if 'parent' in begin: details += to_hex(begin['parent']) + "->" details += name @@ -51,15 +56,15 @@ def complete_task(self, type, begin, end): kind = 'Javascript' record = ( - begin['__file__'].replace("\\", "/") if begin.has_key('__file__') else "", - begin['__line__'] if begin.has_key('__line__') else "0", + begin['__file__'].replace("\\", "/") if '__file__' in begin else "", + begin['__line__'] if '__line__' in begin else "0", kind, "%s | %s" % (details, begin['domain']), name ) record = tuple([cgi.escape(item) for item in record]) - if self.event_map.has_key(record): + if record in self.event_map: index = self.event_map[record] else: index = len(self.events) @@ -74,9 +79,9 @@ def complete_task(self, type, begin, end): args['value'] = begin['delta'] else: # TODO: add multi-value support return - if begin.has_key('args'): + if 'args' in begin: args = begin['args'] - if end.has_key('args'): + if 'args' in end: args.update(end['args']) if args: self.notes.append((start_time, dur, index, args)) @@ -86,20 +91,20 @@ def complete_task(self, type, begin, end): def write_header(self): # at this moment print is redirected to output file - print '' - print '' % (self.start_time, self.end_time) - print '' % (self.end_time - self.start_time) + print('') + print('' % (self.start_time, self.end_time)) + print('' % (self.end_time - self.start_time)) counter = 0 for event in self.events: - print '%s%s%s
%s
%s' \ - % (counter, event[0], event[1], event[2], event[3], event[4]) + print('%s%s%s
%s
%s' \ + % (counter, event[0], event[1], event[2], event[3], event[4])) counter += 1 - print '' + print('') def write_footer(self, file): file.write('\n') for note in self.notes: - args = "\n".join([str(key) + " = " + str(val).replace("{","").replace("}","") for key, val in note[3].iteritems()]) + args = "\n".join([str(key) + " = " + str(val).replace("{","").replace("}","") for (key, val) in iteritems(note[3])]) file.write('%s\n' % (note[0], note[1], note[2], cgi.escape(args))) file.write('\n') @@ -112,7 +117,7 @@ def finish(self): if fi.isfirstline(): self.write_header() wrote_header = True - print line, + print(line,end=' ') if wrote_header: with open(self.file_name, "a") as file: self.write_footer(file) diff --git a/runtool/exporters/Stat.py b/runtool/exporters/Stat.py index c992b72..24a2e88 100644 --- a/runtool/exporters/Stat.py +++ b/runtool/exporters/Stat.py @@ -1,6 +1,10 @@ import cgi from sea_runtool import GraphCombiner +import os +import sys +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) +from python_compatibility_layer import iteritems class Stat(GraphCombiner): def __init__(self, args, tree): @@ -13,8 +17,8 @@ def get_targets(self): def finish(self): GraphCombiner.finish(self) - for domain, data in self.per_domain.iteritems(): - for task_name, task_data in data['tasks'].iteritems(): + for (domain, data) in iteritems(self.per_domain): + for (task_name, task_data) in iteritems(data['tasks']): time = task_data['time'] self.file.write('%s,%s,%s,%s,%s,%d\n' % ( cgi.escape(domain), cgi.escape(task_name), @@ -28,7 +32,7 @@ def join_traces(traces, output, args): # FIXME: implement real joiner sorting = [] for trace in traces: sorting.append((os.path.getsize(trace), trace)) - sorting.sort(key=lambda (size, trace): size, reverse=True) + sorting.sort(key=lambda size_trace: size_trace[0], reverse=True) shutil.copyfile(sorting[0][1], output + ".csv") return output + ".csv" diff --git a/runtool/importers/etw.py b/runtool/importers/etw.py index 3c00f00..0a73eeb 100644 --- a/runtool/importers/etw.py +++ b/runtool/importers/etw.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import sys import glob @@ -7,7 +8,7 @@ sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) import sea import strings - +from python_compatibility_layer import basestring, unicode, iteritems, itervalues class ETWXML: @@ -35,9 +36,9 @@ def iterate_events(self, file): yield elem elem.clear() level -= 1 - except ET.ParseError, exc: - print "\nError: Bad XML file: %s\n", file - print exc.message + except ET.ParseError as exc: + print("\nError: Bad XML file: %s\n", file) + print(exc.message) def as_dict(self, elem): return dict((self.tag_name(child.tag), child) for child in elem.getchildren()) @@ -47,24 +48,24 @@ def parse_system(self, system): system = self.as_dict(system) if not system: return res - if system.has_key('TimeCreated'): + if 'TimeCreated' in system: time_created = system['TimeCreated'] try: res['time'] = time_created.attrib['RawTime'] except KeyError: res['time'] = time_created.attrib['SystemTime'] - if system.has_key('Task'): + if 'Task' in system: task = system['Task'] res['Task'] = task.text - if system.has_key('EventID'): + if 'EventID' in system: EventID = system['EventID'] res['EventID'] = EventID.text - if system.has_key('Opcode'): + if 'Opcode' in system: Opcode = system['Opcode'] res['Opcode'] = Opcode.text provider = system['Provider'] - execution = system['Execution'] if system.has_key('Execution') else None - res['provider'] = provider.attrib['Name'] if provider.attrib.has_key('Name') else provider.attrib['Guid'] if provider.attrib.has_key('Guid') else None + execution = system['Execution'] if 'Execution' in system else None + res['provider'] = provider.attrib['Name'] if 'Name' in provider.attrib else provider.attrib['Guid'] if 'Guid' in provider.attrib else None if execution != None: res['pid'] = execution.attrib['ProcessID'] res['tid'] = execution.attrib['ThreadID'] @@ -83,7 +84,7 @@ def parse_event_data(self, data): def parse_rendering_info(self, info): res = {} info = self.as_dict(info) - for key, data in info.iteritems(): + for (key, data) in iteritems(info): res[key] = data.text.strip() if data.text else "" return res @@ -97,7 +98,7 @@ def parse(self, file): if not system: continue if system['provider'] in self.providers or system['provider'].upper() in self.providers: - if children.has_key('BinaryEventData'): + if 'BinaryEventData' in children: self.callback(system, children['BinaryEventData'].text, self.as_dict(children['ExtendedTracingInfo'])['EventGuid'].text) else: data = None @@ -134,7 +135,7 @@ def parse_system(self, time, args): def receive(self, time, args): system = self.parse_system(time, args) system['EventName'] = system['Task'] - data = dict((key, val) for (key, val) in args.iteritems() if key and not key.startswith('hdr:')) + data = dict((key, val) for (key, val) in iteritems(args) if key and not key.startswith('hdr:')) if not data and 'hdr:EventMessage' in args and args['hdr:EventMessage']: if len(args['hdr:EventMessage']) % 4 == 0: # base64 try: @@ -208,7 +209,7 @@ def volume_to_drive(path, drives={}): continue drives[buffer.value] = drive drives[r'\SystemRoot'] = os.path.expandvars('%systemroot%') - for (volume, drive) in drives.iteritems(): + for (volume, drive) in iteritems(drives): path = path.replace(volume, drive) assert ('systemroot' not in path.lower()) return path @@ -233,7 +234,7 @@ def auto_break_gui_packets(self, call_data, tid, begin): to_remove = [] del packets[id] # the task has ended, removing it from the pipeline time_shift = 0 - for begin_data in sorted(packets.itervalues(), key=lambda data: data['realtime']): # finish all and start again to form melting task queue + for begin_data in sorted(itervalues(packets), key=lambda data: data['realtime']): # finish all and start again to form melting task queue time_shift += 1 # making sure the order of tasks on timeline, probably has to be done in Chrome code rather end_data = begin_data.copy() # the end of previous part of task is also here end_data['time'] = call_data['time'] - time_shift # new begin for every task is here @@ -371,9 +372,9 @@ def MapReasonToState(self, state, wait_reason): """ def get_process_name_by_tid(self, tid): - if self.thread_pids.has_key(tid): + if tid in self.thread_pids: pid = self.thread_pids[tid] - if self.process_names.has_key(pid): + if pid in self.process_names: name = self.process_names[pid]['name'] else: name = "PID:%d" % parse_int(pid) @@ -397,26 +398,26 @@ def MSNT_SystemTrace(self, system, data, info): ver = '.'.join(str(num) for num in struct.unpack('BBHL', struct.pack('Q', int(data['Version'])))[:-1]) self.callbacks.add_metadata('OS', {'Windows': ver, 'Build': int(data['ProviderVersion'])}) except: - print "Error: failed to parse OS version" + print("Error: failed to parse OS version") elif info['EventName'] == 'DiskIo': if info['Opcode'] in ['FileDelete', 'FileRundown']: - if self.files.has_key(data['FileObject']): + if data['FileObject'] in self.files: file = self.files[data['FileObject']] - if file.has_key('pid'): + if 'pid' in file: call_data = {'tid': file['tid'], 'pid': file['pid'], 'domain': 'MSNT_SystemTrace', 'time': self.convert_time(system['time']), 'str': file['name'], 'type':11, 'id': parse_int(data['FileObject'])} self.callbacks.on_event("object_delete", call_data) del self.files[data['FileObject']] elif info['Opcode'] in ['Read', 'Write', 'HardFault', 'FlushBuffers', 'WriteInit', 'ReadInit', 'FlushInit']: - tid = int(data['IssuingThreadId']) if data.has_key('IssuingThreadId') else parse_int(data['TThreadId']) if data.has_key('TThreadId') else None + tid = int(data['IssuingThreadId']) if 'IssuingThreadId' in data else parse_int(data['TThreadId']) if 'TThreadId' in data else None if tid == None: return - if not data.has_key('FileObject'): - if self.irps.has_key(data['Irp']): + if 'FileObject' not in data: + if data['Irp'] in self.irps: data['FileObject'] = self.irps[data['Irp']] else: return - if self.files.has_key(data['FileObject']) and self.thread_pids.has_key(tid): + if data['FileObject'] in self.files and tid in self.thread_pids: file = self.files[data['FileObject']] pid = parse_int(self.thread_pids[tid]) call_data = {'tid': tid, 'pid': pid, 'domain': 'MSNT_SystemTrace', 'time': self.convert_time(system['time']), 'str': file['name'], 'type': 10, 'id': parse_int(data['FileObject'])} @@ -428,13 +429,13 @@ def MSNT_SystemTrace(self, system, data, info): creation['time'] = file['creation'] self.callbacks.on_event("object_new", creation) file['creation'] = None - if data.has_key('Irp'): + if 'Irp' in data: self.irps[data['Irp']] = data['FileObject'] data['OPERATION'] = info['Opcode'] call_data['args'] = {'snapshot': data} self.callbacks.on_event("object_snapshot", call_data) else: - print info['Opcode'] + print(info['Opcode']) elif info['EventName'] == 'FileIo': if info['Opcode'] == 'FileCreate': file_name = self.handle_file_name(data['FileName']) @@ -447,13 +448,13 @@ def MSNT_SystemTrace(self, system, data, info): call_data = {'tid': int(system['tid']), 'pid': int(system['pid']), 'domain': 'MSNT_SystemTrace', 'time': self.convert_time(system['time']), 'str': file_name, 'type': 9, 'id': parse_int(data['FileObject'])} self.callbacks.on_event("object_new", call_data) elif info['Opcode'] in ['Close', 'FileDelete', 'Delete']: - if self.files.has_key(data['FileObject']): + if data['FileObject'] in self.files: file = self.files[data['FileObject']] call_data = {'tid': int(system['tid']), 'pid': int(system['pid']), 'domain': 'MSNT_SystemTrace', 'time': self.convert_time(system['time']), 'str': file['name'], 'type': 11, 'id': parse_int(data['FileObject'])} self.callbacks.on_event("object_delete", call_data) del self.files[data['FileObject']] elif info['Opcode'] not in ['OperationEnd', 'Cleanup', 'QueryInfo']: - if self.files.has_key(data['FileObject']): + if data['FileObject'] in self.files: file = self.files[data['FileObject']] tid = int(system['tid']) pid = int(system['pid']) @@ -461,7 +462,7 @@ def MSNT_SystemTrace(self, system, data, info): file['tid'] = tid file['last_access'] = call_data['time'] file['pid'] = pid - if data.has_key('IrpPtr'): + if 'IrpPtr' in data: self.irps[data['IrpPtr']] = data['FileObject'] data['OPERATION'] = info['Opcode'] call_data['args'] = {'snapshot': data} @@ -553,7 +554,7 @@ def MSNT_SystemTrace(self, system, data, info): pid = parse_int(data['StackProcess']) if pid in self.images: stack = [0] * (len(data) - 3) - for key in data.iterkeys(): + for key in data: num = key[5:] if key.startswith('Stack') and num.isdigit(): stack[int(num) - 1] = parse_int(data[key]) if data[key] else 0 @@ -567,7 +568,7 @@ def MSNT_SystemTrace(self, system, data, info): decoder.handle_record(system, data, info) return if sys.gettrace(): - print 'Unsupported EventName:', info['EventName'], 'Opcode:', info['Opcode'] + print('Unsupported EventName:', info['EventName'], 'Opcode:', info['Opcode']) return def on_stack(self, time, pid, tid, stack): @@ -631,11 +632,11 @@ def on_event(self, system, data, info): adapter = data['pDxgAdapter'] type = parse_int(data['AdapterType']) flags = [] - for key, val in ADAPTER_TYPE.iteritems(): + for (key, val) in iteritems(ADAPTER_TYPE): if type & key: flags.append(val) if adapter in self.adapters: - print "Existing adapter: %s \t new one: %s" % (str(self.adapters[adapter]), str((len(self.adapters), ' '.join(flags)))) + print("Existing adapter: %s \t new one: %s" % (str(self.adapters[adapter]), str((len(self.adapters), ' '.join(flags))))) self.adapters[adapter] = (len(self.adapters), ' '.join(flags)) return @@ -668,7 +669,7 @@ def on_event(self, system, data, info): call_data['tid'] = tid PacketType = int(data['PacketType']) call_data['str'] = DMA_PACKET_TYPE[PacketType] if PacketType < len(DMA_PACKET_TYPE) else str(PacketType) - id = int(data['uliSubmissionId'] if data.has_key('uliSubmissionId') else data['uliCompletionId']) + id = int(data['uliSubmissionId'] if 'uliSubmissionId' in data else data['uliCompletionId']) call_data['id'] = id call_data['str'] += ":%d" % id ulQueueSubmitSequence = int(data['ulQueueSubmitSequence']) @@ -695,13 +696,13 @@ def on_event(self, system, data, info): if call_data['tid'] not in self.thread_names: self.thread_names[call_data['tid']] = ("CPU Queue", call_data['pid']) id = int(data['SubmitSequence']) - if not data.has_key('PacketType'): # workaround, PacketType is not set for Waits - if data.has_key('FenceValue'): + if 'PacketType' not in data: # workaround, PacketType is not set for Waits + if 'FenceValue' in data: call_data['str'] = 'WAIT' else: call_data['str'] = 'Unknown' else: - PacketType = long(data['PacketType']) + PacketType = int(data['PacketType']) call_data['str'] = QUEUE_PACKET_TYPE[PacketType] if PacketType < len(QUEUE_PACKET_TYPE) else str(PacketType) if 'bPresent' in data and data['bPresent'] in ['1', 'true']: call_data['str'] = "PRESENT_" + call_data['str'] @@ -720,7 +721,7 @@ def on_event(self, system, data, info): self.callbacks.on_event("task_end_overlapped", closing) static['queue'][id] = call_data self.auto_break_gui_packets(call_data, call_data['tid'], True) - if data.has_key('FenceValue') and static['fence'].has_key(data['FenceValue']): + if 'FenceValue' in data and data['FenceValue'] in static['fence']: relation = (call_data.copy(), static['fence'][data['FenceValue']], call_data) relation[0]['parent'] = data['FenceValue'] del static['fence'][data['FenceValue']] @@ -787,7 +788,7 @@ def on_event(self, system, data, info): if 'Start' in opcode: static['resident'].setdefault(system['tid'], []).append(data) elif 'Stop' in opcode: - resident = static['resident'][system['tid']] if static['resident'].has_key(system['tid']) else [] + resident = static['resident'][system['tid']] if system['tid'] in static['resident'] else [] if len(resident): saved = resident.pop() else: @@ -803,12 +804,12 @@ def on_event(self, system, data, info): self.thread_names[call_data['tid']] = ("CPU Queue", call_data['pid']) id = parse_int(data['PagingQueuePacket']) call_data['id'] = id - if data.has_key('PagingQueueType'): + if 'PagingQueueType' in data: VidMmOpType = int(data['VidMmOpType']) PagingQueueType = int(data['PagingQueueType']) call_data['str'] = (PAGING_QUEUE_TYPE[PagingQueueType] if PagingQueueType < len(PAGING_QUEUE_TYPE) else str(PagingQueueType)) + ":" + (VIDMM_OPERATION[VidMmOpType] if VidMmOpType in VIDMM_OPERATION else str(VidMmOpType)) static['paging'][id] = call_data - elif static['paging'].has_key(id): # doesn't have type on end + elif id in static['paging']: # doesn't have type on end start = static['paging'][id] call_data['str'] = start['str'] call_data['pid'] = start['pid'] @@ -819,7 +820,7 @@ def on_event(self, system, data, info): return pDmaBuffer = data['pDmaBuffer'] call_data['id'] = parse_int(pDmaBuffer) - if 'Stop' in opcode and static['dmabuff'].has_key(pDmaBuffer): + if 'Stop' in opcode and pDmaBuffer in static['dmabuff']: call_data['args'].update(static['dmabuff'][pDmaBuffer]) del static['dmabuff'][pDmaBuffer] elif call_data['str'] == 'AddDmaBuffer': # Microsoft-Windows-DxgKrnl @@ -902,24 +903,24 @@ def on_event(self, system, data, info): def finish(self): if None in self.statistics: - for (pid, tid), track in self.statistics[None].iteritems(): + for (pid, tid), track in iteritems(self.statistics[None]): thread = self.callbacks.process(pid).thread(tid) if track['zone']['last']: thread.marker('task', track['zone']['name']).set(track['zone']['last']) - for adapter, (id, flags) in self.adapters.iteritems(): + for adapter, (id, flags) in iteritems(self.adapters): pid = -1 - id name = 'GPU Nodes of Adapter #%s%s:' % (str(id), (' (%s)' % flags) if flags else '') self.callbacks("metadata_add", {'domain': 'GPU', 'str': '__process__', 'pid': pid, 'tid': -1, 'data': name, 'delta': -2}) - for (adptr, node), name in self.node_info.iteritems(): + for (adptr, node), name in iteritems(self.node_info): if adapter != adptr: continue self.callbacks("metadata_add", {'domain': 'GPU', 'str': '__thread__', 'pid': pid, 'tid': node, 'data': name}) - for id, file in self.files.iteritems(): - if file.has_key('last_access'): # rest aren't rendered anyways + for (id, file) in iteritems(self.files): + if 'last_access' in file: # rest aren't rendered anyways call_data = {'tid': file['tid'], 'pid': file['pid'], 'domain': 'MSNT_SystemTrace', 'time': file['last_access'], 'str': file['name'], 'type': 11, 'id': parse_int(id)} self.callbacks.on_event("object_delete", call_data) - decoder_set = set([decoder for decoders in self.decoders.itervalues() for decoder in decoders]) + decoder_set = set([decoder for decoders in itervalues(self.decoders) for decoder in decoders]) for decoder in decoder_set: # since one decoder can handle different providers decoder.finalize() @@ -1000,7 +1001,7 @@ def parse(self): # 'MICROSOFT-WINDOWS-SHELL-CORE' None # Win7 events ] - for provider in self.decoders.iterkeys(): + for provider in self.decoders: providers.append(provider) if self.args.input.endswith('.xml'): with open(self.args.input) as file: @@ -1011,7 +1012,7 @@ def parse(self): etwxml = ETWXML(self.on_event, providers) unhandled_providers = etwxml.parse(file) self.finish() - print "Unhandled providers:", str(unhandled_providers) + print("Unhandled providers:", str(unhandled_providers)) elif self.args.input.endswith('.etl'): with Progress(1000, 50, strings.parsing_files % (os.path.basename(self.args.input), format_bytes(os.path.getsize(self.args.input)))) as progress: self.progress = progress @@ -1019,14 +1020,14 @@ def parse(self): self.file = reader reader.parse(self.args.input) self.finish() - for pid, data in self.process_names.iteritems(): + for (pid, data) in iteritems(self.process_names): proc_name = data['name'] if len(data['cmd']) > len(proc_name): proc_name = data['cmd'].replace('\\"', '').replace('"', '') pid = parse_int(pid) self.callbacks.set_process_name(pid, proc_name) self.callbacks.set_process_name(-pid, 'Sampling: ' + proc_name) - for tid, (name, pid) in self.thread_names.iteritems(): + for (tid, (name, pid)) in iteritems(self.thread_names): thread_name = name.replace('\\"', '').replace('"', '') self.callbacks.set_thread_name(pid, tid, thread_name) diff --git a/runtool/importers/mac_log.py b/runtool/importers/mac_log.py index df59b80..83eca87 100644 --- a/runtool/importers/mac_log.py +++ b/runtool/importers/mac_log.py @@ -39,7 +39,7 @@ def transform_log(args): if len(parts) != 5: continue if not parts[0].startswith('Timestamp'): - print "Error: this log format is not supported. Expected log from OSX's 'log stream'" + print("Error: this log format is not supported. Expected log from OSX's 'log stream'") header = parts continue else: @@ -61,4 +61,4 @@ def transform_log(args): 'format': 'log', 'available': True, 'importer': transform_log -}] \ No newline at end of file +}] diff --git a/runtool/importers/osx.py b/runtool/importers/osx.py index 20f7d97..d2b5908 100644 --- a/runtool/importers/osx.py +++ b/runtool/importers/osx.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import sys import glob @@ -6,6 +7,8 @@ sys.path.append(os.path.realpath(os.path.dirname(__file__))) # weird CentOS behaviour workaround from etw import GPUQueue +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) +from python_compatibility_layer import iteritems class DTrace(GPUQueue): def __init__(self, args, gt, callbacks): @@ -109,7 +112,7 @@ def handle_record(self, time, cmd, args): for decoder in self.decoders: decoder.handle_record(time, pid, tid, cmd, args[2:]) else: - print "unsupported cmd:", cmd, args + print("unsupported cmd:", cmd, args) def handle_stack(self, kind, time, pid, tid, stack): pid = int(pid, 16) @@ -153,7 +156,7 @@ def arg(self, time, pid, tid, name, value): if thread.task_stack: thread.task_stack[-1].add_args({name: value}) else: - print "Orphan arg:", name, value + print("Orphan arg:", name, value) def submit_prepare(self, time, id, pid, tid, args): if id not in self.prepares: @@ -195,9 +198,9 @@ def append_stage(self, ring_type, channel, stamp, data): def complete_stage(self, ring_type, channel, latest_stamp, data): stamps = self.event_tracker.setdefault((ring_type, channel), {}) latest_stamp = int(latest_stamp, 16) - to_del = set(stamp for stamp in stamps.iterkeys() if stamp <= latest_stamp) + to_del = set(stamp for stamp in stamps if stamp <= latest_stamp) if len(to_del) < 100: # in old driver the CompleteExecute might be called so rare that it is not reliable at all - for stamp, stages in stamps.iteritems(): + for (stamp, stages) in iteritems(stamps): if stamp <= latest_stamp: verbose = ['%s(%s) %d:' % (ring_type, channel, stamp)] ctx_type = None @@ -219,7 +222,7 @@ def complete_stage(self, ring_type, channel, latest_stamp, data): else: verbose.append(data['cmd']) if self.args.verbose: - print 'verbose:', ' '.join(verbose) + print('verbose:', ' '.join(verbose)) if not changed_context: # not sure what TODO with it yet task = self.complete_gpu(stages[-1], data, ctx_type, old_ctx_id) found_submit = False @@ -327,7 +330,7 @@ def gpu_call(self, time, cmd, pid, tid, args): cpu_time = int(cpu_time, 16) if not cpu_time: if self.args.debug: - print "Warning: zero timestamp: ", cmd, args + print("Warning: zero timestamp: ", cmd, args) return thread = self.callbacks.process(-1).thread(int(ctx_id, 16)) ring = self.map_ring_type(ring_type) @@ -339,33 +342,33 @@ def gpu_call(self, time, cmd, pid, tid, args): cpu_time = int(cpu_time, 16) if not cpu_time: if self.args.debug: - print "Warning: zero timestamp: ", cmd, args + print("Warning: zero timestamp: ", cmd, args) return thread = self.callbacks.process(-1).thread(int(ctx_id, 16)) if (ring_type, channel, stamp) in thread.task_pool: thread.task_pool[(ring_type, channel, stamp)].end(cpu_time) del thread.task_pool[(ring_type, channel, stamp)] else: - print "Unhandled gpu_call:", cmd + print("Unhandled gpu_call:", cmd) def on_gpu_frame(self, time, pid, tid): self.callbacks.on_event("marker", {'pid': pid, 'tid': tid, 'domain': 'gits', 'time': time, 'str': "GPU Frame", 'type': 5, 'data': 'task'}) def finalize(self): - for tid, (name, pid) in self.thread_names.iteritems(): + for (tid, (name, pid)) in iteritems(self.thread_names): for callback in self.callbacks.callbacks: thread_name = name.replace('\\"', '').replace('"', '') callback("metadata_add", {'domain': 'IntelSEAPI', 'str': '__thread__', 'pid': pid, 'tid': tid, 'data': '%s (%d)' % (thread_name, tid)}) - for pid, name in self.pid_names.iteritems(): + for (pid, name) in iteritems(self.pid_names): self.callbacks.set_process_name(pid, name) self.callbacks.set_process_name(-pid, 'Sampling: ' + name) - for context, name in self.contexts.iteritems(): + for (context, name) in iteritems(self.contexts): self.callbacks.set_thread_name(-1, int(context, 16), name) - for pid, proc in self.callbacks.processes.iteritems(): + for (pid, proc) in iteritems(self.callbacks.processes): name = None - for tid, thread in proc.threads.iteritems(): + for (tid, thread) in iteritems(proc.threads): if tid in self.pid_names: name = self.pid_names[tid] break @@ -410,7 +413,7 @@ def transform_dtrace(args): continue parts = line.split('\t') if len(parts) < 4: - print "Warning: weird line:", line + print("Warning: weird line:", line) continue if parts[1] in ['ustack', 'kstack', 'jstack']: reading_stack = [parts[1], int(parts[0], 16), parts[2], parts[3].rstrip(':')] diff --git a/runtool/importers/pprof_importer.py b/runtool/importers/pprof_importer.py index 29a9ef8..13442f6 100644 --- a/runtool/importers/pprof_importer.py +++ b/runtool/importers/pprof_importer.py @@ -10,6 +10,9 @@ from sea_runtool import default_tree, Callbacks, Progress, get_decoders +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) +from python_compatibility_layer import iteritems + IS_AVAILABLE = True profile = None @@ -161,14 +164,14 @@ def preprocess(self, profile): num_labels[key].append(label.numX) if len(labels) > 0: - for key, value in labels.iteritems(): + for (key, value) in iteritems(labels): label = sample.Label.add() label.key = key for val in value: label.value.append(val) if len(num_labels) > 0: - for key, value in num_labels.iteritems(): + for (key, value) in iteritems(num_labels): label = sample.NumLabel.add() label.key = key for val in value: diff --git a/runtool/importers/qnx.py b/runtool/importers/qnx.py index 3f135bb..ddcc7b2 100644 --- a/runtool/importers/qnx.py +++ b/runtool/importers/qnx.py @@ -1,6 +1,10 @@ +from __future__ import print_function import os +import sys from sea_runtool import default_tree, Callbacks, Progress, TaskCombiner, ProgressConst, TaskTypes, format_bytes +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) +from python_compatibility_layer import iteritems class Parser: def __init__(self, args, callbacks): @@ -137,7 +141,7 @@ def on_thread(self): elif self.current['name'] == 'THSEND': pass else: - print self.current['name'] + print(self.current['name']) assert False def on_ker_call(self): @@ -173,10 +177,10 @@ def finish(self): self.handle_event() self.current = {} for callback in self.callbacks.callbacks: - for pid, proc_data in self.proc_map.iteritems(): + for (pid, proc_data) in iteritems(self.proc_map): proc_name = proc_data['name'].replace('\\"', '').replace('"', '') callback("metadata_add", {'domain': 'IntelSEAPI', 'str': '__process__', 'pid': pid, 'tid': -1, 'data': proc_name}) - for tid, thread_data in proc_data['threads'].iteritems(): + for (tid, thread_data) in iteritems(proc_data['threads']): thread_name = thread_data['name'].replace('\\"', '').replace('"', '') callback("metadata_add", {'domain': 'IntelSEAPI', 'str': '__thread__', 'pid': pid, 'tid': tid, 'data': '%s (tid %d)' % (thread_name, tid)}) diff --git a/runtool/python_compatibility_layer.py b/runtool/python_compatibility_layer.py new file mode 100644 index 0000000..860e724 --- /dev/null +++ b/runtool/python_compatibility_layer.py @@ -0,0 +1,22 @@ +import sys + +if sys.version_info[0] > 2: + def itervalues(dictionary): + return dictionary.values() + + def iteritems(dictionary): + return dictionary.items() + + xrange = range + basestring = (str, bytes) + unicode = str +else: + def itervalues(dictionary): + return dictionary.itervalues() + + def iteritems(dictionary): + return dictionary.iteritems() + + xrange = xrange + basestring = basestring + unicode = unicode diff --git a/runtool/sea_runtool.py b/runtool/sea_runtool.py index a6da7b1..2922823 100644 --- a/runtool/sea_runtool.py +++ b/runtool/sea_runtool.py @@ -14,10 +14,10 @@ # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # ******************************************************************************************************************************************************************************************************************************************************************************************** - +from __future__ import print_function import os -import imp import sys +import imp import time import shutil import struct @@ -35,6 +35,8 @@ except: pass +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) +from python_compatibility_layer import iteritems, itervalues, xrange try: sys.setdefaultencoding("utf-8") @@ -135,7 +137,7 @@ def get_decoders(): def parse_args(args): import argparse parser = argparse.ArgumentParser(epilog="After this command line add ! followed by command line of your program") - format_choices = ["mfc", "mfp"] + list(get_exporters().iterkeys()) + format_choices = ["mfc", "mfp"] + list(get_exporters()) if sys.platform == 'win32': format_choices.append("etw") elif sys.platform == 'darwin': @@ -167,7 +169,7 @@ def parse_args(args): parser.add_argument("--debug", action="store_true", help='Internal: validation') parser.add_argument("--profile", action="store_true", help='Internal: profile runtool execution') parser.add_argument("--trace_to", help='Internal: trace runtool execution into given folder') - parser.add_argument("--collector", choices=list(get_collectors().iterkeys()) + ['default']) + parser.add_argument("--collector", choices=list(get_collectors()) + ['default']) parser.add_argument("--strip_aliens", action="store_true", help='Filters out all but target processes') parser.add_argument("--system_wide", action="store_true", help='Includes all captured data(can kill viewer)') parser.add_argument("--remove_args", action="store_true", help='Deflates trace by removing arguments') @@ -184,11 +186,11 @@ def parse_args(args): parsed_args = parser.parse_args(args[:separator]) if parsed_args.input: parser.print_help() - print "Error: Input argument (-i) contradicts launch mode" + print("Error: Input argument (-i) contradicts launch mode") sys.exit(-1) if not parsed_args.output: parser.print_help() - print "Error: No output (-o) given in launch mode" + print("Error: No output (-o) given in launch mode") sys.exit(-1) victim = args[separator + 1:] victim[-1] = victim[-1].strip() # removal of trailing '\r' - when launched from .sh @@ -204,7 +206,7 @@ def parse_args(args): if not parsed_args.output: parsed_args.output = parsed_args.input return parsed_args, None - print "--input argument is required for transformation mode." + print("--input argument is required for transformation mode.") parser.print_help() sys.exit(-1) @@ -244,7 +246,7 @@ def main(): output = join_gt_output(args, output) replacement = ('/', '\\') if sys.platform == 'win32' else ('\\', '/') for path in output: - print os.path.abspath(path).replace(*replacement) + print(os.path.abspath(path).replace(*replacement)) def os_lib_ext(): @@ -276,19 +278,19 @@ def __init__(self, args): def execute(self, cmd): command = '%s "%s"' % (self.execute_prefix, cmd) if self.args.verbose: - print "command:", command + print("command:", command) out, err = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() if err: - print "Error:", err + print("Error:", err) raise Exception(err) return out def copy(self, source, target): if self.args.verbose: - print "%s %s %s" % (self.copy_prefix, source, target) + print("%s %s %s" % (self.copy_prefix, source, target)) out, err = subprocess.Popen("%s %s %s" % (self.copy_prefix, source, target), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() if err: - print "Error:", err + print("Error:", err) raise Exception(err) return out @@ -296,9 +298,9 @@ def copy(self, source, target): def launch_remote(args, victim): remote = Remote(args) - print 'Getting target uname...', + print('Getting target uname...', end=' ') unix = remote.execute("uname") - print ':', unix + print(':', unix) if 'darwin' in unix.lower(): search = os.path.join(args.bindir, '*IntelSEAPI.dylib') files = glob(search) @@ -311,20 +313,20 @@ def launch_remote(args, victim): load_lib = 'INTEL_LIBITTNOTIFY' + bits target = '/tmp/' + os.path.basename(files[0]) - print 'Copying corresponding library...' - print remote.copy(files[0], '%s:%s' % (args.ssh, target)) + print('Copying corresponding library...') + print(remote.copy(files[0], '%s:%s' % (args.ssh, target))) - print 'Making temp dir...', + print('Making temp dir...', end=' ') trace = remote.execute('mktemp -d' + (' -t SEA_XXX' if 'darwin' in unix.lower() else '')).strip() - print ':', trace + print(':', trace) output = args.output args.output = trace + '/nop' - print 'Starting ftrace...' + print('Starting ftrace...') ftrace = get_collectors()['ftrace'](args, remote) - print 'Executing:', ' '.join(victim), '...' + print('Executing:', ' '.join(victim), '...') variables = dict() variables[load_lib] = target @@ -333,23 +335,23 @@ def launch_remote(args, victim): variables['INTEL_SEA_VERBOSE'] = '1' if args.ring: variables['INTEL_SEA_RING'] = str(args.ring) - suffix = ' '.join(['%s=%s' % pair for pair in variables.iteritems()]) - print remote.execute(suffix + ' '.join(victim)) + suffix = ' '.join(['%s=%s' % pair for pair in iteritems(variables)]) + print(remote.execute(suffix + ' '.join(victim))) if ftrace: args.trace = ftrace.stop() args.output = output local_tmp = tempfile.mkdtemp() - print 'Copying result:' - print remote.copy('-r %s:%s' % (args.ssh, trace), local_tmp) + print('Copying result:') + print(remote.copy('-r %s:%s' % (args.ssh, trace), local_tmp)) - print 'Removing temp dir...' + print('Removing temp dir...') remote.execute('rm -r %s' % trace) - print 'Transformation...' + print('Transformation...') files = glob(os.path.join(local_tmp, '*', 'pid-*')) if not files: - print "Error: Nothing captured" + print("Error: Nothing captured") sys.exit(-1) args.input = files[0] if args.trace: @@ -357,7 +359,7 @@ def launch_remote(args, victim): output = transform(args) output = join_gt_output(args, output) shutil.rmtree(local_tmp) - print "result:", output + print("result:", output) def launch(args, victim): @@ -372,11 +374,11 @@ def launch(args, victim): search = os.path.sep.join([args.bindir, "*IntelSEAPI" + bits + os_lib_ext()]) files = glob(search) if not len(files): - print "Warning: didn't find any files for:", search + print("Warning: didn't find any files for:", search) continue paths.append((bits, files[0])) if not len(paths): - print "Error: didn't find any *IntelSEAPI%s files. Please check that you run from bin directory, or use --bindir." % os_lib_ext() + print("Error: didn't find any *IntelSEAPI%s files. Please check that you run from bin directory, or use --bindir." % os_lib_ext()) sys.exit(-1) if macosx: env["DYLD_INSERT_LIBRARIES"] = paths[0][1] @@ -389,7 +391,7 @@ def launch(args, victim): env["INTEL_LIBITTNOTIFY64"] = paths['64'] env["INTEL_JIT_PROFILER64"] = paths['64'] - env["INTEL_SEA_FEATURES"] = os.environ['INTEL_SEA_FEATURES'] if os.environ.has_key('INTEL_SEA_FEATURES') else "" + env["INTEL_SEA_FEATURES"] = os.environ['INTEL_SEA_FEATURES'] if 'INTEL_SEA_FEATURES' in os.environ else "" env["INTEL_SEA_FEATURES"] += (" " + str(args.format)) if args.format else "" env["INTEL_SEA_FEATURES"] += " stacks" if args.stacks else "" env["INTEL_SEA_FEATURES"] += " memcount|memstat" if args.memory else "" @@ -408,14 +410,14 @@ def launch(args, victim): env['VK_LAYER_PATH'] = (os.environ['VK_LAYER_PATH'] + os.pathsep + args.bindir) if 'VK_LAYER_PATH' in os.environ else args.bindir if args.dry: - for key, val in env.iteritems(): + for (key, val) in iteritems(env): if val: - print key + "=" + val + print(key + "=" + val) return if args.verbose: - print "Running:", victim - print "Environment:", str(env) + print("Running:", victim) + print("Environment:", str(env)) os.environ.update(env) @@ -462,20 +464,20 @@ def launch(args, victim): log = open(os.path.join((tempfile.gettempdir() if sys.platform == 'win32' else '/tmp'), datetime.now().strftime('sea_%H_%M_%S__%d_%m_%Y.log')), 'a') tracer.set_output(log) - print "For execution details see:", log.name + print("For execution details see:", log.name) tracer = tracer(args) # turning class into instance if mdapi: mdapi.start() if sys.platform != 'win32': # collector start may be long, so we freeze victim during this time - print "PID:", proc.pid + print("PID:", proc.pid) import signal os.kill(proc.pid, signal.SIGCONT) try: - print "Waiting application to exit..." + print("Waiting application to exit...") proc.wait() - print "Stopping collectors..." + print("Stopping collectors...") except KeyboardInterrupt: os.kill(proc.pid, signal.SIGABRT) @@ -502,7 +504,7 @@ def transform_all(args): traces = set(args.trace if args.trace else []) importers = get_importers() - for ext in importers.iterkeys(): + for ext in importers: for file in glob(os.path.join(args.input, '*.' + ext)): if not any(sub in file for sub in ['.etl.', '.dtrace.', 'merged.']): traces.add(file) @@ -537,10 +539,10 @@ def transform_all(args): replacement = ('/', '\\') if sys.platform == 'win32' else ('\\', '/') for path in output: - print os.path.abspath(path).replace(*replacement), format_bytes(os.path.getsize(path)) + print(os.path.abspath(path).replace(*replacement), format_bytes(os.path.getsize(path))) save_domains() - print "limits:", Callbacks.get_globals()['limits'] + print("limits:", Callbacks.get_globals()['limits']) return output @@ -651,12 +653,12 @@ def parse_jit(tree, path): def sea_reader(args): # reads the structure of .sea format folder into dictionary folder = args.input if not os.path.exists(folder): - print """Error: folder "%s" doesn't exist""" % folder + print("""Error: folder "%s" doesn't exist""" % folder) tree = default_tree(args) pos = folder.rfind("-") # pid of the process is encoded right in the name of the folder tree["pid"] = int(folder[pos + 1:]) folder = folder.replace("\\", "/").rstrip("/") - toplevel = os.walk(folder).next() + toplevel = next(os.walk(folder)) for filename in toplevel[2]: with open("/".join([folder, filename]), "r") as file: if filename.endswith(".str"): # each string_handle_create writes separate file, name is the handle, content is the value @@ -669,7 +671,7 @@ def sea_reader(args): # reads the structure of .sea format folder into dictiona tree["domains"][domain] = {"files": []} for file in os.walk("/".join([folder, domain])).next()[2]: # each thread of this domain has separate file with data if not file.endswith(".sea"): - print "Warning: weird file found:", file + print("Warning: weird file found:", file) continue filename = file[:-4] @@ -699,7 +701,7 @@ def __init__(self, total, steps, message=""): self.shown_steps = 0 self.message = message if verbose_progress: - print message, "[", + print(message, "[", end=' ') sys.stdout.flush() def __enter__(self): @@ -715,7 +717,7 @@ def show_progress(self, show_steps): if self.shown_steps < show_steps: if verbose_progress: for i in range(show_steps - self.shown_steps): - print ".", + print(".", end=' ') sys.stdout.flush() self.shown_steps = show_steps @@ -724,7 +726,7 @@ def __exit__(self, type, value, traceback): g_progress_interceptor(self.message, self.total, self.total) self.show_progress(self.steps) if verbose_progress: - print "]" + print("]") return False @staticmethod @@ -769,7 +771,7 @@ def read_chunk_header(file): def transform(args): if args.verbose: - print "Transform:", str(args) + print("Transform:", str(args)) tree = sea_reader(args) # parse the structure if args.cuts and args.cuts == ['all'] or not args.cuts: return transform2(args, tree) @@ -780,7 +782,7 @@ def transform(args): if args.cuts and current_cut not in args.cuts: continue args.output = (output + "!" + current_cut) if current_cut else output - print "Cut #", current_cut if current_cut else "" + print("Cut #", current_cut if current_cut else "") def skip_fn(path): filename = os.path.split(path)[1] @@ -829,7 +831,7 @@ def finish(self): def check_leaks(self): if not self.prev_memory: return - args = dict([(size, int(count)) for size, count in self.memory.iteritems() if size and int(count)]) + args = dict([(size, int(count)) for (size, count) in iteritems(self.memory) if size and int(count)]) self.process(self.prev_memory['pid']).thread(-1).marker('process', 'Memory Leaks (block size, count)').set(self.prev_memory['time'], args) @@ -841,9 +843,9 @@ def handle_leftovers(self): begin = end.copy() begin['time'] = self.time_bounds[0] self.complete_task(TaskTypes[begin['type']].split("_")[0], begin, end) - for domain, threads in self.domains.iteritems(): - for tid, records in threads['tasks'].iteritems(): - for id, per_id_records in records['byid'].iteritems(): + for (domain, threads) in iteritems(self.domains): + for (tid, records) in iteritems(threads['tasks']): + for (id, per_id_records) in iteritems(records['byid']): for begin in per_id_records: end = begin.copy() end['time'] = self.time_bounds[1] @@ -869,7 +871,7 @@ def get_task(id): tasks = get_tasks(id) if not tasks: # they can be stacked tasks = get_tasks(None) - if not tasks or not tasks[-1].has_key('id') or tasks[-1]['id'] != id: + if not tasks or 'id' not in tasks[-1] or tasks[-1]['id'] != id: return None else: tasks = get_tasks(None) @@ -879,21 +881,21 @@ def get_task(id): return None def find_task(id): - for thread_stacks in domain['tasks'].itervalues(): # look in all threads - if thread_stacks['byid'].has_key(id) and thread_stacks['byid'][id]: + for thread_stacks in itervalues(domain['tasks']): # look in all threads + if id in thread_stacks['byid'] and thread_stacks['byid'][id]: return thread_stacks['byid'][id][-1] else: for item in thread_stacks['stack']: - if item.has_key('id') and item['id'] == id: + if 'id' in item and item['id'] == id: return item def get_stack(tid): stack = [] - for domain in self.domains.itervalues(): - if not domain['tasks'].has_key(tid): + for domain in itervalues(self.domains): + if tid not in domain['tasks']: continue thread = domain['tasks'][tid] - for byid in thread['byid'].itervalues(): + for byid in itervalues(thread['byid']): stack += byid if thread['stack']: stack += thread['stack'] @@ -911,7 +913,7 @@ def get_last_index(tasks, type): return None if fn == "task_begin" or fn == "task_begin_overlapped": - if not (data.has_key('str') or data.has_key('pointer')): + if not ('str' in data or 'pointer' in data): data['str'] = 'Unknown' self.time_bounds[0] = min(self.time_bounds[0], data['time']) if 'delta' in data and data['delta']: # turbo mode, only begins are written @@ -933,12 +935,12 @@ def get_last_index(tasks, type): self.complete_task('task', item, data) else: assert (self.tree["ring_buffer"] or self.tree['cuts']) - if data.has_key('str'): # nothing to show without name + if 'str' in data: # nothing to show without name self.no_begin.append(data) elif fn == "frame_begin": - get_tasks(data['id'] if data.has_key('id') else None).append(data) + get_tasks(data['id'] if 'id' in data else None).append(data) elif fn == "frame_end": - frames = get_tasks(data['id'] if data.has_key('id') else None) + frames = get_tasks(data['id'] if 'id' in data else None) index = get_last_index(frames, 7) if index is not None: item = frames.pop(index) @@ -957,11 +959,11 @@ def get_last_index(tasks, type): else: # global metadata self.global_metadata(data) elif fn == "object_snapshot": - if data.has_key('args'): + if 'args' in data: args = data['args'].copy() else: args = {'snapshot': {}} - if data.has_key('data'): + if 'data' in data: state = data['data'] for pair in state.split(","): (key, value) = tuple(pair.split("=")) @@ -970,7 +972,7 @@ def get_last_index(tasks, type): self.complete_task(fn, data, data) elif fn in ["marker", "counter", "object_new", "object_delete"]: if fn == "marker" and data['data'] == 'task': - markers = get_tasks("marker_" + (data['id'] if data.has_key('id') else "")) + markers = get_tasks("marker_" + (data['id'] if 'id' in data else "")) if markers: item = markers.pop() item['type'] = 7 # frame_begin @@ -994,7 +996,7 @@ def get_last_index(tasks, type): if data['domain'] == 'Memory': size = int(data['str'].split('<')[1].split('>')[0]) prev_value = 0. - if self.memory.has_key(size): + if size in self.memory: prev_value = self.memory[size] delta = data['delta'] - prev_value # data['delta'] has current value of the counter self.total_memory += delta * size @@ -1024,14 +1026,14 @@ def get_last_index(tasks, type): self.compress_counter(cache, data) values.append(data) return - if data.has_key('id') and thread['args'].has_key(data['id']): + if 'id' in data and data['id'] in thread['args']: data['args'] = thread['args'][data['id']] del thread['args'][data['id']] self.complete_task(fn, data, data) elif fn == "relation": self.relation( data, - get_task(data['id'] if data.has_key('id') else None), + get_task(data['id'] if 'id' in data else None), get_task(data['parent']) or find_task(data['parent']) ) else: @@ -1056,7 +1058,7 @@ def handle_special(self, kind, begin, end): return False def flush_counters(self, domain, data): - for name, counter in domain['counters'].iteritems(): + for (name, counter) in iteritems(domain['counters']): common_data = data.copy() common_data['time'] = counter['begin'] + (counter['end'] - counter['begin']) / 2 common_data['str'] = name @@ -1064,9 +1066,9 @@ def flush_counters(self, domain, data): self.complete_task('counter', common_data, common_data) def flush_compressed_counters(self): - for pid, threads in self.memcounters.iteritems(): - for tid, counters in threads.iteritems(): - for name, counter in counters.iteritems(): + for (pid, threads) in iteritems(self.memcounters): + for (tid, counters) in iteritems(threads): + for (name, counter) in iteritems(counters): self.compress_counter(counter, None) @@ -1085,7 +1087,7 @@ def __init__(self, args, tree): self.on_finalize_callbacks = [] if hasattr(self.args, 'user_input') and os.path.isdir(self.args.user_input): tid_map = build_tid_map(self.args, self.args.user_input) - self.allowed_pids = set(tid_map.itervalues()) + self.allowed_pids = set(tid_map.values()) for fmt in args.format: self.callbacks.append(get_exporters()[fmt](args, tree)) @@ -1139,9 +1141,9 @@ def __exit__(self, type, value, traceback): def finalize(self): for decoder in self.sea_decoders: decoder.finalize() - for kind, data in self.tasks_from_samples.iteritems(): - for pid, threads in data.iteritems(): - for tid, tasks in threads.iteritems(): + for (kind, data) in iteritems(self.tasks_from_samples): + for (pid, threads) in iteritems(data): + for (tid, tasks) in iteritems(threads): self.handle_stack(pid, tid, tasks.last_stack_time + TIME_SHIFT_FOR_GT * len(tasks) + 1, [], kind) for function in self.on_finalize_callbacks: function(self) @@ -1277,7 +1279,7 @@ def auto_break_overlapped(self, call_data, begin): to_remove = [] del self.overlapped[id] # the task has ended, removing it from the pipeline time_shift = 0 - for begin_data in sorted(self.overlapped.itervalues(), key=lambda data: data['realtime']): # finish all and start again to form melting task queue + for begin_data in sorted(itervalues(self.overlapped), key=lambda data: data['realtime']): # finish all and start again to form melting task queue time_shift += 1 # making sure the order of tasks on timeline, probably has to be done in Chrome code rather end_data = begin_data.copy() # the end of previous part of task is also here end_data['time'] = call_data['time'] - time_shift # new begin for every task is here @@ -1665,7 +1667,7 @@ def handle_stack(self, pid, tid, time, stack, kind='sampling'): # Find affected tasks, those to the right of most recent of removed. These affected are to be 'restarted' if to_remove: leftmost_time = min(tasks[ptr]['begin'] for ptr in to_remove) - for ptr, task in tasks.iteritems(): + for (ptr, task) in iteritems(tasks): if task['begin'] > leftmost_time and ptr not in to_remove: to_remove.append(ptr) to_add.append(task.copy()) @@ -1730,7 +1732,7 @@ def handle_stack(self, pid, tid, time, stack, kind='sampling'): # task.end(%timestamp%) -class FileWrapper: +class FileWrapper(object): def __init__(self, path, args, tree, domain, tid): self.args = args self.tree = tree @@ -1743,7 +1745,7 @@ def __init__(self, path, args, tree, domain, tid): def __del__(self): self.file.close() - def next(self): + def __next__(self): self.record = self.read() def get_record(self): @@ -1818,7 +1820,7 @@ def transform2(args, tree, skip_fn=None): return callbacks.get_result() wrappers = {} - for domain, content in tree["domains"].iteritems(): # go thru domains + for (domain, content) in iteritems(tree["domains"]): # go thru domains for tid, path in content["files"]: # go thru per thread files parts = split_filename(path) @@ -1826,7 +1828,7 @@ def transform2(args, tree, skip_fn=None): if file_wrapper.get_record(): # record is None if something wrong with file reading wrappers.setdefault(parts['dir'] + '/' + parts['name'], []).append(file_wrapper) - for unordered in wrappers.itervalues(): # chain wrappers by time + for unordered in itervalues(wrappers): # chain wrappers by time ordered = sorted(unordered, key=lambda wrapper: wrapper.get_record()['time']) prev = None for wrapper in ordered: @@ -1836,7 +1838,7 @@ def transform2(args, tree, skip_fn=None): files = [] (left_limit, right_limit) = callbacks.get_limits() - for unordered in wrappers.itervalues(): + for unordered in itervalues(wrappers): for wrapper in unordered: if right_limit and wrapper.get_record()['time'] > right_limit: continue @@ -1848,7 +1850,7 @@ def transform2(args, tree, skip_fn=None): files.append(wrapper) if args.verbose: - print path + print(path) progress = DummyWith() else: size = sum([file.get_size() for file in files]) @@ -1868,24 +1870,24 @@ def transform2(args, tree, skip_fn=None): earliest = file if not record: # all finished break - earliest.next() + next(earliest) if args.verbose: - print "%d\t%s\t%s" % (count, TaskTypes[record['type']], record) + print("%d\t%s\t%s" % (count, TaskTypes[record['type']], record)) elif count % ProgressConst == 0: progress.tick(sum([file.get_pos() for file in files])) callbacks.on_event(TaskTypes[record['type']], record) count += 1 callbacks("metadata_add", {'domain': 'IntelSEAPI', 'str': '__process__', 'pid': tree["pid"], 'tid': -1, 'delta': -1}) - for pid, name in tree['groups'].iteritems(): + for (pid, name) in iteritems(tree['groups']): callbacks("metadata_add", {'domain': 'IntelSEAPI', 'str': '__process__', 'pid': int(pid), 'tid': -1, 'delta': -1, 'data': name}) return callbacks.get_result() def get_module_by_ptr(tree, ptr): - keys = list(tree['modules'].iterkeys()) + keys = list(tree['modules']) keys.sort() # looking for first bigger the address, previous is the module we search for item = keys[0] for key in keys[1:]: @@ -1951,12 +1953,12 @@ def resolve_cmd(args, path, load_addr, ptr, cache={}): err = traceback.format_exc() import gc gc.collect() - print "gc.collect()" + print("gc.collect()") except: err = traceback.format_exc() if err: - print cmd - print err + print(cmd) + print(err) return '' if sys.platform == 'win32': @@ -2050,7 +2052,7 @@ def resolve_stack(args, tree, data): def attachme(): - print "Attach me!" + print("Attach me!") while not sys.gettrace(): pass import time @@ -2102,7 +2104,7 @@ def D3D11_DEPTH_STENCIL_DESC(data): def represent_data(tree, name, data): - for key in struct_decoders.iterkeys(): + for key in struct_decoders: if key in name: return struct_decoders[key](data) if all((31 < ord(chr) < 128) or (chr in ['\t', '\r', '\n']) for chr in data): # string we will show as string @@ -2215,7 +2217,7 @@ def save_domains(): filter = subst_env_vars(filter) with open(filter, 'w') as file: - for key, value in is_domain_enabled.domains.iteritems(): + for (key, value) in iteritems(is_domain_enabled.domains): file.write('%s%s\n' % ('#' if not value else '', key)) @@ -2257,8 +2259,8 @@ def complete_task(self, type, begin, end): if begin['type'] == 0: # non-overlapped only # We expect parents to be reported in the end order (when the end time becomes known) orphans = self.per_thread.setdefault(begin['tid'], []) - left_index = bisect_right(orphans, begin['time'], lambda (b,_): b['time']) # first possible child - right_index = bisect_right(orphans, end['time'], lambda (b,_): b['time']) - 1 # last possible child + left_index = bisect_right(orphans, begin['time'], lambda b__: b__[0]['time']) # first possible child + right_index = bisect_right(orphans, end['time'], lambda b__: b__[0]['time']) - 1 # last possible child for i in xrange(right_index, left_index - 1, -1): # right to left to be able deleting from array orphan = orphans[i] if orphan[1]['time'] < end['time']: # a parent is found! @@ -2289,14 +2291,14 @@ def complete_task(self, type, begin, end): elif 'delete' in type: object['destroy'] = begin['time'] else: - print "Unhandled:", type + print("Unhandled:", type) def finish(self): - for tid, orphans in self.per_thread.iteritems(): + for (tid, orphans) in iteritems(self.per_thread): last_time = 0 for orphan in orphans: if (orphan[1]['time'] < last_time): - print "FIXME: orphan[1]['time'] < last_time" + print("FIXME: orphan[1]['time'] < last_time") last_time = orphan[1]['time'] begin = orphan[0] self.add_relation({'label': 'executes', 'from': self.make_id("threads", str(tid)), @@ -2313,7 +2315,7 @@ def relation(self, data, head, tail): self.add_relation({'label': self.get_name_ex(data), 'from': self.make_id(head['domain'], self.get_name_ex(head)), 'to': self.make_id(tail['domain'], self.get_name_ex(tail)), 'color': 'red'}) def add_relation(self, relation): - key = frozenset(relation.iteritems()) + key = frozenset(iteritems(relation)) if key in self.relations: return self.relations[key] = relation @@ -2410,5 +2412,5 @@ def detect_instances(cls, what): start_time = time.time() main() elapsed = time.time() - start_time - print "Time Elapsed:", str(timedelta(seconds=elapsed)).split('.')[0] + print("Time Elapsed:", str(timedelta(seconds=elapsed)).split('.')[0]) From db3ef71a283d6efeb87fad91c253926b18236984 Mon Sep 17 00:00:00 2001 From: Anatoly Myachev Date: Wed, 23 Jan 2019 06:57:33 +0000 Subject: [PATCH 2/3] removed extra sys.path.append() --- runtool/decoders/Adreno.py | 3 --- runtool/decoders/MSNT_SystemTrace.py | 2 -- runtool/exporters/ChromeTracing.py | 3 +-- runtool/exporters/QtCreatorProfiler.py | 4 ---- runtool/exporters/Stat.py | 4 ---- runtool/importers/osx.py | 2 -- runtool/importers/pprof_importer.py | 2 -- runtool/importers/qnx.py | 3 --- runtool/sea_runtool.py | 5 ++--- 9 files changed, 3 insertions(+), 25 deletions(-) diff --git a/runtool/decoders/Adreno.py b/runtool/decoders/Adreno.py index ca5402b..415a359 100644 --- a/runtool/decoders/Adreno.py +++ b/runtool/decoders/Adreno.py @@ -1,7 +1,4 @@ from __future__ import print_function -import os -import sys -sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) from python_compatibility_layer import xrange TRACK_INDEX, TRACK_NAME = -1, 'GPU' diff --git a/runtool/decoders/MSNT_SystemTrace.py b/runtool/decoders/MSNT_SystemTrace.py index 1b272b9..26301ea 100644 --- a/runtool/decoders/MSNT_SystemTrace.py +++ b/runtool/decoders/MSNT_SystemTrace.py @@ -1,8 +1,6 @@ from __future__ import print_function -import os import sys import socket -sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) from python_compatibility_layer import basestring, iteritems def resolve_host(ip): diff --git a/runtool/exporters/ChromeTracing.py b/runtool/exporters/ChromeTracing.py index 3b8807d..7e5c981 100644 --- a/runtool/exporters/ChromeTracing.py +++ b/runtool/exporters/ChromeTracing.py @@ -11,12 +11,11 @@ import subprocess from datetime import datetime from sea_runtool import TaskCombiner, Progress, resolve_stack, to_hex, ProgressConst, get_importers +from python_compatibility_layer import unicode, iteritems, itervalues MAX_GT_SIZE = 50 * 1024 * 1024 GT_FLOAT_TIME = True -sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) -from python_compatibility_layer import unicode, iteritems, itervalues class GoogleTrace(TaskCombiner): diff --git a/runtool/exporters/QtCreatorProfiler.py b/runtool/exporters/QtCreatorProfiler.py index 260f9e7..5f1c94b 100644 --- a/runtool/exporters/QtCreatorProfiler.py +++ b/runtool/exporters/QtCreatorProfiler.py @@ -1,10 +1,6 @@ from __future__ import print_function import cgi from sea_runtool import TaskCombiner, get_name, to_hex - -import os -import sys -sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) from python_compatibility_layer import iteritems #https://github.com/danimo/qt-creator/blob/master/src/plugins/qmlprofiler/qmlprofilertracefile.cpp #https://github.com/danimo/qt-creator/blob/master/src/plugins/qmlprofiler/qv8profilerdatamodel.cpp diff --git a/runtool/exporters/Stat.py b/runtool/exporters/Stat.py index 24a2e88..1326fae 100644 --- a/runtool/exporters/Stat.py +++ b/runtool/exporters/Stat.py @@ -1,9 +1,5 @@ import cgi from sea_runtool import GraphCombiner - -import os -import sys -sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) from python_compatibility_layer import iteritems class Stat(GraphCombiner): diff --git a/runtool/importers/osx.py b/runtool/importers/osx.py index d2b5908..f0cb959 100644 --- a/runtool/importers/osx.py +++ b/runtool/importers/osx.py @@ -6,8 +6,6 @@ from sea_runtool import default_tree, Callbacks, Progress, TaskCombiner, get_exporters, get_decoders, get_importers, format_bytes sys.path.append(os.path.realpath(os.path.dirname(__file__))) # weird CentOS behaviour workaround from etw import GPUQueue - -sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) from python_compatibility_layer import iteritems class DTrace(GPUQueue): diff --git a/runtool/importers/pprof_importer.py b/runtool/importers/pprof_importer.py index 13442f6..35566f6 100644 --- a/runtool/importers/pprof_importer.py +++ b/runtool/importers/pprof_importer.py @@ -9,8 +9,6 @@ import imp from sea_runtool import default_tree, Callbacks, Progress, get_decoders - -sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) from python_compatibility_layer import iteritems IS_AVAILABLE = True diff --git a/runtool/importers/qnx.py b/runtool/importers/qnx.py index ddcc7b2..b75b5f4 100644 --- a/runtool/importers/qnx.py +++ b/runtool/importers/qnx.py @@ -1,9 +1,6 @@ from __future__ import print_function import os -import sys from sea_runtool import default_tree, Callbacks, Progress, TaskCombiner, ProgressConst, TaskTypes, format_bytes - -sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) from python_compatibility_layer import iteritems class Parser: diff --git a/runtool/sea_runtool.py b/runtool/sea_runtool.py index 2922823..d6a78d7 100644 --- a/runtool/sea_runtool.py +++ b/runtool/sea_runtool.py @@ -16,8 +16,8 @@ # ******************************************************************************************************************************************************************************************************************************************************************************************** from __future__ import print_function import os -import sys import imp +import sys import time import shutil import struct @@ -28,6 +28,7 @@ import traceback import subprocess from glob import glob +from python_compatibility_layer import iteritems, itervalues, xrange sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), 'decoders'))) try: @@ -35,8 +36,6 @@ except: pass -sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) -from python_compatibility_layer import iteritems, itervalues, xrange try: sys.setdefaultencoding("utf-8") From ecc758f46487f28299034d0001f5d5ffb808a4e2 Mon Sep 17 00:00:00 2001 From: Anatoly Myachev Date: Wed, 23 Jan 2019 12:52:31 +0000 Subject: [PATCH 3/3] added next method for python3 compatibility --- runtool/exporters/ChromeTracing.py | 2 +- runtool/importers/etw.py | 2 +- runtool/sea_runtool.py | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/runtool/exporters/ChromeTracing.py b/runtool/exporters/ChromeTracing.py index 7e5c981..2c39b70 100644 --- a/runtool/exporters/ChromeTracing.py +++ b/runtool/exporters/ChromeTracing.py @@ -242,7 +242,7 @@ def global_metadata(self, data): self.file.write( '{"name": "process_labels", "ph":"M", "pid":%d, "tid":%s, "args": {"labels":"%s"}},\n' % (data['pid'], data['tid'], ','.join(data['labels'])) ) - if data['tid'] >= 0 and '%d,%d' % (data['pid'], data['tid']) not in self.tree['threads']: # marking the main thread + if data['tid'] >= 0 and ('%d,%d' % (data['pid'], data['tid'])) not in self.tree['threads']: # marking the main thread self.file.write( '{"name": "thread_name", "ph":"M", "pid":%d, "tid":%s, "args": {"name":"%s"}},\n' % (data['pid'], data['tid'], "
") ) diff --git a/runtool/importers/etw.py b/runtool/importers/etw.py index 0a73eeb..89ab87b 100644 --- a/runtool/importers/etw.py +++ b/runtool/importers/etw.py @@ -37,7 +37,7 @@ def iterate_events(self, file): elem.clear() level -= 1 except ET.ParseError as exc: - print("\nError: Bad XML file: %s\n", file) + print("\nError: Bad XML file: %s\n" % file) print(exc.message) def as_dict(self, elem): diff --git a/runtool/sea_runtool.py b/runtool/sea_runtool.py index d6a78d7..553ca03 100644 --- a/runtool/sea_runtool.py +++ b/runtool/sea_runtool.py @@ -1731,7 +1731,7 @@ def handle_stack(self, pid, tid, time, stack, kind='sampling'): # task.end(%timestamp%) -class FileWrapper(object): +class FileWrapper(): def __init__(self, path, args, tree, domain, tid): self.args = args self.tree = tree @@ -1744,6 +1744,9 @@ def __init__(self, path, args, tree, domain, tid): def __del__(self): self.file.close() + def next(self): + self.__next__() + def __next__(self): self.record = self.read()