From 543733a1c59755cac18f807de3cd4d0df80c7cc8 Mon Sep 17 00:00:00 2001 From: "anodelman%mozilla.com" Date: Thu, 14 Jan 2010 16:09:06 +0000 Subject: [PATCH] bug 519568 (Create new Talos suite: metrics for plugins) counter manager patch (win32 + linux), v OO, p=jmaher/jgriffin, r=anodelman git-svn-id: svn://10.0.0.236/trunk@259389 18797224-902f-48f8-a5cc-f745e15eee43 --- .../performance/talos/cmanager_linux.py | 109 +++++++++++------- .../performance/talos/cmanager_win32.py | 62 ++++++++-- .../performance/talos/ffprocess_linux.py | 18 ++- 3 files changed, 139 insertions(+), 50 deletions(-) diff --git a/mozilla/testing/performance/talos/cmanager_linux.py b/mozilla/testing/performance/talos/cmanager_linux.py index 6d07555cafd..6d10e5b617e 100644 --- a/mozilla/testing/performance/talos/cmanager_linux.py +++ b/mozilla/testing/performance/talos/cmanager_linux.py @@ -47,64 +47,79 @@ import threading import ffprocess -def GetPrivateBytes(pid): +def GetPrivateBytes(pids): """Calculate the amount of private, writeable memory allocated to a process. This code was adapted from 'pmap.c', part of the procps project. """ - mapfile = '/proc/%s/maps' % pid - maps = open(mapfile) + privateBytes = 0 + for pid in pids: + mapfile = '/proc/%s/maps' % pid + maps = open(mapfile) - private = 0 + private = 0 - for line in maps: - # split up - (range,line) = line.split(" ", 1) + for line in maps: + # split up + (range,line) = line.split(" ", 1) - (start,end) = range.split("-") - flags = line.split(" ", 1)[0] + (start,end) = range.split("-") + flags = line.split(" ", 1)[0] - size = int(end, 16) - int(start, 16) + size = int(end, 16) - int(start, 16) - if flags.find("p") >= 0: - if flags.find("w") >= 0: - private += size + if flags.find("p") >= 0: + if flags.find("w") >= 0: + private += size - return private + privateBytes += private + maps.close() + + return privateBytes -def GetResidentSize(pid): +def GetResidentSize(pids): """Retrieve the current resident memory for a given process""" # for some reason /proc/PID/stat doesn't give accurate information # so we use status instead - file = '/proc/%s/status' % pid - status = open(file) + RSS = 0 + for pid in pids: + file = '/proc/%s/status' % pid - for line in status: - if line.find("VmRSS") >= 0: - return int(line.split()[1]) * 1024 + status = open(file) + for line in status: + if line.find("VmRSS") >= 0: + RSS += int(line.split()[1]) * 1024 + + status.close() + + return RSS def GetCpuTime(pid, sampleTime=1): # return all zeros on this platform as per the 7/18/07 perf meeting return 0 -def GetXRes(pid): +def GetXRes(pids): """Returns the total bytes used by X or raises an error if total bytes is not available""" - try: - cmdline = "xrestop -m 1 -b | grep -A 15 " + str(pid) + " | tr -d \"\n\" | sed \"s/.*total bytes.*: ~//g\"" - pipe = subprocess.Popen(cmdline, shell=True, stdout=-1).stdout - data = pipe.read() - pipe.close() - except: - print "Unexpected error:", sys.exc_info() - raise - try: - float(data) - return data - except: - print "Invalid data, not a float" - raise + XRes = 0 + for pid in pids: + try: + cmdline = "xrestop -m 1 -b | grep -A 15 " + str(pid) + " | tr -d \"\n\" | sed \"s/.*total bytes.*: ~//g\"" + pipe = subprocess.Popen(cmdline, shell=True, stdout=-1).stdout + data = pipe.read() + pipe.close() + except: + print "Unexpected error:", sys.exc_info() + raise + try: + float(data) + XRes += data + except: + print "Invalid data, not a float" + raise + + return XRes counterDict = {} counterDict["Private Bytes"] = GetPrivateBytes @@ -123,7 +138,7 @@ class CounterManager(threading.Thread): pollInterval = .25 - def __init__(self, process, counters=None): + def __init__(self, process, counters=None, childProcess="mozilla-runtime"): """Args: counters: A list of counters to monitor. Any counters whose name does not match a key in 'counterDict' will be ignored. @@ -131,8 +146,10 @@ class CounterManager(threading.Thread): self.allCounters = {} self.registeredCounters = {} self.process = process + self.childProcess = childProcess self.runThread = False - self.pid = -1 + self.primaryPid = -1 + self.pidList = [] self._loadCounters() self.registerCounters(counters) @@ -190,6 +207,17 @@ class CounterManager(threading.Thread): """Returns the process currently associated with this CounterManager""" return self.process + def updatePidList(self): + """Updates the list of PIDs we're interested in""" + try: + self.pidList = [self.primaryPid] + childPids = ffprocess.GetPidsByName(self.childProcess) + for pid in childPids: + os.stat('/proc/%s' % pid) + self.pidList.append(pid) + except: + print "WARNING: problem updating child PID's" + def startMonitor(self): """Starts the monitoring process. Throws an exception if any error occurs @@ -197,8 +225,8 @@ class CounterManager(threading.Thread): # TODO: make this function less ugly try: # the last process is the useful one - self.pid = ffprocess.GetPidsByName(self.process)[-1] - os.stat('/proc/%s' % self.pid) + self.primaryPid = ffprocess.GetPidsByName(self.process)[-1] + os.stat('/proc/%s' % self.primaryPid) self.runThread = True self.start() except: @@ -215,13 +243,14 @@ class CounterManager(threading.Thread): until stopMonitor() is called """ while self.runThread: + self.updatePidList() for counter in self.registeredCounters.keys(): # counter[0] is a function that gets the current value for # a counter # counter[1] is a list of recorded values try: self.registeredCounters[counter][1].append( - self.registeredCounters[counter][0](self.pid)) + self.registeredCounters[counter][0](self.pidList)) except: # if a counter throws an exception, remove it #self.unregisterCounters([counter]) diff --git a/mozilla/testing/performance/talos/cmanager_win32.py b/mozilla/testing/performance/talos/cmanager_win32.py index 47424974fcc..aeba10d2fae 100644 --- a/mozilla/testing/performance/talos/cmanager_win32.py +++ b/mozilla/testing/performance/talos/cmanager_win32.py @@ -42,8 +42,9 @@ import win32pdhutil class CounterManager: - def __init__(self, process, counters=None): + def __init__(self, process, counters=None, childProcess="mozilla-runtime"): self.process = process + self.childProcess = childProcess self.registeredCounters = {} self.registerCounters(counters) # PDH might need to be "refreshed" if it has been queried while the browser @@ -51,6 +52,9 @@ class CounterManager: win32pdh.EnumObjects(None, None, 0, 1) def registerCounters(self, counters): + # self.registeredCounters[counter][0] is a counter query handle + # self.registeredCounters[counter][1] is a list of tuples, the first + # member of which is a counter handle, the second a counter path for counter in counters: self.registeredCounters[counter] = [] @@ -62,16 +66,57 @@ class CounterManager: def getRegisteredCounters(self): return keys(self.registeredCounters) + def updateCounterPathsForChildProcesses(self, counter): + # Create a counter path for each instance of the child process that + # is running. If any of these paths are not in our counter list, + # add them to our counter query and append them to the counter list, + # so that we'll begin tracking their statistics. We don't need to + # worry about removing invalid paths from the list, as getCounterValue() + # will generate a value of 0 for those. + hq = self.registeredCounters[counter][0] + win32pdh.EnumObjects(None, None, 0, 1) + counterListLength = len(self.registeredCounters[counter][1]) + expandedCounterPaths = \ + win32pdh.ExpandCounterPath('\\process(%s*)\\%s' % (self.childProcess, counter)) + for expandedPath in expandedCounterPaths: + alreadyInCounterList = False + for singleCounter in self.registeredCounters[counter][1]: + if expandedPath == singleCounter[1]: + alreadyInCounterList = True + if not alreadyInCounterList: + counterHandle = win32pdh.AddCounter(hq, expandedPath) + self.registeredCounters[counter][1].append((counterHandle, expandedPath)) + if counterListLength != len(self.registeredCounters[counter][1]): + try: + win32pdh.CollectQueryData(hq) + except: + return + def getCounterValue(self, counter): + # Update counter paths, to catch any new child processes that might + # have been launched since last call. Then iterate through all + # counter paths for this counter, and return a combined value. + aggregateValue = 0 + self.updateCounterPathsForChildProcesses(counter) + hq = self.registeredCounters[counter][0] + + # This call can throw an exception in the case where all counter paths + # are invalid (i.e., all the processes have terminated). try: - hq = self.registeredCounters[counter][0] - hc = self.registeredCounters[counter][1] win32pdh.CollectQueryData(hq) - type, val = win32pdh.GetFormattedCounterValue(hc, win32pdh.PDH_FMT_LONG) - return val except: return None + for singleCounter in self.registeredCounters[counter][1]: + hc = singleCounter[0] + try: + type, val = win32pdh.GetFormattedCounterValue(hc, win32pdh.PDH_FMT_LONG) + except: + val = 0 + aggregateValue += val + + return aggregateValue + def getProcess(self): return self.process @@ -80,6 +125,7 @@ class CounterManager: # is closed win32pdh.EnumObjects(None, None, 0, 1) + # Add the counter path for the default process. for counter in self.registeredCounters: path = win32pdh.MakeCounterPath((None, 'process', self.process, None, -1, counter)) @@ -89,10 +135,12 @@ class CounterManager: except: win32pdh.CloseQuery(hq) - self.registeredCounters[counter] = [hq, hc] + self.registeredCounters[counter] = [hq, [(hc, path)]] + self.updateCounterPathsForChildProcesses(counter) def stopMonitor(self): for counter in self.registeredCounters: - win32pdh.RemoveCounter(self.registeredCounters[counter][1]) + for singleCounter in self.registeredCounters[counter][1]: + win32pdh.RemoveCounter(singleCounter[0]) win32pdh.CloseQuery(self.registeredCounters[counter][0]) self.registeredCounters.clear() diff --git a/mozilla/testing/performance/talos/ffprocess_linux.py b/mozilla/testing/performance/talos/ffprocess_linux.py index cf62cf52342..96c0b497ad5 100644 --- a/mozilla/testing/performance/talos/ffprocess_linux.py +++ b/mozilla/testing/performance/talos/ffprocess_linux.py @@ -82,6 +82,12 @@ class LinuxProcess(FFProcess): matchingPids = [] + # A list of process names which should not be in the PID list returned + # by this function. This is needed so that we can reliably build a list + # of browser child processes without including any Talos python + # processes, which can pass the name of the child process as a parameter. + processExclusionList = ['bcontroller.py'] + command = ['ps', 'ax'] handle = subprocess.Popen(command, stdout=subprocess.PIPE) @@ -94,9 +100,15 @@ class LinuxProcess(FFProcess): if line.find('defunct') != -1: continue if line.find(process_name) >= 0: - # splits by whitespace, the first one should be the pid - pid = int(line.split()[0]) - matchingPids.append(pid) + shouldExclude = False + # skip this process if it's in the processExclusionList + for excludedProcess in processExclusionList: + if line.find(excludedProcess) >= 0: + shouldExclude = True + if not shouldExclude: + # splits by whitespace, the first one should be the pid + pid = int(line.split()[0]) + matchingPids.append(pid) return matchingPids