Bommels05/Mozilla
Bommels05/Mozilla
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bug 519568 (Create new Talos suite: metrics for plugins) counter manager patch (win32 + linux), v OO, p=jmaher/jgriffin, r=anodelman

Browse Source 
git-svn-id: svn://10.0.0.236/trunk@259389 18797224-902f-48f8-a5cc-f745e15eee43
This commit is contained in:
anodelman%mozilla.com 2010-01-14 16:09:06 +00:00
parent 360df85825
commit 543733a1c5
3 changed files with 139 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
mozilla/testing/performance/talos/cmanager_linux.py
View File

@ -47,64 +47,79 @@ import threading
import ffprocess
def GetPrivateBytes(pid):
def GetPrivateBytes(pids):
"""Calculate the amount of private, writeable memory allocated to a process.
This code was adapted from 'pmap.c', part of the procps project.
"""
mapfile = '/proc/%s/maps' % pid
maps = open(mapfile)
privateBytes = 0
for pid in pids:
mapfile = '/proc/%s/maps' % pid
maps = open(mapfile)
private = 0
private = 0
for line in maps:
# split up
(range,line) = line.split(" ", 1)
for line in maps:
# split up
(range,line) = line.split(" ", 1)
(start,end) = range.split("-")
flags = line.split(" ", 1)[0]
(start,end) = range.split("-")
flags = line.split(" ", 1)[0]
size = int(end, 16) - int(start, 16)
size = int(end, 16) - int(start, 16)
if flags.find("p") >= 0:
if flags.find("w") >= 0:
private += size
if flags.find("p") >= 0:
if flags.find("w") >= 0:
private += size
return private
privateBytes += private
maps.close()
return privateBytes
def GetResidentSize(pid):
def GetResidentSize(pids):
"""Retrieve the current resident memory for a given process"""
# for some reason /proc/PID/stat doesn't give accurate information
# so we use status instead
file = '/proc/%s/status' % pid
status = open(file)
RSS = 0
for pid in pids:
file = '/proc/%s/status' % pid
for line in status:
if line.find("VmRSS") >= 0:
return int(line.split()[1]) * 1024
status = open(file)
for line in status:
if line.find("VmRSS") >= 0:
RSS += int(line.split()[1]) * 1024
status.close()
return RSS
def GetCpuTime(pid, sampleTime=1):
# return all zeros on this platform as per the 7/18/07 perf meeting
return 0
def GetXRes(pid):
def GetXRes(pids):
"""Returns the total bytes used by X or raises an error if total bytes is not available"""
try:
cmdline = "xrestop -m 1 -b | grep -A 15 " + str(pid) + " | tr -d \"\n\" | sed \"s/.*total bytes.*: ~//g\""
pipe = subprocess.Popen(cmdline, shell=True, stdout=-1).stdout
data = pipe.read()
pipe.close()
except:
print "Unexpected error:", sys.exc_info()
raise
try:
float(data)
return data
except:
print "Invalid data, not a float"
raise
XRes = 0
for pid in pids:
try:
cmdline = "xrestop -m 1 -b | grep -A 15 " + str(pid) + " | tr -d \"\n\" | sed \"s/.*total bytes.*: ~//g\""
pipe = subprocess.Popen(cmdline, shell=True, stdout=-1).stdout
data = pipe.read()
pipe.close()
except:
print "Unexpected error:", sys.exc_info()
raise
try:
float(data)
XRes += data
except:
print "Invalid data, not a float"
raise
return XRes
counterDict = {}
counterDict["Private Bytes"] = GetPrivateBytes
@ -123,7 +138,7 @@ class CounterManager(threading.Thread):
pollInterval = .25
def __init__(self, process, counters=None):
def __init__(self, process, counters=None, childProcess="mozilla-runtime"):
"""Args:
counters: A list of counters to monitor. Any counters whose name does
not match a key in 'counterDict' will be ignored.
@ -131,8 +146,10 @@ class CounterManager(threading.Thread):
self.allCounters = {}
self.registeredCounters = {}
self.process = process
self.childProcess = childProcess
self.runThread = False
self.pid = -1
self.primaryPid = -1
self.pidList = []
self._loadCounters()
self.registerCounters(counters)
@ -190,6 +207,17 @@ class CounterManager(threading.Thread):
"""Returns the process currently associated with this CounterManager"""
return self.process
def updatePidList(self):
"""Updates the list of PIDs we're interested in"""
try:
self.pidList = [self.primaryPid]
childPids = ffprocess.GetPidsByName(self.childProcess)
for pid in childPids:
os.stat('/proc/%s' % pid)
self.pidList.append(pid)
except:
print "WARNING: problem updating child PID's"
def startMonitor(self):
"""Starts the monitoring process.
Throws an exception if any error occurs
@ -197,8 +225,8 @@ class CounterManager(threading.Thread):
# TODO: make this function less ugly
try:
# the last process is the useful one
self.pid = ffprocess.GetPidsByName(self.process)[-1]
os.stat('/proc/%s' % self.pid)
self.primaryPid = ffprocess.GetPidsByName(self.process)[-1]
os.stat('/proc/%s' % self.primaryPid)
self.runThread = True
self.start()
except:
@ -215,13 +243,14 @@ class CounterManager(threading.Thread):
until stopMonitor() is called
"""
while self.runThread:
self.updatePidList()
for counter in self.registeredCounters.keys():
# counter[0] is a function that gets the current value for
# a counter
# counter[1] is a list of recorded values
try:
self.registeredCounters[counter][1].append(
self.registeredCounters[counter][0](self.pid))
self.registeredCounters[counter][0](self.pidList))
except:
# if a counter throws an exception, remove it
#self.unregisterCounters([counter])

62
mozilla/testing/performance/talos/cmanager_win32.py
View File

@ -42,8 +42,9 @@ import win32pdhutil
class CounterManager:
def __init__(self, process, counters=None):
def __init__(self, process, counters=None, childProcess="mozilla-runtime"):
self.process = process
self.childProcess = childProcess
self.registeredCounters = {}
self.registerCounters(counters)
# PDH might need to be "refreshed" if it has been queried while the browser
@ -51,6 +52,9 @@ class CounterManager:
win32pdh.EnumObjects(None, None, 0, 1)
def registerCounters(self, counters):
# self.registeredCounters[counter][0] is a counter query handle
# self.registeredCounters[counter][1] is a list of tuples, the first
# member of which is a counter handle, the second a counter path
for counter in counters:
self.registeredCounters[counter] = []
@ -62,16 +66,57 @@ class CounterManager:
def getRegisteredCounters(self):
return keys(self.registeredCounters)
def updateCounterPathsForChildProcesses(self, counter):
# Create a counter path for each instance of the child process that
# is running. If any of these paths are not in our counter list,
# add them to our counter query and append them to the counter list,
# so that we'll begin tracking their statistics. We don't need to
# worry about removing invalid paths from the list, as getCounterValue()
# will generate a value of 0 for those.
hq = self.registeredCounters[counter][0]
win32pdh.EnumObjects(None, None, 0, 1)
counterListLength = len(self.registeredCounters[counter][1])
expandedCounterPaths = \
win32pdh.ExpandCounterPath('\\process(%s*)\\%s' % (self.childProcess, counter))
for expandedPath in expandedCounterPaths:
alreadyInCounterList = False
for singleCounter in self.registeredCounters[counter][1]:
if expandedPath == singleCounter[1]:
alreadyInCounterList = True
if not alreadyInCounterList:
counterHandle = win32pdh.AddCounter(hq, expandedPath)
self.registeredCounters[counter][1].append((counterHandle, expandedPath))
if counterListLength != len(self.registeredCounters[counter][1]):
try:
win32pdh.CollectQueryData(hq)
except:
return
def getCounterValue(self, counter):
# Update counter paths, to catch any new child processes that might
# have been launched since last call. Then iterate through all
# counter paths for this counter, and return a combined value.
aggregateValue = 0
self.updateCounterPathsForChildProcesses(counter)
hq = self.registeredCounters[counter][0]
# This call can throw an exception in the case where all counter paths
# are invalid (i.e., all the processes have terminated).
try:
hq = self.registeredCounters[counter][0]
hc = self.registeredCounters[counter][1]
win32pdh.CollectQueryData(hq)
type, val = win32pdh.GetFormattedCounterValue(hc, win32pdh.PDH_FMT_LONG)
return val
except:
return None
for singleCounter in self.registeredCounters[counter][1]:
hc = singleCounter[0]
try:
type, val = win32pdh.GetFormattedCounterValue(hc, win32pdh.PDH_FMT_LONG)
except:
val = 0
aggregateValue += val
return aggregateValue
def getProcess(self):
return self.process
@ -80,6 +125,7 @@ class CounterManager:
# is closed
win32pdh.EnumObjects(None, None, 0, 1)
# Add the counter path for the default process.
for counter in self.registeredCounters:
path = win32pdh.MakeCounterPath((None, 'process', self.process,
None, -1, counter))
@ -89,10 +135,12 @@ class CounterManager:
except:
win32pdh.CloseQuery(hq)
self.registeredCounters[counter] = [hq, hc]
self.registeredCounters[counter] = [hq, [(hc, path)]]
self.updateCounterPathsForChildProcesses(counter)
def stopMonitor(self):
for counter in self.registeredCounters:
win32pdh.RemoveCounter(self.registeredCounters[counter][1])
for singleCounter in self.registeredCounters[counter][1]:
win32pdh.RemoveCounter(singleCounter[0])
win32pdh.CloseQuery(self.registeredCounters[counter][0])
self.registeredCounters.clear()

18
mozilla/testing/performance/talos/ffprocess_linux.py
View File

@ -82,6 +82,12 @@ class LinuxProcess(FFProcess):
matchingPids = []
# A list of process names which should not be in the PID list returned
# by this function. This is needed so that we can reliably build a list
# of browser child processes without including any Talos python
# processes, which can pass the name of the child process as a parameter.
processExclusionList = ['bcontroller.py']
command = ['ps', 'ax']
handle = subprocess.Popen(command, stdout=subprocess.PIPE)
@ -94,9 +100,15 @@ class LinuxProcess(FFProcess):
if line.find('defunct') != -1:
continue
if line.find(process_name) >= 0:
# splits by whitespace, the first one should be the pid
pid = int(line.split()[0])
matchingPids.append(pid)
shouldExclude = False
# skip this process if it's in the processExclusionList
for excludedProcess in processExclusionList:
if line.find(excludedProcess) >= 0:
shouldExclude = True
if not shouldExclude:
# splits by whitespace, the first one should be the pid
pid = int(line.split()[0])
matchingPids.append(pid)
return matchingPids