From f9c01263928e62edd737eacd3cc8e41f432aba21 Mon Sep 17 00:00:00 2001 From: "anodelman%mozilla.com" Date: Fri, 5 Feb 2010 16:07:29 +0000 Subject: [PATCH] bug 474478 (set up talos on winmo) new files for windows mobile tests (4)/config changes and new command line parameters (5) p=jmaher a=anodelman git-svn-id: svn://10.0.0.236/trunk@259619 18797224-902f-48f8-a5cc-f745e15eee43 --- .../performance/talos/PerfConfigurator.py | 107 +++- .../testing/performance/talos/bcontroller.py | 40 +- .../performance/talos/devicemanager.py | 605 ++++++++++++++++++ .../performance/talos/ffprocess_winmo.py | 219 +++++++ .../testing/performance/talos/run_tests.py | 23 + .../testing/performance/talos/winmo.config | 225 +++++++ 6 files changed, 1196 insertions(+), 23 deletions(-) create mode 100644 mozilla/testing/performance/talos/devicemanager.py create mode 100644 mozilla/testing/performance/talos/ffprocess_winmo.py create mode 100644 mozilla/testing/performance/talos/winmo.config diff --git a/mozilla/testing/performance/talos/PerfConfigurator.py b/mozilla/testing/performance/talos/PerfConfigurator.py index a308461a1c6..53627b77295 100644 --- a/mozilla/testing/performance/talos/PerfConfigurator.py +++ b/mozilla/testing/performance/talos/PerfConfigurator.py @@ -19,6 +19,7 @@ import re import time from datetime import datetime from os import path +import devicemanager masterIniSubpath = "application.ini" defaultTitle = "qm-pxp01" @@ -26,7 +27,7 @@ defaultTitle = "qm-pxp01" help_message = ''' This is the buildbot performance runner's YAML configurator.bean -USAGE: python PerfConfigurator.py --title title --executablePath path --configFilePath cpath --buildid id --branch branch --testDate date --resultsServer server --resultsLink link --activeTests testlist --branchName branchFullName --fast --symbolsPath path --sampleConfig cfile --browserWait seconds +USAGE: python PerfConfigurator.py --title title --executablePath path --configFilePath cpath --buildid id --branch branch --testDate date --resultsServer server --resultsLink link --activeTests testlist --branchName branchFullName --fast --symbolsPath path --sampleConfig cfile --browserWait seconds --remoteDevice ip_of_device --remotePort port_on_device --webServer webserver_ipaddr --deviceRoot rootdir_on_device example testlist: tp:tsspider:tdhtml:twinopen ''' @@ -50,6 +51,15 @@ class PerfConfigurator: activeTests = '' noChrome = False fast = False + remoteDevice = '' + webServer = '' + deviceRoot = '' + _remote = False + port = '' + + def _setupRemote(self, host, port = 27020): + self.testAgent = devicemanager.DeviceManager(host, port) + self._remote = True def _dumpConfiguration(self): """dump class configuration for convenient pickup or perusal""" @@ -68,6 +78,11 @@ class PerfConfigurator: print " - resultsServer = " + self.resultsServer print " - resultsLink = " + self.resultsLink print " - activeTests = " + self.activeTests + if (self._remote == True): + print " - deviceIP = " + self.remoteDevice + print " - devicePort = " + self.port + print " - webServer = " + self.webServer + print " - deviceRoot = " + self.deviceRoot if self.symbolsPath: print " - symbolsPath = " + self.symbolsPath @@ -75,14 +90,26 @@ class PerfConfigurator: """collect a date string to be used in naming the created config file""" currentDateTime = datetime.now() return currentDateTime.strftime("%Y%m%d_%H%M") + + def _getMasterIniContents(self): + """ Open and read the application.ini on the device under test """ + if (self._remote == True): + localfilename = "remoteapp.ini" + parts = self.exePath.split('/') + remoteFile = '/'.join(parts[0:-1]) + '/' + masterIniSubpath + + self.testAgent.getFile(remoteFile, localfilename) + master = open(localfilename) + else: + master = open(path.join(path.dirname(self.exePath), masterIniSubpath)) + + data = master.read() + master.close() + return data.split('\n') def _getCurrentBuildId(self): - master = open(path.join(path.dirname(self.exePath), masterIniSubpath)) - if not master: - raise Configuration("Unable to open " - + path.join(path.dirname(self.exePath), masterIniSubpath)) - masterContents = master.readlines() - master.close() + masterContents = self._getMasterIniContents() + reBuildid = re.compile('BuildID\s*=\s*(\d{10}|\d{12})') for line in masterContents: match = re.match(reBuildid, line) @@ -138,10 +165,21 @@ class PerfConfigurator: newline += "test_name_extension: _nochrome\n" if self.symbolsPath: newline += '\nsymbols_path: %s\n' % self.symbolsPath + if 'deviceip:' in line: + newline = 'deviceip: %s\n' % self.remoteDevice + if 'webserver:' in line: + newline = 'webserver: %s\n' % self.webServer + if 'deviceroot:' in line: + newline = 'deviceroot: %s\n' % self.deviceRoot + if 'deviceport:' in line: + newline = 'deviceport: %s\n' % self.port + if 'remote:' in line: + newline = 'remote: %s\n' % self._remote if 'buildid:' in line: - newline = 'buildid: ' + buildidString + newline = 'buildid: ' + buildidString + '\n' if 'testbranch' in line: newline = 'branch: ' + self.branch + #only change the results_server if the user has provided one if self.resultsServer and ('results_server' in line): newline = 'results_server: ' + self.resultsServer + '\n' @@ -215,13 +253,24 @@ class PerfConfigurator: self.fast = kwargs['fast'] if 'symbolsPath' in kwargs: self.symbolsPath = kwargs['symbolsPath'] + if 'remoteDevice' in kwargs: + self.remoteDevice = kwargs['remoteDevice'] + if 'webServer' in kwargs: + self.webServer = kwargs['webServer'] + if 'deviceRoot' in kwargs: + self.deviceRoot = kwargs['deviceRoot'] + if 'remotePort' in kwargs: + self.port = kwargs['remotePort'] + + if (self.remoteDevice <> ''): + self._setupRemote(self.remoteDevice, self.port) + self.currentDate = self._getCurrentDateString() if not self.buildid: self.buildid = self._getCurrentBuildId() if not self.outputName: self.outputName = self.currentDate + "_config.yml" - class Configuration(Exception): def __init__(self, msg): self.msg = "ERROR: " + msg @@ -249,14 +298,22 @@ def main(argv=None): noChrome = False fast = False symbolsPath = None - + remoteDevice = '' + remotePort = '' + webServer = 'localhost' + deviceRoot = '' + if argv is None: argv = sys.argv try: try: - opts, args = getopt.getopt(argv[1:], "hvue:c:t:b:o:i:d:s:l:a:n", - ["help", "verbose", "useId", "executablePath=", "configFilePath=", "sampleConfig=", "title=", - "branch=", "output=", "id=", "testDate=", "browserWait=", "resultsServer=", "resultsLink=", "activeTests=", "noChrome", "branchName=", "fast", "symbolsPath="]) + opts, args = getopt.getopt(argv[1:], "hvue:c:t:b:o:i:d:s:l:a:n:r:p:w", + ["help", "verbose", "useId", "executablePath=", + "configFilePath=", "sampleConfig=", "title=", + "branch=", "output=", "id=", "testDate=", "browserWait=", + "resultsServer=", "resultsLink=", "activeTests=", + "noChrome", "branchName=", "fast", "symbolsPath=", + "remoteDevice=", "remotePort=", "webServer=", "deviceRoot="]) except getopt.error, msg: raise Usage(msg) @@ -296,6 +353,14 @@ def main(argv=None): activeTests = value if option in ("-n", "--noChrome"): noChrome = True + if option in ("-r", "--remoteDevice"): + remoteDevice = value + if option in ("-p", "--remotePort"): + remotePort = value + if option in ("-w", "--webServer"): + webServer = value + if option in ("--deviceRoot"): + deviceRoot = value if option in ("--fast"): fast = True if option in ("--symbolsPath",): @@ -305,7 +370,15 @@ def main(argv=None): print >> sys.stderr, sys.argv[0].split("/")[-1] + ": " + str(err.msg) print >> sys.stderr, "\t for help use --help" return 2 - + + #remotePort will default to 27020 and is optional. + #webServer can be used without remoteDevice, but is required when using remoteDevice + if (remoteDevice != '' or deviceRoot != ''): + if (webServer == 'localhost' or deviceRoot == '' or remoteDevice == ''): + print "\nERROR: When running Talos on a remote device, you need to provide a webServer, deviceRoot and optionally a remotePort" + print help_message + return 2 + configurator = PerfConfigurator(title=title, executablePath=exePath, configFilePath=configPath, @@ -323,7 +396,11 @@ def main(argv=None): activeTests=activeTests, noChrome=noChrome, fast=fast, - symbolsPath=symbolsPath) + symbolsPath=symbolsPath, + remoteDevice=remoteDevice, + remotePort=remotePort, + webServer=webServer, + deviceRoot=deviceRoot) try: configurator.writeConfigFile() except Configuration, err: diff --git a/mozilla/testing/performance/talos/bcontroller.py b/mozilla/testing/performance/talos/bcontroller.py index e4aa6a89c73..68485a68c28 100644 --- a/mozilla/testing/performance/talos/bcontroller.py +++ b/mozilla/testing/performance/talos/bcontroller.py @@ -51,6 +51,7 @@ import sys import getopt import stat +import devicemanager if platform.system() == "Linux": platform_type = 'linux_' @@ -68,19 +69,23 @@ elif platform.system() == "Darwin": class BrowserWaiter(threading.Thread): - def __init__(self, command, log, mod): + def __init__(self, command, log, mod, deviceManager = None): self.command = command self.log = log self.mod = mod self.endTime = -1 self.returncode = -1 + self.deviceManager = deviceManager threading.Thread.__init__(self) self.start() def run(self): if self.mod: self.command = self.command + eval(self.mod) - self.returncode = os.system(self.command + " > " + self.log) #blocking call to system + + #blocking call to system + self.returncode = os.system(self.command + " > " + self.log) + self.endTime = int(time.time()*1000) def hasTime(self): @@ -94,7 +99,8 @@ class BrowserWaiter(threading.Thread): class BrowserController: - def __init__(self, command, mod, name, child_process, timeout, log): + def __init__(self, command, mod, name, child_process, + timeout, log, host='', port=27020, root=''): self.command = command self.mod = mod self.process_name = name @@ -102,9 +108,14 @@ class BrowserController: self.browser_wait = timeout self.log = log self.timeout = 600 #no output from the browser in 10 minutes = failure + self.host = host + self.port = port + self.root = root + + self.ffprocess = ffprocess def run(self): - self.bwaiter = BrowserWaiter(self.command, self.log, self.mod) + self.bwaiter = BrowserWaiter(self.command, self.log, self.mod, self.ffprocess) noise = 0 prev_size = 0 while not self.bwaiter.hasTime(): @@ -119,8 +130,12 @@ class BrowserController: results_file.close() return time.sleep(1) - open(self.log, "r").close() #HACK FOR WINDOWS: refresh the file information - size = os.path.getsize(self.log) + try: + open(self.log, "r").close() #HACK FOR WINDOWS: refresh the file information + size = os.path.getsize(self.log) + except: + size = 0 + if size > prev_size: prev_size = size noise = 0 @@ -144,10 +159,13 @@ def main(argv=None): timeout = "" log = "" mod = "" + host = "" + deviceRoot = "" + port = 27020 if argv is None: argv = sys.argv - opts, args = getopt.getopt(argv[1:], "c:t:n:p:l:m:", ["command=", "timeout=", "name=", "child_process=", "log=", "mod="]) + opts, args = getopt.getopt(argv[1:], "c:t:n:p:l:m:h:r:o", ["command=", "timeout=", "name=", "child_process=", "log=", "mod=", "host=", "deviceRoot=", "port="]) # option processing for option, value in opts: @@ -163,9 +181,15 @@ def main(argv=None): log = value if option in ("-m", "--mod"): mod = value + if option in ("-h", "--host"): + host = value + if option in ("-r", "--deviceRoot"): + deviceRoot = value + if option in ("-o", "--port"): + port = value if command and timeout and log: - bcontroller = BrowserController(command, mod, name, child_process, timeout, log) + bcontroller = BrowserController(command, mod, name, child_process, timeout, log, host, port, deviceRoot) bcontroller.run() else: print "\nFAIL: no command\n" diff --git a/mozilla/testing/performance/talos/devicemanager.py b/mozilla/testing/performance/talos/devicemanager.py new file mode 100644 index 00000000000..94cd203c4e8 --- /dev/null +++ b/mozilla/testing/performance/talos/devicemanager.py @@ -0,0 +1,605 @@ +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is Test Automation Framework. +# +# The Initial Developer of the Original Code is Joel Maher. +# +# Portions created by the Initial Developer are Copyright (C) 2___ +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Joel Maher (Original Developer) +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +import socket +import time, datetime +import os +import re +import hashlib +import subprocess +from threading import Thread +import traceback +import sys + + +class myProc(Thread): + def __init__(self, hostip, hostport, cmd, new_line = True, sleeptime = 0): + self.cmdline = cmd + self.newline = new_line + self.sleep = sleeptime + self.host = hostip + self.port = hostport + Thread.__init__(self) + + def run(self): + promptre =re.compile('.*\$\>.$') + data = "" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + except: + return None + + try: + s.connect((self.host, int(self.port))) + except: + s.close() + return None + + try: + s.recv(1024) + except: + s.close() + return None + + for cmd in self.cmdline: + if (cmd == 'quit'): break + if self.newline: cmd += '\r\n' + try: + s.send(cmd) + except: + s.close() + return None + + time.sleep(int(self.sleep)) + + found = False + while (found == False): + try: + temp = s.recv(1024) + except: + s.close() + return None + + lines = temp.split('\n') + for line in lines: + if (promptre.match(line)): + found = True + data += temp + + try: + s.send('quit\r\n') + except: + s.close() + return None + try: + s.close() + except: + return None + return data + +class DeviceManager: + host = '' + port = 0 + debug = 2 + _redo = False + dirSlash = "/" + deviceRoot = '/tests' + tempRoot = os.getcwd() + + def __init__(self, host, port = 27020): + self.host = host + self.port = port + self._sock = None + + def sendCMD(self, cmdline, newline = True, sleep = 0): + promptre = re.compile('.*\$\>.$') + + #TODO: any commands that don't output anything and quit need to match this RE + pushre = re.compile('^push .*$') + data = "" + noQuit = False + + if (self._sock == None): + try: + print "reconnecting socket" + self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + except: + self._redo = True + self._sock = None + return None + + try: + self._sock.connect((self.host, int(self.port))) + self._sock.recv(1024) + except: + self._redo = True + self._sock.close() + self._sock = None + return None + + for cmd in cmdline: + if (cmd == 'quit'): break + if newline: cmd += '\r\n' + + try: + self._sock.send(cmd) + except: + self._redo = True + self._sock.close() + self._sock = None + return None + + if (pushre.match(cmd) or cmd == 'rebt'): + noQuit = True + elif noQuit == False: + time.sleep(int(sleep)) + found = False + while (found == False): + if (self.debug >= 3): print "recv'ing..." + + try: + temp = self._sock.recv(1024) + except: + self._redo = True + self._sock.close() + self._sock = None + return None + lines = temp.split('\n') + for line in lines: + if (promptre.match(line)): + found = True + data += temp + + time.sleep(int(sleep)) + if (noQuit == True): + try: + self._sock.close() + self._sock = None + except: + self._redo = True + self._sock = None + return None + + return data + + + #take a data blob and strip instances of the prompt '$>\x00' + def stripPrompt(self, data): + promptre = re.compile('.*\$\>.*') + retVal = [] + lines = data.split('\n') + for line in lines: + try: + while (promptre.match(line)): + pieces = line.split('\x00') + index = pieces.index("$>") + pieces.pop(index) + line = '\x00'.join(pieces) + except(ValueError): + pass + retVal.append(line) + + return '\n'.join(retVal) + + + def pushFile(self, localname, destname): + if (self.validateFile(destname, localname) == True): + return '' + + self.mkDirs(destname) + if (self.debug >= 2): print "sending: push " + destname + + #sleep 5 seconds / MB + filesize = os.path.getsize(localname) + sleepsize = 1024 * 1024 + sleepTime = int(filesize / sleepsize) * 5 + f = open(localname, 'rb') + data = f.read() + f.close() + retVal = self.sendCMD(['push ' + destname + '\r\n', data], newline = False, sleep = sleepTime) + if (retVal == None): + return None + + if (self.validateFile(destname, localname) == False): + if (self.debug >= 2): print "file did not copy as expected" + return None + + return retVal + + def mkDir(self, name): + return self.sendCMD(['mkdr ' + name, 'quit']) + + + #make directory structure on the device + def mkDirs(self, filename): + parts = filename.split(self.dirSlash) + name = "" + for part in parts: + if (part == parts[-1]): break + if (part != ""): + name += self.dirSlash + part + if (self.mkDir(name) == None): + return None + + + #push localDir from host to remoteDir on the device + def pushDir(self, localDir, remoteDir): + if (self.debug >= 2): print "pushing directory: " + localDir + " to " + remoteDir + for root, dirs, files in os.walk(localDir): + parts = root.split(localDir) + for file in files: + remoteRoot = remoteDir + self.dirSlash + parts[1] + remoteRoot = remoteRoot.replace('/', self.dirSlash) + remoteRoot = remoteRoot.replace('\\\\', '\\') + remoteName = remoteRoot + self.dirSlash + file + if (parts[1] == ""): remoteRoot = remoteDir + if (self.pushFile(os.path.join(root, file), remoteName) == None): + time.sleep(5) + self.removeFile(remoteName) + time.sleep(5) + if (self.pushFile(os.path.join(root, file), remoteName) == None): + return None + return True + + + def dirExists(self, dirname): + match = ".*" + dirname.replace('\\', '\\\\') + "$" +# match = ".*" + dirname.replace('/', '\\\\') + "$" + dirre = re.compile(match) + data = self.sendCMD(['cd ' + dirname, 'cwd', 'quit'], sleep = 1) + if (data == None): + return None + retVal = self.stripPrompt(data) + data = retVal.split('\n') + found = False + for d in data: + if (dirre.match(d)): + found = True + + return found + + # Because we always have / style paths we make this a lot easier with some + # assumptions + def fileExists(self, filepath): + s = filepath.split('/') + containingpath = '/'.join(s[:-1]) + listfiles = self.listFiles(containingpath) + for f in listfiles: + if (f == s[-1]): + return True + return False + + #list files on the device, requires cd to directory first + def listFiles(self, rootdir): + if (self.dirExists(rootdir) == False): + return [] + data = self.sendCMD(['cd ' + rootdir, 'ls', 'quit'], sleep=1) + if (data == None): + return None + retVal = self.stripPrompt(data) + return retVal.split('\n') + + def removeFile(self, filename): + if (self.debug>= 2): print "removing file: " + filename + return self.sendCMD(['rm ' + filename, 'quit']) + + + #does a recursive delete of directory on the device: rm -Rf remoteDir + def removeDir(self, remoteDir): + filelist = self.listFiles(remoteDir) + if (filelist == None): + return None + + #TODO: logic is way too simple and basic, make more robust + isFile = re.compile('^([a-zA-Z0-9_\-\. ]+)\.([a-zA-Z0-9]+)$') + for f in filelist: + if (isFile.match(f)): + if (self.removeFile(remoteDir + self.dirSlash + f) == None): + return None + else: + if (self.removeDir(remoteDir + self.dirSlash + f) == None): + return None + + if (self.debug >= 3): print "removing: " + remoteDir + return self.sendCMD(['rmdr ' + remoteDir, 'rm ' + remoteDir, 'quit'], sleep = 1) + + + def getProcessList(self): + data = self.sendCMD(['ps', 'quit'], sleep = 3) + if (data == None): + return None + + retVal = self.stripPrompt(data) + lines = retVal.split('\n') + files = [] + for line in lines: + if (line.strip() != ''): + pidproc = line.strip().split(' ') + if (len(pidproc) == 2): + files += [[pidproc[0], pidproc[1]]] + + return files + + + def getMemInfo(self): + data = self.sendCMD(['mems', 'quit']) + if (data == None): + return None + retVal = self.stripPrompt(data) + #TODO: this is hardcoded for now + fhandle = open("memlog.txt", 'a') + fhandle.write("\n") + fhandle.write(retVal) + fhandle.close() + + def fireProcess(self, appname): + if (self.debug >= 2): print "FIRE PROC: '" + appname + "'" + self.process = myProc(self.host, self.port, ['exec ' + appname, 'quit']) + self.process.start() + + def launchProcess(self, cmd, outputFile = "process.txt", cwd = ''): + if (outputFile == "process.txt"): + outputFile = self.getDeviceRoot() + self.dirSlash + "process.txt" + + cmdline = subprocess.list2cmdline(cmd) + self.fireProcess(cmdline + " > " + outputFile) + handle = outputFile + + return handle + + def communicate(self, process, timeout = 600): + timed_out = True + if (timeout > 0): + total_time = 0 + while total_time < timeout: + time.sleep(1) + if (not self.poll(process)): + timed_out = False + break + total_time += 1 + + if (timed_out == True): + return None + + return [self.getFile(process, "temp.txt"), None] + + + def poll(self, process): + try: + if (not self.process.isAlive()): + return None + return 1 + except: + return None + return 1 + + + + #iterates process list and returns pid if exists, otherwise '' + def processExist(self, appname): + pid = '' + + pieces = appname.split(self.dirSlash) + app = pieces[-1] + procre = re.compile('.*' + app + '.*') + + procList = self.getProcessList() + if (procList == None): + return None + + for proc in procList: + if (procre.match(proc[1])): + pid = proc[0] + break + return pid + + + def killProcess(self, appname): + pid = "0xFEEDFACE" + + while (pid != ''): + pid = self.processExist(appname) + if (pid == None): + return None + + if (pid != ''): + if (self.debug >= 2): print "found pid, now kill: " + pid + if (self.sendCMD(['kill ' + pid, 'quit']) == None): + return None + + return True + + def getTempDir(self): + promptre = re.compile('.*\$\>\x00.*') + retVal = '' + data = self.sendCMD(['tmpd', 'quit']) + if (data == None): + return None + return self.stripPrompt(data).strip('\n') + + + #copy file from device (remoteFile) to host (localFile) + def getFile(self, remoteFile, localFile = ''): + if localFile == '': + localFile = os.path.join(self.tempRoot, "temp.txt") + + promptre = re.compile('.*\$\>\x00.*') + data = self.sendCMD(['cat ' + remoteFile, 'quit'], sleep = 5) + if (data == None): + return None + retVal = self.stripPrompt(data) + fhandle = open(localFile, 'wb') + fhandle.write(retVal) + fhandle.close() + return retVal + + + #copy directory structure from device (remoteDir) to host (localDir) + def getDirectory(self, remoteDir, localDir): + if (self.debug >= 2): print "getting files in '" + remoteDir + "'" + filelist = self.listFiles(remoteDir) + if (filelist == None): + return None + if (self.debug >= 3): print filelist + if not os.path.exists(localDir): + os.makedirs(localDir) + + #TODO: is this a comprehensive file regex? + isFile = re.compile('^([a-zA-Z0-9_\-\. ]+)\.([a-zA-Z0-9]+)$') + for f in filelist: + if (isFile.match(f)): + if (self.getFile(remoteDir + self.dirSlash + f, os.path.join(localDir, f)) == None): + return None + else: + if (self.getDirectory(remoteDir + self.dirSlash + f, os.path.join(localDir, f)) == None): + return None + + + #true/false check if the two files have the same md5 sum + def validateFile(self, remoteFile, localFile): + remoteHash = self.getRemoteHash(remoteFile) + localHash = self.getLocalHash(localFile) + + if (remoteHash == localHash): + return True + + return False + + + #return the md5 sum of a remote file + def getRemoteHash(self, filename): + filename = filename.replace("/", self.dirSlash) + filename = filename.replace("\\\\", "\\") + data = self.sendCMD(['hash ' + filename, 'quit'], sleep = 1) + if (data == None): + return '' + retVal = self.stripPrompt(data) + if (retVal != None): + retVal = retVal.strip('\n') + if (self.debug >= 3): + print "remote hash: '" + retVal + "'" + return retVal + + + #return the md5 sum of a file on the host + def getLocalHash(self, filename): + file = open(filename, 'rb') + if (file == None): + return None + + mdsum = hashlib.md5() + + while 1: + data = file.read(1024) + if not data: + break + mdsum.update(data) + + file.close() + hexval = mdsum.hexdigest() + if (self.debug >= 3): + print "local hash: '" + hexval + "'" + return hexval + + # Gets the device root for the testing area on the device + # For all devices we will use / type slashes and depend on the device-agent + # to sort those out. + # Structure on the device is as follows: + # /tests + # /| --> approot + # /profile + # /xpcshell + # /reftest + # /mochitest + def getDeviceRoot(self): + if (not self.deviceRoot): + if (self.dirExists('/tests')): + self.deviceRoot = '/tests' + else: + self.mkDir('/tests') + self.deviceRoot = '/tests' + return self.deviceRoot + + # Either we will have /tests/fennec or /tests/firefox but we will never have + # both. Return the one that exists + def getAppRoot(self): + if (self.dirExists(self.getDeviceRoot() + '/fennec')): + return self.getDeviceRoot() + '/fennec' + else: + return self.getDeviceRoot() + '/firefox' + + # Gets the directory location on the device for a specific test type + # Type is one of: xpcshell|reftest|mochitest + def getTestRoot(self, type): + if (re.search('xpcshell', type, re.I)): + self.testRoot = self.getDeviceRoot() + '/xpcshell' + elif (re.search('?(i)reftest', type)): + self.testRoot = self.getDeviceRoot() + '/reftest' + elif (re.search('?(i)mochitest', type)): + self.testRoot = self.getDeviceRoot() + '/mochitest' + return self.testRoot + + # Sends a specific process ID a signal code and action. + # For Example: SIGINT and SIGDFL to process x + def signal(self, processID, signalType, signalAction): + # currently not implemented in device agent - todo + pass + + # Get a return code from process ending -- needs support on device-agent + # this is a todo + def getReturnCode(self, processID): + #todo make this real + return 0 + + def unpackFile(self, filename): + self.sendCMD(['cd \\tests', 'unzp ' + filename]) + + + #validate localDir from host to remoteDir on the device + def validateDir(self, localDir, remoteDir): + if (self.debug >= 2): print "validating directory: " + localDir + " to " + remoteDir + for root, dirs, files in os.walk(localDir): + parts = root.split(localDir) + for file in files: + remoteRoot = remoteDir + self.dirSlash + parts[1] + remoteRoot = remoteRoot.replace('/', self.dirSlash) + remoteRoot = remoteRoot.replace('\\\\', '\\') + if (parts[1] == ""): remoteRoot = remoteDir + remoteName = remoteRoot + self.dirSlash + file + if (self.validateFile(remoteName, os.path.join(root, file)) <> True): + return None + return True diff --git a/mozilla/testing/performance/talos/ffprocess_winmo.py b/mozilla/testing/performance/talos/ffprocess_winmo.py new file mode 100644 index 00000000000..67da4fe0a7d --- /dev/null +++ b/mozilla/testing/performance/talos/ffprocess_winmo.py @@ -0,0 +1,219 @@ +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is standalone Firefox Windows Mobile performance test. +# +# Contributor(s): +# Joel Maher (original author) +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** +from ffprocess import FFProcess +import devicemanager +import os +import time +import tempfile +import re + +DEFAULT_PORT = 27020 + +class WinmoProcess(FFProcess): + testAgent = None + rootdir = '' + dirSlash = '' + host = '' + port = '' + + def __init__(self, host, port, rootdir): + if (port == 0): + port = DEFAULT_PORT + if (port == ''): + port = DEFAULT_PORT + if (port == None): + port = DEFAULT_PORT + + self.port = port + self.host = host + self.setupRemote(host, port) + self.rootdir = rootdir + parts = self.rootdir.split("\\") + if (len(parts) > 1): + self.dirSlash = "\\" + else: + self.dirSlash = "/" + + def setupRemote(self, host = '', port = DEFAULT_PORT): + self.testAgent = devicemanager.DeviceManager(host, port) + + def GetRunningProcesses(self): + current_procs = [] + return self.testAgent.getProcessList() + + + def GenerateBrowserCommandLine(self, browser_path, extra_args, profile_dir, url): + """Generates the command line for a process to run Browser + + Args: + browser_path: String containing the path to the browser exe to use + profile_dir: String containing the directory of the profile to run Browser in + url: String containing url to start with. + """ + + profile_arg = '' + if profile_dir: + profile_arg = '-profile %s' % profile_dir + + cmd = '%s %s %s %s' % (browser_path, + extra_args, + profile_arg, + url) + return cmd + + + def ProcessesWithNameExist(self, *process_names): + """Returns true if there are any processes running with the + given name. Useful to check whether a Browser process is still running + + Args: + process_name: String or strings containing the process name, i.e. "firefox" + + Returns: + True if any processes with that name are running, False otherwise. + """ + + # refresh list of processes + data = self.GetRunningProcesses() + if (data == None): + return False + + for process_name in process_names: + try: + procre = re.compile(".*" + process_name + ".*") + for line in data: + if (procre.match(line[1])): + return True + except: + # Might get an exception if there are no instances of the process running. + continue + return False + + + def TerminateAllProcesses(self, *process_names): + """Helper function to terminate all processes with the given process name + + Args: + process_name: String or strings containing the process name, i.e. "firefox" + """ + for process_name in process_names: + try: + self.testAgent.killProcess(process_name) + except: + # Might get an exception if there are no instances of the process running. + continue + + + def NonBlockingReadProcessOutput(self, handle): + """Does a non-blocking read from the output of the process + with the given handle. + + Args: + handle: The process handle returned from os.popen() + + Returns: + A tuple (bytes, output) containing the number of output + bytes read, and the actual output. + """ + + output = "" + try: + output = self.getFile(handle) + return (len(output), output) + except: + return (0, output) + + def getFile(self, handle, localFile = ""): + if (localFile == ""): + localFile = os.path.join(tempfile.mkdtemp(), "temp.txt") + if (os.path.exists(handle)): + #TODO + return "" + + re_nofile = re.compile("error:.*") + data = self.testAgent.getFile(handle, localFile) + if (re_nofile.match(data)): + fileData = '' + if (os.path.isfile(handle)): + results_file = open(handle, "r") + fileData = results_file.read() + results_file.close() + return fileData + return data + + def launchProcess(self, cmd, outputFile = "process.txt", timeout = -1): + if (outputFile == "process.txt"): + outputFile = self.rootdir + self.dirSlash + "process.txt" + self.testAgent.fireProcess(cmd + " > " + outputFile) + handle = outputFile + + timed_out = True + if (timeout > 0): + total_time = 0 + while total_time < timeout: + time.sleep(1) + if (not self.poll(handle)): + timed_out = False + break + total_time += 1 + + if (timed_out == True): + return None + + return handle + + def poll(self, process): + try: + if (not self.testAgent.process.isAlive()): + return None + return 1 + except: + return None + return 1 + + def copyDirToDevice(self, localDir): + head, tail = os.path.split(localDir) + + remoteDir = self.rootdir + self.dirSlash + tail + self.testAgent.pushDir(localDir, remoteDir) + return remoteDir + + def removeDirectory(self, dir): + self.testAgent.removeDir(dir) + + def MakeDirectoryContentsWritable(self, dir): + pass + + def copyFile(self, fromfile, toDir): + toDir = toDir.replace("/", self.dirSlash) + self.testAgent.pushFile(fromfile, toDir + self.dirSlash + os.path.basename(fromfile)) + diff --git a/mozilla/testing/performance/talos/run_tests.py b/mozilla/testing/performance/talos/run_tests.py index 99c877b2ab5..fd9de0610e8 100755 --- a/mozilla/testing/performance/talos/run_tests.py +++ b/mozilla/testing/performance/talos/run_tests.py @@ -352,6 +352,7 @@ def test_file(filename): 'env' : yaml_config['env'], 'dirs' : yaml_config['dirs'], 'init_url' : yaml_config['init_url']} + if 'child_process' in yaml_config: browser_config['child_process'] = yaml_config['child_process'] else: @@ -362,6 +363,28 @@ def test_file(filename): browser_config['test_name_extension'] = yaml_config['test_name_extension'] else: browser_config['test_name_extension'] = '' + + if 'deviceip' in yaml_config: + browser_config['host'] = yaml_config['deviceip'] + else: + browser_config['host'] = '' + if 'deviceport' in yaml_config: + browser_config['port'] = yaml_config['deviceport'] + else: + browser_config['port'] = '' + if 'webserver' in yaml_config: + browser_config['webserver'] = yaml_config['webserver'] + else: + browser_config['webserver'] = '' + if 'deviceroot' in yaml_config: + browser_config['deviceroot'] = yaml_config['deviceroot'] + else: + browser_config['deviceroot'] = '' + if 'remote' in yaml_config: + browser_config['remote'] = yaml_config['remote'] + else: + browser_config['remote'] = False + #normalize paths to work accross platforms browser_config['browser_path'] = os.path.normpath(browser_config['browser_path']) for dir in browser_config['dirs']: diff --git a/mozilla/testing/performance/talos/winmo.config b/mozilla/testing/performance/talos/winmo.config new file mode 100644 index 00000000000..58bc7285b32 --- /dev/null +++ b/mozilla/testing/performance/talos/winmo.config @@ -0,0 +1,225 @@ +# Sample Talos configuration file + +# The title of the report +title: mobile + +#*** output options **** +#uncomment to turn on dump to csv option +#csv_dir: 'output' +#comment out next two lines to disable send to graph server +results_server: 'graphs-stage.mozilla.org' +results_link: '/server/collect.cgi' + +# browser info +process : fennec.exe +browser_path: /tests/fennec +browser_log: browser_output.txt +# arguments to pass to browser +extra_args: '' +#how long the browser takes to open/close +browser_wait: 20 + +branch: mobile + +remote: False +deviceip: +deviceport: +webserver: localhost +deviceroot: + +buildid: testbuildid + +init_url: getInfo.html + +# Preferences to set in the test (use "preferences : {}" for no prefs) +preferences : + browser.EULA.override : true + security.fileuri.strict_origin_policy : false + browser.shell.checkDefaultBrowser : false + browser.warnOnQuit : false + browser.link.open_newwindow : 2 + dom.allow_scripts_to_close_windows : true + dom.disable_open_during_load: false + dom.max_script_run_time : 0 + browser.dom.window.dump.enabled: true + network.proxy.type : 1 + network.proxy.http : localhost + network.proxy.http_port : 80 + dom.disable_window_flip : true + dom.disable_window_move_resize : true + security.enable_java : false + extensions.checkCompatibility : false + extensions.update.notifyUser: false + +# Extensions to install in test (use "extensions: {}" for none) +# Need quotes around guid because of curly braces +# extensions : +# "{12345678-1234-1234-1234-abcd12345678}" : c:\path\to\unzipped\xpi +# foo@sample.com : c:\path\to\other\unzipped\xpi +extensions : {} + +#any directories whose contents need to be installed in the browser before running the tests +# this assumes that the directories themselves already exist in the firefox path +dirs: + chrome : page_load_test/chrome + components : page_load_test/components + +# Environment variables to set during test (use env: {} for none) +env : + NO_EM_RESTART : 1 +# Tests to run +# url : (REQUIRED) url to load into the given firefox browser +# url_mod : (OPTIONAL) a bit of code to be evaled and added to the given url during each cycle of the test +# resolution : (REQUIRED) how long (in seconds) to pause between counter sampling +# cycles : (REQUIRED) how many times to run the test +# timeout : (OPTIONAL) how many seconds the test can run before we consider it failed and quit (default 8 hours) +# pagetimeout : (OPTIONAL) how many seconds each page is allowed to take before considered to be frozen (default 8 hours) +# Must be used in conjuction with the pageloader with the -tpnoisy option +# counters : (REQUIRED) types of system activity to monitor during test run, can be empty +# For possible values of counters argument on Windows, see +# http://technet2.microsoft.com/WindowsServer/en/Library/86b5d116-6fb3-427b-af8c-9077162125fe1033.mspx?mfr=true +# Possible values on Mac: +# counters : ['Private Bytes', 'RSS'] +# Possible values on Linux: +# counters : ['Private Bytes', 'RSS', 'XRes'] +# Standard windows values: +# counters : ['Working Set', 'Private Bytes', '% Processor Time'] + +# to set up a new test it must have the correct configuration options and drop information in a standard format +# the format is seen in the regular expressions in ttest.py +# to see how the data passed from the browser is processed see send_to_graph and send_to_csv in run_tests.py +tests : +- name: ts + url : startup_test/startup_test.html?begin= + url_mod : str(int(time.time()*1000)) + resolution : 1 + cycles : 10 + timeout: 300 + win_counters : [] + linux_counters : [] + mac_counters : [] + shutdown : True + profile_path: base_profile +- name: ts_places_generated_max + url : startup_test/startup_test.html?begin= + url_mod : str(int(time.time()*1000)) + resolution : 1 + cycles : 10 + timeout: 150 + win_counters : [] + linux_counters : [] + mac_counters : [] + shutdown : True + profile_path: places_generated_max +- name: ts_places_generated_min + url : startup_test/startup_test.html?begin= + url_mod : str(int(time.time()*1000)) + resolution : 1 + cycles : 10 + timeout: 150 + win_counters : [] + linux_counters : [] + shutdown : True + profile_path: places_generated_min +- name: ts_places_generated_med + url : startup_test/startup_test.html?begin= + url_mod : str(int(time.time()*1000)) + resolution : 1 + cycles : 10 + timeout: 150 + win_counters : [] + linux_counters : [] + mac_counters : [] + shutdown : True + profile_path: places_generated_med +- name: tp4 + url : '-tp page_load_test/tp4.manifest -tpchrome -tpnoisy -tpformat tinderbox -tpcycles 3' + resolution : 20 + cycles : 1 + pagetimeout : 700 + win_counters : [] + linux_counters : [] + mac_counters : [] + timeout : 14400 + shutdown : True + profile_path: base_profile +- name: tdhtml + url: '-tp page_load_test/dhtml/dhtml.manifest -tpchrome -tpnoisy -tpformat tinderbox -tpcycles 3' + resolution : 1 + cycles : 1 + pagetimeout : 300 + win_counters : [] + linux_counters : [] + mac_counters : [] + shutdown : False + profile_path: base_profile +- name: tgfx + url: '-tp page_load_test/gfx/gfx.manifest -tpchrome -tpnoisy -tpformat tinderbox -tpcycles 3 -tprender' + resolution : 1 + cycles : 1 + pagetimeout : 300 + win_counters : [] + linux_counters : [] + mac_counters : [] + shutdown : False + profile_path: base_profile +- name: tsvg + url: '-tp page_load_test/svg/svg.manifest -tpchrome -tpnoisy -tpformat tinderbox -tpcycles 3' + resolution : 1 + cycles : 1 + pagetimeout : 300 + win_counters : [] + linux_counters : [] + mac_counters : [] + shutdown : False + profile_path: base_profile +- name: twinopen + url: startup_test/twinopen/winopen.xul?phase1=20 + resolution: 1 + cycles : 1 + timeout : 300 + win_counters: [] + linux_counters : [] + mac_counters : [] + shutdown : False + profile_path: base_profile +- name: tjss + url: '-tp page_load_test/jss/jss.manifest -tpchrome -tpnoisy -tpformat tinderbox -tpcycles 1' + resolution : 1 + cycles : 1 + pagetimeout : 300 + win_counters : [] + linux_counters : [] + mac_counters : [] + shutdown : False + profile_path: base_profile +- name: tsspider + url: '-tp page_load_test/sunspider/sunspider.manifest -tpchrome -tpnoisy -tpformat tinderbox -tpcycles 3' + resolution : 1 + cycles : 1 + pagetimeout : 400 + win_counters : [] + linux_counters : [] + mac_counters : [] + shutdown : False + profile_path: base_profile +- name: tpan + url: startup_test/fennecmark.html?test=PanDown + resolution: 1 + cycles : 10 + timeout : 300 + win_counters: [] + linux_counters : [] + mac_counters : [] + shutdown : False + profile_path: base_profile +- name: tzoom + url: startup_test/fennecmark.html?test=Zoom + resolution: 1 + cycles : 10 + timeout : 300 + win_counters: [] + linux_counters : [] + mac_counters : [] + shutdown : False + profile_path: base_profile