diff --git a/mozilla/build/pgo/automation.py.in b/mozilla/build/pgo/automation.py.in index a934775427d..2d72dac1322 100644 --- a/mozilla/build/pgo/automation.py.in +++ b/mozilla/build/pgo/automation.py.in @@ -39,15 +39,28 @@ from datetime import datetime import itertools +import logging import shutil import os +import signal import sys +import threading """ Runs the browser from a script, and provides useful utilities for setting up the browser environment. """ +__all__ = [ + "UNIXISH", + "IS_WIN32", + "runApp", + "Process", + "initializeProfile", + "DIST_BIN", + "DEFAULT_APP", + ] + # Since some tests require cross-domain support in Mochitest, across ports, # domains, subdomains, etc. we use a proxy autoconfig hack to map a bunch of # servers onto localhost:8888. We have to grant them the same privileges as @@ -112,6 +125,19 @@ UNIXISH = not IS_WIN32 and not IS_MAC #expand DEFAULT_APP = "./" + __BROWSER_PATH__ +########### +# LOGGING # +########### + +# We use the logging system here primarily because it'll handle multiple +# threads, which is needed to process the output of the server and application +# processes simultaneously. +log = logging.getLogger() +handler = logging.StreamHandler(sys.stdout) +log.setLevel(logging.INFO) +log.addHandler(handler) + + ################# # SUBPROCESSING # ################# @@ -125,15 +151,17 @@ class Process: def __init__(self, command, args, env): """ - Executes the given command, which must be an absolute path, with the given - arguments in the given environment. + Creates a process representing the execution of the given command, which + must be an absolute path, with the given arguments in the given environment. + The process is then started. """ command = os.path.abspath(command) if IS_WIN32: import subprocess cmd = [command] cmd.extend(args) - self._process = subprocess.Popen(cmd, env = env) + p = subprocess.Popen(cmd, env = env) + self._out = p.stdout else: import popen2 cmd = [] @@ -142,34 +170,48 @@ class Process: cmd.append("'" + command + "'") cmd.extend(map(lambda x: "'" + x + "'", args)) cmd = " ".join(cmd) - self._process = popen2.Popen4(cmd) - self.pid = self._process.pid + p = popen2.Popen4(cmd) + self._out = p.fromchild - def wait(self): - "Waits for this process to finish, then returns the process's status." - if IS_WIN32: - return self._process.wait() - # popen2 is a bit harder to work with because we have to manually redirect - # output to stdout + self._process = p + self.pid = p.pid + + self._thread = threading.Thread(target = lambda: self._run()) + self._thread.start() + + def _run(self): + "Continues execution of this process on a separate thread." p = self._process - stdout = sys.stdout - out = p.fromchild + out = self._out + # read in lines until the process finishes, then read in any last remaining + # buffered lines while p.poll() == -1: line = out.readline().rstrip() if len(line) > 0: - print >> stdout, line - # read in the last lines that happened between the last -1 poll and the - # process finishing + log.info(line) for line in out: line = line.rstrip() if len(line) > 0: - print >> stdout, line - return p.poll() + log.info(line) + self._status = p.poll() + + def wait(self): + "Waits for this process to finish, then returns the process's status." + self._thread.join() + return self._status + + def kill(self): + "Kills this process." + try: + if not IS_WIN32: # XXX + os.kill(self._process.pid, signal.SIGKILL) + except: + pass -####################### -# PROFILE SETUP # -####################### +################# +# PROFILE SETUP # +################# def initializeProfile(profileDir): "Sets up the standard testing profile." @@ -243,9 +285,9 @@ user_pref("network.proxy.autoconfig_url", "%(pacURL)s"); prefsFile.close() -####################### -# RUN THE APP # -####################### +############### +# RUN THE APP # +############### def runApp(testURL, env, app, profileDir, extraArgs): "Run the app, returning the time at which it was started." @@ -270,9 +312,9 @@ def runApp(testURL, env, app, profileDir, extraArgs): args.extend(("-no-remote", "-profile", profileDirectory, testURL)) args.extend(extraArgs) proc = Process(cmd, args, env = env) - print "Application pid: " + str(proc.pid) + log.info("Application pid: %d", proc.pid) status = proc.wait() if status != 0: - print "FAIL Exited with code " + str(status) + " during test run" + log.info("ERROR FAIL Exited with code %d during test run", status) return start diff --git a/mozilla/testing/mochitest/runtests.py.in b/mozilla/testing/mochitest/runtests.py.in index 082b0f32115..f8bb9f1f93d 100644 --- a/mozilla/testing/mochitest/runtests.py.in +++ b/mozilla/testing/mochitest/runtests.py.in @@ -42,11 +42,11 @@ Runs the Mochitest test harness. """ from datetime import datetime +import logging import optparse import os import os.path import re -import signal import sys import time from urllib import quote_plus as encodeURIComponent @@ -82,6 +82,9 @@ PROFILE_DIRECTORY = os.path.abspath("./mochitesttestingprofile") LEAK_REPORT_FILE = PROFILE_DIRECTORY + "/" + "leaks-report.log" +log = logging.getLogger() + + ####################### # COMMANDLINE OPTIONS # ####################### @@ -210,7 +213,7 @@ class MochitestServer: if pid < 0: print "Error starting server." sys.exit(2) - print "Server pid: " + str(pid) + log.info("Server pid: %d", pid) def ensureReady(self, timeout): @@ -229,17 +232,13 @@ class MochitestServer: sys.exit(1) def stop(self): - pid = self._process.pid try: c = urllib2.urlopen(SERVER_SHUTDOWN_URL) c.read() c.close() - os.waitpid(pid, 0) + self._process.wait() except: - if automation.IS_WIN32: - pass # XXX do something here! - else: - os.kill(pid, signal.SIGKILL) + self._process.kill() ################# @@ -333,14 +332,16 @@ Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" start = automation.runApp(testURL, browserEnv, options.app, PROFILE_DIRECTORY, options.browserArgs) + # Server's no longer needed, and perhaps more importantly, anything it might + # spew to console shouldn't disrupt the leak information table we print next. server.stop() if not os.path.exists(LEAK_REPORT_FILE): - print "WARNING refcount logging is off, so leaks can't be detected!" + log.info("WARNING refcount logging is off, so leaks can't be detected!") else: leaks = open(LEAK_REPORT_FILE, "r") for line in leaks: - print line, + log.info(line.rstrip()) leaks.close() threshold = options.leakThreshold @@ -365,12 +366,12 @@ Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" if bytesLeaked > threshold: thresholdExceeded = True prefix = "ERROR FAIL" - print ("ERROR FAIL leaked %(actual)d bytes during test " - "execution (should have leaked no more than " - "%(expect)d bytes)") % { "actual": bytesLeaked, - "expect": threshold } + log.info("ERROR FAIL leaked %d bytes during test execution (should " + "have leaked no more than %d bytes)", + bytesLeaked, threshold) elif bytesLeaked > 0: - print "WARNING leaked %d bytes during test execution" % bytesLeaked + log.info("WARNING leaked %d bytes during test execution", + bytesLeaked) else: numLeaked = int(matches.group("numLeaked")) if numLeaked != 0: @@ -380,27 +381,31 @@ Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" else: instance = "instance" rest = "" - vars = { "prefix": prefix, - "numLeaked": numLeaked, - "instance": instance, - "name": name, - "size": matches.group("size"), - "rest": rest } - print ("%(prefix)s leaked %(numLeaked)d %(instance)s of %(name)s " - "with size %(size)s bytes%(rest)s") % vars + log.info("%(prefix)s leaked %(numLeaked)d %(instance)s of %(name)s " + "with size %(size)s bytes%(rest)s" % + { "prefix": prefix, + "numLeaked": numLeaked, + "instance": instance, + "name": name, + "size": matches.group("size"), + "rest": rest }) if not seenTotal: - print "ERROR FAIL missing output line for total leaks!" + log.info("ERROR FAIL missing output line for total leaks!") leaks.close() # print test run times finish = datetime.now() - print " started: " + str(start) - print "finished: " + str(finish) + log.info(" started: %s", str(start)) + log.info("finished: %s", str(finish)) # delete the profile and manifest os.remove(manifest) + # hanging due to non-halting threads is no fun; assume we hit the errors we + # were going to hit already and exit with a success code + sys.exit(0) + #######################