diff --git a/mozilla/testing/tests/memtest.py b/mozilla/testing/tests/memtest.py
new file mode 100644
index 00000000000..79256dc4ed3
--- /dev/null
+++ b/mozilla/testing/tests/memtest.py
@@ -0,0 +1,536 @@
+#!/usr/bin/python
+#
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is standalone Firefox memory test.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Graydon Hoare (original author)
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+
+import subprocess
+import tempfile
+import os
+import re
+import getpass
+import datetime
+import logging
+import time
+import sys
+import signal
+import shutil
+import glob
+
+def sighandler(signal, frame):
+ print "signal %d, throwing" % signal
+ raise Exception
+
+signal.signal(signal.SIGHUP, sighandler)
+
+
+########################################################################
+
+def start_xvnc(disp, tmpdir):
+ xvnc_stdout = open(os.path.join(tmpdir, "xvnc.stdout"), mode="w")
+ xvnc_stderr = open(os.path.join(tmpdir, "xvnc.stderr"), mode="w")
+ xvnc_proc = subprocess.Popen (["Xvnc",
+ "-fp", "/usr/share/X11/fonts/misc",
+ "-localhost",
+ "-SecurityTypes", "None",
+ "-IdleTimeout", "604800",
+ "-depth", "24",
+ "-ac",
+ (":%d" % disp)],
+ cwd=tmpdir,
+ shell=False,
+ stdout=xvnc_stdout,
+ stderr=xvnc_stderr)
+ time.sleep(3)
+ assert xvnc_proc.poll() == None
+ logging.info("started Xvnc server (pid %d) on display :%d"
+ % (xvnc_proc.pid, disp))
+ return xvnc_proc
+
+########################################################################
+
+def large_variant_filename(variant):
+ return variant + "-large.swf"
+
+def large_variant_html_filename(variant):
+ return variant + "-large.html"
+
+def small_variant_filename(variant):
+ return variant + ".swf"
+
+def start_xvnc_recorder(vnc2swfpath, disp, variant, tmpdir):
+ rec_stdout = open(os.path.join(tmpdir, "vnc2swf.stdout"), mode="w")
+ rec_stderr = open(os.path.join(tmpdir, "vnc2swf.stderr"), mode="w")
+ rec_proc = subprocess.Popen ([os.path.join(vnc2swfpath, "vnc2swf.py"),
+ "-n",
+ "-o", large_variant_filename(variant),
+ "-C", "800x600+0+100",
+ "-r", "2",
+ "-t", "video",
+ "localhost:%d" % disp],
+ cwd=tmpdir,
+ shell=False,
+ stdout=rec_stdout,
+ stderr=rec_stderr)
+ time.sleep(3)
+ assert rec_proc.poll() == None
+ logging.info("started vnc2swf recorder (pid %d) on display :%d"
+ % (rec_proc.pid, disp))
+ return rec_proc
+
+def complete_xvnc_recording(vnc2swfpath, proc, variant, tmpdir):
+ edit_stdout = open(os.path.join(tmpdir, "edit.stdout"), mode="w")
+ edit_stderr = open(os.path.join(tmpdir, "edit.stderr"), mode="w")
+ logging.info("killing vnc2swf recorder (pid %d)" % proc.pid)
+ os.kill(proc.pid, signal.SIGINT)
+ proc.wait()
+ logging.info("down-sampling video")
+ subprocess.Popen([os.path.join(vnc2swfpath, "edit.py"),
+ "-c",
+ "-o", small_variant_filename(variant),
+ "-t", "video",
+ "-s", "0.5",
+ "-r", "32",
+ large_variant_filename(variant)],
+ cwd=tmpdir,
+ shell=False,
+ stdout=edit_stdout,
+ stderr=edit_stderr).wait()
+ #os.unlink(large_variant_html_filename(variant))
+ #os.unlink(large_variant_filename(variant))
+ logging.info("output video is in " + small_variant_filename(variant))
+
+
+########################################################################
+
+def write_vgopts(tmpdir, vgopts):
+ f = open(os.path.join(tmpdir, ".valgrindrc"), "w")
+ for i in vgopts:
+ f.write(i + "\n")
+ f.close()
+
+########################################################################
+
+class Firefox_runner:
+ def __init__(this, batchprefix, homedir, ffdir, timestr, tmpdir, disp):
+ this.tmpdir = tmpdir
+ this.homedir = homedir
+ this.ffdir = ffdir
+ this.profile = batchprefix + timestr
+ this.profile_dir = os.path.join(this.tmpdir, this.profile)
+ this.bin = os.path.join(this.ffdir, "firefox-bin")
+ this.environ = {
+ "PATH" : os.getenv("PATH"),
+ "HOME" : this.homedir,
+ "LD_LIBRARY_PATH" : this.ffdir,
+ "MOZ_NO_REMOTE" : "1",
+# "DISPLAY" : ":0",
+ "DISPLAY" : (":%d" % disp),
+ }
+
+ this.kill_initial_profiles_ini()
+ this.create_tmp_profile()
+ this.write_proxy_pac_file()
+ this.write_user_prefs()
+ this.perform_initial_registration()
+
+ def kill_initial_profiles_ini(this):
+ prof = os.path.join(this.homedir, ".mozilla")
+ shutil.rmtree(prof, True)
+ os.mkdir(prof)
+
+
+ def clear_cache(this):
+ shutil.rmtree(os.path.join(this.profile_dir, "Cache"), True)
+
+ def create_tmp_profile(this):
+ subprocess.Popen ([this.bin, "-CreateProfile", (this.profile + " " + this.profile_dir)],
+ cwd=this.tmpdir, shell=False, env=this.environ).wait()
+
+ def write_proxy_pac_file(this):
+ this.proxypac = os.path.join(this.tmpdir, "proxy.pac")
+ p = open(this.proxypac, "w")
+ p.write("function FindProxyForURL(url, host) {\n"
+ " if (host == \"localhost\")\n"
+ " return \"DIRECT\";\n"
+ " else\n"
+ " return \"PROXY localhost\";\n"
+ "}\n")
+ p.close()
+
+ def write_user_prefs(this):
+ userprefs = open(os.path.join(this.profile_dir, "user.js"), "w")
+ userprefs.write("user_pref(\"network.proxy.autoconfig_url\", \"file://%s\");\n" % this.proxypac)
+ userprefs.write("user_pref(\"network.proxy.type\", 2);\n")
+ userprefs.write("user_pref(\"privacy.popups.firstTime\", false);\n")
+ userprefs.write("user_pref(\"plugin.default_plugin_disabled\", false);\n")
+ userprefs.write("user_pref(\"dom.max_script_run_time\", 0);\n")
+ userprefs.write("user_pref(\"dom.max_script_run_time\", 0);\n")
+ userprefs.write("user_pref(\"dom.allow_scripts_to_close_windows\", true);\n")
+ userprefs.close()
+
+ def perform_initial_registration(this):
+ dummy_file = os.path.join(this.tmpdir, "dummy.html")
+ f = open(dummy_file, "w")
+ f.write("\n")
+ f.close()
+ logging.info("running dummy firefox under profile, for initial component-registration")
+ subprocess.Popen ([this.bin, "-P", this.profile, "file://" + dummy_file],
+ cwd=this.tmpdir, shell=False, env=this.environ).wait()
+
+ def run_normal(this, url):
+ ff_proc = subprocess.Popen ([this.bin, "-P", this.profile, url],
+ cwd=this.tmpdir, shell=False, env=this.environ)
+ logging.info("started 'firefox-bin ... %s' process (pid %d)"
+ % (url, ff_proc.pid))
+ return ff_proc
+
+
+ def run_valgrind(this, vg_tool, url):
+ vg_proc = subprocess.Popen (["valgrind",
+ "--tool=" + vg_tool,
+ "--log-file=valgrind-" + vg_tool + "-log",
+ this.bin, "-P", this.profile, url],
+ cwd=this.tmpdir, shell=False, env=this.environ)
+ logging.info("started 'valgrind --tool=%s firefox-bin ... %s' process (pid %d)"
+ % (vg_tool, url, vg_proc.pid))
+
+ return vg_proc
+
+
+########################################################################
+# homebrew memory monitor until valgrind works properly
+########################################################################
+
+
+class sampler:
+
+ def __init__(self, name):
+ self.name = name
+ self.max = 0
+ self.final = 0
+ self.sum = 0
+ self.samples = 0
+
+ def sample(self):
+ s = self.get_sample()
+ self.samples += 1
+ self.sum += s
+ self.max = max(self.max, s)
+ self.final = s
+
+ def report(self):
+ self.samples = max(self.samples, 1)
+ logging.info("__COUNT_%s_MAX!%d" % (self.name.upper(), self.max))
+ logging.info("__COUNT_%s_AVG!%d" % (self.name.upper(), (self.sum / self.samples)))
+ logging.info("__COUNT_%s_FINAL!%d" % (self.name.upper(), self.final))
+
+
+class proc_maps_heap_sampler(sampler):
+
+ def __init__(self, procpath):
+ sampler.__init__(self, "heap")
+ self.procpath = procpath
+
+ def get_sample(self):
+ map_entry_rx = re.compile("^([0-9a-f]+)-([0-9a-f]+)\s+[-r][-w][-x][-p]\s+(?:\S+\s+){3}\[heap\]$")
+ maps_path = os.path.join(self.procpath, "maps")
+ maps_file = open(maps_path)
+ for line in maps_file.xreadlines():
+ m = map_entry_rx.match(line)
+ if m:
+ (lo,hi) = m.group(1,2)
+ lo = int(lo, 16)
+ hi = int(hi, 16)
+ sz = hi - lo
+ maps_file.close()
+ return sz
+ maps_file.close()
+ return 0
+
+
+class proc_status_sampler(sampler):
+
+ def __init__(self, procpath, entry):
+ sampler.__init__(self, entry)
+ self.status_entry_rx = re.compile("^Vm%s:\s+(\d+) kB" % entry)
+ self.procpath = procpath
+
+ def get_sample(self):
+ status_path = os.path.join(self.procpath, "status")
+ status_file = open(status_path)
+ for line in status_file.xreadlines():
+ m = self.status_entry_rx.match(line)
+ if m:
+ status_file.close()
+ return int(m.group(1)) * 1024
+
+ status_file.close()
+ return 0
+
+
+def wait_collecting_memory_stats(process):
+
+
+ procdir = os.path.join("/proc", str(process.pid))
+ samplers = [ proc_status_sampler(procdir, "RSS"),
+ proc_status_sampler(procdir, "Size"),
+ proc_maps_heap_sampler(procdir) ]
+
+ process.poll()
+ while process.returncode == None:
+
+ for s in samplers:
+ s.sample()
+
+ time.sleep(1)
+ process.poll()
+
+
+ for s in samplers:
+ s.report()
+
+
+########################################################################
+# config variables
+########################################################################
+
+
+disp = 25
+batchprefix = "batch-"
+homedir = "/home/mozvalgrind"
+vnc2swfpath = "%s/pyvnc2swf-0.8.2" % homedir
+url = "file://%s/workload.xml" % homedir
+probe_point = "nsWindow::SetTitle(*"
+num_frames = 10
+vgopts = [ "--memcheck:leak-check=yes",
+ "--memcheck:error-limit=no",
+ ("--memcheck:num-callers=%d" % num_frames),
+# "--memcheck:leak-resolution=high",
+# "--memcheck:show-reachable=yes",
+ "--massif:format=html",
+ ("--massif:depth=%d" % num_frames),
+ "--massif:instrs=yes",
+ "--callgrind:simulate-cache=yes",
+ "--callgrind:simulate-hwpref=yes",
+# ("--callgrind:dump-before=%s" % probe_point),
+ "--callgrind:I1=65536,2,64",
+ "--callgrind:D1=65536,2,64",
+ "--callgrind:L2=524288,8,64",
+ "--verbose"
+ ]
+
+
+######################################################
+# logging results
+######################################################
+
+def archive_dir(dir, sums):
+ res = "current"
+ tar = res + ".tar.gz"
+
+ if os.path.exists(res):
+ shutil.rmtree(res)
+
+ if os.path.exists(tar):
+ os.unlink(tar)
+
+ os.mkdir(res)
+ ix = open(os.path.join(res, "index.html"), "w")
+ ix.write("\n
\n")
+ ix.write("run: %s
\n" % dir)
+
+ # summary info
+ ix.write("Summary info
\n")
+ ix.write("\n")
+ for x in sums:
+ ix.write("| %s | %s |
\n" % (x, sums[x]))
+ ix.write("
\n")
+
+
+ # primary logs
+ ix.write("Primary logs
\n")
+ for log in glob.glob(os.path.join(dir, "valgrind-*-log*")):
+ (dirname, basename) = os.path.split(log)
+ shutil.copy(log, os.path.join(res, basename))
+ ix.write("%s
\n" % (basename, basename))
+
+
+ # massif graphs
+ ix.write("Massif results
\n")
+ ix.write("Click graph to see details
\n")
+ for mp in glob.glob(os.path.join(dir, "massif.*.ps")):
+ (dirname,basename) = os.path.split(mp)
+ (base,ext) = os.path.splitext(basename)
+ png = base + ".png"
+ html = base + ".html"
+ subprocess.call(["convert", "-rotate", "270", mp, os.path.join(res, png)])
+ shutil.copy(os.path.join(dir, html), os.path.join(res, html))
+ ix.write("
\n" % (html, png))
+
+ # run movies
+ ix.write("Movies
\n")
+ for movie in ["memcheck", "massif", "callgrind"]:
+ for ext in [".html", ".swf"]:
+ base = movie + ext
+ if os.path.exists(os.path.join(dir, base)):
+ shutil.copy(os.path.join(dir, base), os.path.join(res, base))
+ if os.path.exists(os.path.join(res, movie + ".html")):
+ ix.write("%s movie
\n" % (movie + ".html", movie))
+
+ # callgrind profile
+ ix.write("Callgrind profiles
\n")
+ for cg in glob.glob(os.path.join(dir, "callgrind.out.*")):
+ (dir, base) = os.path.split(cg)
+ shutil.copy(cg, os.path.join(res, base))
+ ix.write("%s
\n" % (base, base))
+
+ ix.write("\n\n")
+ ix.close()
+
+ for i in glob.glob(os.path.join(res, "*")):
+ os.chmod(i, 0644)
+ os.chmod(res, 0755)
+
+ subprocess.call(["tar", "czvf", tar, res])
+ os.chmod(tar, 0644)
+
+
+def log_result_summaries(tmpdir):
+
+ pats = {
+ "IR" : re.compile("==\d+==\s+I\s+refs:\s+([0-9,]+)"),
+ "ALLOC" : re.compile("==\d+==\s+malloc/free:\s+[0-9,]+\s+allocs,\s+[0-9,]+\s+frees,\s+([0-9,]+)\s+bytes allocated."),
+ "LEAKED" : re.compile("==\d+==\s+(?:definitely|indirectly)\s+lost:\s+([0-9,]+)\s+bytes"),
+ "DUBIOUS" : re.compile("==\d+==\s+possibly\s+lost:\s+([0-9,]+)\s+bytes"),
+ "LIVE" : re.compile("==\d+==\s+still\s+reachable:\s+([0-9,]+)\s+bytes"),
+ }
+
+ sums = {}
+
+ for fname in glob.glob("%s/valgrind-*-log*" % tmpdir):
+ f = open(fname)
+
+ for line in f.xreadlines():
+ for pat in pats:
+ rx = pats[pat]
+ match = rx.search(line)
+ if match:
+ val = int(match.group(1).replace(",", ""))
+ if pat in sums:
+ val = val + sums[pat]
+ sums[pat] = val
+ f.close()
+
+ for pat in sums:
+ logging.info("__COUNT_%s!%d" % (pat, sums[pat]))
+
+ archive_dir(tmpdir, sums)
+
+
+
+########################################################################
+# main
+########################################################################
+
+if len(sys.argv) != 2:
+ print("usage: %s " % sys.argv[0])
+ sys.exit(1)
+
+logging.basicConfig(level=logging.INFO,
+ format='%(asctime)s %(levelname)s %(message)s')
+
+logging.info("running as %s" % getpass.getuser())
+
+logging.info("killing any residual processes in this group")
+subprocess.call(["killall", "-q", "-9", "memcheck", "callgrind", "massif", "valgrind", "firefox-bin"])
+time.sleep(3)
+
+dirs=glob.glob(os.path.join(homedir, batchprefix + "*"))
+dirs.sort()
+if len(dirs) > 5:
+ for olddir in dirs[:-5]:
+ logging.info("cleaning up old directory %s" % olddir)
+ shutil.rmtree(olddir)
+
+
+timestr = datetime.datetime.now().isoformat().replace(":", "-")
+tmpdir = tempfile.mkdtemp(prefix=(batchprefix + timestr + "-"), dir=homedir)
+
+logging.info("started batch run in dir " + tmpdir)
+
+os.chdir(tmpdir)
+ffdir = sys.argv[1]
+
+write_vgopts(tmpdir, vgopts)
+
+xvnc_proc = None
+runner = None
+recorder = None
+
+######################################################
+# note: runit is supervising a single Xvnc on disp 25
+# there is no need to run one here as well
+######################################################
+# xvnc_proc = start_xvnc(disp, tmpdir)
+######################################################
+
+try:
+
+ runner = Firefox_runner(batchprefix, homedir, ffdir, timestr, tmpdir, disp)
+
+ wait_collecting_memory_stats(runner.run_normal(url))
+ runner.clear_cache()
+
+# for variant in ["memcheck", "massif", "callgrind"]:
+# recorder = start_xvnc_recorder(vnc2swfpath, disp, variant, tmpdir)
+# runner.run_valgrind(variant, url).wait()
+# runner.clear_cache()
+# complete_xvnc_recording(vnc2swfpath, recorder, variant, tmpdir)
+
+ log_result_summaries(tmpdir)
+ logging.info("valgrind-firefox processes complete")
+
+finally:
+ if recorder and recorder.poll() == None:
+ logging.info("killing recorder process %d" % recorder.pid)
+ os.kill(recorder.pid, signal.SIGKILL)
+ if xvnc_proc and xvnc_proc.poll() == None:
+ logging.info("killing Xvnc process %d" % xvnc_proc.pid)
+ os.kill(xvnc_proc.pid, signal.SIGKILL)
+
+logging.info("batch run in dir %s complete" % tmpdir)