From d2ca8fb8a3ecef61586112bdb5cf47ab6cd33014 Mon Sep 17 00:00:00 2001 From: "rhelmer%mozilla.com" Date: Mon, 10 Jul 2006 19:24:43 +0000 Subject: [PATCH] add tinderbox standalone tests, b=343899 r=davel git-svn-id: svn://10.0.0.236/trunk@201833 18797224-902f-48f8-a5cc-f745e15eee43 --- .../tinderbox-standalone-tests/Tests.pm | 92 ++++ .../Tests/AliveTest.pm | 39 ++ .../Tests/AliveTestReturnToken.pm | 29 ++ .../Tests/BloatTest.pm | 135 ++++++ .../Tests/BloatTest2.pm | 135 ++++++ .../Tests/CodesizeTest.pm | 127 ++++++ .../Tests/DHTMLPerformanceTest.pm | 38 ++ .../Tests/FileBasedTest.pm | 64 +++ .../Tests/LayoutPerformanceTest.pm | 66 +++ .../Tests/QATest.pm | 50 +++ .../Tests/ReadLeakstatsLog.pm | 42 ++ .../Tests/Regxpcom.pm | 19 + .../Tests/RenderPerformanceTest.pm | 87 ++++ .../Tests/StartupPerformanceTest.pm | 107 +++++ .../Time/PossiblyHiRes.pm | 73 ++++ .../tinderbox-standalone-tests/Util.pm | 405 ++++++++++++++++++ .../tinderbox-standalone-tests/Util/Prefs.pm | 139 ++++++ .../Util/Settings.pm | 53 +++ .../test-mozilla.pl | 16 + 19 files changed, 1716 insertions(+) create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/AliveTest.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/AliveTestReturnToken.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/BloatTest.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/BloatTest2.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/CodesizeTest.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/DHTMLPerformanceTest.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/FileBasedTest.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/LayoutPerformanceTest.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/QATest.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/ReadLeakstatsLog.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/Regxpcom.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/RenderPerformanceTest.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Tests/StartupPerformanceTest.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Time/PossiblyHiRes.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Util.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Util/Prefs.pm create mode 100644 mozilla/testing/tinderbox-standalone-tests/Util/Settings.pm create mode 100755 mozilla/testing/tinderbox-standalone-tests/test-mozilla.pl diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests.pm b/mozilla/testing/tinderbox-standalone-tests/Tests.pm new file mode 100644 index 00000000000..59a046636ad --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests.pm @@ -0,0 +1,92 @@ +package Tests; + +use Util; +use strict; + +my $objdir = ''; + +sub global_prefs { + my ($binary, $embed_binary, $build_dir) = @_; + + my $binary_basename = File::Basename::basename($binary); + my $binary_dir = File::Basename::dirname($binary); + my $embed_binary_basename = File::Basename::basename($embed_binary); + my $embed_binary_dir = File::Basename::dirname($embed_binary); + + my $test_result = 'success'; + + # Windows needs this for file: urls. + my $win32_build_dir = $build_dir; + if ($Settings::OS =~ /^WIN/ && $win32_build_dir !~ m/^.:\//) { + chomp($win32_build_dir = `cygpath -w $win32_build_dir`); + $win32_build_dir =~ s/\\/\//g; + } + + # + # Set prefs to run tests properly. + # + # Chances are we will be timing these tests. Bring gettime() into memory + # by calling it once, before any tests run. + Time::PossiblyHiRes::getTime(); + + my $pref_file = 'prefs.js'; + + # Some tests need browser.dom.window.dump.enabled set to true, so + # that JS dump() will work in optimized builds. + Prefs::set_pref($pref_file, 'browser.dom.window.dump.enabled', 'true'); + + # Set security prefs to allow us to close our own window, + # pageloader test (and possibly other tests) needs this on. + Prefs::set_pref($pref_file, 'dom.allow_scripts_to_close_windows', 'true'); + + # Set security prefs to allow us to resize our windows. + # DHTML and Tgfx perf tests (and possibly other tests) need this off. + Prefs::set_pref($pref_file, 'dom.disable_window_flip', 'false'); + + # Set prefs to allow us to move, resize, and raise/lower the + # current window. Tgfx needs this. + Prefs::set_pref($pref_file, 'dom.disable_window_flip', 'false'); + Prefs::set_pref($pref_file, 'dom.disable_window_move_resize', 'false'); + + # Suppress firefox's popup blocking + if ($Settings::BinaryName =~ /^firefox/) { + Prefs::set_pref($pref_file, 'privacy.popups.firstTime', 'false'); + Prefs::set_pref($pref_file, 'dom.disable_open_during_load', 'false'); + + # Suppress default browser dialog + Prefs::set_pref($pref_file, 'browser.shell.checkDefaultBrowser', 'false'); + + # Suppress session restore dialog + Prefs::set_pref($pref_file, 'browser.sessionstore.resume_from_crash', 'false'); + } + elsif ($Settings::BinaryName eq 'Camino') { + Prefs::set_pref($pref_file, 'camino.check_default_browser', 'false'); + } + + # Suppress security warnings for QA test. + if ($Settings::QATest) { + Prefs::set_pref($pref_file, 'security.warn_submit_insecure', 'true'); + } + + # + # Assume that we want to test modern skin for all tests. + # + if (system("\\grep -s general.skins.selectedSkin \"$pref_file\" > /dev/null")) { + Util::print_log("Setting general.skins.selectedSkin to modern/1.0\n"); + open PREFS, ">>$pref_file" or die "can't open $pref_file ($?)\n"; + print PREFS "user_pref(\"general.skins.selectedSkin\", \"modern/1.0\");\n"; + close PREFS; + } else { + Util::print_log("Modern skin already set.\n"); + } + + if ($Settings::BinaryName eq 'Camino') { + # stdout will be block-buffered and will not be flushed when the test + # timeout expires and the process is killed, this would make tests + # appear to fail. + $ENV{'MOZ_UNBUFFERED_STDIO'} = 1; + } +} + + +1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/AliveTest.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/AliveTest.pm new file mode 100644 index 00000000000..39fc44e2449 --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/AliveTest.pm @@ -0,0 +1,39 @@ +# Start up Mozilla, test passes if Mozilla is still alive +# after $Settings::AliveTestTimeout (seconds). +# +sub AliveTest { + my ($build_dir, $binary) = @_; + my $binary_basename = File::Basename::basename($binary); + my $binary_dir = File::Basename::dirname($binary); + my $binary_log = "$build_dir/MozillaAliveTest.log"; + my $args = [$binary]; + local $_; + + # Print out testname + Util::print_log("\n\nRunning AliveTest test ...\n"); + + # Debug + Util::print_log("build_dir = $build_dir ...\n"); + Util::print_log("binary_dir = $binary_dir ...\n"); + Util::print_log("binary = $binary ...\n"); + + # Print out timeout. + Util::print_log("Timeout = $Settings::AliveTestTimeout seconds.\n"); + + my $result = Util::run_cmd($build_dir, $binary_dir, $args, + $Settings::AliveTestTimeout); + + Util::print_logfile($binary_log, "AliveTest"); + + if ($result->{timed_out}) { + Util::print_log("AliveTest: $binary_basename successfully stayed up". + " for $Settings::AliveTestTimeout seconds.\n"); + return 'success'; + } else { + Util::print_test_errors($result, $binary_basename); + Util::print_log("AliveTest: did not time out\n"); + return 'success'; + } +} + +1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/AliveTestReturnToken.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/AliveTestReturnToken.pm new file mode 100644 index 00000000000..1ea19e294f9 --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/AliveTestReturnToken.pm @@ -0,0 +1,29 @@ +# Same as AliveTest, but look for a token in the log and return +# the value. (used for startup iteration test). +sub AliveTestReturnToken { + my ($test_name, $build_dir, $args, $timeout_secs, $token, $delimiter) = @_; + my $status; + my $rv = 0; + + # Same as in AliveTest, needs to match in order to find the log file. + my $binary = @$args[0]; + my $binary_basename = File::Basename::basename($binary); + my $binary_dir = File::Basename::dirname($binary); + my $binary_log = "$build_dir/$test_name.log"; + + $status = AliveTest($test_name, $build_dir, $args, $timeout_secs); + + # Look for and return token + if ($status) { + $rv = Util::extract_token_from_file($binary_log, $token, $delimiter); + chomp($rv); + chop($rv) if ($rv =~ /\r$/); # cygwin perl doesn't chomp dos-newlinesproperly so use chop. + if ($rv) { + print "AliveTestReturnToken: token value = $rv\n"; + } + } + + return $rv; +} + +return 1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/BloatTest.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/BloatTest.pm new file mode 100644 index 00000000000..ff77a7e7c22 --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/BloatTest.pm @@ -0,0 +1,135 @@ +# Page loader/cycling mechanism (mozilla -f option): +# If you are building optimized, you need to add +# --enable-logrefcnt +# to turn the pageloader code on. These are on by default for debug. +# +sub BloatTest { + my ($binary, $build_dir, $bloat_args, $bloatdiff_label, $timeout_secs) = @_; + my $binary_basename = File::Basename::basename($binary); + my $binary_dir = File::Basename::dirname($binary); + my $binary_log = "$build_dir/bloat-cur.log"; + my $old_binary_log = "$build_dir/bloat-prev.log"; + local $_; + + rename($binary_log, $old_binary_log); + + unless (-e "$binary_dir/bloaturls.txt") { + Util::print_log("Error: bloaturls.txt does not exist.\n"); + return 'testfailed'; + } + + my $platform = $Settings::OS =~ /^WIN/ ? 'windows' : 'unix'; + # If on Windows, make sure the urls file has dos lineendings, or + # mozilla won't parse the file correctly + if ($platform eq 'windows') { + my $bu = "$binary_dir/bloaturls.txt"; + open(IN,"$bu") || die ("$bu: $!\n"); + open(OUT,">$bu.new") || die ("$bu.new: $!\n"); + while () { + if (! m/\r\n$/) { + s/\n$/\r\n/; + } + print OUT "$_"; + } + close(IN); + close(OUT); + File::Copy::move("$bu.new", "$bu") or die("move: $!\n"); + } + + $ENV{XPCOM_MEM_BLOAT_LOG} = 1; # Turn on ref counting to track leaks. + + # Build up binary command, look for profile. + my @args = ($binary_basename); + unless (($Settings::MozProfileName eq "") or + ($Settings::BinaryName eq "TestGtkEmbed")) { + @args = (@args, "-P", $Settings::MozProfileName); + } + @args = (@args, @$bloat_args); + + my $result = Util::run_cmd($build_dir, $binary_dir, \@args, $timeout_secs); + $ENV{XPCOM_MEM_BLOAT_LOG} = 0; + delete $ENV{XPCOM_MEM_BLOAT_LOG}; + + Util::print_logfile($binary_log, "$bloatdiff_label bloat test"); + + if ($result->{timed_out}) { + Util::print_log("Error: bloat test timed out after" + ." $timeout_secs seconds.\n"); + return 'testfailed'; + } elsif ($result->{exit_value}) { + Util::print_test_errors($result, $binary_basename); + Util::print_log("Error: bloat test failed.\n"); + return 'testfailed'; + } + + # Print out bloatdiff stats, also look for TOTAL line for leak/bloat #s. + Util::print_log("\n######################## BLOAT STATISTICS\n"); + my $found_total_line = 0; + my @total_line_array; + open DIFF, "perl $build_dir/../bloatdiff.pl $build_dir/bloat-prev.log $binary_log $bloatdiff_label|" + or die "Unable to run bloatdiff.pl"; + while () { + Util::print_log($_); + + # Look for fist TOTAL line + unless ($found_total_line) { + if (/TOTAL/) { + @total_line_array = split(" ", $_); + $found_total_line = 1; + } + } + } + close DIFF; + Util::print_log("######################## END BLOAT STATISTICS\n\n"); + + + # Scrape for leak/bloat totals from TOTAL line + # TOTAL 23 0% 876224 + my $leaks = $total_line_array[1]; + my $bloat = $total_line_array[3]; + + Util::print_log("leaks = $leaks\n"); + Util::print_log("bloat = $bloat\n"); + + # Figure out what the label prefix is. + my $label_prefix = ""; + my $testname_prefix = ""; + unless($bloatdiff_label eq "") { + $label_prefix = "$bloatdiff_label "; + $testname_prefix = "$bloatdiff_label" . "_"; + } + + # Figure out testnames to send to server + my $leaks_testname = "refcnt_leaks"; + my $bloat_testname = "refcnt_bloat"; + unless($bloatdiff_label eq "") { + $leaks_testname = $testname_prefix . "refcnt_leaks"; + $bloat_testname = $testname_prefix . "refcnt_bloat"; + } + + # Figure out testname labels + my $leaks_testname_label = "refcnt Leaks"; + my $bloat_testname_label = "refcnt Bloat"; + unless($bloatdiff_label eq "") { + $leaks_testname_label = $label_prefix . $leaks_testname_label; + $bloat_testname_label = $label_prefix . $bloat_testname_label; + } + + my $embed_prefix = ""; + if($Settings::BinaryName eq "TestGtkEmbed") { + $embed_prefix = "m"; + } + + Util::print_log_test_result_bytes($leaks_testname, $leaks_testname_label, + $leaks, + $label_prefix . $embed_prefix . 'RLk', 3); + + if($Settings::TestsPhoneHome) { + # Report numbers to server. + Util::send_results_to_server($leaks, "--", $leaks_testname); + } + + return 'success'; +} + +1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/BloatTest2.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/BloatTest2.pm new file mode 100644 index 00000000000..b65acb1807d --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/BloatTest2.pm @@ -0,0 +1,135 @@ +sub BloatTest2 { + my ($binary, $build_dir, $timeout_secs) = @_; + my $binary_basename = File::Basename::basename($binary); + my $binary_dir = File::Basename::dirname($binary); + my $PERL = $^X; + if ($Settings::OS =~ /^WIN/ && $build_dir !~ m/^.:\//) { + chomp($build_dir = `cygpath -w $build_dir`); + $build_dir =~ s/\\/\//g; + $PERL = "perl"; + } + my $binary_log = "$build_dir/bloattest2.log"; + my $malloc_log = "$build_dir/malloc.log"; + my $sdleak_log = "$build_dir/sdleak.log"; + my $old_sdleak_log = "$build_dir/sdleak.log.old"; + my $leakstats_log = "$build_dir/leakstats.log"; + my $old_leakstats_log = "$build_dir/leakstats.log.old"; + my $sdleak_diff_log = "$build_dir/sdleak.diff.log"; + local $_; + + unless (-e "$binary_dir/bloaturls.txt") { + Util::print_log("Error: bloaturls.txt does not exist.\n"); + return 'testfailed'; + } + + my $platform = $Settings::OS =~ /^WIN/ ? 'windows' : 'unix'; + # If on Windows, make sure the urls file has dos lineendings, or + # mozilla won't parse the file correctly + if ($platform eq 'windows') { + my $bu = "$binary_dir/bloaturls.txt"; + open(IN,"$bu") || die ("$bu: $!\n"); + open(OUT,">$bu.new") || die ("$bu.new: $!\n"); + while () { + if (! m/\r\n$/) { + s/\n$/\r\n/; + } + print OUT "$_"; + } + close(IN); + close(OUT); + File::Copy::move("$bu.new", "$bu") or die("move: $!\n"); + } + + rename($sdleak_log, $old_sdleak_log); + + my @args; + if($Settings::BinaryName eq "TestGtkEmbed" || + $Settings::BinaryName =~ /^firefox/) { + @args = ($binary_basename, "-P", $Settings::MozProfileName, + "resource:///res/bloatcycle.html", + "--trace-malloc", $malloc_log); + } else { + @args = ($binary_basename, "-P", $Settings::MozProfileName, + "-f", "bloaturls.txt", + "--trace-malloc", $malloc_log); + } + + # win32 builds crash on multiple runs when --shutdown-leaks is used + @args = (@args, "--shutdown-leaks", $sdleak_log) unless $Settings::OS =~ /^WIN/; + my $result = Util::run_cmd($build_dir, $binary_dir, \@args, $timeout_secs); + + Util::print_logfile($binary_log, "trace-malloc bloat test"); + + if ($result->{timed_out}) { + Util::print_log("Error: bloat test timed out after" + ." $timeout_secs seconds.\n"); + return 'testfailed'; + } elsif ($result->{exit_value}) { + Util::print_test_errors($result, $binary_basename); + Util::print_log("Error: bloat test failed.\n"); + return 'testfailed'; + } + + rename($leakstats_log, $old_leakstats_log); + + if ($Settings::OS =~ /^WIN/) { + @args = ("leakstats", $malloc_log); + } else { + @args = ("run-mozilla.sh", "./leakstats", $malloc_log); + } + $result = Util::run_cmd($build_dir, $binary_dir, \@args, $timeout_secs); + Util::print_logfile($leakstats_log, "trace-malloc bloat test: leakstats"); + + my $newstats = ReadLeakstatsLog($leakstats_log); + my $oldstats; + if (-e $old_leakstats_log) { + $oldstats = ReadLeakstatsLog($old_leakstats_log); + } else { + $oldstats = $newstats; + } + my $leakchange = PercentChange($oldstats->{'leaks'}, $newstats->{'leaks'}); + my $mhschange = PercentChange($oldstats->{'mhs'}, $newstats->{'mhs'}); + + my $leaks_testname_label = "Leaks: total bytes 'malloc'ed and not 'free'd"; + my $maxheap_testname_label = "Maximum Heap: max (bytes 'malloc'ed - bytes 'free'd) over run"; + my $allocs_testname_label = "Allocations: number of calls to 'malloc' and friends"; + + my $embed_prefix = ""; + if($Settings::BinaryName eq "TestGtkEmbed") { + $embed_prefix = "m"; + } + + my $leaks_testname = "trace_malloc_leaks"; + Util::print_log_test_result_bytes($leaks_testname, $leaks_testname_label, + $newstats->{'leaks'}, + $embed_prefix . 'Lk', 3); + + my $maxheap_testname = "trace_malloc_maxheap"; + Util::print_log_test_result_bytes($maxheap_testname, + $maxheap_testname_label, + $newstats->{'mhs'}, + $embed_prefix . 'MH', 3); + + my $allocs_testname = "trace_malloc_allocs"; + Util::print_log_test_result_count($allocs_testname, $allocs_testname_label, + $newstats->{'allocs'}, + $embed_prefix . 'A', 3); + + if ($Settings::TestsPhoneHome) { + # Send results to server. + Util::send_results_to_server($newstats->{'leaks'}, "--", $leaks_testname); + Util::send_results_to_server($newstats->{'mhs'}, "--", $maxheap_testname); + Util::send_results_to_server($newstats->{'allocs'}, "--", $allocs_testname); + } + + if (-e $old_sdleak_log && -e $sdleak_log) { + Util::print_logfile($old_leakstats_log, "previous run of trace-malloc bloat test leakstats"); + @args = ($PERL, "$build_dir/mozilla/tools/trace-malloc/diffbloatdump.pl", "--depth=15", $old_sdleak_log, $sdleak_log); + $result = Util::run_cmd($build_dir, $binary_dir, \@args, $timeout_secs); + Util::print_logfile($sdleak_diff_log, "trace-malloc leak stats differences"); + } + + return 'success'; +} + +1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/CodesizeTest.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/CodesizeTest.pm new file mode 100644 index 00000000000..748cf206e39 --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/CodesizeTest.pm @@ -0,0 +1,127 @@ +# +# Codesize test. Needs: cvs checkout mozilla/tools/codesighs +# +# This test can be run in two modes. One for the whole SeaMonkey +# tree, the other for just the embedding stuff. +# +sub CodesizeTest { + my ($test_name, $build_dir, $isEmbedTest) = @_; + + my $topsrcdir = "$build_dir/mozilla"; + + # test needs this set + $ENV{MOZ_MAPINFO} = "1"; + $ENV{TINDERBOX_OUTPUT} = "1"; + + #chdir(".."); # up one level. + + my $cwd = get_system_cwd(); + Util::print_log("cwd = $cwd\n"); + + my $type; # "auto" or "base" + my $zee; # Letter that shows up on tbox. + my $testNameString; + my $graphName; + + if($isEmbedTest) { + $testNameString = "Embed"; + $type = "base"; # Embed test. + $zee = "mZ"; + $graphName = "codesize_embed"; + } else { + if ($Settings::ProductName eq 'Mozilla') { + $testNameString = "SeaMonkey"; + } else { + $testNameString = $Settings::ProductName; + } + $type = "auto"; # SeaMonkey test. + $zee = "Z"; + $graphName = "codesize"; + } + + my $new_log = "Codesize-" . $type . "-new.log"; + my $old_log = "Codesize-" . $type . "-old.log"; + my $diff_log = "Codesize-" . $type . "-diff.log"; + my $test_log = "$test_name.log"; + + Util::print_log("\$build_dir = $build_dir\n"); + + # Clear the logs from the last run, so we can properly test for success. + unlink("$build_dir/$new_log"); + unlink("$build_dir/$diff_log"); + unlink("$build_dir/$test_log"); + + my $bash_cmd = "$topsrcdir/tools/codesighs/"; + if ($Settings::CodesizeManifest ne '') { + $type = ''; + } + if ($Settings::OS =~ /^WIN/ && $Settings::Compiler ne "gcc") { + $bash_cmd .= $type . "summary.win.bash"; + } else { + # Assume Linux for non-windows for now. + $bash_cmd .= $type . "summary.unix.bash"; + } + + my $cmd = ["bash", $bash_cmd]; + push(@{$cmd}, "-o", "$objdir") if ($Settings::ObjDir ne ""); + if ($Settings::CodesizeManifest ne '') { + push(@{$cmd}, "mozilla/$Settings::CodesizeManifest"); + } + push(@{$cmd}, $new_log, $old_log, $diff_log); + + my $test_result = + FileBasedTest($test_name, + "$build_dir", + "$build_dir", # run top of tree, not in dist. + $cmd, + $Settings::CodesizeTestTimeout, + "FAILED", # Fake out failure mode, test file instead. + 0, 0); # Timeout means failure. + + # Set status based on file creation. + if (-e "$build_dir/$new_log") { + Util::print_log("found $build_dir/$new_log\n"); + $test_result = 'success'; + + # Print diff data to tbox log. + if (-e "$build_dir/$diff_log") { + Util::print_logfile("$build_dir/$diff_log", "codesize diff log"); + } + + # + # Extract data. + # + my $z_data = Util::extract_token_from_file("$build_dir/$test_log", "__codesize", ":"); + chomp($z_data); + Util::print_log_test_result_bytes($graphName, + "$testNameString: Code + data size of all shared libs & executables", + $z_data, $zee, 4); + + if($Settings::TestsPhoneHome) { + Util::send_results_to_server($z_data, "--", $graphName); + } + + my $zdiff_data = Util::extract_token_from_file("$build_dir/$test_log", "__codesizeDiff", ":"); + chomp($zdiff_data); + + # Print out Zdiff if not zero. Testing "zero" by looking for "+0 ". + my $zdiff_sample = substr($zdiff_data,0,3); + if (not ($zdiff_sample eq "+0 ")) { + Util::print_log("\n"); + } + + # Testing only! Moves the old log to some unique name for testing. + # my $time = POSIX::strftime "%d%H%M%S", localtime; + # rename("$build_dir/$old_log", "$build_dir/$old_log.$time"); + + # Get ready for next cycle. + rename("$build_dir/$new_log", "$build_dir/$old_log"); + + + } else { + Util::print_log("Error: $build_dir/$new_log not found.\n"); + $test_result = 'buildfailed'; + } +} + +1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/DHTMLPerformanceTest.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/DHTMLPerformanceTest.pm new file mode 100644 index 00000000000..ba7732e1d9a --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/DHTMLPerformanceTest.pm @@ -0,0 +1,38 @@ +sub DHTMLPerformanceTest { + my ($build_dir, $binary) = @_; + my $dhtml_test_result; + my $binary_basename = File::Basename::basename($binary); + my $binary_dir = File::Basename::dirname($binary); + my $binary_log = "$build_dir/DHTMLPerformanceTest.log"; + my $url = "http://www.mozilla.org/performance/test-cases/dhtml/runTests.html"; + my $timeout_secs = 3; + my $args = [$binary, $url]; + + # Settle OS. + system('/bin/sh -c "sync; sleep 5"'); + + my $result = Util::run_cmd($build_dir, $binary_dir, $args, $timeout_secs); + + my $dhtml_time = Util::extract_token($result->{output}, + "_x_x_mozilla_dhtml", + ","); + + if($dhtml_time) { + $dhtml_test_result = 'success'; + } else { + $dhtml_test_result = 'testfailed'; + Util::print_log("DHTMLTest: test failed\n"); + } + + if($dhtml_test_result eq 'success') { + Util::print_log_test_result_ms('dhtml', 'DHTML time', + $dhtml_time, 'Tdhtml'); + if ($Settings::TestsPhoneHome) { + Util::send_results_to_server($dhtml_time, "--", "dhtml"); + } + } + + return $dhtml_test_result; +} + +1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/FileBasedTest.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/FileBasedTest.pm new file mode 100644 index 00000000000..9955e679f41 --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/FileBasedTest.pm @@ -0,0 +1,64 @@ +# Run a generic test that writes output to stdout, save that output to a +# file, parse the file looking for failure token and report status based +# on that. A hack, but should be useful for many tests. +# +# test_name = Name of test we're gonna run, in $Settings::DistBin. +# testExecString = How to run the test +# testTimeoutSec = Timeout for hung tests, minimum test time. +# statusToken = What string to look for in test output to +# determine test status. +# statusTokenMeansPass = Default use of status token is to look for +# failure string. If this is set to 1, then invert logic to look for +# success string. +# +# timeout_is_ok = Don't report test failure if test times out. +# +# Note: I tried to merge this function with AliveTest(), +# the process flow control got too confusing :( -mcafee +# +sub FileBasedTest { + my ($test_name, $build_dir, $binary_dir, $test_args, $timeout_secs, + $status_token, $status_token_means_pass, $timeout_is_ok) = @_; + local $_; + + # Assume the app is the first argument in the array. + my ($binary_basename) = @$test_args[0]; + my $binary_log = "$build_dir/$test_name.log"; + + # Print out test name + Util::print_log("\n\nRunning $test_name ...\n"); + + my $result = Util::run_cmd($build_dir, $binary_dir, $test_args, + $timeout_secs); + + Util::print_logfile($binary_log, $result->{output}); + Util::print_logfile($binary_log, $test_name); + + if (($result->{timed_out}) and (!$timeout_is_ok)) { + Util::print_log("Error: $test_name timed out after $timeout_secs seconds.\n"); + return 'testfailed'; + } elsif ($result->{exit_value} != 0) { + Util::print_log("Error: $test_name exited with status $result->{exit_value}\n"); + Util::print_test_errors($result, $test_name); + return 'testfailed'; + } else { + Util::print_log("$test_name exited normally\n"); + } + + my $found_token = file_has_token($binary_log, $status_token); + if ($found_token) { + Util::print_log("Found status token in log file: $status_token\n"); + } else { + Util::print_log("Status token, $status_token, not found\n"); + } + + if (($status_token_means_pass and $found_token) or + (not $status_token_means_pass and not $found_token)) { + return 'success'; + } else { + Util::print_log("Error: $test_name has failed.\n"); + return 'testfailed'; + } +} # FileBasedTest + +1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/LayoutPerformanceTest.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/LayoutPerformanceTest.pm new file mode 100644 index 00000000000..40def24312f --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/LayoutPerformanceTest.pm @@ -0,0 +1,66 @@ +sub LayoutPerformanceTest { + my ($build_dir, $binary) = @_; + + my $layout_test_result; + my $layout_time_details; + my $binary_basename = File::Basename::basename($binary); + my $binary_dir = File::Basename::dirname($binary); + #my $url = "http://$Settings::pageload_server/page-loader/loader.pl?delay=1000&nocache=0&maxcyc=4&timeout=$Settings::LayoutPerformanceTestPageTimeout&auto=1"; + my $timeout_secs = 4500; + my $url = 'file:///home/rhelmer/src/mozilla/tools/performance/pageload/cycler.html'; + $args = [$binary, $url]; + + # Settle OS. + system('/bin/sh -c "sync; sleep 5"'); + + + my $result = Util::run_cmd($build_dir, $binary_dir, $args, $timeout_secs); + + my $layout_time = Util::extract_token( + $result->{output}, + "_x_x_mozilla_page_load", + ","); + + if($layout_time) { + chomp($layout_time); + my @times = split(',', $layout_time); + $layout_time = $times[0]; # Set layout time to first number. + print("Layout time: $layout_time"); + } else { + Util::print_log("TinderboxPrint:Tp:[CRASH]\n"); + } + + # Set test status. + if($layout_time) { + $layout_test_result = 'success'; + Util::print_log("LayoutPerformanceTest: test succeeded\n"); + } else { + $layout_test_result = 'testfailed'; + Util::print_log("LayoutPerformanceTest: test failed\n"); + } + + if($layout_test_result eq 'success') { + my $tp_prefix = ""; + if($Settings::BinaryName eq "TestGtkEmbed") { + $tp_prefix = "m"; + } + + Util::print_log_test_result_ms('pageload', + 'Avg of the median per url pageload time', + $layout_time, 'Tp'); + + # Pull out detail data from log. + my $raw_data = Util::extract_token($result->{output}, + "_x_x_mozilla_page_load_details", + ","); + chomp($raw_data); + + if($Settings::TestsPhoneHome) { + Util::send_results_to_server($layout_time, $raw_data, "pageload"); + } + } + + return $layout_test_result; +} + +1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/QATest.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/QATest.pm new file mode 100644 index 00000000000..a524a6d45ba --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/QATest.pm @@ -0,0 +1,50 @@ + +# Client-side JavaScript, DOM Core/HTML/Views, and Form Submission tests. +# Currently only available inside netscape firewall. +sub QATest { + my ($test_name, $build_dir, $binary_dir, $args) = @_; + my $binary_log = "$build_dir/$test_name.log"; + my $url = "http://geckoqa.mcom.com/ngdriver/cgi-bin/ngdriver.cgi?findsuites=suites&tbox=1"; + + # Settle OS. + system('/bin/sh -c "sync; sleep 5"'); + + my $rv; + + $rv = AliveTest("QATest_raw", $build_dir, + [@$args, $url], + $Settings::QATestTimeout); + + # XXXX testing. -mcafee + $rv = 'success'; + + + # Post-process log of test output. + my $mode = "express"; + open QATEST, "perl $build_dir/../qatest.pl $build_dir/QATest_raw.log $mode |" + or die "Unable to run qatest.pl"; + my $qatest_html = ""; + while () { + chomp; + #$_ =~ s/\"/"/g; #### it doesn't like this line + # $_ =~ s/\012//g; + ### $_ =~ s/\s+\S/ /g; # compress whitespace. + $qatest_html .= $_; + } + close QATEST; + Util::print_log("\n"); + + # This works. + #$qatest_html = "
 PassedFailedTotalDied% Passed% Failed
DHTML90901000
DOM VIEWS20201000
Total:1101101000
"; + + # Testing output + open TEST_OUTPUT, ">qatest_out.log"; + print TEST_OUTPUT $qatest_html; + close TEST_OUTPUT; + + Util::print_log("TinderboxPrint:QA\n"); + + return $rv; # Hard-coded for now. +} + +1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/ReadLeakstatsLog.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/ReadLeakstatsLog.pm new file mode 100644 index 00000000000..91fd50feae2 --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/ReadLeakstatsLog.pm @@ -0,0 +1,42 @@ +# Page loader (-f option): +# If you are building optimized, you need to add +# --enable-trace-malloc --enable-perf-metrics +# to turn the pageloader code on. If you are building debug you only +# need +# --enable-trace-malloc +# + +sub ReadLeakstatsLog($) { + my ($filename) = @_; + my $leaks = 0; + my $leaked_allocs = 0; + my $mhs = 0; + my $bytes = 0; + my $allocs = 0; + + open LEAKSTATS, "$filename" + or die "unable to open $filename"; + while () { + chop; + my $line = $_; + if ($line =~ /Leaks: (\d+) bytes, (\d+) allocations/) { + $leaks = $1; + $leaked_allocs = $2; + } elsif ($line =~ /Maximum Heap Size: (\d+) bytes/) { + $mhs = $1; + } elsif ($line =~ /(\d+) bytes were allocated in (\d+) allocations./) { + $bytes = $1; + $allocs = $2; + } + } + + return { + 'leaks' => $leaks, + 'leaked_allocs' => $leaked_allocs, + 'mhs' => $mhs, + 'bytes' => $bytes, + 'allocs' => $allocs + }; +} + +1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/Regxpcom.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/Regxpcom.pm new file mode 100644 index 00000000000..a8148a3a6c0 --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/Regxpcom.pm @@ -0,0 +1,19 @@ +sub Regxpcom { + # + # Before running tests, run regxpcom so that we don't crash when + # people change contractids on us (since we don't autoreg opt builds) + # + unlink("$binary_dir/components/compreg.dat") or warn "$binary_dir/components/compreg.dat not removed\n"; + if($Settings::RegxpcomTest) { + my $args; + if ($Settings::BinaryName =~ /^(firefox|thunderbird)/) { + $args = [$binary, "-register"]; + } else { + $args = ["$binary_dir/regxpcom"]; + } + AliveTest("regxpcom", $binary_dir, $args, + $Settings::RegxpcomTestTimeout); + } +} + +1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/RenderPerformanceTest.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/RenderPerformanceTest.pm new file mode 100644 index 00000000000..c2e5746b050 --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/RenderPerformanceTest.pm @@ -0,0 +1,87 @@ + +# +# Trender test +# + +sub RenderPerformanceTest { + my ($test_name, $build_dir, $binary_dir, $args) = @_; + my $render_test_result; + my $render_time; + my $render_gfx_time; + my $render_details; + my $binary_log = "$build_dir/$test_name.log"; + my $url; + + # Find Trender.xml + if (-f "/cygdrive/c/builds/tinderbox/Trender/Trender.xml") { + $url = "file:///C:/builds/tinderbox/Trender/Trender.xml#tinderbox=1"; + } elsif (-f "/builds/tinderbox/Trender/Trender.xml") { + $url = "file:///builds/tinderbox/Trender/Trender.xml#tinderbox=1"; + } else { + Util::print_log("TinderboxPrint:Trender:[NOTFOUND]\n"); + return 'testfailed'; + } + + # Settle OS. + system('/bin/sh -c "sync; sleep 5"'); + + $render_test_result = FileBasedTest($test_name, $build_dir, $binary_dir, + [@$args, $url], + $Settings::RenderTestTimeout, + "_x_x_mozilla_trender", 1, 1); + + # double check to make sure the test didn't really succeed + # even though the scripts think it failed. Prevents various breakage + # (e.g. when a timeout happens on the mac, killing the process returns + # a bogus result code). FileBasedTest checks the status code + # before the token + my $found_token = file_has_token($binary_log, "_x_x_mozilla_trender"); + if ($found_token) { + $render_test_result = 'success'; + } + + if ($render_test_result eq 'testfailed') { + Util::print_log("TinderboxPrint:Trender:[FAILED]\n"); + return 'testfailed'; + } + + $render_time = Util::extract_token_from_file($binary_log, "_x_x_mozilla_trender", ","); + if ($render_time) { + chomp($render_time); + my @times = split(',', $render_time); + $render_time = $times[0]; + } + $render_time =~ s/[\r\n]//g; + + $render_gfx_time = Util::extract_token_from_file($binary_log, "_x_x_mozilla_trender_gfx", ","); + if ($render_gfx_time) { + chomp($render_gfx_time); + my @times = split(',', $render_gfx_time); + $render_gfx_time = $times[0]; + } + $render_gfx_time =~ s/[\r\n]//g; + + if (!$render_time || !$render_gfx_time) { + Util::print_log("TinderboxPrint:Trender:[FAILED]\n"); + return 'testfailed'; + } + + Util::print_log_test_result_ms('render', 'Avg page render time in ms', + $render_time, 'Tr'); + + Util::print_log_test_result_ms('rendergfx', 'Avg gfx render time in ms', + $render_gfx_time, 'Tgfx'); + + if($Settings::TestsPhoneHome) { + # Pull out detail data from log; this includes results for all sets + my $raw_data = Util::extract_token_from_file($binary_log, "_x_x_mozilla_trender_details", ","); + chomp($raw_data); + + Util::send_results_to_server($render_time, $raw_data, "render"); + Util::send_results_to_server($render_gfx_time, $raw_data, "rendergfx"); + } + + return 'success'; +} + +1; diff --git a/mozilla/testing/tinderbox-standalone-tests/Tests/StartupPerformanceTest.pm b/mozilla/testing/tinderbox-standalone-tests/Tests/StartupPerformanceTest.pm new file mode 100644 index 00000000000..3735baaca39 --- /dev/null +++ b/mozilla/testing/tinderbox-standalone-tests/Tests/StartupPerformanceTest.pm @@ -0,0 +1,107 @@ +# Startup performance test. Time how fast it takes the browser +# to start up. Some help from John Morrison to get this going. +# +# Needs user_pref("browser.dom.window.dump.enabled", 1); +# (or CPPFLAGS=-DMOZ_ENABLE_JS_DUMP in mozconfig since we +# don't have profiles for tbox right now.) +# +# $startup_url needs ?begin=