#!/usr/bin/perl -w require 5.000; # This script has split some functions off into a util # script so they can be re-used by other scripts. require "build-seamonkey-util.pl"; use strict; use POSIX qw(sys_wait_h strftime); use Cwd; use File::Basename; # for basename(); use Config; # for $Config{sig_name} and $Config{sig_num} $::Version = '$Revision: 1.80 $ '; sub PrintUsage { die <) { s/^\s*\$(\S*)/\$Settings::$1/; eval; } } sub PrintExampleConfig { local $_; print "#- tinder-config.pl - Tinderbox configuration file.\n"; print "#- Uncomment the variables you need to set.\n"; print "#- The default values are the same as the commented variables.\n"; print "\n"; while () { s/^\$/\#\$/; print; } } sub ParseArgs { PrintUsage() if $#ARGV == -1; my $args = {}; while (my $arg = shift @ARGV) { $Settings::BuildDepend = 0, next if $arg eq '--clobber'; $Settings::BuildDepend = 1, next if $arg eq '--depend'; PrintExampleConfig(), exit if $arg eq '--example-config'; PrintUsage(), exit if $arg eq '--help' or $arg eq '-h'; $args->{ReportStatus} = 0, next if $arg eq '--noreport'; $args->{ReportFinalStatus} = 0, next if $arg eq '--nofinalreport'; $args->{RunTest} = 0, next if $arg eq '--notest'; $args->{TestOnly} = 1, next if $arg eq '--testonly'; $args->{BuildOnce} = 1, next if $arg eq '--once'; $args->{UseTimeStamp} = 0, next if $arg eq '--notimestamp'; my %args_with_options = qw( --display DisplayServer -tag BuildTag -t BuildTree --mozconfig MozConfigFileName ); if (defined $args_with_options{$arg}) { my $arg_arg = shift @ARGV; PrintUsage() if $arg_arg eq '' or $arg_arg =~ /^-/; $args->{$args_with_options{$arg}} = $arg_arg; } elsif ($arg eq '--version' or $arg eq '-v') { die "$0: version" . substr($::Version,9,6) . "\n"; } else { PrintUsage(); } } return $args; } sub ApplyArgs { my ($args) = @_; my ($variable_name, $value); while (($variable_name, $value) = each %{$args}) { eval "\$Settings::$variable_name = \"$value\";"; } } sub ConditionalArgs { $ENV{CVSROOT} = ":pserver:$ENV{USER}%netscape.com\@cvs.mozilla.org:/cvsroot"; $Settings::CVSCO .= " -r $Settings::BuildTag" unless $Settings::BuildTag eq ''; } sub print_log { my ($text) = @_; print LOG $text; print $text; } sub run_shell_command { my ($shell_command) = @_; local $_; chomp($shell_command); print_log "$shell_command\n"; open CMD, "$shell_command 2>&1|" or die "open: $!\n"; print_log $_ while ; close CMD; } sub adjust_start_time { # Allows the start time to match up with the update times of a mirror. my ($start_time) = @_; # Since we are not pulling for cvs-mirror anymore, just round times # to 1 minute intervals to make them nice and even. my $cycle = 1 * 60; # Updates every 1 minutes. my $begin = 0 * 60; # Starts 0 minutes after the hour. my $lag = 0 * 60; # Takes 0 minute to update. return int(($start_time - $begin - $lag) / $cycle) * $cycle + $begin; } sub BuildIt { # $Settings::DirName is set in buildl-seamonkey-utils.pl mkdir $Settings::DirName, 0777; chdir $Settings::DirName or die "Couldn't enter $Settings::DirName"; my $build_dir = getcwd(); my $binary_basename = "$Settings::BinaryName"; my $binary_dir = "$build_dir/$Settings::Topsrcdir/dist/bin"; my $full_binary_name = "$binary_dir/$binary_basename"; my $exit_early = 0; my $start_time = 0; # Bypass profile at startup. $ENV{MOZ_BYPASS_PROFILE_AT_STARTUP} = 1; print "Starting dir is : $build_dir\n"; while (not $exit_early) { # $BuildSleep is the minimum amount of time a build is allowed to take. # It prevents sending too many messages to the tinderbox server when # something is broken. my $sleep_time = ($Settings::BuildSleep * 60) - (time - $start_time); if (not $Settings::TestOnly and $sleep_time > 0) { print "\n\nSleeping $sleep_time seconds ...\n"; sleep $sleep_time; } $start_time = time(); my $cvsco = ''; if ($Settings::UseTimeStamp) { $start_time = adjust_start_time($start_time); my $time_str = strftime("%m/%d/%Y %H:%M", localtime($start_time)); $ENV{MOZ_CO_DATE} = "$time_str"; $cvsco = "$Settings::CVSCO -D '$time_str'"; } else { $cvsco = "$Settings::CVSCO -A"; } mail_build_started_message($start_time) if $Settings::ReportStatus; chdir $build_dir; my $logfile = "$Settings::DirName.log"; print "Opening $logfile\n"; open LOG, ">$logfile" or die "Cannot open logfile, $logfile: $?\n"; print_log "current dir is -- " . $ENV{HOST} . ":$build_dir\n"; print_log "Build Administrator is $Settings::BuildAdministrator\n"; PrintEnv(); unless (-e "mozilla/client.mk") { run_shell_command "$Settings::CVS $cvsco mozilla/client.mk"; } chdir $Settings::Topsrcdir or die "chdir $Settings::Topsrcdir: $!\n"; DeleteBinary($full_binary_name); # Delete the binary before rebuilding # Build it unless ($Settings::TestOnly) { # Do not build if testing smoke tests. my $targets = ''; $targets = 'checkout realclean build' unless $Settings::BuildDepend; run_shell_command "$Settings::Make -f client.mk $targets"; } my $build_status; if (BinaryExists($full_binary_name)) { if ($Settings::RunTest) { $build_status = run_tests($full_binary_name, $build_dir); } else { print_log "$binary_basename binary exists, build successful.\n"; print_log "Skipping test.\n"; $build_status = 'success'; } } else { print_log "Error: $binary_basename binary missing, build FAILED\n"; $exit_early++ if $Settings::TestOnly; $build_status = 'busted'; } close LOG; chdir $build_dir; mail_build_finished_message($start_time, $build_status, $logfile) if $Settings::ReportStatus; $exit_early++ if $Settings::BuildOnce; } } sub mail_build_started_message { my ($start_time) = @_; open LOG, "|$Settings::mail $Settings::Tinderbox_server"; PrintUsage() if $Settings::BuildTree =~ /^\s+$/i; print_log "\n"; print_log "tinderbox: tree: $Settings::BuildTree\n"; print_log "tinderbox: builddate: $start_time\n"; print_log "tinderbox: status: building\n"; print_log "tinderbox: build: $Settings::BuildName\n"; print_log "tinderbox: errorparser: unix\n"; print_log "tinderbox: buildfamily: unix\n"; print_log "tinderbox: version: $::Version\n"; print_log "tinderbox: END\n"; print_log "\n"; close LOG; } sub mail_build_finished_message { my ($start_time, $build_status, $logfile) = @_; # Rewrite LOG to OUTLOG, shortening lines. open OUTLOG, ">$logfile.last" or die "Unable to open logfile, $logfile: $!"; # Put the status at the top of the log, so the server will not # have to search through the entire log to find it. print OUTLOG "tinderbox: tree: $Settings::BuildTree\n"; print OUTLOG "tinderbox: builddate: $start_time\n"; print OUTLOG "tinderbox: status: $build_status\n"; print OUTLOG "tinderbox: build: $Settings::BuildName\n"; print OUTLOG "tinderbox: errorparser: unix\n"; print OUTLOG "tinderbox: buildfamily: unix\n"; print OUTLOG "tinderbox: version: $::Version\n"; print OUTLOG "tinderbox: END\n"; # Make sendmail happy. # Split lines longer than 1000 charaters into 1000 character lines. # If any line is a dot on a line by itself, replace it with a blank # line. This prevents cases where a . occurs in the log file. # Sendmail interprets that as the end of the mail, and truncates the # log before it gets to Tinderbox. (terry weismann, chris yeh) open LOG, "$logfile" or die "Couldn't open logfile, $logfile: $!"; while () { my $length = length($_); for (my $offset = 0; $offset < $length ; $offset += 1000) { my $chars_left = $length - $offset; my $output_length = $chars_left < 1000 ? $chars_left : 1000; my $output = substr $_, $offset, $output_length; $output =~ s/^\.$//g; $output =~ s/\n//g; print OUTLOG "$output\n"; } } close OUTLOG; close LOG; unlink($logfile); if ($Settings::ReportStatus and $Settings::ReportFinalStatus) { system "$Settings::mail $Settings::Tinderbox_server " ." < $logfile.last"; } } sub run_tests { my ($binary, $build_dir) = @_; my $binary_basename = basename($binary); my $binary_dir = dirname($binary); # Mozilla alive test print_log "Running AliveTest ...\n"; my $test_result = AliveTest($build_dir, $binary, 45); # Viewer alive test if ($Settings::ViewerTest and $test_result eq 'success') { print_log "Running ViewerTest ...\n"; $test_result = AliveTest($build_dir, "$binary_dir/viewer", 45); } # Bloat test if ($Settings::BloatStats or $Settings::BloatTest and $test_result eq 'success') { print_log "Running BloatTest ...\n"; $test_result = BloatTest($binary, $build_dir); } # MailNews test needs this preference set: # user_pref("signed.applets.codebase_principal_support",true); # First run gives two dialogs; they set this preference: # user_pref("security.principal.X0","[Codebase http://www.mozilla.org/quality/mailnews/APITest.html] UniversalBrowserRead=1 UniversalXPConnect=1"); if ($Settings::MailNewsTest and $test_result eq 'success') { print_log "Running MailNewsTest ...\n"; my $cmd = "$binary_basename " ."http://www.mozilla.org/quality/mailnews/APITest.html"; $test_result = FileBasedTest("MailNewsTest", $binary_dir, $cmd, 90, "MAILNEWS TEST: Passed", 1); } # Editor test if (($Settings::EditorTest or $Settings::DomToTextConversionTest) and $test_result eq 'success') { print_log "Running DomToTextConversionTest ...\n"; $test_result = FileBasedTest("DomToTextConversionTest", $build_dir, $binary_dir, "TestOutSinks", 45, "FAILED", 0); } return $test_result; } sub BinaryExists { my ($binary) = @_; my ($binary_basename) = basename($binary); if (not -e $binary) { print_log "$binary does not exist.\n"; 0; } elsif (not -s _) { print_log "$binary is zero-size.\n"; 0; } elsif (not -x _) { print_log "$binary is not executable.\n"; 0; } else { print_log "$binary_basename exists, is nonzero, and executable.\n"; 1; } } sub DeleteBinary { my ($binary) = @_; my ($binary_basename) = basename($binary); if (BinaryExists($binary)) { # Don't delete binary if we're only running tests. unless ($Settings::TestOnly) { print_log "Deleting binary: $binary_basename\n"; print_log "unlinking $binary\n"; unlink $binary or print_log "Error: Unlinking $binary failed\n"; } } else { print_log "No binary detected; none deleted.\n"; } } sub PrintEnv { local $_; foreach my $key (sort keys %ENV) { print_log "$key=$ENV{$key}\n"; } if (defined $ENV{MOZCONFIG} and -e $ENV{MOZCONFIG}) { print_log "-->mozconfig<----------------------------------------\n"; open CONFIG, "$ENV{MOZCONFIG}"; print_log $_ while ; close CONFIG; print_log "-->end mozconfig<----------------------------------------\n"; } if ($Settings::Compiler ne '') { print_log "===============================\n"; if ($Settings::Compiler eq 'gcc' or $Settings::Compiler eq 'egcc') { my $comptmp = `$Settings::Compiler --version`; chomp($comptmp); print_log "Compiler is -- $Settings::Compiler ($comptmp)\n"; } else { print_log "Compiler is -- $Settings::Compiler\n"; } print_log "===============================\n"; } } # Parse a file for $token, given a file handle. sub file_has_token { my ($filename, $token) = @_; local $_; my $has_token = 0; open TESTLOG, "<$filename" or die "Cannot open file, $filename: $!"; while () { if (/$token/) { $has_token = 1; last; } } close TESTLOG; return $has_token; } sub kill_process { my ($target_pid) = @_; my $start_time = time; # Try to kill and wait 10 seconds, then try a kill -9 for my $sig ('TERM', 'KILL') { kill $sig => $target_pid; my $interval_start = time; while (time - $interval_start < 10) { my $pid = waitpid($target_pid, WNOHANG()); if (($pid == $target_pid and WIFEXITED($?)) or $pid == -1) { my $secs = time - $start_time; $secs = $secs == 1 ? '1 second' : "$secs seconds"; print_log "Process killed. Took $secs to die.\n"; return; } sleep 1; } } die "Unable to kill process: $target_pid"; } BEGIN { my %sig_num = (); my @sig_name = (); sub signal_name { # Find the name of a signal number my ($number) = @_; unless (@sig_name) { unless($Config{sig_name} && $Config{sig_num}) { die "No sigs?"; } else { my @names = split ' ', $Config{sig_name}; @sig_num{@names} = split ' ', $Config{sig_num}; foreach (@names) { $sig_name[$sig_num{$_}] ||= $_; } } } return $sig_name[$number]; } } sub fork_and_log { # Fork a sub process and log the output. my ($home, $dir, $cmd, $logfile) = @_; my $pid = fork; # Fork off a child process. unless ($pid) { # child $ENV{HOME} = $home; chdir $dir; open STDOUT, ">$logfile"; open STDERR, ">&STDOUT"; select STDOUT; $| = 1; # make STDOUT unbuffered select STDERR; $| = 1; # make STDERR unbuffered exec $cmd; die "Could not exec()"; } return $pid; } sub wait_for_pid { # Wait for a process to exit or kill it if it takes too long. my ($pid, $timeout_secs) = @_; my ($exit_value, $signal_num, $dumped_core, $timed_out) = (0,0,0,0); $SIG{ALRM} = sub { die "timeout" }; eval { alarm $timeout_secs; while (1) { my $wait_pid = waitpid($pid, WNOHANG()); last if ($wait_pid == $pid and WIFEXITED($?)) or $wait_pid == -1; sleep 1; } $exit_value = $? >> 8; $signal_num = $? >> 127; $dumped_core = $? & 128; alarm 0; # Clear the alarm }; if ($@) { if ($@ =~ /timeout/) { kill_process($pid); $timed_out = 1; } else { # Died for some other reason. alarm(0); die; # Propagate the error up. } } my $signal_name = $signal_num ? signal_name($signal_num) : ''; return { timed_out=>$timed_out, exit_value=>$exit_value, signal_name=>$signal_name, dumped_core=>$dumped_core }; } sub run_test { my ($home_dir, $binary_dir, $binary, $logfile, $timeout_secs) = @_; my $now = localtime(); print_log "Begin: $now\n"; print_log "$binary\n"; my $pid = fork_and_log($home_dir, $binary_dir, $binary, $logfile); my $result = wait_for_pid($pid, $timeout_secs); $now = localtime(); print_log "End: $now\n"; return $result; } sub print_test_errors { my ($result, $name) = @_; if (not $result->{timed_out} and $result->{exit_value} != 0) { if ($result->{signal_name} ne '') { print_log "Error: $name: received SIG$result->{signal_name}\n"; } print_log "Error: $name: exited with status $result->{exit_value}\n"; if ($result->{dumped_core}) { print_log "Error: $name: dumped core.\n"; } } } sub print_logfile { my ($logfile, $test_name) = @_; print_log "----------- Output from $test_name ------------- \n"; open READRUNLOG, "$logfile"; print_log " $_" while ; close READRUNLOG; print_log "----------- End Output from $test_name --------- \n"; } # Start up Mozilla, test passes if Mozilla is still alive # after $timeout_secs (seconds). # sub AliveTest { my ($build_dir, $binary, $timeout_secs) = @_; my $binary_basename = basename($binary); my $binary_dir = dirname($binary); my $binary_log = "$build_dir/runlog"; local $_; my $result = run_test($build_dir, $binary_dir, $binary_basename, $binary_log, $timeout_secs); print_logfile($binary_log, "$binary_basename alive test"); if ($result->{timed_out}) { print_log "$binary_basename successfully stayed up" ." for $timeout_secs seconds.\n"; return 'success'; } else { print_test_errors($result, $binary_basename); print_log "Turn the tree orange now.\n"; return 'testfailed'; } } # Run a generic test that writes output to stdout, save that output to a # file, parse the file looking for failure token and report status based # on that. A hack, but should be useful for many tests. # # test_name = Name of test we're gonna run, in dist/bin. # testExecString = How to run the test # testTimeoutSec = Timeout for hung tests, minimum test time. # statusToken = What string to look for in test output to # determine test status. # statusTokenMeansPass = Default use of status token is to look for # failure string. If this is set to 1, then invert logic to look for # success string. # # Note: I tried to merge this function with AliveTest(), # the process flow control got too confusing :( -mcafee # sub FileBasedTest { my ($test_name, $build_dir, $binary_dir, $test_command, $timeout_secs, $status_token, $status_token_means_pass) = @_; local $_; # Assume the app is the first argument in the execString. my ($binary_basename) = (split /\s+/, $test_command)[0]; my $binary_log = "$build_dir/$test_name.log"; my $result = run_test($build_dir, $binary_dir, $test_command, $binary_log, $timeout_secs); print_logfile($binary_log, $test_name); if ($result->{timed_out}) { print_log "Error: $test_name timed out after $timeout_secs seconds.\n"; return 'testfailed'; } elsif ($result->{exit_value} != 0) { print_test_errors($result, $test_name); return 'testfailed'; } else { print_log "$test_name exited normally\n"; } my $found_token = file_has_token($binary_log, $status_token); if ($found_token) { print_log "Found status token in log file: $status_token\n"; } else { print_log "Status token, $status_token, not found\n"; } if (($status_token_means_pass and $found_token) or (not $status_token_means_pass and not $found_token)) { return 'success'; } else { print_log "Error: $test_name has failed. Turn the tree orange now.\n"; return 'testfailed'; } } # FileBasedTest sub BloatTest { my ($binary, $build_dir) = @_; my $binary_basename = basename($binary); my $binary_dir = dirname($binary); my $binary_log = "$build_dir/bloat-cur.log"; my $old_binary_log = "$build_dir/bloat-prev.log"; my $timeout_secs = 120; local $_; rename($binary_log, $old_binary_log); unless (-e "$binary_dir/bloaturls.txt") { print_log "Error: bloaturls.txt does not exist.\n"; return 'testfailed'; } $ENV{XPCOM_MEM_BLOAT_LOG} = 1; # Turn on ref counting to track leaks. my $cmd = "$binary_basename -f bloaturls.txt"; my $result = run_test($build_dir, $binary_dir, $cmd, $binary_log, $timeout_secs); delete $ENV{XPCOM_MEM_BLOAT_LOG}; print_logfile($binary_log, "bloat test"); if ($result->{timed_out}) { print_log "Error: bloat test timed out after" ." $timeout_secs seconds.\n"; return 'testfailed'; } elsif ($result->{exit_value}) { print_test_errors($result, $binary_basename); return 'testfailed'; } print_log "\n######################## BLOAT STATISTICS\n"; open DIFF, "$build_dir/../bloatdiff.pl $build_dir/bloat-prev.log $binary_log|" or die "Unable to run bloatdiff.pl"; print_log $_ while ; close DIFF; print_log "######################## END BLOAT STATISTICS\n\n"; return 'success'; } __END__ #- PLEASE FILL THIS IN WITH YOUR PROPER EMAIL ADDRESS $BuildAdministrator = "$ENV{USER}\@$ENV{HOST}"; #- You'll need to change these to suit your machine's needs $BaseDir = '/builds/tinderbox/SeaMonkey'; $DisplayServer = ':0.0'; #- Default values of command-line opts #- $BuildDepend = 1; # Depend or Clobber $ReportStatus = 1; # Send results to server, or not $ReportFinalStatus = 1; # Finer control over $ReportStatus. $UseTimeStamp = 1; # Use the CVS 'pull-by-timestamp' option, or not $BuildOnce = 0; # Build once, don't send results to server $RunTest = 1; # Run the smoke tests on successful build, or not $TestOnly = 0; # Only run tests, don't pull/build # Extra tests $BloatTest = 0; $BloatStats = 0; $DomToTextConversionTest = 0; $EditorTest = 0; $MailNewsTest = 0; $ViewerTest = 0; $MozConfigFileName = 'mozconfig'; #- Set these to what makes sense for your system $Make = 'gmake'; # Must be GNU make $MakeOverrides = ''; $mail = '/bin/mail'; $CVS = 'cvs -q'; $CVSCO = 'checkout -P'; #- Set these proper values for your tinderbox server $Tinderbox_server = 'tinderbox-daemon@tinderbox.mozilla.org'; #- #- The rest should not need to be changed #- #- Minimum wait period from start of build to start of next build in minutes. $BuildSleep = 10; #- Until you get the script working. When it works, #- change to the tree you're actually building $BuildTree = 'MozillaTest'; $BuildName = ''; $BuildTag = ''; $BuildConfigDir = 'mozilla/config'; $Topsrcdir = 'mozilla'; $BinaryName = 'mozilla-bin'; $ShellOverride = ''; # Only used if the default shell is too stupid $ConfigureArgs = ''; $ConfigureEnvArgs = ''; $Compiler = 'gcc'; $NSPRArgs = ''; $ShellOverride = '';