From 51d3050eafa215cb5bc7ce3090b47225080d7d75 Mon Sep 17 00:00:00 2001 From: "ted.mielczarek%gmail.com" Date: Wed, 27 Jun 2007 10:25:16 +0000 Subject: [PATCH] bug 383083 - post-process symbol files to add source file revision numbers. r=bsmedberg git-svn-id: svn://10.0.0.236/trunk@228830 18797224-902f-48f8-a5cc-f745e15eee43 --- mozilla/Makefile.in | 23 +- .../toolkit/airbag/tools/make_symbol_store.pl | 133 ------- mozilla/toolkit/airbag/tools/symbolstore.py | 332 ++++++++++++++++++ 3 files changed, 341 insertions(+), 147 deletions(-) delete mode 100755 mozilla/toolkit/airbag/tools/make_symbol_store.pl create mode 100755 mozilla/toolkit/airbag/tools/symbolstore.py diff --git a/mozilla/Makefile.in b/mozilla/Makefile.in index 2601ac9c1b0..ba6964968ba 100644 --- a/mozilla/Makefile.in +++ b/mozilla/Makefile.in @@ -137,32 +137,27 @@ endif # MOZILLA_OFFICIAL endif # WINNT ifeq ($(OS_ARCH),WINNT) -SYM_FIND_CMD := /bin/find . -path dist -prune -o -name "*.exe" \ - -o -name "*.dll" -o -name "*.EXE" | sed "s/\.[^\.]*$$/\.pdb/" # we want to copy PDB files on Windows MAKE_SYM_STORE_ARGS := -c DUMP_SYMS_BIN := $(topsrcdir)/toolkit/airbag/tools/win32/dump_syms.exe +# PDB files don't get moved to dist, so we need to scan the whole objdir +MAKE_SYM_STORE_PATH := . endif ifeq ($(OS_ARCH),Darwin) # need to pass arch flags for universal builds ifdef UNIVERSAL_BINARY MAKE_SYM_STORE_ARGS := -a "ppc i386" -SYM_DIST := $(DIST)/universal +MAKE_SYM_STORE_PATH := $(DIST)/universal else MAKE_SYM_STORE_ARGS := -a $(OS_TEST) -SYM_DIST := $(DIST) +MAKE_SYM_STORE_PATH := $(DIST)/bin endif -# |file| is stupid on universal binaries, it produces one line of output -# for the file, and one line of output for each architecture contained within. -SYM_FIND_CMD := find -L $(SYM_DIST) -type f -a -perm -100 -o -name "*.dylib" \ - | xargs file -L | grep "Mach-O" | grep -v "for architecture" | cut -f1 -d':' DUMP_SYMS_BIN := $(DIST)/host/bin/dump_syms endif ifeq ($(OS_ARCH),Linux) MAKE_SYM_STORE_ARGS := -SYM_FIND_CMD := find -L $(DIST)/bin -type f -a -perm -100 -o -name "*.so" \ - | xargs file -L | grep "ELF" | cut -f1 -d':' DUMP_SYMS_BIN := $(DIST)/host/bin/dump_syms +MAKE_SYM_STORE_PATH := $(DIST)/bin endif ifdef MOZ_SYMBOLS_EXTRA_BUILDID @@ -173,10 +168,10 @@ buildsymbols: ifdef MOZ_AIRBAG echo building symbol store mkdir -p $(DIST)/crashreporter-symbols/$(BUILDID) - $(SYM_FIND_CMD) | \ - xargs $(topsrcdir)/toolkit/airbag/tools/make_symbol_store.pl \ - $(MAKE_SYM_STORE_ARGS) $(DUMP_SYMS_BIN) \ - $(DIST)/crashreporter-symbols/$(BUILDID) > \ + $(PYTHON) $(topsrcdir)/toolkit/airbag/tools/symbolstore.py \ + $(MAKE_SYM_STORE_ARGS) -s $(topsrcdir) $(DUMP_SYMS_BIN) \ + $(DIST)/crashreporter-symbols/$(BUILDID) \ + $(MAKE_SYM_STORE_PATH) > \ $(DIST)/crashreporter-symbols/$(BUILDID)/$(MOZ_APP_NAME)-$(MOZ_APP_VERSION)-$(OS_ARCH)-$(BUILDID)$(EXTRA_BUILDID)-symbols.txt echo packing symbols mkdir -p $(topsrcdir)/../$(BUILDID) diff --git a/mozilla/toolkit/airbag/tools/make_symbol_store.pl b/mozilla/toolkit/airbag/tools/make_symbol_store.pl deleted file mode 100755 index 750fc4db95f..00000000000 --- a/mozilla/toolkit/airbag/tools/make_symbol_store.pl +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/perl -w -# ***** BEGIN LICENSE BLOCK ***** -# Version: MPL 1.1/GPL 2.0/LGPL 2.1 -# -# The contents of this file are subject to the Mozilla Public License Version -# 1.1 (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.mozilla.org/MPL/ -# -# Software distributed under the License is distributed on an "AS IS" basis, -# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -# for the specific language governing rights and limitations under the -# License. -# -# The Original Code is mozilla.org code. -# -# The Initial Developer of the Original Code is -# Ted Mielczarek -# Portions created by the Initial Developer are Copyright (C) 2006 -# the Initial Developer. All Rights Reserved. -# -# Contributor(s): -# -# Alternatively, the contents of this file may be used under the terms of -# either the GNU General Public License Version 2 or later (the "GPL"), or -# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), -# in which case the provisions of the GPL or the LGPL are applicable instead -# of those above. If you wish to allow use of your version of this file only -# under the terms of either the GPL or the LGPL, and not to allow others to -# use your version of this file under the terms of the MPL, indicate your -# decision by deleting the provisions above and replace them with the notice -# and other provisions required by the GPL or the LGPL. If you do not delete -# the provisions above, a recipient may use your version of this file under -# the terms of any one of the MPL, the GPL or the LGPL. -# -# ***** END LICENSE BLOCK ***** -# -# Usage: make_symbol_store.pl -# -# Runs dump_syms on each debug info file specified on the command line, -# then places the resulting symbol file in the proper directory -# structure in the symbol store path. Accepts multiple files -# on the command line, so can be called as part of a pipe using -# find | xargs make_symbol_store.pl -# Parameters accepted: -# -c : Copy debug info files to the same directory structure -# as sym files -# -a "" : Run dump_syms -a for each space separated -# cpu architecture in (only on OS X) - -use FileHandle; -use File::Path; -use File::Copy; -use File::Basename; - -print "Usage: make_symbol_store.pl " . - " \n" - and exit if scalar @ARGV < 3; - -# Given a symbol file generated by dump_syms, -# and a directory to store the resulting symbol path, -# move the symbol file into the directory structure -# expected by the breakpad processor. For details, see: -# http://google-breakpad.googlecode.com/svn/trunk/src/processor/simple_symbol_supplier.h -sub rename_symbol_file -{ - my ($symbol_file, $dest_path) = @_; - my $fh = FileHandle->new($symbol_file, "r"); - return "" unless $fh; - - my $line = <$fh>; - return "" unless $line; - $line =~ s/\s*$//; - $fh->close(); - return "" unless $line =~ m/^MODULE/; - - # the first line of a sym file looks like: - # MODULE os cpu identifier debug_file - my ($guid,$dbgfile) = (split(/ +/, $line))[3..4]; - my $newpath = $dest_path . "/" . $dbgfile . "/" . $guid; - eval { mkpath($newpath) }; - return "" if $@; - - if(move($symbol_file, $newpath)) { - my $out = $newpath; - my ($f) = fileparse($symbol_file); - $out =~ s/^$dest_path//; - $out =~ s|^/||; - print "$out/$f\n"; - return $newpath; - } - - return ""; -} - -my $copy_dbg = 0; -my @archs = (''); -while (@ARGV && $ARGV[0] =~ m/^-/) { - my $arg = shift; - if ($arg eq '-c') { - $copy_dbg = 1; - } - elsif ($arg eq '-a') { - @archs = (split(/\s+/, shift)); - } -} - -my $dump_syms = shift; -my $symbol_path = shift; -foreach my $dbgfile (@ARGV) { - next unless -f $dbgfile; - # get filename without path or .pdb extension, if it exists - my ($sf) = fileparse($dbgfile, ".pdb"); - my $symfile = $symbol_path . "/" . $sf . ".sym"; - foreach my $arch (@archs) { - my $a = ''; - $a = "-a $arch" if $arch ne ''; - system("${dump_syms} ${a} ${dbgfile} > ${symfile}"); - # remove empty sym file, probably no debug symbols in that file - if (-s $symfile == 0) { - unlink($symfile); - next; - } - my $newpath = rename_symbol_file $symfile, $symbol_path; - if ($copy_dbg && $newpath ne "") { - my $out = $newpath; - $out =~ s/^$symbol_path//; - $out =~ s|^/||; - print "$out/${sf}.pdb\n"; - copy($dbgfile, $newpath); - } - } -} diff --git a/mozilla/toolkit/airbag/tools/symbolstore.py b/mozilla/toolkit/airbag/tools/symbolstore.py new file mode 100755 index 00000000000..7ccfb4b8e92 --- /dev/null +++ b/mozilla/toolkit/airbag/tools/symbolstore.py @@ -0,0 +1,332 @@ +#!/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is mozilla.org code. +# +# The Initial Developer of the Original Code is +# The Mozilla Foundation +# Portions created by the Initial Developer are Copyright (C) 2007 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Ted Mielczarek +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** +# +# Usage: symbolstore.py +# +# Runs dump_syms on each debug info file specified on the command line, +# then places the resulting symbol file in the proper directory +# structure in the symbol store path. Accepts multiple files +# on the command line, so can be called as part of a pipe using +# find | xargs symbolstore.pl +# But really, you might just want to pass it . +# +# Parameters accepted: +# -c : Copy debug info files to the same directory structure +# as sym files +# -a "" : Run dump_syms -a for each space separated +# cpu architecture in (only on OS X) +# -s : Use as the top source directory to +# generate relative filenames. + +import sys +import os +import re +import shutil +from optparse import OptionParser + +# Utility functions + +def GetCVSRevision(file): + """Given a full path to a file, look in CVS/Entries + for the CVS revision number""" + (path, filename) = os.path.split(file) + entries = os.path.join(path, "CVS", "Entries") + if not os.path.isfile(entries): + return None + f = open(entries, "r") + for line in f: + parts = line.split("/") + if len(parts) > 1 and parts[1] == filename: + return parts[2] + print >> sys.stderr, "Failed to get CVS Revision for %s" % filename + return None + +def GetCVSRoot(file): + """Given a full path to a file, look in CVS/Root + for the CVS Root""" + (path, filename) = os.path.split(file) + root = os.path.join(path, "CVS", "Root") + if not os.path.isfile(root): + return None + f = open(root, "r") + root_name = f.readline().strip() + f.close() + parts = root_name.split("@") + if len(parts) > 1: + # we don't want the extra colon + return parts[1].replace(":","") + print >> sys.stderr, "Failed to get CVS Root for %s" % filename + return None + +def GetVCSFilename(file, srcdir): + """Given a full path to a file, and the top source directory, + look for version control information about this file, and return + a specially formatted filename that contains the VCS type, + VCS location, relative filename, and revision number, formatted like: + vcs:vcs location:filename:revision + For example: + cvs:cvs.mozilla.org/cvsroot:mozilla/browser/app/nsBrowserApp.cpp:1.36""" + (path, filename) = os.path.split(file) + cvsdir = os.path.join(path, "CVS") + if os.path.isdir(cvsdir): + rev = GetCVSRevision(file) + root = GetCVSRoot(file) + if rev is not None and root is not None: + if srcdir is not None: + # strip the base path off + # but we actually want the last dir in srcdir + # also, we want forward slashes on win32 paths + file = os.path.normpath(file) + file = file.replace(srcdir, "", 1) + (head, tail) = os.path.split(srcdir) + if tail == "": + tail = os.path.basename(head) + file = tail + file + file = file.replace("\\", "/") + return "cvs:%s:%s:%s" % (root, file, rev) + file = file.replace("\\", "/") + return file + +def GetPlatformSpecificDumper(**kwargs): + """This function simply returns a instance of a subclass of Dumper + that is appropriate for the current platform.""" + return {'win32': Dumper_Win32, + 'cygwin': Dumper_Win32, + 'linux2': Dumper_Linux, + 'darwin': Dumper_Mac}[sys.platform](**kwargs) + +class Dumper: + """This class can dump symbols from a file with debug info, and + store the output in a directory structure that is valid for use as + a Breakpad symbol server. Requires a path to a dump_syms binary-- + |dump_syms| and a directory to store symbols in--|symbol_path|. + Optionally takes a list of processor architectures to process from + each debug file--|archs|, the full path to the top source + directory--|srcdir|, for generating relative source file names, + and an option to copy debug info files alongside the dumped + symbol files--|copy_debug|, mostly useful for creating a + Microsoft Symbol Server from the resulting output. + + You don't want to use this directly if you intend to call + ProcessDir. Instead, call GetPlatformSpecificDumper to + get an instance of a subclass.""" + def __init__(self, dump_syms, symbol_path, + archs=None, srcdir=None, copy_debug=False): + self.dump_syms = dump_syms + self.symbol_path = symbol_path + if archs is None: + # makes the loop logic simpler + self.archs = [''] + else: + self.archs = ['-a %s' % a for a in archs.split()] + if srcdir is not None: + self.srcdir = os.path.normpath(srcdir) + else: + self.srcdir = None + self.copy_debug = copy_debug + + # subclasses override this + def ShouldProcess(self, file): + return False + + def RunFileCommand(self, file): + """Utility function, returns the output of file(1)""" + try: + # we use -L to read the targets of symlinks, + # and -b to print just the content, not the filename + return os.popen("file -Lb " + file).read() + except: + return "" + + # This is a no-op except on Win32 + def FixFilenameCase(self, file): + return file + + def Process(self, file_or_dir): + "Process a file or all the (valid) files in a directory." + if os.path.isdir(file_or_dir): + return self.ProcessDir(file_or_dir) + elif os.path.isfile(file_or_dir): + return self.ProcessFile(file_or_dir) + # maybe it doesn't exist? + return False + + def ProcessDir(self, dir): + """Process all the valid files in this directory. Valid files + are determined by calling ShouldProcess.""" + result = True + for root, dirs, files in os.walk(dir): + for f in files: + fullpath = os.path.join(root, f) + if self.ShouldProcess(fullpath): + if not self.ProcessFile(fullpath): + result = False + return result + + def ProcessFile(self, file): + """Dump symbols from this file into a symbol file, stored + in the proper directory structure in |symbol_path|.""" + result = False + for arch in self.archs: + try: + cmd = os.popen("%s %s %s" % (self.dump_syms, arch, file), "r") + module_line = cmd.next() + if module_line.startswith("MODULE"): + # MODULE os cpu guid debug_file + (guid, debug_file) = (module_line.split())[3:5] + # strip off .pdb extensions, and append .sym + sym_file = re.sub("\.pdb$", "", debug_file) + ".sym" + # we do want forward slashes here + rel_path = os.path.join(debug_file, + guid, + sym_file).replace("\\", "/") + full_path = os.path.normpath(os.path.join(self.symbol_path, + rel_path)) + try: + os.makedirs(os.path.dirname(full_path)) + except OSError: # already exists + pass + f = open(full_path, "w") + f.write(module_line) + # now process the rest of the output + for line in cmd: + if line.startswith("FILE"): + # FILE index filename + (x, index, filename) = line.split(None, 2) + filename = self.FixFilenameCase(filename.rstrip()) + filename = GetVCSFilename(filename, self.srcdir) + f.write("FILE %s %s\n" % (index, filename)) + else: + # pass through all other lines unchanged + f.write(line) + f.close() + cmd.close() + # we output relative paths so callers can get a list of what + # was generated + print rel_path + if self.copy_debug: + rel_path = os.path.join(debug_file, + guid, + debug_file).replace("\\", "/") + print rel_path + full_path = os.path.normpath(os.path.join(self.symbol_path, + rel_path)) + shutil.copyfile(file, full_path) + result = True + except StopIteration: + pass + except: + print >> sys.stderr, "Unexpected error: ", sys.exc_info()[0] + raise + return result + +# Platform-specific subclasses. For the most part, these just have +# logic to determine what files to extract symbols from. + +class Dumper_Win32(Dumper): + def ShouldProcess(self, file): + """This function will allow processing of pdb files that have dll + or exe files with the same base name next to them.""" + if file.endswith(".pdb"): + (path,ext) = os.path.splitext(file) + if os.path.isfile(path + ".exe") or os.path.isfile(path + ".dll"): + return True + return False + + def FixFilenameCase(self, file): + """Recent versions of Visual C++ put filenames into + PDB files as all lowercase. If the file exists + on the local filesystem, fix it.""" + (path, filename) = os.path.split(file) + if not os.path.isdir(path): + return file + lc_filename = filename.lower() + for f in os.listdir(path): + if f.lower() == lc_filename: + return os.path.join(path, f) + return file + +class Dumper_Linux(Dumper): + def ShouldProcess(self, file): + """This function will allow processing of files that are + executable, or end with the .so extension, and additionally + file(1) reports as being ELF files. It expects to find the file + command in PATH.""" + if file.endswith(".so") or os.access(file, os.X_OK): + return self.RunFileCommand(file).startswith("ELF") + return False + +class Dumper_Mac(Dumper): + def ShouldProcess(self, file): + """This function will allow processing of files that are + executable, or end with the .dylib extension, and additionally + file(1) reports as being Mach-O files. It expects to find the file + command in PATH.""" + if file.endswith(".dylib") or os.access(file, os.X_OK): + return self.RunFileCommand(file).startswith("Mach-O") + return False + +# Entry point if called as a standalone program +def main(): + parser = OptionParser(usage="usage: %prog [options] ") + parser.add_option("-c", "--copy", + action="store_true", dest="copy_debug", default=False, + help="Copy debug info files into the same directory structure as symbol files") + parser.add_option("-a", "--archs", + action="store", dest="archs", + help="Run dump_syms -a for each space separated cpu architecture in ARCHS (only on OS X)") + parser.add_option("-s", "--srcdir", + action="store", dest="srcdir", + help="Use SRCDIR to determine relative paths to source files") + (options, args) = parser.parse_args() + + if len(args) < 3: + parser.error("not enough arguments") + exit(1) + + dumper = GetPlatformSpecificDumper(dump_syms=args[0], + symbol_path=args[1], + copy_debug=options.copy_debug, + archs=options.archs, + srcdir=options.srcdir) + for arg in args[2:]: + dumper.Process(arg) + +# run main if run directly +if __name__ == "__main__": + main()