From b841cdc7d6ff8c117eb49ce22af2da868daf2527 Mon Sep 17 00:00:00 2001 From: "dbaron%dbaron.org" Date: Sat, 18 Jan 2003 16:24:55 +0000 Subject: [PATCH] Add memory dump reader and new tool (leaksoup) for analyzing leak graph at shutdown. b=179212 Partly reviewed by brendan. git-svn-id: svn://10.0.0.236/trunk@136600 18797224-902f-48f8-a5cc-f745e15eee43 --- mozilla/tools/trace-malloc/Makefile.in | 15 +- mozilla/tools/trace-malloc/adreader.cpp | 161 +++++++++ mozilla/tools/trace-malloc/adreader.h | 129 +++++++ mozilla/tools/trace-malloc/leaksoup.cpp | 431 ++++++++++++++++++++++++ 4 files changed, 733 insertions(+), 3 deletions(-) create mode 100644 mozilla/tools/trace-malloc/adreader.cpp create mode 100644 mozilla/tools/trace-malloc/adreader.h create mode 100644 mozilla/tools/trace-malloc/leaksoup.cpp diff --git a/mozilla/tools/trace-malloc/Makefile.in b/mozilla/tools/trace-malloc/Makefile.in index 0a85d33ca79..3d9d5ca8c44 100644 --- a/mozilla/tools/trace-malloc/Makefile.in +++ b/mozilla/tools/trace-malloc/Makefile.in @@ -27,6 +27,7 @@ VPATH = @srcdir@ include $(DEPTH)/config/autoconf.mk REQUIRES = xpcom \ + string \ tracemalloc \ $(NULL) @@ -36,7 +37,14 @@ SIMPLECSRCS += \ tmstats.c \ $(NULL) -SIMPLE_PROGRAMS = $(SIMPLECSRCS:.c=$(BIN_SUFFIX)) +SIMPLECPPSRCS = \ + leaksoup.cpp \ + $(NULL) + +SIMPLE_PROGRAMS = \ + $(SIMPLECSRCS:.c=$(BIN_SUFFIX)) \ + $(SIMPLECPPSRCS:.cpp=$(BIN_SUFFIX)) \ + $(NULL) CSRCS = \ spacetrace.c \ @@ -57,6 +65,7 @@ EXTRA_DSO_LIBS = tracemalloc LIBS += \ $(LIBS_DIR) \ tmreader.$(OBJ_SUFFIX) \ + adreader.$(OBJ_SUFFIX) \ $(EXTRA_DSO_LIBS) \ $(XPCOM_LIBS) \ $(NSPR_LIBS) \ @@ -82,8 +91,8 @@ DEFINES += \ $(NULL) endif -EXTRA_DEPS = tmreader.$(OBJ_SUFFIX) -GARBAGE += tmreader.$(OBJ_SUFFIX) +EXTRA_DEPS = tmreader.$(OBJ_SUFFIX) adreader.$(OBJ_SUFFIX) +GARBAGE += tmreader.$(OBJ_SUFFIX) adreader.$(OBJ_SUFFIX) include $(topsrcdir)/config/rules.mk diff --git a/mozilla/tools/trace-malloc/adreader.cpp b/mozilla/tools/trace-malloc/adreader.cpp new file mode 100644 index 00000000000..b1798cdadcd --- /dev/null +++ b/mozilla/tools/trace-malloc/adreader.cpp @@ -0,0 +1,161 @@ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is the trace-malloc Allocation Dump Reader (adreader). + * + * The Initial Developer of the Original Code is L. David Baron. + * Portions created by the Initial Developer are Copyright (C) 2002 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * L. David Baron (original author) + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +#include "adreader.h" + +#include +#include + +ADLog::ADLog() + : mEntryCount(0) +{ + mEntries.mNext = static_cast(&mEntries); + mEntries.mPrev = static_cast(&mEntries); +} + +ADLog::~ADLog() +{ + for (const_iterator entry = begin(), entry_end = end(); + entry != entry_end; ++entry) { + free((void*) (*entry)->type); + free((char*) (*entry)->data); + free((char*) (*entry)->allocation_stack); + } + + for (EntryBlock *b = mEntries.mNext, *next; b != &mEntries; b = next) { + next = b->mNext; + delete b; + } +} + +bool +ADLog::Read(const char* aFileName) +{ + FILE *in = fopen(aFileName, "r"); + if (!in) { + return false; + } + + while (!feof(in)) { + unsigned int ptr; + char typebuf[256]; + int datasize; + int res = fscanf(in, "%x %s (%d)\n", &ptr, typebuf, &datasize); + if (res == EOF) + break; + if (res != 3) { + return false; + } + + char *data = (char*)malloc(((datasize+3)/4)*4); + + for (char *cur_data = data, + *cur_data_end = data + ((datasize+3)/4)*4; + cur_data != cur_data_end; cur_data += 4) { + res = fscanf(in, " %x\n", (unsigned int*)cur_data); + if (res != 1) { + return false; + } + } + + char stackbuf[100000]; + stackbuf[0] = '\0'; + + char *stack = stackbuf; + int len; + do { + fgets(stack, sizeof(stackbuf) - (stack - stackbuf), in); + len = strlen(stack); + stack += len; + } while (len > 1); + + if (mEntryCount % ADLOG_ENTRY_BLOCK_SIZE == 0) { + EntryBlock *new_block = new EntryBlock(); + new_block->mNext = static_cast(&mEntries); + new_block->mPrev = mEntries.mPrev; + mEntries.mPrev->mNext = new_block; + mEntries.mPrev = new_block; + } + + size_t typelen = strlen(typebuf); + char *type = (char*)malloc(typelen-1); + strncpy(type, typebuf+1, typelen-2); + type[typelen-2] = '\0'; + + Entry *entry = + &mEntries.mPrev->entries[mEntryCount % ADLOG_ENTRY_BLOCK_SIZE]; + entry->address = (Pointer)ptr; + entry->type = type; + entry->datasize = datasize; + entry->data = data; + entry->allocation_stack = strdup(stackbuf); + + ++mEntryCount; + } + + return true; +} + +ADLog::const_iterator::const_iterator(ADLog::EntryBlock *aBlock, + size_t aOffset) +{ + SetBlock(aBlock); + mCur = mBlockStart + aOffset; +} + +ADLog::const_iterator& +ADLog::const_iterator::operator++() +{ + ++mCur; + if (mCur == mBlockEnd) { + SetBlock(mBlock->mNext); + mCur = mBlockStart; + } + + return *this; +} + +ADLog::const_iterator& +ADLog::const_iterator::operator--() +{ + if (mCur == mBlockStart) { + SetBlock(mBlock->mPrev); + mCur = mBlockEnd; + } + --mCur; + + return *this; +} diff --git a/mozilla/tools/trace-malloc/adreader.h b/mozilla/tools/trace-malloc/adreader.h new file mode 100644 index 00000000000..f3f40284dbb --- /dev/null +++ b/mozilla/tools/trace-malloc/adreader.h @@ -0,0 +1,129 @@ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is the trace-malloc Allocation Dump Reader (adreader). + * + * The Initial Developer of the Original Code is L. David Baron. + * Portions created by the Initial Developer are Copyright (C) 2002 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * L. David Baron (original author) + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +#include + +#define ADLOG_ENTRY_BLOCK_SIZE 4096 + +class ADLog { + +public: + + // Use typedef in case somebody wants to process 64-bit output on a + // 32-bit machine someday. + typedef const char* Pointer; + + struct Entry { + Pointer address; + const char *type; + + const char *data; // The contents of the memory. + size_t datasize; + + const char *allocation_stack; + }; + + ADLog(); + ~ADLog(); + + /* + * Returns false on failure and true on success. + */ + bool Read(const char *aFilename); + +private: + // Link structure for a circularly linked list. + struct EntryBlock; + struct EntryBlockLink { + EntryBlock *mPrev; + EntryBlock *mNext; + }; + + struct EntryBlock : public EntryBlockLink { + Entry entries[ADLOG_ENTRY_BLOCK_SIZE]; + }; + + size_t mEntryCount; + EntryBlockLink mEntries; + +public: + + class const_iterator { + private: + // Only |ADLog| member functions can construct iterators. + friend class ADLog; + const_iterator(EntryBlock *aBlock, size_t aOffset); + + public: + const Entry* operator*() { return mCur; } + const Entry* operator->() { return mCur; } + + const_iterator& operator++(); + const_iterator& operator--(); + + bool operator==(const const_iterator& aOther) const { + return mCur == aOther.mCur; + } + + bool operator!=(const const_iterator& aOther) const { + return mCur != aOther.mCur; + } + + private: + void SetBlock(EntryBlock *aBlock) { + mBlock = aBlock; + mBlockStart = aBlock->entries; + mBlockEnd = aBlock->entries + ADLOG_ENTRY_BLOCK_SIZE; + } + + EntryBlock *mBlock; + Entry *mCur, *mBlockStart, *mBlockEnd; + + // Not to be implemented. + const_iterator operator++(int); + const_iterator operator--(int); + }; + + const_iterator begin() { + return const_iterator(mEntries.mNext, 0); + } + const_iterator end() { + return const_iterator(mEntries.mPrev, + mEntryCount % ADLOG_ENTRY_BLOCK_SIZE); + } + + size_t count() { return mEntryCount; } +}; diff --git a/mozilla/tools/trace-malloc/leaksoup.cpp b/mozilla/tools/trace-malloc/leaksoup.cpp new file mode 100644 index 00000000000..11a3257423c --- /dev/null +++ b/mozilla/tools/trace-malloc/leaksoup.cpp @@ -0,0 +1,431 @@ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is leaksoup3, a memory graph analysis tool that + * finds roots based on strongly connected components (SCCs). + * + * The Initial Developer of the Original Code is L. David Baron. + * Portions created by the Initial Developer are Copyright (C) 2002 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * L. David Baron (original author) + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +#include "adreader.h" + +#include +#include "plhash.h" + +#include "nsVoidArray.h" +#include "nsQuickSort.h" + +/* + * Read in an allocation dump, presumably one taken at shutdown (using + * the --shutdown-leaks=file option, which must be used along with + * --trace-malloc=tmlog), and treat the memory in the dump as leaks. + * Find the leak roots, including cycles that are roots, by finding the + * strongly connected components in the graph. Print output to stdout + * as HTML. + */ + +struct AllocationNode { + const ADLog::Entry *entry; + + // Other |AllocationNode| objects whose memory has a pointer to + // this object. + nsAutoVoidArray pointers_to; + + // The reverse. + nsAutoVoidArray pointers_from; + + // Early on in the algorithm, the pre-order index from a DFS. + // Later on, set to the index of the strongly connected index to + // which this node belongs. + PRUint32 index; + + PRPackedBool reached; + PRPackedBool is_root; +}; + +static PLHashNumber hash_pointer(const void *key) +{ + return (PLHashNumber) key; +} + +static int sort_by_index(const void* e1, const void* e2, void*) +{ + const AllocationNode *n1 = *NS_STATIC_CAST(const AllocationNode*const*, e1); + const AllocationNode *n2 = *NS_STATIC_CAST(const AllocationNode*const*, e2); + return n1->index - n2->index; +} + +static int sort_by_reverse_index(const void* e1, const void* e2, void*) +{ + const AllocationNode *n1 = *NS_STATIC_CAST(const AllocationNode*const*, e1); + const AllocationNode *n2 = *NS_STATIC_CAST(const AllocationNode*const*, e2); + return n2->index - n1->index; +} + +static void print_escaped(FILE *aStream, const char* aData) +{ + char c; + char buf[1000]; + char *p = buf; + while ((c = *aData++)) { + switch (c) { +#define CH(char) *p++ = char + case '<': + CH('&'); CH('l'); CH('t'); CH(';'); + break; + case '>': + CH('&'); CH('g'); CH('t'); CH(';'); + break; + case '&': + CH('&'); CH('a'); CH('m'); CH('p'); CH(';'); + break; + default: + CH(c); + break; +#undef CH + } + if (p + 10 > buf + sizeof(buf)) { + *p = '\0'; + fputs(buf, aStream); + p = buf; + } + } + *p = '\0'; + fputs(buf, aStream); +} + +int main(int argc, char **argv) +{ + if (argc != 2) { + fprintf(stderr, + "Expected usage: %s \n" + " sd-leak-file: Output of --shutdown-leaks= option.\n", + argv[0]); + return 1; + } + + ADLog log; + if (!log.Read(argv[1])) { + fprintf(stderr, + "%s: Error reading input file %s.\n", argv[0], argv[1]); + } + + const size_t count = log.count(); + + PLHashTable *memory_map = + PL_NewHashTable(count * 8, hash_pointer, PL_CompareValues, + PL_CompareValues, 0, 0); + if (!memory_map) { + fprintf(stderr, "%s: Out of memory.\n", argv[0]); + return 1; + } + + // Create one |AllocationNode| object for each log entry, and create + // entries in the hashtable pointing to it for each byte it occupies. + AllocationNode *nodes = new AllocationNode[count]; + if (!nodes) { + fprintf(stderr, "%s: Out of memory.\n", argv[0]); + return 1; + } + + { + AllocationNode *cur_node = nodes; + for (ADLog::const_iterator entry = log.begin(), entry_end = log.end(); + entry != entry_end; ++entry, ++cur_node) { + const ADLog::Entry *e = cur_node->entry = *entry; + cur_node->reached = PR_FALSE; + + for (ADLog::Pointer p = e->address, + p_end = e->address + e->datasize; + p != p_end; ++p) { + PLHashEntry *e = PL_HashTableAdd(memory_map, p, cur_node); + if (!e) { + fprintf(stderr, "%s: Out of memory.\n", argv[0]); + return 1; + } + } + } + } + + // Construct graph based on pointers. + for (AllocationNode *node = nodes, *node_end = nodes + count; + node != node_end; ++node) { + const ADLog::Entry *e = node->entry; + for (const char *d = e->data, *d_end = e->data + e->datasize - + e->datasize % sizeof(ADLog::Pointer); + d != d_end; d += sizeof(ADLog::Pointer)) { + AllocationNode *target = (AllocationNode*) + PL_HashTableLookup(memory_map, *(void**)d); + if (target) { + target->pointers_from.AppendElement(node); + node->pointers_to.AppendElement(target); + } + } + } + + // Do a depth-first search on the graph (i.e., by following + // |pointers_to|) and assign the post-order index to |index|. + { + PRUint32 dfs_index = 0; + nsVoidArray stack; + + for (AllocationNode *n = nodes, *n_end = nodes+count; n != n_end; ++n) { + if (n->reached) { + continue; + } + stack.AppendElement(n); + + do { + PRUint32 pos = stack.Count() - 1; + AllocationNode *n = + NS_STATIC_CAST(AllocationNode*, stack[pos]); + if (n->reached) { + n->index = dfs_index++; + stack.RemoveElementAt(pos); + } else { + n->reached = PR_TRUE; + + // When doing post-order processing, we have to be + // careful not to put reached nodes into the stack. + nsVoidArray &pf = n->pointers_to; + for (PRInt32 i = pf.Count() - 1; i >= 0; --i) { + if (!NS_STATIC_CAST(AllocationNode*, pf[i])->reached) { + stack.AppendElement(pf[i]); + } + } + } + } while (stack.Count() > 0); + } + } + + // Sort the nodes by their DFS index, in reverse, so that the first + // node is guaranteed to be in a root SCC. + AllocationNode **sorted_nodes = new AllocationNode*[count]; + if (!sorted_nodes) { + fprintf(stderr, "%s: Out of memory.\n", argv[0]); + return 1; + } + + { + for (size_t i = 0; i < count; ++i) { + sorted_nodes[i] = nodes + i; + } + NS_QuickSort(sorted_nodes, count, sizeof(AllocationNode*), + sort_by_reverse_index, 0); + } + + // Put the nodes into their strongly-connected components. + PRUint32 num_sccs = 0; + { + for (size_t i = 0; i < count; ++i) { + nodes[i].reached = PR_FALSE; + } + nsVoidArray stack; + for (AllocationNode **sn = sorted_nodes, + **sn_end = sorted_nodes + count; sn != sn_end; ++sn) { + if ((*sn)->reached) { + continue; + } + + // We found a new strongly connected index. + stack.AppendElement(*sn); + do { + PRUint32 pos = stack.Count() - 1; + AllocationNode *n = + NS_STATIC_CAST(AllocationNode*, stack[pos]); + stack.RemoveElementAt(pos); + + if (!n->reached) { + n->reached = PR_TRUE; + n->index = num_sccs; + stack.AppendElements(n->pointers_from); + } + } while (stack.Count() > 0); + ++num_sccs; + } + } + + // Identify which nodes are leak roots by using DFS, and watching + // for component transitions. + PRUint32 num_root_nodes = count; + { + for (size_t i = 0; i < count; ++i) { + nodes[i].is_root = PR_TRUE; + } + + nsVoidArray stack; + for (AllocationNode *n = nodes, *n_end = nodes+count; n != n_end; ++n) { + if (!n->is_root) { + continue; + } + + // Loop through pointers_to, and add any that are in a + // different SCC to stack: + for (int i = n->pointers_to.Count() - 1; i >= 0; --i) { + AllocationNode *target = + NS_STATIC_CAST(AllocationNode*, n->pointers_to[i]); + if (n->index != target->index) { + stack.AppendElement(target); + } + } + + while (stack.Count() > 0) { + PRUint32 pos = stack.Count() - 1; + AllocationNode *n = + NS_STATIC_CAST(AllocationNode*, stack[pos]); + stack.RemoveElementAt(pos); + + if (n->is_root) { + n->is_root = PR_FALSE; + --num_root_nodes; + stack.AppendElements(n->pointers_to); + } + } + } + } + + // Sort the nodes by their SCC index. + NS_QuickSort(sorted_nodes, count, sizeof(AllocationNode*), + sort_by_index, 0); + + // Print output. + { + printf("\n" + "\n" + "\n" + "Leak analysis\n" + "\n" + "\n"); + printf("\n\n" + "

Generated %d entries (%d in root SCCs) and %d SCCs.

\n\n", + count, num_root_nodes, num_sccs); + + for (size_t i = 0; i < count; ++i) { + nodes[i].reached = PR_FALSE; + } + + // Loop over the sorted nodes twice, first printing the roots + // and then the non-roots. + for (PRBool root_type = PR_TRUE; + root_type == PR_TRUE || root_type == PR_FALSE; --root_type) { + if (root_type) { + printf("\n\n" + "
\n" + "

Root components

\n"); + } else { + printf("\n\n" + "
\n" + "

Non-root components

\n"); + } + PRUint32 component = (PRUint32)-1; + PRBool one_object_component; + for (const AllocationNode *const* sn = sorted_nodes, + *const* sn_end = sorted_nodes + count; + sn != sn_end; ++sn) { + const AllocationNode *n = *sn; + if (n->is_root != root_type) + continue; + const ADLog::Entry *e = n->entry; + + if (n->index != component) { + component = n->index; + one_object_component = + sn + 1 == sn_end || (*(sn+1))->index != component; + if (!one_object_component) + printf("\n\n

Component %d

\n", + component, component); + } + + if (one_object_component) { + printf("\n\n
\n", component); + printf("

Object %d " + "(single-object component %d)

\n", + n-nodes, n-nodes, component); + } else { + printf("\n\n

Object %d

\n", + n-nodes, n-nodes); + } + printf("
\n");
+                printf("%p <%s> (%d)\n",
+                       e->address, e->type, e->datasize);
+                for (size_t d = 0; d < e->datasize;
+                     d += sizeof(ADLog::Pointer)) {
+                    AllocationNode *target = (AllocationNode*)
+                        PL_HashTableLookup(memory_map, *(void**)(e->data + d));
+                    if (target) {
+                        printf("        0x%08X <%s>\n",
+                               target - nodes,
+                               *(unsigned int*)(e->data + d),
+                               target->entry->type);
+                    } else {
+                        printf("        0x%08X\n",
+                               *(unsigned int*)(e->data + d));
+                    }
+                }
+
+                if (n->pointers_from.Count()) {
+                    printf("\nPointers from:\n");
+                    for (PRUint32 i = 0, i_end = n->pointers_from.Count();
+                         i != i_end; ++i) {
+                        AllocationNode *t = NS_STATIC_CAST(AllocationNode*,
+                                                          n->pointers_from[i]);
+                        const ADLog::Entry *te = t->entry;
+                        printf("    %s (Object %d, ",
+                               t - nodes, te->type, t - nodes);
+                        if (t == n) {
+                            printf("self)\n");
+                        } else {
+                            printf("%p)\n", te->address);
+                        }
+                    }
+                }
+
+                print_escaped(stdout, e->allocation_stack);
+
+                printf("
\n"); + if (one_object_component) { + printf("
\n"); + } + } + printf("
\n"); + } + printf("\n" + "\n"); + } + + delete [] sorted_nodes; + delete [] nodes; + + return 0; +}