From c6ac89905601c6a3a41cb959ea6a9de69102f7cf Mon Sep 17 00:00:00 2001 From: "brettw%gmail.com" Date: Mon, 10 Apr 2006 23:04:40 +0000 Subject: [PATCH] Bug 328598 r=bryner Expire history as you browse for faster shutdown times. git-svn-id: svn://10.0.0.236/branches/MOZILLA_1_8_BRANCH@194068 18797224-902f-48f8-a5cc-f745e15eee43 --- .../places/public/nsIFaviconService.idl | 20 +- .../places/public/nsINavHistoryService.idl | 14 + .../browser/components/places/src/Makefile.in | 1 + .../places/src/nsFaviconService.cpp | 18 - .../components/places/src/nsFaviconService.h | 3 - .../components/places/src/nsNavBookmarks.cpp | 8 + .../components/places/src/nsNavHistory.cpp | 162 +---- .../components/places/src/nsNavHistory.h | 14 +- .../places/src/nsNavHistoryExpire.cpp | 639 ++++++++++++++++++ .../places/src/nsNavHistoryExpire.h | 114 ++++ .../places/src/nsNavHistoryResult.cpp | 27 + .../places/src/nsNavHistoryResult.h | 5 +- 12 files changed, 863 insertions(+), 162 deletions(-) create mode 100644 mozilla/browser/components/places/src/nsNavHistoryExpire.cpp create mode 100644 mozilla/browser/components/places/src/nsNavHistoryExpire.h diff --git a/mozilla/browser/components/places/public/nsIFaviconService.idl b/mozilla/browser/components/places/public/nsIFaviconService.idl index 0e0ba48ecdb..8e19215de07 100644 --- a/mozilla/browser/components/places/public/nsIFaviconService.idl +++ b/mozilla/browser/components/places/public/nsIFaviconService.idl @@ -103,23 +103,27 @@ interface nsIFaviconService : nsISupports * You can set the data even if you haven't called SetFaviconUrlForPage * yet. It will be stored but will not be associated with any page. * However, any favicons not associated with a visited web page, bookmark, - * or "place:" URI will be expired when history cleanup is done (typically - * at app shutdown, but also possibly if the user clears their cache or - * history). It is best to call this function first for notification purposes. + * or "place:" URI will be expired when history cleanup is done. This might + * be done at any time on a timer, so you should not let the message loop + * run between calls or your icon may get deleted. + * + * It is best to set the favicon data, and then associate it with a page. + * This will make the notifications more efficient since the icon will + * already have data when the set favicon observer messages goes out. * * The expiration time is stored. This will be used if you call * SetAndLoadFaviconForPage to see whether the data needs reloading. * - * Do not use this function for chrome: URLs. You should reference the chrome - * image yourself. The GetFaviconLinkForIcon/Page will ignore any associated - * data if the favicon URI is "chrome:" and just return the same chrome URI. + * Do not use this function for chrome: icon URLs. You should reference the + * chrome image yourself. The GetFaviconLinkForIcon/Page will ignore any + * associated data if the favicon URI is "chrome:" and just return the same + * chrome URI. * * This function does NOT send out notifications that the data has changed. * Potentially, many pages could be referencing the favicon and they could * be visible in a history view or toolbar. But sending out those * notifications is very intensive. Those pages will keep the old icon - * until they have been refreshed by other means. Favicons don't change much - * anyway. + * until they have been refreshed by other means. * * @param aFavicon * URI of the favicon whose data is being set. diff --git a/mozilla/browser/components/places/public/nsINavHistoryService.idl b/mozilla/browser/components/places/public/nsINavHistoryService.idl index 451053a4fbe..fdd42c4e020 100644 --- a/mozilla/browser/components/places/public/nsINavHistoryService.idl +++ b/mozilla/browser/components/places/public/nsINavHistoryService.idl @@ -757,6 +757,20 @@ interface nsINavHistoryObserver : nsISupports */ const PRUint32 ATTRIBUTE_FAVICON = 3; // favicon updated, aString = favicon annotation URI void onPageChanged(in nsIURI aURI, in PRUint32 aWhat, in AString aValue); + + /** + * Called when a history entry expires. You will recieve notifications that + * a specific visit has expired with the time of that visit. When the last + * visit for a history entry expires, the history entry itself is deleted + * and aWholeEntry is set. (If your observer only cares about URLs and not + * specific visits, it needs only to listen for aWholeEntry notifications). + * + * It is possible for a history entry to be deleted that has no visits if + * something is out of sync or after a bookmark is deleted that has no + * visits (thus freeing the history entry). In these cases, aVisitTime will + * be 0. + */ + void onPageExpired(in nsIURI aURI, in PRTime aVisitTime, in boolean aWholeEntry); }; diff --git a/mozilla/browser/components/places/src/Makefile.in b/mozilla/browser/components/places/src/Makefile.in index 66cc43b26e1..994eff339f7 100644 --- a/mozilla/browser/components/places/src/Makefile.in +++ b/mozilla/browser/components/places/src/Makefile.in @@ -84,6 +84,7 @@ CPPSRCS = \ nsFaviconService.cpp \ nsNavHistory.cpp \ nsNavHistoryAutoComplete.cpp \ + nsNavHistoryExpire.cpp \ nsNavHistoryQuery.cpp \ nsNavHistoryResult.cpp \ nsNavBookmarks.cpp \ diff --git a/mozilla/browser/components/places/src/nsFaviconService.cpp b/mozilla/browser/components/places/src/nsFaviconService.cpp index 003c5ad7f7e..0232b01b98c 100644 --- a/mozilla/browser/components/places/src/nsFaviconService.cpp +++ b/mozilla/browser/components/places/src/nsFaviconService.cpp @@ -191,24 +191,6 @@ nsFaviconService::InitTables(mozIStorageConnection* aDBConn) } -// nsFaviconservice::VacuumFavicons -// -// Called by the history service after history entries have been expired. -// This will delete all favicon entries that are not referenced. -// -// FIXME: This is very slow, and slows down shutdown. See bug 328598. - -nsresult // static -nsFaviconService::VacuumFavicons(mozIStorageConnection* aDBConn) -{ - return aDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "DELETE FROM moz_favicon WHERE id IN " - "(SELECT f.id FROM moz_favicon f " - "LEFT OUTER JOIN moz_history h ON f.id = h.favicon " - "WHERE h.favicon IS NULL)")); -} - - // nsFaviconService::SetFaviconUrlForPage NS_IMETHODIMP diff --git a/mozilla/browser/components/places/src/nsFaviconService.h b/mozilla/browser/components/places/src/nsFaviconService.h index c0506b6230c..4004a874772 100644 --- a/mozilla/browser/components/places/src/nsFaviconService.h +++ b/mozilla/browser/components/places/src/nsFaviconService.h @@ -57,9 +57,6 @@ public: // called by nsNavHistory::Init static nsresult InitTables(mozIStorageConnection* aDBConn); - // called by nsNavHistory::VacuumDB - static nsresult VacuumFavicons(mozIStorageConnection* aDBConn); - /** * Returns a cached pointer to the favicon service for consumers in the * places directory. diff --git a/mozilla/browser/components/places/src/nsNavBookmarks.cpp b/mozilla/browser/components/places/src/nsNavBookmarks.cpp index 9edeb53bbc9..6bd49c2b5d9 100644 --- a/mozilla/browser/components/places/src/nsNavBookmarks.cpp +++ b/mozilla/browser/components/places/src/nsNavBookmarks.cpp @@ -2316,3 +2316,11 @@ nsNavBookmarks::OnPageChanged(nsIURI *aURI, PRUint32 aWhat, } return NS_OK; } + +NS_IMETHODIMP +nsNavBookmarks::OnPageExpired(nsIURI* aURI, PRTime aVisitTime, + PRBool aWholeEntry) +{ + // pages that are bookmarks shouldn't expire, so we don't need to handle it + return NS_OK; +} diff --git a/mozilla/browser/components/places/src/nsNavHistory.cpp b/mozilla/browser/components/places/src/nsNavHistory.cpp index abd18a5d669..69c350e4da9 100644 --- a/mozilla/browser/components/places/src/nsNavHistory.cpp +++ b/mozilla/browser/components/places/src/nsNavHistory.cpp @@ -216,6 +216,7 @@ nsNavHistory* nsNavHistory::gHistoryService; nsNavHistory::nsNavHistory() : mNowValid(PR_FALSE), mExpireNowTimer(nsnull), + mExpire(this), mBatchesInProgress(0), mAutoCompleteOnlyTyped(PR_FALSE) { @@ -448,6 +449,15 @@ nsNavHistory::InitDB(PRBool *aDoImport) NS_ENSURE_SUCCESS(rv, rv); } + // FIXME: this should be moved inside the moz_history table creation block. + // It is left outside and the return value is ignored because alpha 1 did not + // have this index. When it is likely that all alpha users have run a more + // recent build, we can move this to only happen on init so that startup time + // is faster. This index is used for favicon expiration, see + // nsNavHistoryExpire::ExpireItems + mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "CREATE INDEX moz_history_faviconindex ON moz_history (favicon)")); + // moz_historyvisit rv = mDBConn->TableExists(NS_LITERAL_CSTRING("moz_historyvisit"), &tableExists); NS_ENSURE_SUCCESS(rv, rv); @@ -1005,105 +1015,6 @@ PRBool nsNavHistory::IsURIStringVisited(const nsACString& aURIString) } -// nsNavHistory::VacuumDB -// -// Deletes everything from the database that is old. It is called on app -// shutdown and when you clear history. -// -// "Old" is anything older than now - aTimeAgo. On app shutdown, aTimeAgo is -// the time in preferences for history expiration. If aTimeAgo is 0 -// (everything older than now), all history will be deleted (everything -// older than now). -// -// We DO NOT notify observers that the items are being deleted. I can't think -// of any reason why anybody would need to know (we are shutting down, after -// all), and if there is an observer still attached, it could significantly -// prolong shutdown. -// -// The statement: -// DELETE FROM a WHERE a.id in (SELECT id FROM a LEFT OUTER JOIN b ON a.id = b.bar WHERE b.bid IS NULL); -// matches the visits and history entries. An outer join means that we -// generate a row for every item in the left-hand table (in this case, the -// history table), even when there is no corresponding right-hand table. -// Then we just pick those items where there was no corresponding right-hand -// one (i.e., a moz_history entry with no visits), and delete it. -// -// We never delete "place:" URIs. These are used to associate information -// with queries and bookmark folders that are never technically visited. -// In some cases, you can get dangling ones, (referencing a folder that -// was deleted, for example) but that should be rare and insignificant. -// -// Some times we may want to vacuum which will defragment the database and -// return any unused space to the filesystem. This operation can be -// extremely slow (sometimes 1 minute) and we don't do it now. Deleted data -// is overwritten with 0s by sqlite since we use SQLITE_SECURE_DELETE -// (in the sqlite makefile). Perhaps we should expose a way to do that. -// -// Implementation note if we do a vacuum: There can not be any active -// statements for the vacuuming to work. This includes the dummy database. -// To make this work you would have to complete the dummy statement (and -// possibly detach the connection), do the vacuum, and then re-attach -// everything. - -nsresult -nsNavHistory::VacuumDB(PRTime aTimeAgo) -{ - nsresult rv; - - // go ahead and commit on error, only some things will get deleted, which, - // if you are trying to clear your history, is probably better than nothing. - mozStorageTransaction transaction(mDBConn, PR_TRUE); - - if (aTimeAgo) { - // find the threshold for deleting old visits - PRTime now = GetNow(); - PRInt64 expirationTime = now - aTimeAgo; - if (expirationTime < 0) - expirationTime = 0; - - // delete expired visits - nsCOMPtr visitDelete; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "DELETE FROM moz_historyvisit WHERE visit_date < ?1"), - getter_AddRefs(visitDelete)); - NS_ENSURE_SUCCESS(rv, rv); - rv = visitDelete->BindInt64Parameter(0, expirationTime); - NS_ENSURE_SUCCESS(rv, rv); - rv = visitDelete->Execute(); - NS_ENSURE_SUCCESS(rv, rv); - } else { - // delete all visits - nsCOMPtr visitDelete; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "DELETE FROM moz_historyvisit"), getter_AddRefs(visitDelete)); - NS_ENSURE_SUCCESS(rv, rv); - rv = visitDelete->Execute(); - NS_ENSURE_SUCCESS(rv, rv); - } - - // delete history entries with no visits that are not bookmarked - // also never delete any "place:" URIs (see function header comment) - rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "DELETE FROM moz_history WHERE id IN (SELECT id FROM moz_history h " - "LEFT OUTER JOIN moz_historyvisit v ON h.id = v.page_id " - "LEFT OUTER JOIN moz_bookmarks b ON h.id = b.item_child " - "WHERE v.visit_id IS NULL " - "AND b.item_child IS NULL " - "AND SUBSTR(url,0,6) <> 'place:')")); - NS_ENSURE_SUCCESS(rv, rv); - - // FIXME: delete annotations for expiring pages - - // delete favicons that no pages reference - rv = nsFaviconService::VacuumFavicons(mDBConn); - NS_ENSURE_SUCCESS(rv, rv); - - transaction.Commit(); - - return NS_OK; -} - - // nsNavHistory::LoadPrefs nsresult @@ -2395,20 +2306,17 @@ NS_IMETHODIMP nsNavHistory::RemoveAllPages() { // expire everything - VacuumDB(0); + mExpire.ClearHistory(); - // compress DB (compression is slow, but since the user requested it, they - // either want the disk space or to cover their tracks). The dummy statement - // must be stopped because vacuuming will change everything and invalidate - // the cache. + // Compress DB. Currently commented out because compression is very slow. + // Deleted data will be overwritten with 0s by sqlite. Note that we have to + // stop the dummy statement before doing this. +#if 0 StopDummyStatement(); nsresult rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("VACUUM")); StartDummyStatement(); NS_ENSURE_SUCCESS(rv, rv); - - // notify observers - ENUMERATE_WEAKARRAY(mObservers, nsINavHistoryObserver, OnClearHistory()) - +#endif return NS_OK; } @@ -2536,9 +2444,12 @@ nsNavHistory::AddURI(nsIURI *aURI, PRBool aRedirect, if (mExpireDays == 0) return NS_OK; + PRTime now = PR_Now(); + + nsresult rv; #ifdef LAZY_ADD LazyMessage message; - nsresult rv = message.Init(LazyMessage::Type_AddURI, aURI); + rv = message.Init(LazyMessage::Type_AddURI, aURI); NS_ENSURE_SUCCESS(rv, rv); message.isRedirect = aRedirect; message.isToplevel = aToplevel; @@ -2546,11 +2457,17 @@ nsNavHistory::AddURI(nsIURI *aURI, PRBool aRedirect, rv = aReferrer->Clone(getter_AddRefs(message.referrer)); NS_ENSURE_SUCCESS(rv, rv); } - message.time = PR_Now(); - return AddLazyMessage(message); + message.time = now; + rv = AddLazyMessage(message); + NS_ENSURE_SUCCESS(rv, rv); #else - return AddURIInternal(aURI, PR_Now(), aRedirect, aToplevel, aReferrer); + rv = AddURIInternal(aURI, now, aRedirect, aToplevel, aReferrer); + NS_ENSURE_SUCCESS(rv, rv); #endif + + mExpire.OnAddURI(now); + + return NS_OK; } @@ -2862,22 +2779,8 @@ nsNavHistory::Observe(nsISupports *aSubject, const char *aTopic, if (NS_SUCCEEDED(rv)) prefService->SavePrefFile(nsnull); - // Prevent Int64 overflow for people that type in huge numbers. - // This number is 2^63 / 24 / 60 / 60 / 1000000 (reversing the math below) - PRInt64 expireDays = mExpireDays; - const PRInt64 maxDays = 106751991; - if (mExpireDays > maxDays) - expireDays = maxDays; - - // compute how long ago to expire from - const PRInt64 secsPerDay = 24*60*60; - const PRInt64 usecsPerSec = 1000000; - const PRInt64 usecsPerDay = secsPerDay * usecsPerSec; - const PRInt64 expireUsecsAgo = expireDays * usecsPerDay; - - // FIXME: should we compress sometimes? It's slow, so we shouldn't do it - // every time. - VacuumDB(expireUsecsAgo); + // notify expiring system that we're quitting, it may want to do stuff + mExpire.OnQuit(); } else if (nsCRT::strcmp(aTopic, gXpcomShutdown) == 0) { nsresult rv; nsCOMPtr observerService = @@ -2886,7 +2789,10 @@ nsNavHistory::Observe(nsISupports *aSubject, const char *aTopic, observerService->RemoveObserver(this, gXpcomShutdown); observerService->RemoveObserver(this, gQuitApplicationMessage); } else if (nsCRT::strcmp(aTopic, "nsPref:changed") == 0) { + PRInt32 oldDays = mExpireDays; LoadPrefs(); + if (oldDays != mExpireDays) + mExpire.OnExpirationChanged(); } return NS_OK; diff --git a/mozilla/browser/components/places/src/nsNavHistory.h b/mozilla/browser/components/places/src/nsNavHistory.h index 8b6890f5b79..b8b46e03963 100644 --- a/mozilla/browser/components/places/src/nsNavHistory.h +++ b/mozilla/browser/components/places/src/nsNavHistory.h @@ -73,6 +73,7 @@ #include "nsINavBookmarksService.h" #include "nsMaybeWeakPtr.h" +#include "nsNavHistoryExpire.h" #include "nsNavHistoryResult.h" #include "nsNavHistoryQuery.h" @@ -166,9 +167,11 @@ public: PRBool aAutoCreate); /** - * Returns a pointer to the storage connection used by history. This connection - * object is also used by the annotation service and bookmarks, so that - * things can be grouped into transactions across these components. + * Returns a pointer to the storage connection used by history. This + * connection object is also used by the annotation service and bookmarks, so + * that things can be grouped into transactions across these components. + * + * NOT ADDREFed. * * This connection can only be used in the thread that created it the * history service! @@ -394,7 +397,6 @@ protected: PRBool FindLastVisit(nsIURI* aURI, PRInt64* aVisitID, PRInt64* aSessionID); PRBool IsURIStringVisited(const nsACString& url); - nsresult VacuumDB(PRTime aTimeAgo); nsresult LoadPrefs(); // Current time optimization @@ -403,6 +405,10 @@ protected: nsCOMPtr mExpireNowTimer; static void expireNowTimerCallback(nsITimer* aTimer, void* aClosure); + // expiration + friend class nsNavHistoryExpire; + nsNavHistoryExpire mExpire; + #ifdef LAZY_ADD // lazy add committing struct LazyMessage { diff --git a/mozilla/browser/components/places/src/nsNavHistoryExpire.cpp b/mozilla/browser/components/places/src/nsNavHistoryExpire.cpp new file mode 100644 index 00000000000..76ce5124938 --- /dev/null +++ b/mozilla/browser/components/places/src/nsNavHistoryExpire.cpp @@ -0,0 +1,639 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Mozilla History System + * + * The Initial Developer of the Original Code is + * Google Inc. + * Portions created by the Initial Developer are Copyright (C) 2006 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Brett Wilson (original author) + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +/** + * This class handles expiration of history for nsNavHistory. There is a 1-1 + * mapping between nsNavHistory class and a nsNavHistoryExpire class, the + * code is separated for better understandability. + */ + +#include "nsNavHistory.h" +#include "mozStorageHelper.h" +#include "nsNetUtil.h" + +struct nsNavHistoryExpireRecord { + nsNavHistoryExpireRecord(mozIStorageStatement* statement); + + PRInt64 visitID; + PRInt64 pageID; + PRTime visitDate; + nsCString uri; + PRInt64 faviconID; + PRBool hidden; + PRBool bookmarked; + PRBool erased; // set to true if/when the history entry is erased +}; + +// Number of things we'll expire at once. Runtime of expiration is approximately +// linear with the number of things we expire at once. This number was picked so +// we expire "several" things at once, but still run quickly. Just doing 3 +// expirations at once isn't much faster than 6 due to constant overhead of +// running the query. +#define EXPIRATION_COUNT_PER_RUN 6 + +// The time in ms to wait after AddURI to try expiration of pages. Short is +// actually better. If expiration takes an unusually long period of time, it +// will interfere with video playback in the browser, for example. Such a blip +// is not likely to be noticable when the page has just appeared. +#define PARTIAL_EXPIRATION_TIMEOUT 3500 + +// The time in ms to wait after the initial expiration run for additional ones +#define SUBSEQUENT_EXIPRATION_TIMEOUT 20000 + +// Number of expirations we'll do after the most recent page is loaded before +// stopping. We don't want to keep the computer chugging forever expiring +// annotations if the user stopped using the browser. +// +// This current value of one prevents history expiration while the page is +// being shown, because expiration may interfere with media playback. +#define MAX_SEQUENTIAL_RUNS 1 + + +// nsNavHistoryExpire::nsNavHistoryExpire +// +// Warning: don't do anything with aHistory in the constructor, since +// this is a member of the nsNavHistory, it is still being constructed +// when this is called. + +nsNavHistoryExpire::nsNavHistoryExpire(nsNavHistory* aHistory) : + mHistory(aHistory), + mSequentialRuns(0), + mTimerSet(PR_FALSE), + mAnyEmptyRuns(PR_FALSE), + mNextExpirationTime(0), + mAddCount(0), + mExpiredItems(0), + mExpireRuns(0) +{ + +} + + +// nsNavHistoryExpire::~nsNavHistoryExpire + +nsNavHistoryExpire::~nsNavHistoryExpire() +{ + +} + + +// nsNavHistoryExpire::OnAddURI +// +// Called by history when a URI is added to history. This starts the timer +// for when we are going to expire. +// +// The current time is passed in by the history service as an optimization. +// The AddURI function has already computed the proper time, and getting the +// time again from the OS is nontrivial. + +void +nsNavHistoryExpire::OnAddURI(PRTime aNow) +{ + mAddCount ++; + mSequentialRuns = 0; + + if (mTimer && mTimerSet) { + mTimer->Cancel(); + mTimerSet = PR_FALSE; + } + + if (mNextExpirationTime != 0 && aNow < mNextExpirationTime) + return; // we know there's nothing to expire yet + + StartTimer(PARTIAL_EXPIRATION_TIMEOUT); +} + + +// nsNavHistoryExpire::OnQuit +// +// Here we check for some edge cases and fix them + +void +nsNavHistoryExpire::OnQuit() +{ + mozIStorageConnection* connection = mHistory->GetStorageConnection(); + if (! connection) { + NS_NOTREACHED("No connection"); + return; + } + + // Handle degenerate runs: + ExpireForDegenerateRuns(); + + // vacuum up dangling items + ExpireHistoryParanoid(connection); + ExpireFaviconsParanoid(connection); + ExpireAnnotationsParanoid(connection); +} + + +// nsNavHistoryExpire::ClearHistory +// +// Performance: ExpireItems sends notifications. We may want to disable this +// for clear history cases. However, my initial tests show that the +// notifications are not a significant part of clear history time. + +nsresult +nsNavHistoryExpire::ClearHistory() +{ + PRBool keepGoing; + + mozIStorageConnection* connection = mHistory->GetStorageConnection(); + NS_ENSURE_TRUE(connection, NS_ERROR_OUT_OF_MEMORY); + + ExpireItems(0, &keepGoing); + + ExpireHistoryParanoid(connection); + ExpireFaviconsParanoid(connection); + ExpireAnnotationsParanoid(connection); + + ENUMERATE_WEAKARRAY(mHistory->mObservers, nsINavHistoryObserver, + OnClearHistory()) + + return NS_OK; +} + + +// nsNavHistoryExpire::OnExpirationChanged +// +// Called when the expiration length in days has changed. We clear any +// next expiration time, meaning that we'll try to expire stuff next time, +// and recompute the value if there's still nothing to expire. + +void +nsNavHistoryExpire::OnExpirationChanged() +{ + mNextExpirationTime = 0; +} + + +// nsNavHistoryExpire::DoPartialExpiration + +nsresult +nsNavHistoryExpire::DoPartialExpiration() +{ + mSequentialRuns ++; + + PRBool keepGoing; + ExpireItems(EXPIRATION_COUNT_PER_RUN, &keepGoing); + + if (keepGoing && mSequentialRuns < MAX_SEQUENTIAL_RUNS) + StartTimer(SUBSEQUENT_EXIPRATION_TIMEOUT); + return NS_OK; +} + + +// nsNavHistoryExpire::ExpireItems +// +// Here, we try to expire aNumToExpire items and their associated data, +// If we expired things and then stopped because we hit this limit, +// aKeepGoing will be set indicating we should keep expiring. If we ran +// out of things to expire, it will be unset indicating we should wait. +// +// As a special case, aNumToExpire can be 0 and we'll expire everything +// in history. + +nsresult +nsNavHistoryExpire::ExpireItems(PRUint32 aNumToExpire, PRBool* aKeepGoing) +{ + // mark how many times we've been able to run + mExpireRuns ++; + + mozIStorageConnection* connection = mHistory->GetStorageConnection(); + NS_ENSURE_TRUE(connection, NS_ERROR_OUT_OF_MEMORY); + + // This transaction is important for performance. It makes the DB flush + // everything to disk in one larger operation rather than many small ones. + // Note that this transaction always commits. + mozStorageTransaction transaction(connection, PR_TRUE); + + *aKeepGoing = PR_TRUE; + + PRInt64 expireTime; + if (aNumToExpire == 0) { + // special case: erase all history + expireTime = 0; + } else { + expireTime = PR_Now() - GetExpirationTimeAgo(); + } + + // find some visits to expire + nsTArray expiredVisits; + nsresult rv = FindVisits(expireTime, aNumToExpire, connection, + expiredVisits); + NS_ENSURE_SUCCESS(rv, rv); + + // if we didn't find the as many things to expire as we could have, then + // we should note the next time we need to expire. + if (expiredVisits.Length() < aNumToExpire) { + *aKeepGoing = PR_FALSE; + ComputeNextExpirationTime(connection); + + if (expiredVisits.Length() == 0) { + // Nothing to expire. Set the flag so we know we don't have to do any + // work on shutdown. + mAnyEmptyRuns = PR_TRUE; + return NS_OK; + } + } + mExpiredItems += expiredVisits.Length(); + + rv = EraseVisits(connection, expiredVisits); + NS_ENSURE_SUCCESS(rv, rv); + + rv = EraseHistory(connection, expiredVisits); + NS_ENSURE_SUCCESS(rv, rv); + + // send observer messages + nsCOMPtr uri; + for (PRUint32 i = 0; i < expiredVisits.Length(); i ++) { + rv = NS_NewURI(getter_AddRefs(uri), expiredVisits[i].uri); + if (NS_FAILED(rv)) continue; + + // FIXME bug 325241 provide a way to observe hidden elements + if (expiredVisits[i].hidden) continue; + + ENUMERATE_WEAKARRAY(mHistory->mObservers, nsINavHistoryObserver, + OnPageExpired(uri, expiredVisits[i].visitDate, + expiredVisits[i].erased)); + } + + // don't worry about errors here, it doesn't affect out ability to continue + EraseFavicons(connection, expiredVisits); + EraseAnnotations(connection, expiredVisits); + + return NS_OK; +} + + +// nsNavHistoryExpireRecord::nsNavHistoryExpireRecord +// +// Statement should be the one created in FindVisits. The parameters must +// agree. + +nsNavHistoryExpireRecord::nsNavHistoryExpireRecord( + mozIStorageStatement* statement) +{ + visitID = statement->AsInt64(0); + pageID = statement->AsInt64(1); + visitDate = statement->AsInt64(2); + statement->GetUTF8String(3, uri); + faviconID = statement->AsInt64(4); + hidden = (statement->AsInt32(5) > 0); + bookmarked = (statement->AsInt32(6) > 0); + erased = PR_FALSE; +} + + +// nsNavHistoryExpire::FindVisits +// +// aExpireThreshold is the time at which we will delete visits before. +// If it is zero, we will not use a threshold and will match everything. +// +// aNumToExpire is the maximum number of visits to find. If it is 0, then +// we will get all matching visits. + +nsresult +nsNavHistoryExpire::FindVisits(PRTime aExpireThreshold, PRUint32 aNumToExpire, + mozIStorageConnection* aConnection, + nsTArray& aRecords) +{ + nsresult rv; + + // get info for expiring visits, special case no threshold so there is no + // SQL parameter + nsCOMPtr selectStatement; + nsCString sql; + sql.AssignLiteral("SELECT " + "v.visit_id, v.page_id, v.visit_date, h.url, h.favicon, h.hidden, b.item_child " + "FROM moz_historyvisit v LEFT JOIN moz_history h ON v.page_id = h.id " + "LEFT OUTER JOIN moz_bookmarks b on v.page_id = b.item_child"); + if (aExpireThreshold != 0) + sql.AppendLiteral(" WHERE visit_date < ?1"); + rv = aConnection->CreateStatement(sql, getter_AddRefs(selectStatement)); + NS_ENSURE_SUCCESS(rv, rv); + if (aExpireThreshold != 0) { + rv = selectStatement->BindInt64Parameter(0, aExpireThreshold); + NS_ENSURE_SUCCESS(rv, rv); + } + + PRBool hasMore = PR_FALSE; + while (NS_SUCCEEDED(selectStatement->ExecuteStep(&hasMore)) && hasMore && + (aNumToExpire == 0 || aRecords.Length() < aNumToExpire)) { + nsNavHistoryExpireRecord record(selectStatement); + aRecords.AppendElement(record); + } + return NS_OK; +} + + +// nsNavHistoryExpire::EraseVisits + +nsresult +nsNavHistoryExpire::EraseVisits(mozIStorageConnection* aConnection, + const nsTArray& aRecords) +{ + nsCOMPtr deleteStatement; + nsresult rv = aConnection->CreateStatement(NS_LITERAL_CSTRING( + "DELETE FROM moz_historyvisit WHERE visit_id = ?1"), + getter_AddRefs(deleteStatement)); + NS_ENSURE_SUCCESS(rv, rv); + PRUint32 i; + for (i = 0; i < aRecords.Length(); i ++) { + deleteStatement->BindInt64Parameter(0, aRecords[i].visitID); + rv = deleteStatement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + return NS_OK; +} + + +// nsNavHistoryExpire::EraseHistory +// +// This erases records in moz_history when there are no more visits. +// We need to be careful not to delete bookmarks and place:URIs. +// +// This will modify the input by setting the erased flag on each of the +// array elements according to whether the history item was erased or not. + +nsresult +nsNavHistoryExpire::EraseHistory(mozIStorageConnection* aConnection, + nsTArray& aRecords) +{ + nsCOMPtr deleteStatement; + nsresult rv = aConnection->CreateStatement(NS_LITERAL_CSTRING( + "DELETE FROM moz_history WHERE id = ?1"), + getter_AddRefs(deleteStatement)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr selectStatement; + rv = aConnection->CreateStatement(NS_LITERAL_CSTRING( + "SELECT page_id FROM moz_historyvisit WHERE page_id = ?1"), + getter_AddRefs(selectStatement)); + NS_ENSURE_SUCCESS(rv, rv); + + for (PRUint32 i = 0; i < aRecords.Length(); i ++) { + if (aRecords[i].bookmarked) + continue; // don't delete bookmarked entries + if (StringBeginsWith(aRecords[i].uri, NS_LITERAL_CSTRING("place:"))) + continue; // don't delete "place" URIs + + // check that there are no visits + rv = selectStatement->BindInt64Parameter(0, aRecords[i].pageID); + NS_ENSURE_SUCCESS(rv, rv); + PRBool hasVisit = PR_FALSE; + rv = selectStatement->ExecuteStep(&hasVisit); + selectStatement->Reset(); + if (hasVisit) continue; + + aRecords[i].erased = PR_TRUE; + rv = deleteStatement->BindInt64Parameter(0, aRecords[i].pageID); + rv = deleteStatement->Execute(); + } + return NS_OK; +} + + +// nsNavHistoryExpire::EraseFavicons + +nsresult +nsNavHistoryExpire::EraseFavicons(mozIStorageConnection* aConnection, + const nsTArray& aRecords) +{ + // see if this favicon still has an entry + nsCOMPtr selectStatement; + nsresult rv = aConnection->CreateStatement(NS_LITERAL_CSTRING( + "SELECT id FROM moz_history where favicon = ?1"), + getter_AddRefs(selectStatement)); + NS_ENSURE_SUCCESS(rv, rv); + + // delete a favicon + nsCOMPtr deleteStatement; + rv = aConnection->CreateStatement(NS_LITERAL_CSTRING( + "DELETE FROM moz_favicon WHERE id = ?1"), + getter_AddRefs(deleteStatement)); + NS_ENSURE_SUCCESS(rv, rv); + + for (PRUint32 i = 0; i < aRecords.Length(); i ++) { + if (! aRecords[i].erased) + continue; // main entry not expired + if (aRecords[i].faviconID == 0) + continue; // no favicon + selectStatement->BindInt64Parameter(0, aRecords[i].faviconID); + + // see if there are any history entries and skip if so + PRBool hasEntry; + if (NS_SUCCEEDED(selectStatement->ExecuteStep(&hasEntry)) && hasEntry) { + selectStatement->Reset(); + continue; // favicon still referenced + } + selectStatement->Reset(); + + // delete the favicon, ignoring errors. We could have the same favicon + // referenced twice in our list, and we'd try to delete it twice. + deleteStatement->BindInt64Parameter(0, aRecords[i].faviconID); + deleteStatement->Execute(); + } + return NS_OK; +} + + +// nsNavHistoryExpire::EraseAnnotations + +nsresult +nsNavHistoryExpire::EraseAnnotations(mozIStorageConnection* aConnection, + const nsTArray& aRecords) +{ + // FIXME bug 319455 expire annotations + return NS_OK; +} + + +// nsNavHistoryExpire::ExpireHistoryParanoid +// +// Deletes any dangling history entries that aren't associated with any +// visits or bookmarks. Also, special case "place:" URIs. + +nsresult +nsNavHistoryExpire::ExpireHistoryParanoid(mozIStorageConnection* aConnection) +{ + // delete history entries with no visits that are not bookmarked + // also never delete any "place:" URIs (see function header comment) + nsresult rv = aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "DELETE FROM moz_history WHERE id IN (SELECT id FROM moz_history h " + "LEFT OUTER JOIN moz_historyvisit v ON h.id = v.page_id " + "LEFT OUTER JOIN moz_bookmarks b ON h.id = b.item_child " + "WHERE v.visit_id IS NULL " + "AND b.item_child IS NULL " + "AND SUBSTR(url,0,6) <> 'place:')")); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + + +// nsNavHistoryExpire::ExpireFaviconsParanoid +// +// Deletes any dangling favicons that aren't associated with any pages. + +nsresult +nsNavHistoryExpire::ExpireFaviconsParanoid(mozIStorageConnection* aConnection) +{ + return aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "DELETE FROM moz_favicon WHERE id IN " + "(SELECT f.id FROM moz_favicon f " + "LEFT OUTER JOIN moz_history h ON f.id = h.favicon " + "WHERE h.favicon IS NULL)")); +} + + +// nsNavHistoryExpire::ExpireAnnotationsParanoid + +nsresult +nsNavHistoryExpire::ExpireAnnotationsParanoid(mozIStorageConnection* aConnection) +{ + // FIXME bug 319455 expire annotations + // Also remember to expire unused names in moz_anno_name + return NS_OK; +} + + +// nsNavHistoryExpire::ExpireForDegenerateRuns +// +// This checks for potentiall degenerate runs. For example, a tinderbox +// loads many web pages quickly and we'll never have a chance to expire. +// Particularly crazy users might also do this. If we detect this, then we +// want to force some expiration so history doesn't keep increasing. +// +// Returns true if we did anything. + +PRBool +nsNavHistoryExpire::ExpireForDegenerateRuns() +{ + // If there were any times that we didn't have anything to expire, this is + // not a degenerate run. + if (mAnyEmptyRuns) + return PR_FALSE; + + // If very few URIs were added this run, or we expired more items than we + // added, don't worry about it + if (mAddCount < 10 || mAddCount < mExpiredItems) + return PR_FALSE; + + // This run looks suspicious, try to expire up to the number of items + // we may have missed this session. + PRBool keepGoing; + ExpireItems(mAddCount - mExpiredItems, &keepGoing); + return PR_TRUE; +} + + +// nsNavHistoryExpire::ComputeNextExpirationTime +// +// This computes mNextExpirationTime. See that var in the header file. +// It is passed the number of microseconds that things expire in. + +void +nsNavHistoryExpire::ComputeNextExpirationTime( + mozIStorageConnection* aConnection) +{ + mNextExpirationTime = 0; + + nsCOMPtr statement; + nsresult rv = aConnection->CreateStatement(NS_LITERAL_CSTRING( + "SELECT MIN(visit_date) FROM moz_historyvisit"), + getter_AddRefs(statement)); + NS_ASSERTION(NS_SUCCEEDED(rv), "Could not create statement"); + if (NS_FAILED(rv)) return; + + PRBool hasMore; + rv = statement->ExecuteStep(&hasMore); + if (NS_FAILED(rv) || ! hasMore) + return; // no items, we'll leave mNextExpirationTime = 0 and try to expire + // again next time + + PRTime minTime = statement->AsInt64(0); + mNextExpirationTime = minTime + GetExpirationTimeAgo(); +} + + +// nsNavHistoryExpire::StartTimer + +nsresult +nsNavHistoryExpire::StartTimer(PRUint32 aMilleseconds) +{ + if (! mTimer) + mTimer = do_CreateInstance("@mozilla.org/timer;1"); + NS_ENSURE_STATE(mTimer); // returns on error + nsresult rv = mTimer->InitWithFuncCallback(TimerCallback, this, + aMilleseconds, + nsITimer::TYPE_ONE_SHOT); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + + +// nsNavHistoryExpire::TimerCallback + +void // static +nsNavHistoryExpire::TimerCallback(nsITimer* aTimer, void* aClosure) +{ + nsNavHistoryExpire* that = NS_STATIC_CAST(nsNavHistoryExpire*, aClosure); + that->mTimerSet = PR_FALSE; + that->DoPartialExpiration(); +} + + +// nsNavHistoryExpire::GetExpirationTimeAgo + +PRTime +nsNavHistoryExpire::GetExpirationTimeAgo() +{ + PRInt64 expireDays = mHistory->mExpireDays; + + // Prevent Int64 overflow for people that type in huge numbers. + // This number is 2^63 / 24 / 60 / 60 / 1000000 (reversing the math below) + const PRInt64 maxDays = 106751991; + if (expireDays > maxDays) + expireDays = maxDays; + + // compute how long ago to expire from + const PRInt64 secsPerDay = 24*60*60; + const PRInt64 usecsPerSec = 1000000; + const PRInt64 usecsPerDay = secsPerDay * usecsPerSec; + return expireDays * usecsPerDay; +} diff --git a/mozilla/browser/components/places/src/nsNavHistoryExpire.h b/mozilla/browser/components/places/src/nsNavHistoryExpire.h new file mode 100644 index 00000000000..da5f18188a6 --- /dev/null +++ b/mozilla/browser/components/places/src/nsNavHistoryExpire.h @@ -0,0 +1,114 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Mozilla History System + * + * The Initial Developer of the Original Code is + * Google Inc. + * Portions created by the Initial Developer are Copyright (C) 2006 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Brett Wilson (original author) + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +/** + * This class handles expiration of history for nsNavHistory. There is a 1-1 + * mapping between nsNavHistory class and a nsNavHistoryExpire class, the + * code is separated for better understandability. + */ + +class mozIStorageConnection; +class nsNavHistory; +class nsNavHistoryExpireRecord; + +class nsNavHistoryExpire +{ +public: + nsNavHistoryExpire(nsNavHistory* aHistory); + ~nsNavHistoryExpire(); + + void OnAddURI(PRTime aNow); + void OnQuit(); + nsresult ClearHistory(); + void OnExpirationChanged(); + +protected: + + nsNavHistory* mHistory; + + // Number of partial expirations since last AddURI call. + PRUint32 mSequentialRuns; + + nsCOMPtr mTimer; + PRBool mTimerSet; + + // Set when we try to expire something and find there is nothing to expire. + // This short-curcuits the shutdown logic by indicating that there probably + // isn't anything important we need to expire. + PRBool mAnyEmptyRuns; + + // When we have found nothing to expire, we compute the time the next item + // will expire. This is that time so we won't try to expire anything until + // then. It is 0 when we don't need to wait to expire stuff. + PRTime mNextExpirationTime; + void ComputeNextExpirationTime(mozIStorageConnection* aConnection); + + // global statistics + PRUint32 mAddCount; + PRUint32 mExpiredItems; + PRUint32 mExpireRuns; + + nsresult DoPartialExpiration(); + + nsresult ExpireItems(PRUint32 aNumToExpire, PRBool* aKeepGoing); + + // parts of ExpireItems + nsresult FindVisits(PRTime aExpireThreshold, PRUint32 aNumToExpire, + mozIStorageConnection* aConnection, + nsTArray& aRecords); + nsresult EraseVisits(mozIStorageConnection* aConnection, + const nsTArray& aRecords); + nsresult EraseHistory(mozIStorageConnection* aConnection, + nsTArray& aRecords); + nsresult EraseFavicons(mozIStorageConnection* aConnection, + const nsTArray& aRecords); + nsresult EraseAnnotations(mozIStorageConnection* aConnection, + const nsTArray& aRecords); + + // paranoid checks + nsresult ExpireHistoryParanoid(mozIStorageConnection* aConnection); + nsresult ExpireFaviconsParanoid(mozIStorageConnection* aConnection); + nsresult ExpireAnnotationsParanoid(mozIStorageConnection* aConnection); + + PRBool ExpireForDegenerateRuns(); + + nsresult StartTimer(PRUint32 aMilleseconds); + static void TimerCallback(nsITimer* aTimer, void* aClosure); + + PRTime GetExpirationTimeAgo(); +}; diff --git a/mozilla/browser/components/places/src/nsNavHistoryResult.cpp b/mozilla/browser/components/places/src/nsNavHistoryResult.cpp index b4e81fb0bf1..5af6b3515ff 100755 --- a/mozilla/browser/components/places/src/nsNavHistoryResult.cpp +++ b/mozilla/browser/components/places/src/nsNavHistoryResult.cpp @@ -2205,6 +2205,19 @@ nsNavHistoryQueryResultNode::OnPageChanged(nsIURI *aURI, PRUint32 aWhat, } +// nsNavHistoryQueryResultNode::OnPageExpired +// +// Do nothing. Perhaps we want to handle this case. If so, add the call to +// the result to enumerate the history observers. + +NS_IMETHODIMP +nsNavHistoryQueryResultNode::OnPageExpired(nsIURI* aURI, PRTime aVisitTime, + PRBool aWholeEntry) +{ + return NS_OK; +} + + // nsNavHistoryQueryResultNode bookmark observers // // These are the bookmark observer functions for query nodes. They listen @@ -3713,6 +3726,20 @@ nsNavHistoryResult::OnPageChanged(nsIURI *aURI, return NS_OK; } + +// nsNavHistoryResult;:OnPageExpired (nsINavHistoryObserver) +// +// Don't do anything when pages expire. Perhaps we want to find the item +// to delete it. + +NS_IMETHODIMP +nsNavHistoryResult::OnPageExpired(nsIURI* aURI, PRTime aVisitTime, + PRBool aWholeEntry) +{ + return NS_OK; +} + + // nsNavHistoryResultTreeViewer ************************************************ NS_IMPL_ADDREF(nsNavHistoryResultTreeViewer) diff --git a/mozilla/browser/components/places/src/nsNavHistoryResult.h b/mozilla/browser/components/places/src/nsNavHistoryResult.h index b81b2befed7..4b2e9d9b652 100644 --- a/mozilla/browser/components/places/src/nsNavHistoryResult.h +++ b/mozilla/browser/components/places/src/nsNavHistoryResult.h @@ -104,7 +104,10 @@ private: NS_IMETHOD OnDeleteURI(nsIURI *aURI); \ NS_IMETHOD OnClearHistory(); \ NS_IMETHOD OnPageChanged(nsIURI *aURI, PRUint32 aWhat, \ - const nsAString &aValue); + const nsAString &aValue); \ + NS_IMETHOD OnPageExpired(nsIURI* aURI, PRTime aVisitTime, \ + PRBool aWholeEntry); + // nsNavHistoryResultNode //