From 3167f3782a5a6182bfa634630911892a30fd4ce3 Mon Sep 17 00:00:00 2001 From: "dietrich%mozilla.com" Date: Wed, 23 Jan 2008 20:21:35 +0000 Subject: [PATCH] Bug 394038: make url bar autocomplete frecency algorithm global (r=sspitzer) git-svn-id: svn://10.0.0.236/trunk@243845 18797224-902f-48f8-a5cc-f745e15eee43 --- mozilla/browser/app/profile/firefox.js | 36 + .../places/public/nsILivemarkService.idl | 4 + .../places/src/nsAnnotationService.cpp | 2 +- .../components/places/src/nsNavBookmarks.cpp | 64 +- .../components/places/src/nsNavHistory.cpp | 840 ++++++++++++++++-- .../components/places/src/nsNavHistory.h | 61 +- .../places/src/nsNavHistoryAutoComplete.cpp | 297 ++++--- .../places/src/nsNavHistoryExpire.cpp | 73 +- .../places/tests/unit/test_000_frecency.js | 254 ++++++ 9 files changed, 1429 insertions(+), 202 deletions(-) create mode 100644 mozilla/toolkit/components/places/tests/unit/test_000_frecency.js diff --git a/mozilla/browser/app/profile/firefox.js b/mozilla/browser/app/profile/firefox.js index d9cb39d51e3..8d14ffd73b3 100644 --- a/mozilla/browser/app/profile/firefox.js +++ b/mozilla/browser/app/profile/firefox.js @@ -599,6 +599,42 @@ pref("browser.places.createdSmartBookmarks", false); // XXX to be removed after beta 2 (bug 391419) pref("browser.places.migratePostDataAnnotations", true); +// the (maximum) number of the recent visits to sample +// when calculating frecency +pref("places.frecency.numVisits", 10); + +// Perform frecency recalculation after this amount of idle, repeating. +// A value of zero disables updating of frecency on idle. +// Default is 1 minute (60000ms). +pref("places.frecency.updateIdleTime", 60000); + +// buckets (in days) for frecency calculation +pref("places.frecency.firstBucketCutoff", 4); +pref("places.frecency.secondBucketCutoff", 14); +pref("places.frecency.thirdBucketCutoff", 31); +pref("places.frecency.fourthBucketCutoff", 90); + +// weights for buckets for frecency calculations +pref("places.frecency.firstBucketWeight", 100); +pref("places.frecency.secondBucketWeight", 70); +pref("places.frecency.thirdBucketWeight", 50); +pref("places.frecency.fourthBucketWeight", 30); +pref("places.frecency.defaultBucketWeight", 10); + +// bonus (in percent) for visit transition types for frecency calculations +pref("places.frecency.embedVisitBonus", 0); +pref("places.frecency.linkVisitBonus", 120); +pref("places.frecency.typedVisitBonus", 200); +pref("places.frecency.bookmarkVisitBonus", 140); +pref("places.frecency.downloadVisitBonus", 0); +pref("places.frecency.permRedirectVisitBonus", 0); +pref("places.frecency.tempRedirectVisitBonus", 0); +pref("places.frecency.defaultVisitBonus", 0); + +// bonus (in percent) for place types for frecency calculations +pref("places.frecency.unvisitedBookmarkBonus", 140); +pref("places.frecency.unvisitedTypedBonus", 200); + // Controls behavior of the "Add Exception" dialog launched from SSL error pages // 0 - don't pre-populate anything // 1 - pre-populate site URL, but don't fetch certificate diff --git a/mozilla/toolkit/components/places/public/nsILivemarkService.idl b/mozilla/toolkit/components/places/public/nsILivemarkService.idl index 09375a0264c..d3963a52f0d 100644 --- a/mozilla/toolkit/components/places/public/nsILivemarkService.idl +++ b/mozilla/toolkit/components/places/public/nsILivemarkService.idl @@ -161,3 +161,7 @@ interface nsILivemarkService : nsISupports */ void reloadLivemarkFolder(in long long folderID); }; + +%{C++ +#define LMANNO_FEEDURI "livemark/feedURI" +%} diff --git a/mozilla/toolkit/components/places/src/nsAnnotationService.cpp b/mozilla/toolkit/components/places/src/nsAnnotationService.cpp index 93b9f127074..f5680a3bc85 100644 --- a/mozilla/toolkit/components/places/src/nsAnnotationService.cpp +++ b/mozilla/toolkit/components/places/src/nsAnnotationService.cpp @@ -1691,7 +1691,7 @@ nsAnnotationService::CopyItemAnnotations(PRInt64 aSourceItemId, PRInt64 aDestItemId, PRBool aOverwriteDest) { - // XXX: Implment Me! + // XXX: Implement Me! return NS_ERROR_NOT_IMPLEMENTED; } diff --git a/mozilla/toolkit/components/places/src/nsNavBookmarks.cpp b/mozilla/toolkit/components/places/src/nsNavBookmarks.cpp index 0fc6612cbfb..11fd94245be 100644 --- a/mozilla/toolkit/components/places/src/nsNavBookmarks.cpp +++ b/mozilla/toolkit/components/places/src/nsNavBookmarks.cpp @@ -50,6 +50,7 @@ #include "nsAutoLock.h" #include "nsIUUIDGenerator.h" #include "prprf.h" +#include "nsILivemarkService.h" const PRInt32 nsNavBookmarks::kFindBookmarksIndex_ID = 0; const PRInt32 nsNavBookmarks::kFindBookmarksIndex_Type = 1; @@ -796,8 +797,6 @@ nsNavBookmarks::AdjustIndices(PRInt64 aFolder, { NS_ASSERTION(aStartIndex <= aEndIndex, "start index must be <= end index"); - mozIStorageConnection *dbConn = DBConn(); - nsCAutoString buffer; buffer.AssignLiteral("UPDATE moz_bookmarks SET position = position + "); buffer.AppendInt(aDelta); @@ -910,6 +909,43 @@ nsNavBookmarks::InsertBookmark(PRInt64 aFolder, nsIURI *aItem, PRInt32 aIndex, NS_ENSURE_SUCCESS(rv, rv); *aNewBookmarkId = rowId; + // XXX + // 0n import / fx 2 migration, is the frecency work going to slow us down? + // We might want to skip this stuff, as well as the frecency work + // caused by GetUrlIdFor() which calls InternalAddNewPage(). + // If we do skip this, after import, we will + // need to call FixInvalidFrecenciesForExcludedPlaces(). + // We might need to call it anyways, if items aren't properly annotated + // as livemarks feeds yet. + + nsCAutoString url; + rv = aItem->GetSpec(url); + NS_ENSURE_SUCCESS(rv, rv); + + // prevent place: queries from showing up in the URL bar autocomplete results + PRBool isBookmark = !IsQueryURI(url); + + if (isBookmark) { + // if it is a livemark item (the parent is a livemark), + // we pass in false for isBookmark. otherwise, unvisited livemark + // items will appear in URL autocomplete before we visit them. + PRBool parentIsLivemark; + nsCOMPtr lms = + do_GetService(NS_LIVEMARKSERVICE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = lms->IsLivemark(aFolder, &parentIsLivemark); + NS_ENSURE_SUCCESS(rv, rv); + + isBookmark = !parentIsLivemark; + } + + // when we created the moz_place entry for the new bookmark + // (a side effect of calling GetUrlIdFor()) frecency -1; + // now we re-calculate the frecency for this moz_place entry. + rv = History()->UpdateFrecency(childID, isBookmark); + NS_ENSURE_SUCCESS(rv, rv); + rv = SetItemLastModified(aFolder, PR_Now()); NS_ENSURE_SUCCESS(rv, rv); @@ -1004,6 +1040,17 @@ nsNavBookmarks::RemoveItem(PRInt64 aItemId) rv = UpdateBookmarkHashOnRemove(placeId); NS_ENSURE_SUCCESS(rv, rv); + // XXX is this too expensive when updating livemarks? + // UpdateBookmarkHashOnRemove() does a sanity check using + // IsBookmarkedInDatabase(), so it might not have actually + // removed the bookmark. should we have a boolean out param + // for if we actually removed it, and use that to decide if we call + // UpdateFrecency() and the rest of this code? + if (itemType == TYPE_BOOKMARK) { + rv = History()->UpdateFrecency(placeId, PR_FALSE /* isBookmark */); + NS_ENSURE_SUCCESS(rv, rv); + } + ENUMERATE_WEAKARRAY(mObservers, nsINavBookmarkObserver, OnItemRemoved(aItemId, folderId, childIndex)) @@ -2144,6 +2191,19 @@ nsNavBookmarks::ChangeBookmarkURI(PRInt64 aBookmarkId, nsIURI *aNewURI) rv = transaction.Commit(); NS_ENSURE_SUCCESS(rv, rv); + // upon changing the uri for a bookmark, update the frecency for the new place + // no need to check if this is a livemark, because... + rv = History()->UpdateFrecency(placeId, PR_TRUE /* isBookmark */); + NS_ENSURE_SUCCESS(rv, rv); + +#if 0 + // upon changing the uri for a bookmark, update the frecency for the old place + // XXX todo, we need to get the oldPlaceId (fk) before changing it above + // and then here, we need to determine if that oldPlaceId is still a bookmark (and not a livemark) + rv = History()->UpdateFrecency(oldPlaceId, PR_FALSE /* isBookmark */); + NS_ENSURE_SUCCESS(rv, rv); +#endif + nsCAutoString spec; rv = aNewURI->GetSpec(spec); NS_ENSURE_SUCCESS(rv, rv); diff --git a/mozilla/toolkit/components/places/src/nsNavHistory.cpp b/mozilla/toolkit/components/places/src/nsNavHistory.cpp index eac0bcfae52..d3a0d2d2ebd 100644 --- a/mozilla/toolkit/components/places/src/nsNavHistory.cpp +++ b/mozilla/toolkit/components/places/src/nsNavHistory.cpp @@ -85,6 +85,9 @@ #include "nsAppDirectoryServiceDefs.h" #include "nsAutoLock.h" #include "nsIIdleService.h" +#include "nsILivemarkService.h" + +#include "nsMathUtils.h" // for NS_ceilf() // Microsecond timeout for "recent" events such as typed and bookmark following. // If you typed it more than this time ago, it's not recent. @@ -110,6 +113,27 @@ #define PREF_AUTOCOMPLETE_ONLY_TYPED "urlbar.matchOnlyTyped" #define PREF_AUTOCOMPLETE_ENABLED "urlbar.autocomplete.enabled" #define PREF_DB_CACHE_PERCENTAGE "history_cache_percentage" +#define PREF_FRECENCY_NUM_VISITS "places.frecency.numVisits" +#define PREF_FRECENCY_UPDATE_IDLE_TIME "places.frecency.updateIdleTime" +#define PREF_FRECENCY_FIRST_BUCKET_CUTOFF "places.frecency.firstBucketCutoff" +#define PREF_FRECENCY_SECOND_BUCKET_CUTOFF "places.frecency.secondBucketCutoff" +#define PREF_FRECENCY_THIRD_BUCKET_CUTOFF "places.frecency.thirdBucketCutoff" +#define PREF_FRECENCY_FOURTH_BUCKET_CUTOFF "places.frecency.fourthBucketCutoff" +#define PREF_FRECENCY_FIRST_BUCKET_WEIGHT "places.frecency.firstBucketWeight" +#define PREF_FRECENCY_SECOND_BUCKET_WEIGHT "places.frecency.secondBucketWeight" +#define PREF_FRECENCY_THIRD_BUCKET_WEIGHT "places.frecency.thirdBucketWeight" +#define PREF_FRECENCY_FOURTH_BUCKET_WEIGHT "places.frecency.fourthBucketWeight" +#define PREF_FRECENCY_DEFAULT_BUCKET_WEIGHT "places.frecency.defaultBucketWeight" +#define PREF_FRECENCY_EMBED_VISIT_BONUS "places.frecency.embedVisitBonus" +#define PREF_FRECENCY_LINK_VISIT_BONUS "places.frecency.linkVisitBonus" +#define PREF_FRECENCY_TYPED_VISIT_BONUS "places.frecency.typedVisitBonus" +#define PREF_FRECENCY_BOOKMARK_VISIT_BONUS "places.frecency.bookmarkVisitBonus" +#define PREF_FRECENCY_DOWNLOAD_VISIT_BONUS "places.frecency.downloadVisitBonus" +#define PREF_FRECENCY_PERM_REDIRECT_VISIT_BONUS "places.frecency.permRedirectVisitBonus" +#define PREF_FRECENCY_TEMP_REDIRECT_VISIT_BONUS "places.frecency.tempRedirectVisitBonus" +#define PREF_FRECENCY_DEFAULT_VISIT_BONUS "places.frecency.defaultVisitBonus" +#define PREF_FRECENCY_UNVISITED_BOOKMARK_BONUS "places.frecency.unvisitedBookmarkBonus" +#define PREF_FRECENCY_UNVISITED_TYPED_BONUS "places.frecency.unvisitedTypedBonus" #define PREF_BROWSER_IMPORT_BOOKMARKS "browser.places.importBookmarksHTML" #define PREF_BROWSER_IMPORT_DEFAULTS "browser.places.importDefaults" #define PREF_BROWSER_CREATEDSMARTBOOKMARKS "browser.places.createdSmartBookmarks" @@ -156,11 +180,7 @@ #endif // LAZY_ADD -// check idle timer every 5 minutes -#define IDLE_TIMER_TIMEOUT (300 * PR_MSEC_PER_SEC) - -// *** CURRENTLY DISABLED *** -// Perform vacuum after 15 minutes of idle time, repeating. +// Perform "long idle" tasks after 15 minutes of idle time, repeating. // 15 minutes = 900 seconds = 900000 milliseconds #define LONG_IDLE_TIME_IN_MSECS (900000) @@ -168,12 +188,22 @@ // 5 minutes = 300 seconds = 300000 milliseconds #define EXPIRE_IDLE_TIME_IN_MSECS (300000) +// Number of records to update frecency for when idle. +// Amount is low, since we're doing work often (see mFrecencyUpdateIdleTime), +// the idea being to constantly keep the first few chunks up to date. +#define COUNT_TO_RECALCULATE_FRECENCY_ON_IDLE 10 + // Amount of items to expire at idle time. #define MAX_EXPIRE_RECORDS_ON_IDLE 200 // Limit the number of items in the history for performance reasons #define EXPIRATION_CAP_SITES 40000 +// DB migration types +#define DB_MIGRATION_NONE 0 +#define DB_MIGRATION_CREATED 1 +#define DB_MIGRATION_UPDATED 2 + NS_IMPL_ADDREF(nsNavHistory) NS_IMPL_RELEASE(nsNavHistory) @@ -258,6 +288,7 @@ const PRInt32 nsNavHistory::kAutoCompleteIndex_Title = 1; const PRInt32 nsNavHistory::kAutoCompleteIndex_FaviconURL = 2; const PRInt32 nsNavHistory::kAutoCompleteIndex_ItemId = 3; const PRInt32 nsNavHistory::kAutoCompleteIndex_ParentId = 4; +const PRInt32 nsNavHistory::kAutoCompleteIndex_BookmarkTitle = 5; static nsDataHashtable* gTldTypes; static const char* gQuitApplicationMessage = "quit-application"; @@ -328,14 +359,14 @@ nsNavHistory::Init() NS_ENSURE_SUCCESS(rv, rv); // init db and statements - PRBool doImport; - rv = InitDB(&doImport); + PRInt16 migrationType; + rv = InitDB(&migrationType); if (NS_FAILED(rv)) { // if unable to initialize the db, force-re-initialize it: // InitDBFile will backup the old db and create a new one. rv = InitDBFile(PR_TRUE); NS_ENSURE_SUCCESS(rv, rv); - rv = InitDB(&doImport); + rv = InitDB(&migrationType); } NS_ENSURE_SUCCESS(rv, rv); @@ -399,6 +430,9 @@ nsNavHistory::Init() // prefs LoadPrefs(); + // initialize idle timer + InitializeIdleTimer(); + // recent events hash tables NS_ENSURE_TRUE(mRecentTyped.Init(128), NS_ERROR_OUT_OF_MEMORY); NS_ENSURE_TRUE(mRecentBookmark.Init(128), NS_ERROR_OUT_OF_MEMORY); @@ -424,7 +458,7 @@ nsNavHistory::Init() observerService->AddObserver(this, gQuitApplicationMessage, PR_FALSE); observerService->AddObserver(this, gXpcomShutdown, PR_FALSE); - if (doImport) { + if (migrationType == DB_MIGRATION_CREATED) { nsCOMPtr historyFile; rv = NS_GetSpecialDirectory(NS_APP_HISTORY_50_FILE, getter_AddRefs(historyFile)); @@ -433,6 +467,12 @@ nsNavHistory::Init() } } + // In case we've either imported or done a migration from a pre-frecency build, + // calculate the first cutoff period's frecencies now. + // Swallow errors here to not block initialization. + if (migrationType != DB_MIGRATION_NONE) + (void)RecalculateFrecencies(); + // Don't add code that can fail here! Do it up above, before we add our // observers. @@ -557,11 +597,11 @@ nsNavHistory::InitDBFile(PRBool aForceInit) #define PLACES_SCHEMA_VERSION 6 nsresult -nsNavHistory::InitDB(PRBool *aDoImport) +nsNavHistory::InitDB(PRInt16 *aMadeChanges) { nsresult rv; PRBool tableExists; - *aDoImport = PR_FALSE; + *aMadeChanges = DB_MIGRATION_NONE; // IMPORTANT NOTE: // setting page_size must happen first, see bug #401985 for details @@ -574,16 +614,6 @@ nsNavHistory::InitDB(PRBool *aDoImport) rv = mDBConn->ExecuteSimpleSQL(pageSizePragma); NS_ENSURE_SUCCESS(rv, rv); - if (!mIdleTimer) { - mIdleTimer = do_CreateInstance("@mozilla.org/timer;1", &rv); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mIdleTimer->InitWithFuncCallback(IdleTimerCallback, this, - IDLE_TIMER_TIMEOUT, - nsITimer::TYPE_REPEATING_SLACK); - NS_ENSURE_SUCCESS(rv, rv); - } - mozStorageTransaction transaction(mDBConn, PR_FALSE); // Initialize the other places services' database tables. We do this before @@ -667,10 +697,6 @@ nsNavHistory::InitDB(PRBool *aDoImport) rv = UpdateSchemaVersion(); NS_ENSURE_SUCCESS(rv, rv); } - else { - rv = EnsureCurrentSchema(mDBConn); - NS_ENSURE_SUCCESS(rv, rv); - } // Get the page size. This may be different than was set above if the database // file already existed and has a different page size. @@ -714,7 +740,7 @@ nsNavHistory::InitDB(PRBool *aDoImport) // moz_places if (!tableExists) { - *aDoImport = PR_TRUE; + *aMadeChanges = DB_MIGRATION_CREATED; rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("CREATE TABLE moz_places (" "id INTEGER PRIMARY KEY, " "url LONGVARCHAR, " @@ -723,7 +749,8 @@ nsNavHistory::InitDB(PRBool *aDoImport) "visit_count INTEGER DEFAULT 0, " "hidden INTEGER DEFAULT 0 NOT NULL, " "typed INTEGER DEFAULT 0 NOT NULL, " - "favicon_id INTEGER)")); + "favicon_id INTEGER, " + "frecency INTEGER DEFAULT -1 NOT NULL)")); NS_ENSURE_SUCCESS(rv, rv); rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( @@ -742,6 +769,10 @@ nsNavHistory::InitDB(PRBool *aDoImport) rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( "CREATE INDEX moz_places_visitcount ON moz_places (visit_count)")); NS_ENSURE_SUCCESS(rv, rv); + + rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "CREATE INDEX moz_places_frecencyindex ON moz_places (frecency)")); + NS_ENSURE_SUCCESS(rv, rv); } // moz_historyvisits @@ -773,6 +804,12 @@ nsNavHistory::InitDB(PRBool *aDoImport) NS_ENSURE_SUCCESS(rv, rv); } + PRBool migrated = PR_FALSE; + rv = EnsureCurrentSchema(mDBConn, &migrated); + NS_ENSURE_SUCCESS(rv, rv); + if (*aMadeChanges != DB_MIGRATION_CREATED) + *aMadeChanges = DB_MIGRATION_UPDATED; + rv = transaction.Commit(); NS_ENSURE_SUCCESS(rv, rv); @@ -786,6 +823,29 @@ nsNavHistory::InitDB(PRBool *aDoImport) return NS_OK; } +nsresult +nsNavHistory::InitializeIdleTimer() +{ + if (mIdleTimer) { + mIdleTimer->Cancel(); + mIdleTimer = nsnull; + } + nsresult rv; + mIdleTimer = do_CreateInstance("@mozilla.org/timer;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + + PRInt32 idleTimerTimeout = PR_MIN(LONG_IDLE_TIME_IN_MSECS, + EXPIRE_IDLE_TIME_IN_MSECS); + if (mFrecencyUpdateIdleTime) + idleTimerTimeout = PR_MIN(idleTimerTimeout, mFrecencyUpdateIdleTime); + + rv = mIdleTimer->InitWithFuncCallback(IdleTimerCallback, this, + idleTimerTimeout, + nsITimer::TYPE_REPEATING_SLACK); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + // nsNavHistory::UpdateSchemaVersion // // Called by the individual services' InitTables() @@ -856,8 +916,8 @@ nsNavHistory::InitStatements() // mDBAddNewPage (see InternalAddNewPage) rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( "INSERT OR REPLACE INTO moz_places " - "(url, title, rev_host, hidden, typed, visit_count) " - "VALUES (?1, ?2, ?3, ?4, ?5, ?6)"), + "(url, title, rev_host, hidden, typed, visit_count, frecency) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"), getter_AddRefs(mDBAddNewPage)); NS_ENSURE_SUCCESS(rv, rv); @@ -926,6 +986,92 @@ nsNavHistory::InitStatements() getter_AddRefs(mFoldersWithAnnotationQuery)); NS_ENSURE_SUCCESS(rv, rv); + // mDBVisitsForFrecency + // NOTE: we are not limiting to visits with "visit_type NOT IN (0,4)" + // because if we do that, mDBVisitsForFrecency would return no visits + // for places with only embed (or undefined) visits. That would + // cause use to estimate a frecency based on what information we do have, + // see CalculateFrecencyInternal(). that would result in a + // non-zero frecency for a place with only + // embedded visits, instead of a frecency of 0. + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT visit_date, visit_type FROM moz_historyvisits " + "WHERE place_id = ?1 ORDER BY visit_date DESC LIMIT ") + + nsPrintfCString("%d", mNumVisitsForFrecency), + getter_AddRefs(mDBVisitsForFrecency)); + NS_ENSURE_SUCCESS(rv, rv); + + // find places with invalid frecencies (frecency = -1) + // invalid frecencies can happen in these scenarios: + // 1) we've done "clear private data" + // 2) we've expired or deleted visits + // 3) we've migrated from an older version, before global frecency + // + // from older versions, unmigrated bookmarks might be hidden, + // so we can't exclude hidden places (by doing "WHERE hidden <> 1") + // from our query, as we want to calculate the frecency for those + // places and unhide them (if they are not livemark items and not + // place: queries.) + // + // Note, we are not limiting ourselves to places with visits + // because we may not have any if the place is a bookmark and + // we expired or deleted all the visits. + // we sort by visit count (even though it might not be correct + // as those visits could have been removed or expired.) + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT id, visit_count, hidden, typed, frecency, url " + "FROM moz_places WHERE frecency = -1 " + "ORDER BY visit_count DESC LIMIT ") + + nsPrintfCString("%d", COUNT_TO_RECALCULATE_FRECENCY_ON_IDLE), + getter_AddRefs(mDBInvalidFrecencies)); + NS_ENSURE_SUCCESS(rv, rv); + + // this query is designed to find places with high frecency and + // an "old" max visit_date. + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT h.id, h.visit_count, h.hidden, h.typed, h.frecency, h.url " + "FROM moz_places h WHERE " + SQL_STR_FRAGMENT_MAX_VISIT_DATE( "h.id" ) + " < ?1 ORDER BY h.frecency DESC LIMIT ") + + nsPrintfCString("%d", COUNT_TO_RECALCULATE_FRECENCY_ON_IDLE), + getter_AddRefs(mDBOldFrecencies)); + NS_ENSURE_SUCCESS(rv, rv); + + // mDBUpdateFrecencyAndHidden + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "UPDATE moz_places SET frecency = ?2, hidden = ?3 WHERE id = ?1"), + getter_AddRefs(mDBUpdateFrecencyAndHidden)); + NS_ENSURE_SUCCESS(rv, rv); + + // mDBGetPlaceVisitStats + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT typed, hidden, frecency FROM moz_places WHERE id = ?1"), + getter_AddRefs(mDBGetPlaceVisitStats)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT b.parent FROM moz_places h JOIN moz_bookmarks b " + " on b.fk = h.id WHERE b.type = 1 and h.id = ?1"), + getter_AddRefs(mDBGetBookmarkParentsForPlace)); + NS_ENSURE_SUCCESS(rv, rv); + + // when calculating frecency, we want the visit count to be + // all the visits. + rv = mDBConn->CreateStatement( + NS_LITERAL_CSTRING("SELECT COUNT(*) FROM moz_historyvisits " + "WHERE place_id = ?1"), + getter_AddRefs(mDBVisitCountForFrecency)); + NS_ENSURE_SUCCESS(rv, rv); + + // this query is used to calculate the visit_count column + // which we use in the UI. to the end user, we should not count + // embedded (or undefined) visits. + rv = mDBConn->CreateStatement( + NS_LITERAL_CSTRING("SELECT COUNT(*) FROM moz_historyvisits " + "WHERE visit_type NOT IN(0,4) AND place_id = ?1"), + getter_AddRefs(mDBTrueVisitCount)); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; } @@ -1038,7 +1184,7 @@ nsNavHistory::MigrateV6Up(mozIStorageConnection* aDBConn) } nsresult -nsNavHistory::EnsureCurrentSchema(mozIStorageConnection* aDBConn) +nsNavHistory::EnsureCurrentSchema(mozIStorageConnection* aDBConn, PRBool* aDidMigrate) { // We need to do a one-time change of the moz_historyvisits.pageindex // to speed up finding last visit date when joinin with moz_places. @@ -1049,6 +1195,7 @@ nsNavHistory::EnsureCurrentSchema(mozIStorageConnection* aDBConn) NS_ENSURE_SUCCESS(rv, rv); if (oldIndexExists) { + *aDidMigrate = PR_TRUE; // wrap in a transaction for safety and performance mozStorageTransaction pageindexTransaction(aDBConn, PR_FALSE); @@ -1066,6 +1213,44 @@ nsNavHistory::EnsureCurrentSchema(mozIStorageConnection* aDBConn) rv = pageindexTransaction.Commit(); NS_ENSURE_SUCCESS(rv, rv); } + + // for existing profiles, we may not have a frecency column + nsCOMPtr statement; + rv = aDBConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT frecency FROM moz_places"), getter_AddRefs(statement)); + + if (NS_FAILED(rv)) { + *aDidMigrate = PR_TRUE; + // wrap in a transaction for safety and performance + mozStorageTransaction frecencyTransaction(aDBConn, PR_FALSE); + + // add frecency column to moz_places, default to -1 + // so that all the frecencies are invalid and we'll + // recalculate them on idle. + rv = aDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "ALTER TABLE moz_places ADD frecency INTEGER DEFAULT -1 NOT NULL")); + NS_ENSURE_SUCCESS(rv, rv); + + // create index for the frecency column + // XXX multi column index with typed, and visit_count? + rv = aDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "CREATE INDEX IF NOT EXISTS " + "moz_places_frecencyindex ON moz_places (frecency)")); + NS_ENSURE_SUCCESS(rv, rv); + + // XXX todo + // forcibly call the "on idle" timer here to do a little work + // but the rest will happen on idle. + + // for place: items and unvisited livemark items, we need to set + // the frecency to 0 so that they don't show up in url bar autocomplete + rv = FixInvalidFrecenciesForExcludedPlaces(); + NS_ENSURE_SUCCESS(rv, rv); + + rv = frecencyTransaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + } + return NS_OK; } @@ -1097,6 +1282,9 @@ nsNavHistory::CleanUpOnQuit() rv = mDBConn->ExecuteSimpleSQL( NS_LITERAL_CSTRING("DROP INDEX IF EXISTS moz_places_visitcount")); NS_ENSURE_SUCCESS(rv, rv); + rv = mDBConn->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("DROP INDEX IF EXISTS moz_places_frecencyindex")); + NS_ENSURE_SUCCESS(rv, rv); // 2. remove any duplicate URIs rv = RemoveDuplicateURIs(); @@ -1116,7 +1304,8 @@ nsNavHistory::CleanUpOnQuit() "visit_count INTEGER DEFAULT 0, " "hidden INTEGER DEFAULT 0 NOT NULL, " "typed INTEGER DEFAULT 0 NOT NULL, " - "favicon_id INTEGER)")); + "favicon_id INTEGER, " + "frecency INTEGER DEFAULT -1 NOT NULL)")); NS_ENSURE_SUCCESS(rv, rv); // 5. recreate the indexes @@ -1134,11 +1323,14 @@ nsNavHistory::CleanUpOnQuit() rv = mDBConn->ExecuteSimpleSQL( NS_LITERAL_CSTRING("CREATE INDEX moz_places_visitcount ON moz_places (visit_count)")); NS_ENSURE_SUCCESS(rv, rv); + rv = mDBConn->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("CREATE INDEX moz_places_frecencyindex ON moz_places (frecency)")); + NS_ENSURE_SUCCESS(rv, rv); // 6. copy all data into moz_places rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( "INSERT INTO moz_places " - "SELECT id, url, title, rev_host, visit_count, hidden, typed, favicon_id " + "SELECT id, url, title, rev_host, visit_count, hidden, typed, favicon_id, frecency " "FROM moz_places_backup")); NS_ENSURE_SUCCESS(rv, rv); @@ -1252,7 +1444,7 @@ nsNavHistory::GetUrlIdFor(nsIURI* aURI, PRInt64* aEntryID, statementResetter.Abandon(); nsString voidString; voidString.SetIsVoid(PR_TRUE); - return InternalAddNewPage(aURI, voidString, PR_TRUE, PR_FALSE, 0, aEntryID); + return InternalAddNewPage(aURI, voidString, PR_TRUE, PR_FALSE, 0, PR_TRUE, aEntryID); } else { // Doesn't exist: don't do anything, entry ID was already set to 0 above return NS_OK; @@ -1271,7 +1463,9 @@ nsNavHistory::GetUrlIdFor(nsIURI* aURI, PRInt64* aEntryID, nsresult nsNavHistory::InternalAddNewPage(nsIURI* aURI, const nsAString& aTitle, PRBool aHidden, PRBool aTyped, - PRInt32 aVisitCount, PRInt64* aPageID) + PRInt32 aVisitCount, + PRBool aCalculateFrecency, + PRInt64* aPageID) { mozStorageStatementScoper scoper(mDBAddNewPage); nsresult rv = BindStatementURI(mDBAddNewPage, 0, aURI); @@ -1313,6 +1507,22 @@ nsNavHistory::InternalAddNewPage(nsIURI* aURI, const nsAString& aTitle, rv = mDBAddNewPage->BindInt32Parameter(5, aVisitCount); NS_ENSURE_SUCCESS(rv, rv); + nsCAutoString url; + rv = aURI->GetSpec(url); + NS_ENSURE_SUCCESS(rv, rv); + + // frecency + PRInt32 frecency = -1; + if (aCalculateFrecency) { + rv = CalculateFrecency(-1 /* no page id, since this page doesn't exist */, + aTyped, aVisitCount, url, + &frecency); + NS_ENSURE_SUCCESS(rv, rv); + } + + rv = mDBAddNewPage->BindInt32Parameter(6, frecency); + NS_ENSURE_SUCCESS(rv, rv); + rv = mDBAddNewPage->Execute(); NS_ENSURE_SUCCESS(rv, rv); @@ -1325,7 +1535,6 @@ nsNavHistory::InternalAddNewPage(nsIURI* aURI, const nsAString& aTitle, return NS_OK; } - // nsNavHistory::InternalAddVisit // // Just a wrapper for inserting a new visit in the DB. @@ -1460,6 +1669,53 @@ nsNavHistory::LoadPrefs() NS_ENSURE_SUCCESS(rv, rv); } #endif + + // get the frecency prefs + nsCOMPtr prefs(do_GetService("@mozilla.org/preferences-service;1")); + if (prefs) { + prefs->GetIntPref(PREF_FRECENCY_NUM_VISITS, + &mNumVisitsForFrecency); + prefs->GetIntPref(PREF_FRECENCY_UPDATE_IDLE_TIME, + &mFrecencyUpdateIdleTime); + prefs->GetIntPref(PREF_FRECENCY_FIRST_BUCKET_CUTOFF, + &mFirstBucketCutoffInDays); + prefs->GetIntPref(PREF_FRECENCY_SECOND_BUCKET_CUTOFF, + &mSecondBucketCutoffInDays); + prefs->GetIntPref(PREF_FRECENCY_THIRD_BUCKET_CUTOFF, + &mThirdBucketCutoffInDays); + prefs->GetIntPref(PREF_FRECENCY_FOURTH_BUCKET_CUTOFF, + &mFourthBucketCutoffInDays); + prefs->GetIntPref(PREF_FRECENCY_EMBED_VISIT_BONUS, + &mEmbedVisitBonus); + prefs->GetIntPref(PREF_FRECENCY_LINK_VISIT_BONUS, + &mLinkVisitBonus); + prefs->GetIntPref(PREF_FRECENCY_TYPED_VISIT_BONUS, + &mTypedVisitBonus); + prefs->GetIntPref(PREF_FRECENCY_BOOKMARK_VISIT_BONUS, + &mBookmarkVisitBonus); + prefs->GetIntPref(PREF_FRECENCY_DOWNLOAD_VISIT_BONUS, + &mDownloadVisitBonus); + prefs->GetIntPref(PREF_FRECENCY_PERM_REDIRECT_VISIT_BONUS, + &mPermRedirectVisitBonus); + prefs->GetIntPref(PREF_FRECENCY_TEMP_REDIRECT_VISIT_BONUS, + &mTempRedirectVisitBonus); + prefs->GetIntPref(PREF_FRECENCY_DEFAULT_VISIT_BONUS, + &mDefaultVisitBonus); + prefs->GetIntPref(PREF_FRECENCY_UNVISITED_BOOKMARK_BONUS, + &mUnvisitedBookmarkBonus); + prefs->GetIntPref(PREF_FRECENCY_UNVISITED_TYPED_BONUS, + &mUnvisitedTypedBonus); + prefs->GetIntPref(PREF_FRECENCY_FIRST_BUCKET_WEIGHT, + &mFirstBucketWeight); + prefs->GetIntPref(PREF_FRECENCY_SECOND_BUCKET_WEIGHT, + &mSecondBucketWeight); + prefs->GetIntPref(PREF_FRECENCY_THIRD_BUCKET_WEIGHT, + &mThirdBucketWeight); + prefs->GetIntPref(PREF_FRECENCY_FOURTH_BUCKET_WEIGHT, + &mFourthBucketWeight); + prefs->GetIntPref(PREF_FRECENCY_DEFAULT_BUCKET_WEIGHT, + &mDefaultWeight); + } return NS_OK; } @@ -1798,6 +2054,62 @@ nsNavHistory::GetHasHistoryEntries(PRBool* aHasEntries) return dbSelectStatement->ExecuteStep(aHasEntries); } +nsresult +nsNavHistory::FixInvalidFrecenciesForExcludedPlaces() +{ + // for every moz_place that has an invalid frecency (-1) and + // begins with "place:" or is an unvisited child of a livemark feed, + // set frecency to 0 so that it is excluded from url bar autocomplete. + nsCOMPtr dbUpdateStatement; + nsresult rv = mDBConn->CreateStatement( + NS_LITERAL_CSTRING("UPDATE moz_places SET frecency = 0 WHERE id IN (" + "SELECT h.id FROM moz_places h JOIN moz_bookmarks b ON h.id = b.fk " + "WHERE frecency = -1 " + // place is not a livemark feed item + "AND (b.parent IN (" + "SELECT annos.item_id FROM moz_anno_attributes attrs " + "JOIN moz_items_annos annos ON attrs.id = annos.anno_attribute_id " + "WHERE attrs.name = ?1) " + // place has no visits (that are not invalid or embedded) + "AND (SELECT visit_date FROM moz_historyvisits " + "WHERE place_id = h.id AND visit_type NOT IN (0,4) LIMIT 1) is null) " + "OR SUBSTR(h.url,0,6) = 'place:')"), + getter_AddRefs(dbUpdateStatement)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = dbUpdateStatement->BindUTF8StringParameter(0, NS_LITERAL_CSTRING(LMANNO_FEEDURI)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = dbUpdateStatement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +nsNavHistory::CalculateVisitCount(PRInt64 aPlaceId, PRBool aForFrecency, PRInt32 *aVisitCount) +{ + nsCOMPtr dbSelectStatement = + aForFrecency ? mDBVisitCountForFrecency : mDBTrueVisitCount; + + mozStorageStatementScoper scope(dbSelectStatement); + + nsresult rv = dbSelectStatement->BindInt64Parameter(0, aPlaceId); + NS_ENSURE_SUCCESS(rv, rv); + + PRBool hasVisits = PR_TRUE; + rv = dbSelectStatement->ExecuteStep(&hasVisits); + NS_ENSURE_SUCCESS(rv, rv); + + if (hasVisits) { + rv = dbSelectStatement->GetInt32(0, aVisitCount); + NS_ENSURE_SUCCESS(rv, rv); + } + else + *aVisitCount = 0; + + return NS_OK; +} // nsNavHistory::MarkPageAsFollowedBookmark // @@ -1817,9 +2129,10 @@ nsNavHistory::MarkPageAsFollowedBookmark(nsIURI* aURI) return NS_OK; nsCAutoString uriString; - aURI->GetSpec(uriString); + nsresult rv = aURI->GetSpec(uriString); + NS_ENSURE_SUCCESS(rv, rv); - // if URL is already in the queue, then we need to remove the old one + // if URL is already in the bookmark queue, then we need to remove the old one PRInt64 unusedEventTime; if (mRecentBookmark.Get(uriString, &unusedEventTime)) mRecentBookmark.Remove(uriString); @@ -1843,10 +2156,8 @@ nsNavHistory::MarkPageAsFollowedBookmark(nsIURI* aURI) nsresult nsNavHistory::CanAddURI(nsIURI* aURI, PRBool* canAdd) { - nsresult rv; - - nsCString scheme; - rv = aURI->GetScheme(scheme); + nsCAutoString scheme; + nsresult rv = aURI->GetScheme(scheme); NS_ENSURE_SUCCESS(rv, rv); // first check the most common cases (HTTP, HTTPS) to allow in to avoid most @@ -1902,6 +2213,8 @@ nsNavHistory::SetPageDetails(nsIURI* aURI, const nsAString& aTitle, rv = statement->BindInt64Parameter(0, pageID); NS_ENSURE_SUCCESS(rv, rv); + // XXX should we be calculating / setting frecency here? + // for the titles, be careful to interpret isVoid as NULL SQL command so that // we can tell the difference between "set to empty" and "unset" if (aTitle.IsVoid()) @@ -1937,11 +2250,9 @@ nsNavHistory::AddVisit(nsIURI* aURI, PRTime aTime, nsIURI* aReferringURI, PRInt32 aTransitionType, PRBool aIsRedirect, PRInt64 aSessionID, PRInt64* aVisitID) { - nsresult rv; - // Filter out unwanted URIs, silently failing PRBool canAdd = PR_FALSE; - rv = CanAddURI(aURI, &canAdd); + nsresult rv = CanAddURI(aURI, &canAdd); NS_ENSURE_SUCCESS(rv, rv); if (!canAdd) { *aVisitID = 0; @@ -1967,8 +2278,8 @@ nsNavHistory::AddVisit(nsIURI* aURI, PRTime aTime, nsIURI* aReferringURI, rv = mDBGetPageVisitStats->ExecuteStep(&alreadyVisited); PRInt64 pageID = 0; - PRBool hidden; - PRBool typed; + PRBool hidden; // XXX fix me, this should not be a PRBool, as we later do BindInt32Parameter() + PRBool typed; // XXX fix me, this should not be a PRBool, as we later do BindInt32Parameter() PRBool newItem = PR_FALSE; // used to send out notifications at the end if (alreadyVisited) { // Update the existing entry... @@ -2005,26 +2316,39 @@ nsNavHistory::AddVisit(nsIURI* aURI, PRTime aTime, nsIURI* aReferringURI, // the history UI (sidebar, history menu, url bar autocomplete, etc) hidden = oldHiddenState; if (hidden && (!aIsRedirect || aTransitionType == TRANSITION_TYPED) && - aTransitionType != nsINavHistoryService::TRANSITION_EMBED) + aTransitionType != TRANSITION_EMBED) hidden = PR_FALSE; // unhide typed = oldTypedState || (aTransitionType == TRANSITION_TYPED); - // some items may have a visit count of 0 which will not count for link + PRInt32 trueVisitCount = 0; + + // we can't trust the visit_count in the moz_places, because... + rv = CalculateVisitCount(pageID, PR_FALSE /* not for frecency */, + &trueVisitCount); + + // some items may have a true visit count of 0 which will not count for link // visiting, so be sure to note this transition - if (oldVisitCount == 0) + if (trueVisitCount == 0) newItem = PR_TRUE; // update with new stats mozStorageStatementScoper updateScoper(mDBUpdatePageVisitStats); rv = mDBUpdatePageVisitStats->BindInt64Parameter(0, pageID); NS_ENSURE_SUCCESS(rv, rv); - rv = mDBUpdatePageVisitStats->BindInt32Parameter(1, oldVisitCount + 1); + + // only increment the visit_count if the transition was not EMBED + // XXX what about TRANSITION_DOWNLOAD? (bug 412217) + if (aTransitionType != TRANSITION_EMBED) + trueVisitCount++; + + rv = mDBUpdatePageVisitStats->BindInt32Parameter(1, trueVisitCount); NS_ENSURE_SUCCESS(rv, rv); rv = mDBUpdatePageVisitStats->BindInt32Parameter(2, hidden); NS_ENSURE_SUCCESS(rv, rv); rv = mDBUpdatePageVisitStats->BindInt32Parameter(3, typed); NS_ENSURE_SUCCESS(rv, rv); + rv = mDBUpdatePageVisitStats->Execute(); NS_ENSURE_SUCCESS(rv, rv); } else { @@ -2045,7 +2369,7 @@ nsNavHistory::AddVisit(nsIURI* aURI, PRTime aTime, nsIURI* aReferringURI, // set as visited once, no title nsString voidString; voidString.SetIsVoid(PR_TRUE); - rv = InternalAddNewPage(aURI, voidString, hidden, typed, 1, &pageID); + rv = InternalAddNewPage(aURI, voidString, hidden, typed, 1, PR_TRUE, &pageID); NS_ENSURE_SUCCESS(rv, rv); } @@ -2063,18 +2387,23 @@ nsNavHistory::AddVisit(nsIURI* aURI, PRTime aTime, nsIURI* aReferringURI, rv = InternalAddVisit(pageID, referringVisitID, aSessionID, aTime, aTransitionType, aVisitID); + transaction.Commit(); + + // Update frecency (*after* the visit info is in the db) + // Swallow errors here, since if we've gotten this far, it's more + // important to notify the observers below. + (void)UpdateFrecency(pageID, PR_FALSE); // Notify observers: The hidden detection code must match that in // GetQueryResults to maintain consistency. // FIXME bug 325241: make a way to observe hidden URLs - transaction.Commit(); if (! hidden && aTransitionType != TRANSITION_EMBED) { ENUMERATE_WEAKARRAY(mObservers, nsINavHistoryObserver, OnVisit(aURI, *aVisitID, aTime, aSessionID, referringVisitID, aTransitionType)); } - // Normally docshell send the link visited observer notification for us (this + // Normally docshell sends the link visited observer notification for us (this // will tell all the documents to update their visited link coloring). // However, for redirects (since we implement nsIGlobalHistory3) and downloads // (since we implement nsIDownloadHistory) this will not happen and we need to @@ -2804,6 +3133,32 @@ nsNavHistory::RemovePage(nsIURI *aURI) rv = statement->Execute(); NS_ENSURE_SUCCESS(rv, rv); } + else { + // because the moz_place was annotated (or it was a bookmark), + // we didn't delete it, but we did delete the moz_visits + // so we need to reset the frecency. Note, we don't + // reset the visit_count, as we use that in our "on idle" + // query to figure out which places to recalcuate frecency first. + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "UPDATE moz_places SET frecency = -1 WHERE id = ?1"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindInt64Parameter(0, placeId); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + // placeId could have a livemark item, so setting the frecency to -1 + // would cause it to show up in the url bar autocomplete + // call FixInvalidFrecenciesForExcludedPlaces() to handle that scenario + // XXX this might be dog slow, further degrading delete perf. + rv = FixInvalidFrecenciesForExcludedPlaces(); + NS_ENSURE_SUCCESS(rv, rv); + + // XXX todo + // forcibly call the "on idle" timer here to do a little work + // but the rest will happen on idle. + } // Observers: Be sure to finish transaction before calling observers. Note also // that we always call the observers even though we aren't sure something @@ -2960,6 +3315,29 @@ nsNavHistory::RemovePagesFromHost(const nsACString& aHost, PRBool aEntireDomain) NS_LITERAL_CSTRING(")")); NS_ENSURE_SUCCESS(rv, rv); + // reset the frecency for the places we did not delete. Note, we don't + // reset the visit_count, as we use that in our "on idle" + // query to figure out which places to recalcuate frecency first. + if (!deletedPlaceIdsBookmarked.IsEmpty() || + !deletedPlaceIdsWithAnno.IsEmpty()) { + rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "UPDATE moz_places SET frecency = -1 WHERE id IN (") + + deletedPlaceIdsBookmarked + + NS_LITERAL_CSTRING(") OR id IN (") + deletedPlaceIdsWithAnno + + NS_LITERAL_CSTRING(")")); + NS_ENSURE_SUCCESS(rv, rv); + + // the places could be livemark items, so setting the frecency to -1 + // would cause them show up in the url bar autocomplete + // call FixInvalidFrecenciesForExcludedPlaces() to handle that scenario + rv = FixInvalidFrecenciesForExcludedPlaces(); + NS_ENSURE_SUCCESS(rv, rv); + + // XXX todo + // forcibly call the "on idle" timer here to do a little work + // but the rest will happen on idle. + } + transaction.Commit(); // notify observers @@ -3095,7 +3473,8 @@ nsNavHistory::MarkPageAsTyped(nsIURI *aURI) return NS_OK; nsCAutoString uriString; - aURI->GetSpec(uriString); + nsresult rv = aURI->GetSpec(uriString); + NS_ENSURE_SUCCESS(rv, rv); // if URL is already in the typed queue, then we need to remove the old one PRInt64 unusedEventTime; @@ -3496,6 +3875,12 @@ nsNavHistory::OnIdle() rv = idleService->GetIdleTime(&idleTime); NS_ENSURE_SUCCESS(rv, rv); + // If we've been idle for more than mFrecencyUpdateIdleTime + // recalculate some frecency values. A value of zero indicates that + // frecency recalculation on idle is disabled. + if (mFrecencyUpdateIdleTime && idleTime > mFrecencyUpdateIdleTime) + (void)RecalculateFrecencies(); + // If we've been idle for more than EXPIRE_IDLE_TIME_IN_MSECS // keep the expiration engine chugging along. // Note: This is done prior to a possible vacuum, to optimize space reduction @@ -5125,7 +5510,8 @@ nsNavHistory::AddPageWithVisit(nsIURI *aURI, } PRInt64 pageID; - rv = InternalAddNewPage(aURI, aTitle, aHidden, aTyped, aVisitCount, &pageID); + // we don't want to calculate frecency here for each uri we import + rv = InternalAddNewPage(aURI, aTitle, aHidden, aTyped, aVisitCount, PR_FALSE, &pageID); NS_ENSURE_SUCCESS(rv, rv); if (aLastVisitDate != -1) { @@ -5600,3 +5986,347 @@ nsresult BindStatementURI(mozIStorageStatement* statement, PRInt32 index, NS_ENSURE_SUCCESS(rv, rv); return NS_OK; } + +nsresult +nsNavHistory::UpdateFrecency(PRInt64 aPlaceId, PRBool aIsBookmarked) +{ + mozStorageStatementScoper statsScoper(mDBGetPlaceVisitStats); + nsresult rv = mDBGetPlaceVisitStats->BindInt64Parameter(0, aPlaceId); + NS_ENSURE_SUCCESS(rv, rv); + + PRBool hasResults = PR_FALSE; + rv = mDBGetPlaceVisitStats->ExecuteStep(&hasResults); + NS_ENSURE_SUCCESS(rv, rv); + + if (!hasResults) { + NS_WARNING("attempting to update frecency for a bogus place"); + // before I added the check for itemType == TYPE_BOOKMARK + // I hit this with aPlaceId of 0 (on import) + return NS_OK; + } + + PRInt32 typed = 0; + rv = mDBGetPlaceVisitStats->GetInt32(0, &typed); + NS_ENSURE_SUCCESS(rv, rv); + + PRInt32 hidden = 0; + rv = mDBGetPlaceVisitStats->GetInt32(1, &hidden); + NS_ENSURE_SUCCESS(rv, rv); + + PRInt32 oldFrecency = 0; + rv = mDBGetPlaceVisitStats->GetInt32(2, &oldFrecency); + NS_ENSURE_SUCCESS(rv, rv); + + PRInt32 visitCountForFrecency = 0; + + // because visit_count excludes visit with visit_type NOT IN(0,4) + // we can't use it for calculating frecency, so we must + // calculate it. + rv = CalculateVisitCount(aPlaceId, PR_TRUE /* for frecency */, + &visitCountForFrecency); + NS_ENSURE_SUCCESS(rv, rv); + + PRInt32 newFrecency = 0; + rv = CalculateFrecencyInternal(aPlaceId, typed, visitCountForFrecency, + aIsBookmarked, &newFrecency); + NS_ENSURE_SUCCESS(rv, rv); + + // save ourselves the UPDATE if the frecency hasn't changed + // One way this can happen is with livemarks. + // when we added the livemark, the frecency was 0. + // On refresh, when we remove and then add the livemark items, + // the frecency (for a given moz_places) will not have changed + // (if we've never visited that place). + if (newFrecency == oldFrecency) + return NS_OK; + + mozStorageStatementScoper updateScoper(mDBUpdateFrecencyAndHidden); + rv = mDBUpdateFrecencyAndHidden->BindInt64Parameter(0, aPlaceId); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mDBUpdateFrecencyAndHidden->BindInt32Parameter(1, newFrecency); + NS_ENSURE_SUCCESS(rv, rv); + + // if we calculated a non-zero frecency we should unhide this place + // so that previously hidden (non-livebookmark item) bookmarks + // will now appear in autocomplete + // if we calculated a zero frecency, we re-use the old hidden value. + rv = mDBUpdateFrecencyAndHidden->BindInt32Parameter(2, + newFrecency ? 0 /* not hidden */ : hidden); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mDBUpdateFrecencyAndHidden->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +nsNavHistory::CalculateFrecencyInternal(PRInt64 aPlaceId, PRInt32 aTyped, PRInt32 aVisitCount, PRBool aIsBookmarked, PRInt32 *aFrecency) +{ + PRTime normalizedNow = NormalizeTimeRelativeToday(GetNow()); + + float pointsForSampledVisits = 0.0; + + if (aPlaceId != -1) { + PRInt32 numSampledVisits = 0; + + mozStorageStatementScoper scoper(mDBVisitsForFrecency); + nsresult rv = mDBVisitsForFrecency->BindInt64Parameter(0, aPlaceId); + NS_ENSURE_SUCCESS(rv, rv); + + // mDBVisitsForFrecency is limited by the browser.frecency.numVisits pref + PRBool hasMore = PR_FALSE; + while (NS_SUCCEEDED(mDBVisitsForFrecency->ExecuteStep(&hasMore)) + && hasMore) { + numSampledVisits++; + + PRInt32 visitType = mDBVisitsForFrecency->AsInt32(1); + + PRInt32 bonus = 0; + + switch (visitType) { + case nsINavHistoryService::TRANSITION_EMBED: + bonus = mEmbedVisitBonus; + break; + case nsINavHistoryService::TRANSITION_LINK: + bonus = mLinkVisitBonus; + break; + case nsINavHistoryService::TRANSITION_TYPED: + bonus = mTypedVisitBonus; + break; + case nsINavHistoryService::TRANSITION_BOOKMARK: + bonus = mBookmarkVisitBonus; + break; + case nsINavHistoryService::TRANSITION_DOWNLOAD: + bonus = mDownloadVisitBonus; + break; + case nsINavHistoryService::TRANSITION_REDIRECT_PERMANENT: + bonus = mPermRedirectVisitBonus; + break; + case nsINavHistoryService::TRANSITION_REDIRECT_TEMPORARY: + bonus = mTempRedirectVisitBonus; + break; + default: + // 0 == undefined (see bug #375777 for details) + if (visitType) + NS_WARNING("new transition but no weight for frecency"); + bonus = mDefaultVisitBonus; + break; + } + + // if bonus was zero, we can skip the work to determine the weight + if (bonus) { + PRTime visitDate = mDBVisitsForFrecency->AsInt64(0); + PRInt64 ageInDays = GetAgeInDays(normalizedNow, visitDate); + + PRInt32 weight = 0; + + if (ageInDays <= mFirstBucketCutoffInDays) + weight = mFirstBucketWeight; + else if (ageInDays <= mSecondBucketCutoffInDays) + weight = mSecondBucketWeight; + else if (ageInDays <= mThirdBucketCutoffInDays) + weight = mThirdBucketWeight; + else if (ageInDays <= mFourthBucketCutoffInDays) + weight = mFourthBucketWeight; + else + weight = mDefaultWeight; + + pointsForSampledVisits += weight * (bonus / 100.0); + } + } + + if (numSampledVisits) { + // fix for bug #412219 + if (!pointsForSampledVisits) { + // For URIs with zero points in the sampled recent visits + // but "browsing" type visits outside the sampling range, + // set frecency to -1, so that they're still shown in autocomplete. + PRInt32 trueVisitCount = 0; + rv = CalculateVisitCount(aPlaceId, PR_FALSE /* not for frecency */, + &trueVisitCount); + if (NS_SUCCEEDED(rv) && trueVisitCount) + *aFrecency = -1; + else + *aFrecency = 0; + } + else { + // Estimate frecency using the last few visits. + // Use NS_ceilf() so that we don't round down to 0, which + // would cause us to completely ignore the place during autocomplete. + *aFrecency = (PRInt32) NS_ceilf(aVisitCount * NS_ceilf(pointsForSampledVisits) / numSampledVisits); + } + +#ifdef DEBUG_FRECENCY + printf("CalculateFrecency() for place %lld: %d = %d * %f / %d\n", aPlaceId, *aFrecency, aVisitCount, pointsForSampledVisits, numSampledVisits); +#endif + + return NS_OK; + } + } + + // XXX the code below works well for guessing the frecency on import, and we'll correct later once we have + // visits. + // what if we don't have visits and we never visit? we could end up with a really high value + // that keeps coming up in ac results? only do this on import? something to figure out. + PRInt32 bonus = 0; + + // not the same logic above, as a single visit could not be both + // a bookmark visit and a typed visit. but when estimating a frecency + // for a place that doesn't have any visits, this will make it so + // something bookmarked and typed will have a higher frecency than + // something just typed or just bookmarked. + if (aIsBookmarked) + bonus += mUnvisitedBookmarkBonus; + if (aTyped) + bonus += mUnvisitedTypedBonus; + + // assume "now" as our ageInDays, so use the first bucket. + // note, when we recalculate "old frecencies" (see mDBOldFrecencies) + // this frecency value could be off by an order of + // (mFirstBucketWeight / mDefaultBucketWeight) + pointsForSampledVisits = mFirstBucketWeight * (bonus / (float)100.0); + + // for a unvisited bookmark, produce a non-zero frecency + // so that unvisited bookmarks show up in URL bar autocomplete + if (!aVisitCount && aIsBookmarked) + aVisitCount = 1; + + // use NS_ceilf() so that we don't round down to 0, which + // would cause us to completely ignore the place during autocomplete + *aFrecency = (PRInt32) NS_ceilf(aVisitCount * NS_ceilf(pointsForSampledVisits)); +#ifdef DEBUG_FRECENCY + printf("CalculateFrecency() for unvisited: frecency %d = %f points (b: %d, t: %d) * visit count %d\n", *aFrecency, pointsForSampledVisits, aIsBookmarked, aTyped, aVisitCount); +#endif + return NS_OK; +} + +nsresult +nsNavHistory::CalculateFrecency(PRInt64 aPlaceId, PRInt32 aTyped, PRInt32 aVisitCount, nsCAutoString &aURL, PRInt32 *aFrecency) +{ + *aFrecency = 0; + + nsresult rv; + + nsCOMPtr lms = + do_GetService(NS_LIVEMARKSERVICE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + PRBool isBookmark = PR_FALSE; + + // determine if the place is a (non-livemark item) bookmark and prevent + // place: queries from showing up in the URL bar autocomplete results + if (!IsQueryURI(aURL) && aPlaceId != -1) { + mozStorageStatementScoper scope(mDBGetBookmarkParentsForPlace); + + rv = mDBGetBookmarkParentsForPlace->BindInt64Parameter(0, aPlaceId); + NS_ENSURE_SUCCESS(rv, rv); + + // this query can return multiple parent folders because + // it is possible for the user to bookmark something that + // is also a livemark item + PRBool hasMore = PR_FALSE; + while (NS_SUCCEEDED(mDBGetBookmarkParentsForPlace->ExecuteStep(&hasMore)) + && hasMore) { + PRInt64 folderId; + rv = mDBGetBookmarkParentsForPlace->GetInt64(0, &folderId); + NS_ENSURE_SUCCESS(rv, rv); + + PRBool parentIsLivemark; + rv = lms->IsLivemark(folderId, &parentIsLivemark); + NS_ENSURE_SUCCESS(rv, rv); + + // we found one parent that is not a livemark feed, so we can stop + if (!parentIsLivemark) { + isBookmark = PR_TRUE; + break; + } + } + } + + rv = CalculateFrecencyInternal(aPlaceId, aTyped, aVisitCount, + isBookmark, aFrecency); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + +nsresult +nsNavHistory::RecalculateFrecencies() +{ + mozStorageTransaction transaction(mDBConn, PR_TRUE); + + nsresult rv = RecalculateFrecenciesInternal(mDBInvalidFrecencies, -1); + NS_ENSURE_SUCCESS(rv, rv); + + // if the last visit date was "recent" (in the top bucket) + // don't bother recalculating as it was recalculated recently + // and we are looking for "older" places. + PRTime startOfFirstBucket = GetNow() - + (USECS_PER_DAY * (mFirstBucketCutoffInDays + 1)); + + rv = RecalculateFrecenciesInternal(mDBOldFrecencies, startOfFirstBucket); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + +nsresult +nsNavHistory::RecalculateFrecenciesInternal(mozIStorageStatement *aStatement, PRInt64 aBindParameter) +{ + nsresult rv; + + mozStorageStatementScoper scoper(aStatement); + if (aBindParameter != -1) { + rv = aStatement->BindInt64Parameter(0, aBindParameter); + NS_ENSURE_SUCCESS(rv, rv); + } + + PRBool hasMore = PR_FALSE; + while (NS_SUCCEEDED(aStatement->ExecuteStep(&hasMore)) && hasMore) { + PRInt64 placeId = aStatement->AsInt64(0); + // for frecency, we don't use visit_count, aStatement->AsInt32(1) + PRInt32 hidden = aStatement->AsInt32(2); + PRInt32 typed = aStatement->AsInt32(3); + PRInt32 oldFrecency = aStatement->AsInt32(4); + + nsCAutoString url; + aStatement->GetUTF8String(5, url); + + PRInt32 newFrecency = 0; + PRInt32 visitCountForFrecency = 0; + + // because visit_count excludes visit with visit_type NOT IN(0,4) + // we can't use it for calculating frecency so we must calculate it. + rv = CalculateVisitCount(placeId, PR_TRUE /* for frecency */, + &visitCountForFrecency); + NS_ENSURE_SUCCESS(rv, rv); + + rv = CalculateFrecency(placeId, typed, visitCountForFrecency, + url, &newFrecency); + NS_ENSURE_SUCCESS(rv, rv); + + // save ourselves the UPDATE if the frecency hasn't changed + if (newFrecency == oldFrecency) + continue; + + mozStorageStatementScoper updateScoper(mDBUpdateFrecencyAndHidden); + rv = mDBUpdateFrecencyAndHidden->BindInt64Parameter(0, placeId); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mDBUpdateFrecencyAndHidden->BindInt32Parameter(1, newFrecency); + NS_ENSURE_SUCCESS(rv, rv); + + // if we calculated a non-zero frecency we should unhide this place + // so that previously hidden (non-livebookmark items) bookmarks + // will now appear in autocomplete. if we calculated a zero frecency, + // we re-use the old hidden value. + rv = mDBUpdateFrecencyAndHidden->BindInt32Parameter(2, + newFrecency ? 0 /* not hidden */ : hidden); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mDBUpdateFrecencyAndHidden->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} diff --git a/mozilla/toolkit/components/places/src/nsNavHistory.h b/mozilla/toolkit/components/places/src/nsNavHistory.h index 80aad7c991d..d78c647902f 100644 --- a/mozilla/toolkit/components/places/src/nsNavHistory.h +++ b/mozilla/toolkit/components/places/src/nsNavHistory.h @@ -185,6 +185,12 @@ public: nsresult GetUrlIdFor(nsIURI* aURI, PRInt64* aEntryID, PRBool aAutoCreate); + nsresult CalculateVisitCount(PRInt64 aPlaceId, PRBool aForFrecency, PRInt32 *aVisitCount); + + nsresult UpdateFrecency(PRInt64 aPageID, PRBool isBookmark); + + nsresult FixInvalidFrecenciesForExcludedPlaces(); + /** * Returns a pointer to the storage connection used by history. This * connection object is also used by the annotation service and bookmarks, so @@ -390,14 +396,28 @@ protected: nsCOMPtr mDBUrlToUrlResult; // kGetInfoIndex_* results nsCOMPtr mDBBookmarkToUrlResult; // kGetInfoIndex_* results + nsresult RecalculateFrecencies(); + nsresult RecalculateFrecenciesInternal(mozIStorageStatement *aStatement, PRInt64 aBindParameter); + + nsresult CalculateFrecency(PRInt64 aPageID, PRInt32 aTyped, PRInt32 aVisitCount, nsCAutoString &aURL, PRInt32 *aFrecency); + nsresult CalculateFrecencyInternal(PRInt64 aPageID, PRInt32 aTyped, PRInt32 aVisitCount, PRBool aIsBookmarked, PRInt32 *aFrecency); + nsCOMPtr mDBVisitsForFrecency; + nsCOMPtr mDBInvalidFrecencies; + nsCOMPtr mDBOldFrecencies; + nsCOMPtr mDBUpdateFrecencyAndHidden; + nsCOMPtr mDBGetPlaceVisitStats; + nsCOMPtr mDBGetBookmarkParentsForPlace; + nsCOMPtr mDBVisitCountForFrecency; + nsCOMPtr mDBTrueVisitCount; + nsresult InitDBFile(PRBool aForceInit); nsresult BackupDBFile(); - nsresult InitDB(PRBool *aDoImport); + nsresult InitDB(PRInt16 *aMadeChanges); nsresult InitStatements(); nsresult ForceMigrateBookmarksDB(mozIStorageConnection *aDBConn); nsresult MigrateV3Up(mozIStorageConnection *aDBConn); nsresult MigrateV6Up(mozIStorageConnection *aDBConn); - nsresult EnsureCurrentSchema(mozIStorageConnection* aDBConn); + nsresult EnsureCurrentSchema(mozIStorageConnection* aDBConn, PRBool *aMadeChanges); nsresult CleanUpOnQuit(); #ifdef IN_MEMORY_LINKS @@ -418,7 +438,8 @@ protected: PRInt64* aSessionID, PRInt64* aRedirectBookmark); nsresult InternalAddNewPage(nsIURI* aURI, const nsAString& aTitle, PRBool aHidden, PRBool aTyped, - PRInt32 aVisitCount, PRInt64* aPageID); + PRInt32 aVisitCount, PRBool aCalculateFrecency, + PRInt64* aPageID); nsresult InternalAddVisit(PRInt64 aPageID, PRInt64 aReferringVisit, PRInt64 aSessionID, PRTime aTime, PRInt32 aTransitionType, PRInt64* aVisitID); @@ -587,6 +608,7 @@ protected: static const PRInt32 kAutoCompleteIndex_FaviconURL; static const PRInt32 kAutoCompleteIndex_ItemId; static const PRInt32 kAutoCompleteIndex_ParentId; + static const PRInt32 kAutoCompleteIndex_BookmarkTitle; nsCOMPtr mDBAutoCompleteQuery; // kAutoCompleteIndex_* results nsCOMPtr mDBTagAutoCompleteQuery; // kAutoCompleteIndex_* results @@ -596,20 +618,21 @@ protected: nsCOMPtr mAutoCompleteTimer; nsString mCurrentSearchString; + nsString mCurrentSearchStringEscaped; + #ifdef MOZ_XUL nsCOMPtr mCurrentListener; nsCOMPtr mCurrentResult; #endif + nsDataHashtable mCurrentResultURLs; - PRTime mCurrentChunkEndTime; - PRTime mCurrentOldestVisit; - PRBool mFirstChunk; + PRInt32 mCurrentChunkOffset; nsDataHashtable mLivemarkFeedItemIds; nsDataHashtable mLivemarkFeedURIs; nsresult AutoCompleteTypedSearch(); - nsresult AutoCompleteFullHistorySearch(); + nsresult AutoCompleteFullHistorySearch(PRBool* aHasMoreResults); nsresult AutoCompleteTagsSearch(); nsresult PerformAutoComplete(); @@ -621,12 +644,36 @@ protected: PRInt32 mExpireDaysMax; PRInt32 mExpireSites; + // frecency prefs + PRInt32 mNumVisitsForFrecency; + PRInt32 mFrecencyUpdateIdleTime; + PRInt32 mFirstBucketCutoffInDays; + PRInt32 mSecondBucketCutoffInDays; + PRInt32 mThirdBucketCutoffInDays; + PRInt32 mFourthBucketCutoffInDays; + PRInt32 mFirstBucketWeight; + PRInt32 mSecondBucketWeight; + PRInt32 mThirdBucketWeight; + PRInt32 mFourthBucketWeight; + PRInt32 mDefaultWeight; + PRInt32 mEmbedVisitBonus; + PRInt32 mLinkVisitBonus; + PRInt32 mTypedVisitBonus; + PRInt32 mBookmarkVisitBonus; + PRInt32 mDownloadVisitBonus; + PRInt32 mPermRedirectVisitBonus; + PRInt32 mTempRedirectVisitBonus; + PRInt32 mDefaultVisitBonus; + PRInt32 mUnvisitedBookmarkBonus; + PRInt32 mUnvisitedTypedBonus; + // in nsNavHistoryQuery.cpp nsresult TokensToQueries(const nsTArray& aTokens, nsCOMArray* aQueries, nsNavHistoryQueryOptions* aOptions); nsCOMPtr mIdleTimer; + nsresult InitializeIdleTimer(); static void IdleTimerCallback(nsITimer* aTimer, void* aClosure); nsresult OnIdle(); diff --git a/mozilla/toolkit/components/places/src/nsNavHistoryAutoComplete.cpp b/mozilla/toolkit/components/places/src/nsNavHistoryAutoComplete.cpp index 1e6e0656b5a..cb23af9f48c 100644 --- a/mozilla/toolkit/components/places/src/nsNavHistoryAutoComplete.cpp +++ b/mozilla/toolkit/components/places/src/nsNavHistoryAutoComplete.cpp @@ -23,6 +23,8 @@ * Brett Wilson * Joe Hewitt * Blake Ross + * Seth Spitzer + * Dietrich Ayala * * Alternatively, the contents of this file may be used under the terms of * either the GNU General Public License Version 2 or later (the "GPL"), or @@ -42,11 +44,18 @@ /** * Autocomplete algorithm: * - * The current algorithm searches history from now backwards to the oldest - * visit in chunks of time (AUTOCOMPLETE_SEARCH_CHUNK). We currently - * do SQL LIKE searches of the search term in the url and title - * within in each chunk of time. the results are ordered by visit_count - * (and then visit_date), giving us poor man's "frecency". + * Searches moz_places by frecency (in descending order) + * in chunks (AUTOCOMPLETE_SEARCH_CHUNK_SIZE). We currently + * do SQL LIKE searches of the search term in the place title, place url + * and bookmark titles (since a "place" can have multiple bookmarks) + * within in each chunk. The results are ordered by frecency. + * Note, we exclude places with no frecency (0) because + * frecency = 0 means "don't show this in autocomplete". place: queries should + * have that, as should unvisited children of livemark feeds (that aren't + * bookmarked elsewhere). + * + * But places with frecency (-1) are included, as that means that these items + * have not had their frecency calculated yet (will happen on idle). */ #include "nsNavHistory.h" @@ -63,12 +72,14 @@ #include "nsUnicharUtils.h" #include "nsNavBookmarks.h" #include "nsPrintfCString.h" +#include "nsILivemarkService.h" #define NS_AUTOCOMPLETESIMPLERESULT_CONTRACTID \ "@mozilla.org/autocomplete/simple-result;1" // This is the maximum results we'll return for a "typed" search // This happens in response to clicking the down arrow next to the URL. +// XXX todo, check if doing rich autocomplete, and if so, limit to the max results? #define AUTOCOMPLETE_MAX_PER_TYPED 100 // nsNavHistory::InitAutoComplete @@ -100,33 +111,48 @@ nsresult nsNavHistory::CreateAutoCompleteQueries() { nsCString sql = NS_LITERAL_CSTRING( - "SELECT h.url, h.title, f.url, b.id, b.parent " + "SELECT h.url, h.title, f.url, b.id, b.parent, b.title " "FROM moz_places h " - "JOIN moz_historyvisits v ON h.id = v.place_id " "LEFT OUTER JOIN moz_bookmarks b ON b.fk = h.id " "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE v.visit_date >= ?1 AND v.visit_date <= ?2 AND h.hidden <> 1 AND " - " v.visit_type NOT IN(0,4) AND "); + "WHERE h.frecency <> 0 AND "); if (mAutoCompleteOnlyTyped) sql += NS_LITERAL_CSTRING("h.typed = 1 AND "); + // NOTE: + // after migration or clear all private data, we might end up with + // a lot of places with frecency = -1 (until idle) + // + // XXX bug 412736 + // in the case of a frecency tie, break it with h.typed and h.visit_count + // which is better than nothing. but this is slow, so not doing it yet. sql += NS_LITERAL_CSTRING( - "(h.title LIKE ?3 ESCAPE '/' OR h.url LIKE ?3 ESCAPE '/') " - "GROUP BY h.id ORDER BY h.typed DESC, h.visit_count DESC, MAX(v.visit_date) DESC;"); + "(b.title LIKE ?1 ESCAPE '/' OR " + "h.title LIKE ?1 ESCAPE '/' OR " + "h.url LIKE ?1 ESCAPE '/') " + "ORDER BY h.frecency DESC LIMIT ?2 OFFSET ?3"); - nsresult rv = mDBConn->CreateStatement(sql, getter_AddRefs(mDBAutoCompleteQuery)); + nsresult rv = mDBConn->CreateStatement(sql, + getter_AddRefs(mDBAutoCompleteQuery)); NS_ENSURE_SUCCESS(rv, rv); + // NOTE: + // after migration or clear all private data, we might end up with + // a lot of places with frecency = -1 (until idle) + // + // XXX bug 412736 + // in the case of a frecency tie, break it with h.typed and h.visit_count + // which is better than nothing. but this is slow, so not doing it yet. sql = NS_LITERAL_CSTRING( "SELECT h.url, h.title, f.url, b.id, b.parent " "FROM moz_places h " "JOIN moz_bookmarks b ON b.fk = h.id " - "LEFT OUTER JOIN moz_historyvisits v ON h.id = v.place_id " "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE " + "WHERE h.frecency <> 0 AND " "(b.parent in (SELECT t.id FROM moz_bookmarks t WHERE t.parent = ?1 AND LOWER(t.title) = LOWER(?2))) " - "GROUP BY h.id ORDER BY h.visit_count DESC, MAX(v.visit_date) DESC;"); + "ORDER BY h.frecency DESC"); + rv = mDBConn->CreateStatement(sql, getter_AddRefs(mDBTagAutoCompleteQuery)); NS_ENSURE_SUCCESS(rv, rv); @@ -152,25 +178,16 @@ nsNavHistory::StartAutoCompleteTimer(PRUint32 aMilliseconds) return NS_OK; } -static const PRInt64 USECS_PER_DAY = LL_INIT(20, 500654080); - -// search in day chunks. too big, and the UI will be unresponsive +// number of places to search per chunk +// too big, and the UI will be unresponsive // as we will be off searching the database. -// too short, and because of AUTOCOMPLETE_SEARCH_TIMEOUT +// too small, and because of AUTOCOMPLETE_SEARCH_TIMEOUT // results won't come back in fast enough to feel snappy. -// because we sort within chunks by visit_count (then visit_date) -// choose 4 days so that Friday's searches are in the first chunk -// and those will affect Monday's results -#define AUTOCOMPLETE_SEARCH_CHUNK (USECS_PER_DAY * 4) +#define AUTOCOMPLETE_SEARCH_CHUNK_SIZE 100 // wait this many milliseconds between searches -// too short, and the UI will be unresponsive -// as we will be off searching the database. -// too big, and results won't come back in fast enough to feel snappy. #define AUTOCOMPLETE_SEARCH_TIMEOUT 100 -#define LMANNO_FEEDURI "livemark/feedURI" - // nsNavHistory::AutoCompleteTimerCallback void // static @@ -196,15 +213,14 @@ nsNavHistory::PerformAutoComplete() rv = AutoCompleteTypedSearch(); else { // only search tags on the first chunk, - // but before we search history, as we want tagged + // but before we search places, as we want tagged // items to show up first. - if (mFirstChunk) { + if (!mCurrentChunkOffset) { rv = AutoCompleteTagsSearch(); NS_ENSURE_SUCCESS(rv, rv); } - rv = AutoCompleteFullHistorySearch(); - moreChunksToSearch = (mCurrentChunkEndTime >= mCurrentOldestVisit); + rv = AutoCompleteFullHistorySearch(&moreChunksToSearch); } NS_ENSURE_SUCCESS(rv, rv); @@ -229,11 +245,10 @@ nsNavHistory::PerformAutoComplete() mCurrentListener->OnSearchResult(this, mCurrentResult); - // if we're not done searching, adjust our end time and - // search the next earlier chunk of time + // if we're not done searching, adjust our current offset + // and search the next chunk if (moreChunksToSearch) { - mFirstChunk = PR_FALSE; - mCurrentChunkEndTime -= AUTOCOMPLETE_SEARCH_CHUNK; + mCurrentChunkOffset += AUTOCOMPLETE_SEARCH_CHUNK_SIZE; rv = StartAutoCompleteTimer(AUTOCOMPLETE_SEARCH_TIMEOUT); NS_ENSURE_SUCCESS(rv, rv); } else { @@ -262,15 +277,20 @@ nsNavHistory::StartSearch(const nsAString & aSearchString, mCurrentSearchString = aSearchString; // remove whitespace, see bug #392141 for details mCurrentSearchString.Trim(" \r\n\t\b"); + + nsresult rv = mDBAutoCompleteQuery->EscapeStringForLIKE(mCurrentSearchString, PRUnichar('/'), mCurrentSearchStringEscaped); + NS_ENSURE_SUCCESS(rv, rv); + mCurrentListener = aListener; - nsresult rv; // determine if we can start by searching through the previous search results. - // if we can't, we need to reset mCurrentChunkEndTime and mCurrentOldestVisit. + // if we can't, we need to reset mCurrentChunkOffset // if we can, we will search through our previous search results and then resume - // searching using the previous mCurrentChunkEndTime and mCurrentOldestVisit values. + // searching using the previous mCurrentOldestVisit and mLast values. PRBool searchPrevious = PR_FALSE; - if (aPreviousResult) { + // XXX Re-use of previous search results is disabled due to bug 412730. + //if (aPreviousResult) { + if (0) { nsAutoString prevSearchString; aPreviousResult->GetSearchString(prevSearchString); @@ -386,30 +406,8 @@ nsNavHistory::StartSearch(const nsAString & aSearchString, } } else if (!mCurrentSearchString.IsEmpty()) { - // reset to mCurrentChunkEndTime - mCurrentChunkEndTime = PR_Now(); - mCurrentOldestVisit = 0; - mFirstChunk = PR_TRUE; - - // determine our earliest visit - nsCOMPtr dbSelectStatement; - rv = mDBConn->CreateStatement( - NS_LITERAL_CSTRING("SELECT MIN(visit_date) id FROM moz_historyvisits WHERE visit_type NOT IN(0,4)"), - getter_AddRefs(dbSelectStatement)); - NS_ENSURE_SUCCESS(rv, rv); - PRBool hasMinVisit; - rv = dbSelectStatement->ExecuteStep(&hasMinVisit); - NS_ENSURE_SUCCESS(rv, rv); - - if (hasMinVisit) { - rv = dbSelectStatement->GetInt64(0, &mCurrentOldestVisit); - NS_ENSURE_SUCCESS(rv, rv); - } - - if (!mCurrentOldestVisit) { - // if we have no visits, use a reasonable value - mCurrentOldestVisit = PR_Now() - USECS_PER_DAY; - } + // reset mCurrentChunkOffset + mCurrentChunkOffset = 0; } // fire right away, we already waited to start searching @@ -435,32 +433,43 @@ nsNavHistory::StopSearch() // nsNavHistory::AutoCompleteTypedSearch // // Called when there is no search string. This happens when you press -// down arrow from the URL bar: the most recent things you typed are listed. +// down arrow from the URL bar: the most "frecent" things you typed are listed. // -// Ordering here is simpler because there are no boosts for typing, and there -// is no URL information to use. The ordering just comes out of the DB by -// visit count (primary) and time since last visited (secondary). nsresult nsNavHistory::AutoCompleteTypedSearch() { nsCOMPtr dbSelectStatement; + // NOTE: + // after migration or clear all private data, we might end up with + // a lot of places with frecency = -1 (until idle) + // + // XXX bug 412736 + // in the case of a frecency tie, break it with h.typed and h.visit_count + // which is better than nothing. but this is slow, so not doing it yet. + // + // GROUP BY h.id to prevent duplicates. For mDBAutoCompleteQuery, + // we use the mCurrentResultURLs hash to accomplish this. + // + // NOTE: because we are grouping by h.id, b.id and b.parent + // get collapsed, so if something is both a livemark and a bookmark + // we might not show it as a "star" if the parentId we return is + // the one for the livemark item, and not the bookmark item. + // XXX bug 412734 nsCString sql = NS_LITERAL_CSTRING( "SELECT h.url, h.title, f.url, b.id, b.parent " "FROM moz_places h " "LEFT OUTER JOIN moz_bookmarks b ON b.fk = h.id " "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id " - "JOIN moz_historyvisits v ON h.id = v.place_id WHERE (h.id IN " - "(SELECT DISTINCT h.id from moz_historyvisits v, moz_places h WHERE " - "v.place_id = h.id AND h.typed = 1 AND v.visit_type NOT IN(0,4) " - "ORDER BY v.visit_date DESC LIMIT "); + "WHERE h.frecency <> 0 AND h.typed = 1 " + "GROUP BY h.id ORDER BY h.frecency DESC LIMIT "); sql.AppendInt(AUTOCOMPLETE_MAX_PER_TYPED); - sql += NS_LITERAL_CSTRING(")) GROUP BY h.id ORDER BY MAX(v.visit_date) DESC"); nsFaviconService* faviconService = nsFaviconService::GetFaviconService(); NS_ENSURE_TRUE(faviconService, NS_ERROR_OUT_OF_MEMORY); - nsresult rv = mDBConn->CreateStatement(sql, getter_AddRefs(dbSelectStatement)); + nsresult rv = mDBConn->CreateStatement(sql, + getter_AddRefs(dbSelectStatement)); NS_ENSURE_SUCCESS(rv, rv); PRBool hasMore = PR_FALSE; @@ -472,12 +481,13 @@ nsresult nsNavHistory::AutoCompleteTypedSearch() PRInt64 itemId = 0; dbSelectStatement->GetInt64(kAutoCompleteIndex_ItemId, &itemId); PRInt64 parentId = 0; - dbSelectStatement->GetInt64(kAutoCompleteIndex_ParentId, &parentId); + if (itemId) + dbSelectStatement->GetInt64(kAutoCompleteIndex_ParentId, &parentId); PRBool dummy; // don't show rss feed items as bookmarked, // but do show rss feed URIs as bookmarked. - PRBool isBookmark = (itemId != 0 && + PRBool isBookmark = (itemId && !mLivemarkFeedItemIds.Get(parentId, &dummy)) || mLivemarkFeedURIs.Get(entryURL, &dummy); @@ -543,9 +553,8 @@ nsNavHistory::AutoCompleteTagsSearch() "SELECT h.url, h.title, f.url, b.id, b.parent " "FROM moz_places h " "JOIN moz_bookmarks b ON b.fk = h.id " - "LEFT OUTER JOIN moz_historyvisits v ON h.id = v.place_id " "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE " + "WHERE h.frecency <> 0 AND " "(b.parent in " " (SELECT t.id FROM moz_bookmarks t WHERE t.parent = ?1 AND ("); @@ -562,7 +571,7 @@ nsNavHistory::AutoCompleteTagsSearch() } tagQuery += NS_LITERAL_CSTRING("))) " - "GROUP BY h.id ORDER BY h.visit_count DESC, MAX(v.visit_date) DESC;"); + "GROUP BY h.id ORDER BY h.frecency DESC"); rv = mDBConn->CreateStatement(tagQuery, getter_AddRefs(tagAutoCompleteQuery)); NS_ENSURE_SUCCESS(rv, rv); @@ -586,24 +595,34 @@ nsNavHistory::AutoCompleteTagsSearch() // Determine the result of the search while (NS_SUCCEEDED(tagAutoCompleteQuery->ExecuteStep(&hasMore)) && hasMore) { - nsAutoString entryURL, entryTitle, entryFavicon; + nsAutoString entryURL; rv = tagAutoCompleteQuery->GetString(kAutoCompleteIndex_URL, entryURL); NS_ENSURE_SUCCESS(rv, rv); - rv = tagAutoCompleteQuery->GetString(kAutoCompleteIndex_Title, entryTitle); - NS_ENSURE_SUCCESS(rv, rv); - rv = tagAutoCompleteQuery->GetString(kAutoCompleteIndex_FaviconURL, entryFavicon); - NS_ENSURE_SUCCESS(rv, rv); - PRInt64 itemId = 0; - rv = tagAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ItemId, &itemId); - NS_ENSURE_SUCCESS(rv, rv); - PRInt64 parentId = 0; - rv = tagAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ParentId, &parentId); - NS_ENSURE_SUCCESS(rv, rv); PRBool dummy; // prevent duplicates. this can happen when chunking as we - // may have already seen this URL from an earlier chunk of time + // may have already seen this URL. + // NOTE: because we use mCurrentResultURLs to remove duplicates, + // the first url wins. + // so we might not show it as a "star" if the parentId we get first is + // the one for the livemark item, and not the bookmark item, + // we may not show the "star" even though we should. + // XXX bug 412734 if (!mCurrentResultURLs.Get(entryURL, &dummy)) { + nsAutoString entryTitle, entryFavicon; + rv = tagAutoCompleteQuery->GetString(kAutoCompleteIndex_Title, entryTitle); + NS_ENSURE_SUCCESS(rv, rv); + rv = tagAutoCompleteQuery->GetString(kAutoCompleteIndex_FaviconURL, entryFavicon); + NS_ENSURE_SUCCESS(rv, rv); + PRInt64 itemId = 0; + rv = tagAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ItemId, &itemId); + NS_ENSURE_SUCCESS(rv, rv); + PRInt64 parentId = 0; + if (itemId) { + rv = tagAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ParentId, &parentId); + NS_ENSURE_SUCCESS(rv, rv); + } + // new item, append to our results and put it in our hash table. nsCAutoString faviconSpec; faviconService->GetFaviconSpecForIconString( @@ -621,28 +640,26 @@ nsNavHistory::AutoCompleteTagsSearch() // nsNavHistory::AutoCompleteFullHistorySearch // -// Search history for visits that have a url or title that contains mCurrentSearchString -// and are within our current chunk of time: -// between (mCurrentChunkEndTime - AUTOCOMPLETE_SEARCH_CHUNK) and (mCurrentChunkEndTime) +// Search for places that have a title, url, +// or bookmark title(s) that contains mCurrentSearchString +// and are within our current chunk of "frecency". +// +// @param aHasMoreResults is false if the query found no matching items // nsresult -nsNavHistory::AutoCompleteFullHistorySearch() +nsNavHistory::AutoCompleteFullHistorySearch(PRBool* aHasMoreResults) { mozStorageStatementScoper scope(mDBAutoCompleteQuery); - nsresult rv = mDBAutoCompleteQuery->BindInt64Parameter(0, mCurrentChunkEndTime - AUTOCOMPLETE_SEARCH_CHUNK); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBAutoCompleteQuery->BindInt64Parameter(1, mCurrentChunkEndTime); - NS_ENSURE_SUCCESS(rv, rv); - - nsString escapedSearchString; - rv = mDBAutoCompleteQuery->EscapeStringForLIKE(mCurrentSearchString, PRUnichar('/'), escapedSearchString); - NS_ENSURE_SUCCESS(rv, rv); - // prepend and append with % for "contains" - rv = mDBAutoCompleteQuery->BindStringParameter(2, NS_LITERAL_STRING("%") + escapedSearchString + NS_LITERAL_STRING("%")); + nsresult rv = mDBAutoCompleteQuery->BindStringParameter(0, NS_LITERAL_STRING("%") + mCurrentSearchStringEscaped + NS_LITERAL_STRING("%")); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mDBAutoCompleteQuery->BindInt32Parameter(1, AUTOCOMPLETE_SEARCH_CHUNK_SIZE); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mDBAutoCompleteQuery->BindInt32Parameter(2, mCurrentChunkOffset); NS_ENSURE_SUCCESS(rv, rv); nsFaviconService* faviconService = nsFaviconService::GetFaviconService(); @@ -652,36 +669,62 @@ nsNavHistory::AutoCompleteFullHistorySearch() // Determine the result of the search while (NS_SUCCEEDED(mDBAutoCompleteQuery->ExecuteStep(&hasMore)) && hasMore) { - nsAutoString entryURL, entryTitle, entryFavicon; + *aHasMoreResults = PR_TRUE; + nsAutoString entryURL; rv = mDBAutoCompleteQuery->GetString(kAutoCompleteIndex_URL, entryURL); NS_ENSURE_SUCCESS(rv, rv); - rv = mDBAutoCompleteQuery->GetString(kAutoCompleteIndex_Title, entryTitle); - NS_ENSURE_SUCCESS(rv, rv); - rv = mDBAutoCompleteQuery->GetString(kAutoCompleteIndex_FaviconURL, entryFavicon); - NS_ENSURE_SUCCESS(rv, rv); - PRInt64 itemId = 0; - rv = mDBAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ItemId, &itemId); - NS_ENSURE_SUCCESS(rv, rv); - PRInt64 parentId = 0; - rv = mDBAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ParentId, &parentId); - NS_ENSURE_SUCCESS(rv, rv); - - PRBool dummy; - // don't show rss feed items as bookmarked, - // but do show rss feed URIs as bookmarked. - PRBool isBookmark = (itemId != 0 && - !mLivemarkFeedItemIds.Get(parentId, &dummy)) || - mLivemarkFeedURIs.Get(entryURL, &dummy); // prevent duplicates. this can happen when chunking as we - // may have already seen this URL from an earlier chunk of time + // may have already seen this URL from our tag search or an earlier + // chunk. + PRBool dummy; if (!mCurrentResultURLs.Get(entryURL, &dummy)) { + nsAutoString entryTitle, entryFavicon, entryBookmarkTitle; + rv = mDBAutoCompleteQuery->GetString(kAutoCompleteIndex_Title, entryTitle); + NS_ENSURE_SUCCESS(rv, rv); + rv = mDBAutoCompleteQuery->GetString(kAutoCompleteIndex_FaviconURL, entryFavicon); + NS_ENSURE_SUCCESS(rv, rv); + PRInt64 itemId = 0; + rv = mDBAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ItemId, &itemId); + NS_ENSURE_SUCCESS(rv, rv); + + PRInt64 parentId = 0; + // only bother to fetch parent id and bookmark title + // if we have a bookmark (itemId != 0) + if (itemId) { + rv = mDBAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ParentId, &parentId); + NS_ENSURE_SUCCESS(rv, rv); + rv = mDBAutoCompleteQuery->GetString(kAutoCompleteIndex_BookmarkTitle, entryBookmarkTitle); + NS_ENSURE_SUCCESS(rv, rv); + } + + // don't show rss feed items as bookmarked, + // but do show rss feed URIs as bookmarked. + // + // NOTE: because we use mCurrentResultURLs to remove duplicates, + // the first url wins. + // so we might not show it as a "star" if the parentId we get first is + // the one for the livemark item, and not the bookmark item, + // we may not show the "star" even though we should. + // XXX bug 412734 + PRBool isBookmark = (itemId && + !mLivemarkFeedItemIds.Get(parentId, &dummy)) || + mLivemarkFeedURIs.Get(entryURL, &dummy); + // new item, append to our results and put it in our hash table. nsCAutoString faviconSpec; faviconService->GetFaviconSpecForIconString( NS_ConvertUTF16toUTF8(entryFavicon), faviconSpec); - rv = mCurrentResult->AppendMatch(entryURL, entryTitle, - NS_ConvertUTF8toUTF16(faviconSpec), isBookmark ? NS_LITERAL_STRING("bookmark") : NS_LITERAL_STRING("favicon")); + + // if the search string is in the bookmark title, show that in the + // result (instead of the page title) + PRBool matchInBookmarkTitle = itemId && + CaseInsensitiveFindInReadable(mCurrentSearchString, entryBookmarkTitle); + + rv = mCurrentResult->AppendMatch(entryURL, + matchInBookmarkTitle ? entryBookmarkTitle : entryTitle, + NS_ConvertUTF8toUTF16(faviconSpec), + isBookmark ? NS_LITERAL_STRING("bookmark") : NS_LITERAL_STRING("favicon")); NS_ENSURE_SUCCESS(rv, rv); mCurrentResultURLs.Put(entryURL, PR_TRUE); diff --git a/mozilla/toolkit/components/places/src/nsNavHistoryExpire.cpp b/mozilla/toolkit/components/places/src/nsNavHistoryExpire.cpp index a5ed35e2253..bf84a62195a 100644 --- a/mozilla/toolkit/components/places/src/nsNavHistoryExpire.cpp +++ b/mozilla/toolkit/components/places/src/nsNavHistoryExpire.cpp @@ -259,6 +259,26 @@ nsNavHistoryExpire::ClearHistory() if (NS_FAILED(rv)) NS_WARNING("ExpireAnnotationsParanoid failed."); + // for all remaining places, reset the frecency + // Note, we don't reset the visit_count, as we use that in our "on idle" + // query to figure out which places to recalcuate frecency first. + rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "UPDATE moz_places SET frecency = -1")); + if (NS_FAILED(rv)) + NS_WARNING("failed to recent frecency"); + + // some of the remaining places could be place: urls or + // unvisited livemark items, so setting the frecency to -1 + // will cause them to show up in the url bar autocomplete + // call FixInvalidFrecenciesForExcludedPlaces() to handle that scenario + rv = mHistory->FixInvalidFrecenciesForExcludedPlaces(); + if (NS_FAILED(rv)) + NS_WARNING("failed to fix invalid frecencies"); + + // XXX todo + // forcibly call the "on idle" timer here to do a little work + // but the rest will happen on idle. + ENUMERATE_WEAKARRAY(mHistory->mObservers, nsINavHistoryObserver, OnClearHistory()) @@ -506,21 +526,54 @@ nsNavHistoryExpire::EraseVisits(mozIStorageConnection* aConnection, const nsTArray& aRecords) { // build a comma separated string of visit ids to delete + // also build a comma separated string of place ids to reset frecency and + // visit_count. nsCString deletedVisitIds; + nsCString placeIds; + nsTArray deletedPlaceIdsArray, deletedVisitIdsArray; for (PRUint32 i = 0; i < aRecords.Length(); i ++) { - // Do not add comma separator for the first entry - if (! deletedVisitIds.IsEmpty()) - deletedVisitIds.AppendLiteral(","); - deletedVisitIds.AppendInt(aRecords[i].visitID); + // Do not add comma separator for the first visit id + if (deletedVisitIdsArray.IndexOf(aRecords[i].visitID) == -1) { + if (!deletedVisitIds.IsEmpty()) + deletedVisitIds.AppendLiteral(","); + deletedVisitIds.AppendInt(aRecords[i].visitID); + } + + // Do not add comma separator for the first place id + if (deletedPlaceIdsArray.IndexOf(aRecords[i].placeID) == -1) { + if (!placeIds.IsEmpty()) + placeIds.AppendLiteral(","); + placeIds.AppendInt(aRecords[i].placeID); + } } if (deletedVisitIds.IsEmpty()) return NS_OK; - return aConnection->ExecuteSimpleSQL( + nsresult rv = aConnection->ExecuteSimpleSQL( NS_LITERAL_CSTRING("DELETE FROM moz_historyvisits WHERE id IN (") + deletedVisitIds + NS_LITERAL_CSTRING(")")); + NS_ENSURE_SUCCESS(rv, rv); + + if (placeIds.IsEmpty()) + return NS_OK; + + // reset the frecencies for these places. + // Note, we don't reset the visit_count, as we use that in our "on idle" + // query to figure out which places to recalcuate frecency first. + rv = aConnection->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("UPDATE moz_places SET " + "frecency = -1 WHERE id IN (") + + placeIds + + NS_LITERAL_CSTRING(")")); + NS_ENSURE_SUCCESS(rv, rv); + + // XXX todo + // forcibly call the "on idle" timer here to do a little work + // but the rest will happen on idle. + + return NS_OK; } @@ -548,7 +601,7 @@ nsNavHistoryExpire::EraseHistory(mozIStorageConnection* aConnection, // avoid trying to delete the same place id twice if (deletedPlaceIdsArray.IndexOf(aRecords[i].placeID) == -1) { // Do not add comma separator for the first entry - if (! deletedPlaceIds.IsEmpty()) + if (!deletedPlaceIds.IsEmpty()) deletedPlaceIds.AppendLiteral(","); deletedPlaceIdsArray.AppendElement(aRecords[i].placeID); deletedPlaceIds.AppendInt(aRecords[i].placeID); @@ -583,12 +636,12 @@ nsNavHistoryExpire::EraseFavicons(mozIStorageConnection* aConnection, nsTArray deletedFaviconIdsArray; for (PRUint32 i = 0; i < aRecords.Length(); i ++) { // IF main entry not expired OR no favicon DO NOT DELETE - if (! aRecords[i].erased || aRecords[i].faviconID == 0) + if (!aRecords[i].erased || aRecords[i].faviconID == 0) continue; // avoid trying to delete the same favicon id twice if (deletedFaviconIdsArray.IndexOf(aRecords[i].faviconID) == -1) { // Do not add comma separator for the first entry - if (! deletedFaviconIds.IsEmpty()) + if (!deletedFaviconIds.IsEmpty()) deletedFaviconIds.AppendLiteral(","); deletedFaviconIdsArray.AppendElement(aRecords[i].faviconID); deletedFaviconIds.AppendInt(aRecords[i].faviconID); @@ -893,7 +946,7 @@ nsNavHistoryExpire::ComputeNextExpirationTime( PRBool hasMore; rv = statement->ExecuteStep(&hasMore); - if (NS_FAILED(rv) || ! hasMore) + if (NS_FAILED(rv) || !hasMore) return; // no items, we'll leave mNextExpirationTime = 0 and try to expire // again next time @@ -907,7 +960,7 @@ nsNavHistoryExpire::ComputeNextExpirationTime( nsresult nsNavHistoryExpire::StartTimer(PRUint32 aMilleseconds) { - if (! mTimer) + if (!mTimer) mTimer = do_CreateInstance("@mozilla.org/timer;1"); NS_ENSURE_STATE(mTimer); // returns on error nsresult rv = mTimer->InitWithFuncCallback(TimerCallback, this, diff --git a/mozilla/toolkit/components/places/tests/unit/test_000_frecency.js b/mozilla/toolkit/components/places/tests/unit/test_000_frecency.js new file mode 100644 index 00000000000..b54dac5bb32 --- /dev/null +++ b/mozilla/toolkit/components/places/tests/unit/test_000_frecency.js @@ -0,0 +1,254 @@ +version(180); +/* -*- Mode: Java; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Bug 378079 unit test code. + * + * The Initial Developer of the Original Code is POTI Inc. + * Portions created by the Initial Developer are Copyright (C) 2007 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Matt Crocker + * Seth Spitzer + * Dietrich Ayala + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +/* + +Autocomplete Frecency Tests + +- add a visit for each score permutation +- search +- test number of matches +- test each item's location in results + +*/ + +var histsvc = Cc["@mozilla.org/browser/nav-history-service;1"]. + getService(Ci.nsINavHistoryService); +var bmsvc = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. + getService(Ci.nsINavBookmarksService); +var prefs = Cc["@mozilla.org/preferences-service;1"]. + getService(Ci.nsIPrefBranch); + +function add_visit(aURI, aVisitDate, aVisitType) { + var isRedirect = aVisitType == histsvc.TRANSITION_REDIRECT_PERMANENT || + aVisitType == histsvc.TRANSITION_REDIRECT_TEMPORARY; + var placeID = histsvc.addVisit(aURI, aVisitDate, null, + aVisitType, isRedirect, 0); + do_check_true(placeID > 0); + return placeID; +} + +var bucketPrefs = [ + [ "firstBucketCutoff", "firstBucketWeight"], + [ "secondBucketCutoff", "secondBucketWeight"], + [ "thirdBucketCutoff", "thirdBucketWeight"], + [ "fourthBucketCutoff", "fourthBucketWeight"], + [ null, "defaultBucketWeight"] +]; + +var bonusPrefs = { + embedVisitBonus: Ci.nsINavHistoryService.TRANSITION_EMBED, + linkVisitBonus: Ci.nsINavHistoryService.TRANSITION_LINK, + typedVisitBonus: Ci.nsINavHistoryService.TRANSITION_TYPED, + bookmarkVisitBonus: Ci.nsINavHistoryService.TRANSITION_BOOKMARK, + downloadVisitBonus: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD, + permRedirectVisitBonus: Ci.nsINavHistoryService.TRANSITION_REDIRECT_PERMANENT, + tempRedirectVisitBonus: Ci.nsINavHistoryService.TRANSITION_REDIRECT_TEMPORARY, + defaultVisitBonus: 0, + unvisitedBookmarkBonus: 0 +// XXX todo +// unvisitedTypedBonus: 0 +}; + +// create test data +var searchTerm = "frecency"; +var results = []; +var matchCount = 0; +var now = Date.now(); +var prefPrefix = "places.frecency."; +bucketPrefs.every(function(bucket) { + let [cutoffName, weightName] = bucket; + // get pref values + var weight = 0, cutoff = 0, bonus = 0; + try { + weight = prefs.getIntPref(prefPrefix + weightName); + } catch(ex) {} + try { + cutoff = prefs.getIntPref(prefPrefix + cutoffName); + } catch(ex) {} + + if (cutoff < 1) + return true; + + // generate a date within the cutoff period + var dateInPeriod = (now - ((cutoff - 1) * 86400 * 1000)) * 1000; + + for (let [bonusName, visitType] in Iterator(bonusPrefs)) { + var frecency = -1; + var calculatedURI = null; + var matchTitle = ""; + var bonusValue = prefs.getIntPref(prefPrefix + bonusName); + // unvisited (only for first cutoff date bucket) + if (bonusName == "unvisitedBookmarkBonus" || bonusName == "unvisitedTypedBonus") { + if (cutoffName == "firstBucketCutoff") { + var points = Math.ceil(bonusValue / parseFloat(100.0) * weight); + var visitCount = 1; //bonusName == "unvisitedBookmarkBonus" ? 1 : 0; + frecency = Math.ceil(visitCount * points); + calculatedURI = uri("http://" + searchTerm + ".com/" + + bonusName + ":" + bonusValue + "/cutoff:" + cutoff + + "/weight:" + weight + "/frecency:" + frecency); + if (bonusName == "unvisitedBookmarkBonus") { + matchTitle = searchTerm + "UnvisitedBookmark"; + bmsvc.insertBookmark(bmsvc.unfiledBookmarksFolder, calculatedURI, bmsvc.DEFAULT_INDEX, matchTitle); + } + else { + matchTitle = searchTerm + "UnvisitedTyped"; + histsvc.setPageDetails(calculatedURI, matchTitle, 1, false, true); + } + } + } + else { + // visited + var points = Math.ceil(1 * ((bonusValue / parseFloat(100.000000)).toFixed(6) * weight) / 1); + if (!points) { + if (visitType == Ci.nsINavHistoryService.TRANSITION_EMBED || bonusName == "defaultVisitBonus") + frecency = 0; + else + frecency = -1; + } + else + frecency = points; + calculatedURI = uri("http://" + searchTerm + ".com/" + + bonusName + ":" + bonusValue + "/cutoff:" + cutoff + + "/weight:" + weight + "/frecency:" + frecency); + if (visitType == Ci.nsINavHistoryService.TRANSITION_BOOKMARK) { + matchTitle = searchTerm + "Bookmarked"; + bmsvc.insertBookmark(bmsvc.unfiledBookmarksFolder, calculatedURI, bmsvc.DEFAULT_INDEX, matchTitle); + } + else + matchTitle = calculatedURI.spec.substr(calculatedURI.spec.lastIndexOf("/")+1); + add_visit(calculatedURI, dateInPeriod, visitType); + } + + if (calculatedURI && frecency) + results.push([calculatedURI, frecency, matchTitle]); + } + return true; +}); + +// sort results by frecency +results.sort(function(a,b) a[1] - b[1]); +results.reverse(); + +//results.every(function(el) { dump("result: " + el[1] + ": " + el[0].spec + " (" + el[2] + ")\n"); return true; }) + +function AutoCompleteInput(aSearches) { + this.searches = aSearches; +} +AutoCompleteInput.prototype = { + constructor: AutoCompleteInput, + + searches: null, + + minResultsForPopup: 0, + timeout: 10, + searchParam: "", + textValue: "", + disableAutoComplete: false, + completeDefaultIndex: false, + + get searchCount() { + return this.searches.length; + }, + + getSearchAt: function(aIndex) { + return this.searches[aIndex]; + }, + + onSearchComplete: function() {}, + + popupOpen: false, + + popup: { + setSelectedIndex: function(aIndex) {}, + invalidate: function() {}, + + // nsISupports implementation + QueryInterface: function(iid) { + if (iid.equals(Ci.nsISupports) || + iid.equals(Ci.nsIAutoCompletePopup)) + return this; + + throw Components.results.NS_ERROR_NO_INTERFACE; + } + }, + + // nsISupports implementation + QueryInterface: function(iid) { + if (iid.equals(Ci.nsISupports) || + iid.equals(Ci.nsIAutoCompleteInput)) + return this; + + throw Components.results.NS_ERROR_NO_INTERFACE; + } +} + +function run_test() { + var controller = Components.classes["@mozilla.org/autocomplete/controller;1"]. + getService(Components.interfaces.nsIAutoCompleteController); + + // Make an AutoCompleteInput that uses our searches + // and confirms results on search complete + var input = new AutoCompleteInput(["history"]); + + controller.input = input; + + // Search is asynchronous, so don't let the test finish immediately + do_test_pending(); + + input.onSearchComplete = function() { + do_check_eq(controller.searchStatus, + Ci.nsIAutoCompleteController.STATUS_COMPLETE_MATCH); + + // test that all records with non-zero frecency were matched + do_check_eq(controller.matchCount, results.length); + + // test that matches are sorted by frecency + for (var i = 0; i < controller.matchCount; i++) { + do_check_eq(controller.getValueAt(i), results[i][0].spec); + do_check_eq(controller.getCommentAt(i), results[i][2]); + } + + do_test_finished(); + }; + + controller.startSearch(searchTerm); +}