597 lines
20 KiB
JavaScript
597 lines
20 KiB
JavaScript
/* ***** BEGIN LICENSE BLOCK *****
|
|
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
|
|
*
|
|
* The contents of this file are subject to the Mozilla Public License Version
|
|
* 1.1 (the "License"); you may not use this file except in compliance with
|
|
* the License. You may obtain a copy of the License at
|
|
* http://www.mozilla.org/MPL/
|
|
*
|
|
* Software distributed under the License is distributed on an "AS IS" basis,
|
|
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
|
|
* for the specific language governing rights and limitations under the
|
|
* License.
|
|
*
|
|
* The Original Code is Google Safe Browsing.
|
|
*
|
|
* The Initial Developer of the Original Code is Google Inc.
|
|
* Portions created by the Initial Developer are Copyright (C) 2006
|
|
* the Initial Developer. All Rights Reserved.
|
|
*
|
|
* Contributor(s):
|
|
* Niels Provos <niels@google.com> (original author)
|
|
* Fritz Schneider <fritz@google.com>
|
|
*
|
|
* Alternatively, the contents of this file may be used under the terms of
|
|
* either the GNU General Public License Version 2 or later (the "GPL"), or
|
|
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
|
|
* in which case the provisions of the GPL or the LGPL are applicable instead
|
|
* of those above. If you wish to allow use of your version of this file only
|
|
* under the terms of either the GPL or the LGPL, and not to allow others to
|
|
* use your version of this file under the terms of the MPL, indicate your
|
|
* decision by deleting the provisions above and replace them with the notice
|
|
* and other provisions required by the GPL or the LGPL. If you do not delete
|
|
* the provisions above, a recipient may use your version of this file under
|
|
* the terms of any one of the MPL, the GPL or the LGPL.
|
|
*
|
|
* ***** END LICENSE BLOCK ***** */
|
|
|
|
|
|
// A class that manages lists, namely white and black lists for
|
|
// phishing or malware protection. The ListManager knows how to fetch,
|
|
// update, and store lists, and knows the "kind" of list each is (is
|
|
// it a whitelist? a blacklist? etc). However it doesn't know how the
|
|
// lists are serialized or deserialized (the wireformat classes know
|
|
// this) nor the specific format of each list. For example, the list
|
|
// could be a map of domains to "1" if the domain is phishy. Or it
|
|
// could be a map of hosts to regular expressions to match, who knows?
|
|
// Answer: the trtable knows. List are serialized/deserialized by the
|
|
// wireformat reader from/to trtables, and queried by the listmanager.
|
|
//
|
|
// There is a single listmanager for the whole application.
|
|
//
|
|
// The listmanager is used only in privacy mode; in advanced protection
|
|
// mode a remote server is queried.
|
|
//
|
|
// How to add a new table:
|
|
// 1) get it up on the server
|
|
// 2) add it to tablesKnown
|
|
// 3) if it is not a known table type (trtable.js), add an implementation
|
|
// for it in trtable.js
|
|
// 4) add a check for it in the phishwarden's isXY() method, for example
|
|
// isBlackURL()
|
|
//
|
|
// TODO: obviously the way this works could use a lot of improvement. In
|
|
// particular adding a list should just be a matter of adding
|
|
// its name to the listmanager and an implementation to trtable
|
|
// (or not if a talbe of that type exists). The format and semantics
|
|
// of the list comprise its name, so the listmanager should easily
|
|
// be able to figure out what to do with what list (i.e., no
|
|
// need for step 4).
|
|
// TODO more comprehensive update tests, for example add unittest check
|
|
// that the listmanagers tables are properly written on updates
|
|
|
|
/**
|
|
* The base pref name for where we keep table version numbers.
|
|
* We add append the table name to this and set the value to
|
|
* the version. E.g., tableversion.goog-black-enchash may have
|
|
* a value of 1.1234.
|
|
*/
|
|
const kTableVersionPrefPrefix = "urlclassifier.tableversion.";
|
|
|
|
/**
|
|
* Pref name for the update server.
|
|
* TODO: Add support for multiple providers.
|
|
*/
|
|
const kUpdateServerUrl = "urlclassifier.provider.0.updateURL";
|
|
|
|
|
|
/**
|
|
* A ListManager keeps track of black and white lists and knows
|
|
* how to update them.
|
|
*
|
|
* @constructor
|
|
*/
|
|
function PROT_ListManager() {
|
|
this.debugZone = "listmanager";
|
|
G_debugService.enableZone(this.debugZone);
|
|
|
|
this.currentUpdateChecker_ = null; // set when we toggle updates
|
|
this.rpcPending_ = false;
|
|
this.prefs_ = new G_Preferences();
|
|
|
|
this.updateserverURL_ = this.prefs_.getPref(kUpdateServerUrl, null);
|
|
|
|
// The lists we know about and the parses we can use to read
|
|
// them. Default all to the earlies possible version (1.-1); this
|
|
// version will get updated when successfully read from disk or
|
|
// fetch updates.
|
|
this.tablesKnown_ = {};
|
|
this.isTesting_ = false;
|
|
|
|
if (this.isTesting_) {
|
|
// populate with some tables for unittesting
|
|
this.tablesKnown_ = {
|
|
// A major version of zero means local, so don't ask for updates
|
|
"test1-foo-domain" : new PROT_VersionParser("test1-foo-domain", 0, -1),
|
|
"test2-foo-domain" : new PROT_VersionParser("test2-foo-domain", 0, -1),
|
|
"test-white-domain" :
|
|
new PROT_VersionParser("test-white-domain", 0, -1, true /* require mac*/),
|
|
"test-mac-domain" :
|
|
new PROT_VersionParser("test-mac-domain", 0, -1, true /* require mac */)
|
|
};
|
|
|
|
// expose the object for unittesting
|
|
this.wrappedJSObject = this;
|
|
}
|
|
|
|
this.tablesData = {};
|
|
|
|
// Lazily create urlCrypto (see tr-fetcher.js)
|
|
this.urlCrypto_ = null;
|
|
}
|
|
|
|
/**
|
|
* Register a new table table
|
|
* @param tableName - the name of the table
|
|
* @param opt_requireMac true if a mac is required on update, false otherwise
|
|
* @returns true if the table could be created; false otherwise
|
|
*/
|
|
PROT_ListManager.prototype.registerTable = function(tableName,
|
|
opt_requireMac) {
|
|
var table = new PROT_VersionParser(tableName, 1, -1, opt_requireMac);
|
|
if (!table)
|
|
return false;
|
|
this.tablesKnown_[tableName] = table;
|
|
this.tablesData[tableName] = newUrlClassifierTable(tableName);
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Enable updates for some tables
|
|
* @param tables - an array of table names that need updating
|
|
*/
|
|
PROT_ListManager.prototype.enableUpdate = function(tableName) {
|
|
var changed = false;
|
|
var table = this.tablesKnown_[tableName];
|
|
if (table) {
|
|
G_Debug(this, "Enabling table updates for " + tableName);
|
|
table.needsUpdate = true;
|
|
changed = true;
|
|
}
|
|
|
|
if (changed === true)
|
|
this.maybeToggleUpdateChecking();
|
|
}
|
|
|
|
/**
|
|
* Disables updates for some tables
|
|
* @param tables - an array of table names that no longer need updating
|
|
*/
|
|
PROT_ListManager.prototype.disableUpdate = function(tableName) {
|
|
var changed = false;
|
|
var table = this.tablesKnown_[tableName];
|
|
if (table) {
|
|
G_Debug(this, "Disabling table updates for " + tableName);
|
|
table.needsUpdate = false;
|
|
changed = true;
|
|
}
|
|
|
|
if (changed === true)
|
|
this.maybeToggleUpdateChecking();
|
|
}
|
|
|
|
/**
|
|
* Determine if we have some tables that need updating.
|
|
*/
|
|
PROT_ListManager.prototype.requireTableUpdates = function() {
|
|
for (var type in this.tablesKnown_) {
|
|
// All tables with a major of 0 are internal tables that we never
|
|
// update remotely.
|
|
if (this.tablesKnown_[type].major == 0)
|
|
continue;
|
|
|
|
// Tables that need updating even if other tables dont require it
|
|
if (this.tablesKnown_[type].needsUpdate)
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Start managing the lists we know about. We don't do this automatically
|
|
* when the listmanager is instantiated because their profile directory
|
|
* (where we store the lists) might not be available.
|
|
*/
|
|
PROT_ListManager.prototype.maybeStartManagingUpdates = function() {
|
|
if (this.isTesting_)
|
|
return;
|
|
|
|
// We might have been told about tables already, so see if we should be
|
|
// actually updating.
|
|
this.maybeToggleUpdateChecking();
|
|
}
|
|
|
|
/**
|
|
* Determine if we have any tables that require updating. Different
|
|
* Wardens may call us with new tables that need to be updated.
|
|
*/
|
|
PROT_ListManager.prototype.maybeToggleUpdateChecking = function() {
|
|
// If we are testing or dont have an application directory yet, we should
|
|
// not start reading tables from disk or schedule remote updates
|
|
if (this.isTesting_)
|
|
return;
|
|
|
|
// We update tables if we have some tables that want updates. If there
|
|
// are no tables that want to be updated - we dont need to check anything.
|
|
if (this.requireTableUpdates() === true) {
|
|
G_Debug(this, "Starting managing lists");
|
|
this.loadTableVersions_();
|
|
// Multiple warden can ask us to reenable updates at the same time, but we
|
|
// really just need to schedule a single update.
|
|
if (!this.currentUpdateChecker_)
|
|
this.currentUpdateChecker_ =
|
|
new G_Alarm(BindToObject(this.checkForUpdates, this), 3000);
|
|
this.startUpdateChecker();
|
|
} else {
|
|
G_Debug(this, "Stopping managing lists (if currently active)");
|
|
this.stopUpdateChecker(); // Cancel pending updates
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Start periodic checks for updates. Idempotent.
|
|
*/
|
|
PROT_ListManager.prototype.startUpdateChecker = function() {
|
|
this.stopUpdateChecker();
|
|
|
|
// Schedule a check for updates every so often
|
|
// TODO(tc): PREF NEW
|
|
var sixtyMinutes = 60 * 60 * 1000;
|
|
this.updateChecker_ = new G_Alarm(BindToObject(this.checkForUpdates, this),
|
|
sixtyMinutes, true /* repeat */);
|
|
}
|
|
|
|
/**
|
|
* Stop checking for updates. Idempotent.
|
|
*/
|
|
PROT_ListManager.prototype.stopUpdateChecker = function() {
|
|
if (this.updateChecker_) {
|
|
this.updateChecker_.cancel();
|
|
this.updateChecker_ = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Provides an exception free way to look up the data in a table. We
|
|
* use this because at certain points our tables might not be loaded,
|
|
* and querying them could throw.
|
|
*
|
|
* @param table Name of the table that we want to consult
|
|
* @param key Key for table lookup
|
|
* @returns false or the value in the table corresponding to key.
|
|
* If the table name does not exist, we return false, too.
|
|
*/
|
|
PROT_ListManager.prototype.safeExists = function(table, key) {
|
|
var result = false;
|
|
try {
|
|
var map = this.tablesData[table];
|
|
result = map.exists(key);
|
|
} catch(e) {
|
|
result = false;
|
|
G_Debug(this, "safeExists masked failure for " + table + ", key " + key + ": " + e);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Provides an exception free way to insert data into a table.
|
|
* @param table Name of the table that we want to consult
|
|
* @param key Key for table insert
|
|
* @param value Value for table insert
|
|
* @returns true if the value could be inserted, false otherwise
|
|
*/
|
|
PROT_ListManager.prototype.safeInsert = function(table, key, value) {
|
|
if (!this.tablesKnown_[table]) {
|
|
G_Debug(this, "Unknown table: " + table);
|
|
return false;
|
|
}
|
|
if (!this.tablesData[table])
|
|
this.tablesData[table] = newUrlClassifierTable(table);
|
|
try {
|
|
this.tablesData[table].insert(key, value);
|
|
} catch (e) {
|
|
G_Debug(this, "Cannot insert key " + key + " value " + value);
|
|
G_Debug(this, e);
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Provides an exception free way to remove data from a table.
|
|
* @param table Name of the table that we want to consult
|
|
* @param key Key for table erase
|
|
* @returns true if the value could be removed, false otherwise
|
|
*/
|
|
PROT_ListManager.prototype.safeRemove = function(table, key) {
|
|
if (!this.tablesKnown_[table]) {
|
|
G_Debug(this, "Unknown table: " + table);
|
|
return false;
|
|
}
|
|
|
|
if (!this.tablesData[table])
|
|
return false;
|
|
|
|
return this.tablesData[table].remove(key);
|
|
}
|
|
|
|
/**
|
|
* We store table versions in user prefs. This method pulls the values out of
|
|
* the user prefs and into the tablesKnown objects.
|
|
*/
|
|
PROT_ListManager.prototype.loadTableVersions_ = function() {
|
|
// Pull values out of prefs.
|
|
var prefBase = kTableVersionPrefPrefix;
|
|
for (var table in this.tablesKnown_) {
|
|
var version = this.prefs_.getPref(prefBase + table, "1.-1");
|
|
G_Debug(this, "loadTableVersion " + table + ": " + version);
|
|
var tokens = version.split(".");
|
|
G_Assert(this, tokens.length == 2, "invalid version number");
|
|
|
|
this.tablesKnown_[table].major = tokens[0];
|
|
this.tablesKnown_[table].minor = tokens[1];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get lines of the form "[goog-black-enchash 1.1234]" or
|
|
* "[goog-white-url 1.1234 update]" and set the version numbers in the user
|
|
* pref.
|
|
* @param updateString String containing the response from the server.
|
|
* @return Array a list of table names
|
|
*/
|
|
PROT_ListManager.prototype.setTableVersions_ = function(updateString) {
|
|
var updatedTables = [];
|
|
|
|
var prefBase = kTableVersionPrefPrefix;
|
|
var startPos = updateString.indexOf('[');
|
|
var endPos;
|
|
while (startPos != -1) {
|
|
// [ needs to be the start of a new line
|
|
if (0 == startPos || ('\n' == updateString[startPos - 1] &&
|
|
'\n' == updateString[startPos - 2])) {
|
|
endPos = updateString.indexOf('\n', startPos);
|
|
if (endPos != -1) {
|
|
var line = updateString.substring(startPos, endPos);
|
|
var versionParser = new PROT_VersionParser("dummy");
|
|
|
|
if (versionParser.fromString(line)) {
|
|
var tableName = versionParser.type;
|
|
var version = versionParser.major + '.' + versionParser.minor;
|
|
G_Debug(this, "Set table version for " + tableName + ": " + version);
|
|
this.prefs_.setPref(prefBase + tableName, version);
|
|
this.tablesKnown_[tableName].ImportVersion(versionParser);
|
|
|
|
updatedTables.push(tableName);
|
|
}
|
|
}
|
|
}
|
|
// This could catch option params, but that's ok. The double newline
|
|
// check will skip over it.
|
|
startPos = updateString.indexOf('[', startPos + 1);
|
|
}
|
|
return updatedTables;
|
|
}
|
|
|
|
/**
|
|
* Prepares a URL to fetch upates from. Format is a squence of
|
|
* type:major:minor, fields
|
|
*
|
|
* @param url The base URL to which query parameters are appended; assumes
|
|
* already has a trailing ?
|
|
* @returns the URL that we should request the table update from.
|
|
*/
|
|
PROT_ListManager.prototype.getRequestURL_ = function(url) {
|
|
url += "version=";
|
|
var firstElement = true;
|
|
var requestMac = false;
|
|
|
|
for (var type in this.tablesKnown_) {
|
|
// All tables with a major of 0 are internal tables that we never
|
|
// update remotely.
|
|
if (this.tablesKnown_[type].major == 0)
|
|
continue;
|
|
|
|
// Check if the table needs updating
|
|
if (this.tablesKnown_[type].needsUpdate == false)
|
|
continue;
|
|
|
|
if (!firstElement) {
|
|
url += ","
|
|
} else {
|
|
firstElement = false;
|
|
}
|
|
url += type + ":" + this.tablesKnown_[type].toUrl();
|
|
|
|
if (this.tablesKnown_[type].requireMac)
|
|
requestMac = true;
|
|
}
|
|
|
|
// Request a mac only if at least one of the tables to be updated requires
|
|
// it
|
|
if (requestMac) {
|
|
// Add the wrapped key for requesting macs
|
|
if (!this.urlCrypto_)
|
|
this.urlCrypto_ = new PROT_UrlCrypto();
|
|
|
|
url += "&wrkey=" +
|
|
encodeURIComponent(this.urlCrypto_.getManager().getWrappedKey());
|
|
}
|
|
|
|
G_Debug(this, "getRequestURL returning: " + url);
|
|
return url;
|
|
}
|
|
|
|
/**
|
|
* Updates our internal tables from the update server
|
|
*
|
|
* @returns true when a new request was scheduled, false if an old request
|
|
* was still pending.
|
|
*/
|
|
PROT_ListManager.prototype.checkForUpdates = function() {
|
|
// Allow new updates to be scheduled from maybeToggleUpdateChecking()
|
|
this.currentUpdateChecker_ = null;
|
|
|
|
if (this.rpcPending_) {
|
|
G_Debug(this, 'checkForUpdates: old callback is still pending...');
|
|
return false;
|
|
}
|
|
|
|
if (!this.updateserverURL_) {
|
|
G_Debug(this, 'checkForUpdates: no update server url');
|
|
return false;
|
|
}
|
|
G_Debug(this, 'checkForUpdates: scheduling request..');
|
|
this.rpcPending_ = true;
|
|
this.xmlFetcher_ = new PROT_XMLFetcher();
|
|
this.xmlFetcher_.get(this.getRequestURL_(this.updateserverURL_),
|
|
BindToObject(this.rpcDone, this));
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* A callback that is executed when the XMLHTTP request is finished
|
|
*
|
|
* @param data String containing the returned data
|
|
*/
|
|
PROT_ListManager.prototype.rpcDone = function(data) {
|
|
G_Debug(this, "Called rpcDone");
|
|
this.rpcPending_ = false;
|
|
this.xmlFetcher_ = null;
|
|
|
|
if (!data || !data.length) {
|
|
G_Debug(this, "No data. Returning");
|
|
return;
|
|
}
|
|
|
|
// Only use the the data if the mac matches.
|
|
data = this.checkMac_(data);
|
|
|
|
if (data.length == 0) {
|
|
return;
|
|
}
|
|
|
|
// List updates (local lists) don't work yet. See bug 336203.
|
|
throw Exception("dbservice not yet implemented.");
|
|
|
|
var dbUpdateSrv = Cc["@mozilla.org/url-classifier/dbservice;1"]
|
|
.getService(Ci.nsIUrlClassifierDBService);
|
|
|
|
// Update the tables on disk.
|
|
try {
|
|
dbUpdateSrv.updateTables(data);
|
|
} catch (e) {
|
|
// dbUpdateSrv will throw an error if the background thread is already
|
|
// working. In this case, we just wait for the next scheduled update.
|
|
G_Debug(this, "Skipping update, write thread busy.");
|
|
return;
|
|
}
|
|
|
|
// While the update is being processed by a background thread, we need
|
|
// to also update the table versions.
|
|
var tableNames = this.setTableVersions_(data);
|
|
G_Debug(this, "Updated tables: " + tableNames);
|
|
|
|
for (var t = 0, name = null; name = tableNames[t]; ++t) {
|
|
// Create the table object if it doesn't exist.
|
|
if (!this.tablesData[name])
|
|
this.tablesData[name] = newUrlClassifierTable(name);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Given the server response, extract out the new table lines and table
|
|
* version numbers. If the table has a mac, also check to see if it matches
|
|
* the data.
|
|
*
|
|
* @param data String update string from the server
|
|
* @return String The same update string sans tables with invalid macs.
|
|
*/
|
|
PROT_ListManager.prototype.checkMac_ = function(data) {
|
|
var dataTables = data.split('\n\n');
|
|
var returnString = "";
|
|
|
|
for (var table = null, t = 0; table = dataTables[t]; ++t) {
|
|
var firstLineEnd = table.indexOf("\n");
|
|
while (firstLineEnd == 0) {
|
|
// Skip leading blank lines
|
|
table = table.substring(1);
|
|
firstLineEnd = table.indexOf("\n");
|
|
}
|
|
if (firstLineEnd == -1) {
|
|
continue;
|
|
}
|
|
|
|
var versionLine = table.substring(0, firstLineEnd);
|
|
var versionParser = new PROT_VersionParser("dummy");
|
|
if (!versionParser.fromString(versionLine)) {
|
|
// Failed to parse the version string, skip this table.
|
|
G_Debug(this, "Failed to parse version string");
|
|
continue;
|
|
}
|
|
|
|
if (versionParser.mac && versionParser.macval.length > 0) {
|
|
// Includes a mac, so we check it.
|
|
var updateData = table.substring(firstLineEnd + 1) + '\n';
|
|
if (!this.urlCrypto_)
|
|
this.urlCrypto_ = new PROT_UrlCrypto();
|
|
|
|
var computedMac = this.urlCrypto_.computeMac(updateData);
|
|
if (computedMac != versionParser.macval) {
|
|
G_Debug(this, "mac doesn't match: " + computedMac + " != " +
|
|
versionParser.macval)
|
|
continue;
|
|
}
|
|
} else {
|
|
// No mac in the return. Check to see if it's required. If it is
|
|
// required, skip this data.
|
|
if (this.tablesKnown_[versionParser.type] &&
|
|
this.tablesKnown_[versionParser.type].requireMac) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Everything looks ok, add it to the return string.
|
|
returnString += table + "\n\n";
|
|
}
|
|
|
|
return returnString;
|
|
}
|
|
|
|
PROT_ListManager.prototype.QueryInterface = function(iid) {
|
|
if (iid.equals(Components.interfaces.nsISupports) ||
|
|
iid.equals(Components.interfaces.nsIUrlListManager))
|
|
return this;
|
|
|
|
Components.returnCode = Components.results.NS_ERROR_NO_INTERFACE;
|
|
return null;
|
|
}
|
|
|
|
// A simple factory function that creates nsIUrlClassifierTable instances based
|
|
// on a name. The name is a string of the format
|
|
// provider_name-semantic_type-table_type. For example, goog-white-enchash
|
|
// or goog-black-url.
|
|
function newUrlClassifierTable(name) {
|
|
G_Debug("protfactory", "Creating a new nsIUrlClassifierTable: " + name);
|
|
var tokens = name.split('-');
|
|
var type = tokens[2];
|
|
var table = Cc['@mozilla.org/url-classifier/table;1?type=' + type]
|
|
.createInstance(Ci.nsIUrlClassifierTable);
|
|
table.name = name;
|
|
return table;
|
|
}
|