Mercurial Repositories > eric / changeset
changeset
Merged the Safe Browsing extension into the default branch for further processing.
Fri, 04 Aug 2017 18:41:50 +0200
- author
- Detlev Offenbach <detlev@die-offenbachs.de>
- date
- Fri, 04 Aug 2017 18:41:50 +0200
- changeset 5830
- e91a1a8c0a5d
- parent 5828
- c8deff89c20c (current diff)
- parent 5829
- d3448873ced3 (diff)
- child 5831
- 536d97e3f1a1
Merged the Safe Browsing extension into the default branch for further processing.
--- a/Preferences/ConfigurationPages/WebBrowserPage.ui Fri Aug 04 18:29:30 2017 +0200 +++ b/Preferences/ConfigurationPages/WebBrowserPage.ui Fri Aug 04 18:41:50 2017 +0200 @@ -7,7 +7,7 @@ <x>0</x> <y>0</y> <width>616</width> - <height>1610</height> + <height>2000</height> </rect> </property> <layout class="QVBoxLayout" name="verticalLayout_2"> @@ -479,8 +479,8 @@ <property name="title"> <string>Security</string> </property> - <layout class="QHBoxLayout" name="horizontalLayout_7"> - <item> + <layout class="QGridLayout" name="gridLayout_9"> + <item row="0" column="0"> <widget class="QCheckBox" name="xssAuditingCheckBox"> <property name="toolTip"> <string>Select to enable XSS auditing</string> @@ -494,7 +494,7 @@ </property> </widget> </item> - <item> + <item row="0" column="1"> <widget class="QCheckBox" name="insecureContentsCheckBox"> <property name="toolTip"> <string>Select to allow HTTPS pages to run JavaScript, CSS, plugins or web-sockets from HTTP URLs</string>
--- a/Preferences/__init__.py Fri Aug 04 18:29:30 2017 +0200 +++ b/Preferences/__init__.py Fri Aug 04 18:41:50 2017 +0200 @@ -1119,6 +1119,10 @@ "SessionAutoSave": True, "SessionAutoSaveInterval": 15, # interval in seconds "SessionLastActivePath": "", + # Google Safe Browsing + "SafeBrowsingEnabled": True, + "SafeBrowsingApiKey": "", # API key + "SafeBrowsingFilterPlatform": True, } if QWebEngineSettings: webBrowserDefaults["HelpViewerType"] = 1 # eric browser @@ -2919,6 +2923,7 @@ "SpellCheckEnabled", "ShowToolbars", "MenuBarVisible", "BookmarksToolBarVisible", "StatusBarVisible", "SessionAutoSave", "LoadTabOnActivation", + "SafeBrowsingEnabled", "SafeBrowsingFilterPlatform", ]: return toBool(prefClass.settings.value( "WebBrowser/" + key, prefClass.webBrowserDefaults[key]))
--- a/WebBrowser/Download/DownloadManager.py Fri Aug 04 18:29:30 2017 +0200 +++ b/WebBrowser/Download/DownloadManager.py Fri Aug 04 18:41:50 2017 +0200 @@ -167,9 +167,29 @@ download data. @type QWebEngineDownloadItem """ - if downloadItem.url().isEmpty(): + url = downloadItem.url() + if url.isEmpty(): return + # Safe Browsing + threatLists = WebBrowserWindow.safeBrowsingManager().lookupUrl(url) + if threatLists: + threatMessages = WebBrowserWindow.safeBrowsingManager()\ + .getThreatMessages(threatLists) + res = E5MessageBox.warning( + WebBrowserWindow.getWindow(), + self.tr("Suspicuous URL detected"), + self.tr("<p>The URL <b>{0}</b> was found in the Safe Browsing" + " database.</p>{1}").format(url.toString(), + "".join(threatMessages)), + E5MessageBox.StandardButtons( + E5MessageBox.Abort | + E5MessageBox.Ignore), + E5MessageBox.Abort) + if res == E5MessageBox.Abort: + downloadItem.cancel() + return + from .DownloadItem import DownloadItem itm = DownloadItem(downloadItem, parent=self) self.__addItem(itm)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/WebBrowser/SafeBrowsing/SafeBrowsingAPIClient.py Fri Aug 04 18:41:50 2017 +0200 @@ -0,0 +1,412 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing the low level interface for Google Safe Browsing. +""" + +from __future__ import unicode_literals +try: + str = unicode # __IGNORE_EXCEPTION__ +except NameError: + pass + +import json +import base64 + +from PyQt5.QtCore import pyqtSignal, QObject, QDateTime, QUrl, QByteArray, \ + QCoreApplication, QEventLoop +from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply + +from WebBrowser.WebBrowserWindow import WebBrowserWindow + + +class SafeBrowsingAPIClient(QObject): + """ + Class implementing the low level interface for Google Safe Browsing. + + @signal networkError(str) emitted to indicate a network error + """ + ClientId = "eric6_API_client" + ClientVersion = "1.0.0" + + GsbUrlTemplate = "https://safebrowsing.googleapis.com/v4/{0}?key={1}" + + networkError = pyqtSignal(str) + + def __init__(self, apiKey, fairUse=True, parent=None): + """ + Constructor + + @param apiKey API key to be used + @type str + @param fairUse flag indicating to follow the fair use policy + @type bool + @param parent reference to the parent object + @type QObject + """ + self.__apiKey = apiKey + self.__fairUse = fairUse + + self.__nextRequestNoSoonerThan = QDateTime() + self.__failCount = 0 + + def setApiKey(self, apiKey): + """ + Public method to set the API key. + + @param apiKey API key to be set + @type str + """ + self.__apiKey = apiKey + + def getThreatLists(self): + """ + Public method to retrieve all available threat lists. + + @return list of threat lists + @rtype list of dict containing 'threatType', 'platformType' and + 'threatEntryType' + """ + url = QUrl(self.GsbUrlTemplate.format("threatLists", self.__apiKey)) + req = QNetworkRequest(url) + reply = WebBrowserWindow.networkManager().get(req) + + while reply.isRunning(): + QCoreApplication.processEvents(QEventLoop.AllEvents, 200) + # max. 200 ms processing + + res = None + if reply.error() != QNetworkReply.NoError: + self.networkError.emit(reply.errorString()) + else: + result = self.__extractData(reply) + res = result["threatLists"] + + reply.deleteLater() + return res + + def getThreatsUpdate(self, clientStates): + """ + Public method to fetch hash prefix updates for the given threat list. + + @param clientStates dictionary of client states with keys like + (threatType, platformType, threatEntryType) + @type dict + @return list of threat updates + @rtype list of dict + """ + requestBody = { + "client": { + "clientId": self.ClientId, + "clientVersion": self.ClientVersion, + }, + "listUpdateRequests": [], + } + + for (threatType, platformType, threatEntryType), currentState in \ + clientStates.items(): + requestBody["listUpdateRequests"].append( + { + "threatType": threatType, + "platformType": platformType, + "threatEntryType": threatEntryType, + "state": currentState, + "constraints": { + "supportedCompressions": ["RAW"], + } + } + ) + + data = QByteArray(json.dumps(requestBody).encode("utf-8")) + url = QUrl(self.GsbUrlTemplate.format("threatListUpdates:fetch", + self.__apiKey)) + req = QNetworkRequest(url) + req.setHeader(QNetworkRequest.ContentTypeHeader, "application/json") + reply = WebBrowserWindow.networkManager().post(req, data) + + while reply.isRunning(): + QCoreApplication.processEvents(QEventLoop.AllEvents, 200) + # max. 200 ms processing + + res = None + if reply.error() != QNetworkReply.NoError: + self.networkError.emit(reply.errorString()) + else: + result = self.__extractData(reply) + res = result["listUpdateResponses"] + + reply.deleteLater() + return res + + def getFullHashes(self, prefixes, clientState): + """ + Public method to find full hashes matching hash prefixes. + + @param prefixes list of hash prefixes to find + @type list of str (Python 2) or list of bytes (Python 3) + @param clientState dictionary of client states with keys like + (threatType, platformType, threatEntryType) + @type dict + @return dictionary containing the list of found hashes and the + negative cache duration + @rtype dict + """ + requestBody = { + "client": { + "clientId": self.ClientId, + "clientVersion": self.ClientVersion, + }, + "clientStates": [], + "threatInfo": { + "threatTypes": [], + "platformTypes": [], + "threatEntryTypes": [], + "threatEntries": [], + }, + } + + for prefix in prefixes: + requestBody["threatInfo"]["threatEntries"].append( + {"hash": base64.b64encode(prefix).decode("ascii")}) + + for (threatType, platformType, threatEntryType), currentState in \ + clientState.items(): + requestBody["clientStates"].append(currentState) + if threatType not in requestBody["threatInfo"]["threatTypes"]: + requestBody["threatInfo"]["threatTypes"].append(threatType) + if platformType not in \ + requestBody["threatInfo"]["platformTypes"]: + requestBody["threatInfo"]["platformTypes"].append( + platformType) + if threatEntryType not in \ + requestBody["threatInfo"]["threatEntryTypes"]: + requestBody["threatInfo"]["threatEntryTypes"].append( + threatEntryType) + + data = QByteArray(json.dumps(requestBody).encode("utf-8")) + url = QUrl(self.GsbUrlTemplate.format("fullHashes:find", + self.__apiKey)) + req = QNetworkRequest(url) + req.setHeader(QNetworkRequest.ContentTypeHeader, "application/json") + reply = WebBrowserWindow.networkManager().post(req, data) + + while reply.isRunning(): + QCoreApplication.processEvents(QEventLoop.AllEvents, 200) + # max. 200 ms processing + + res = None + if reply.error() != QNetworkReply.NoError: + self.networkError.emit(reply.errorString()) + else: + res = self.__extractData(reply) + + reply.deleteLater() + return res + + def __extractData(self, reply): + """ + Private method to extract the data of a network reply. + + @param reply reference to the network reply object + @type QNetworkReply + @return extracted data + @type list or dict + """ + result = json.loads(str(reply.readAll(), "utf-8")) + self.__setWaitDuration(result.get("minimumWaitDuration")) + return result + + def __setWaitDuration(self, minimumWaitDuration): + """ + Private method to set the minimum wait duration. + + @param minimumWaitDuration duration to be set + @type str + """ + if not self.__fairUse or minimumWaitDuration is None: + self.__nextRequestNoSoonerThan = QDateTime() + else: + waitDuration = int(float(minimumWaitDuration.rstrip("s"))) + self.__nextRequestNoSoonerThan = \ + QDateTime.currentDateTime().addSecs(waitDuration) + + def fairUseDelayExpired(self): + """ + Public method to check, if the fair use wait period has expired. + + @return flag indicating expiration + @rtype bool + """ + return ( + self.__fairUse and + QDateTime.currentDateTime() >= self.__nextRequestNoSoonerThan + ) or not self.__fairUse + + def getFairUseDelayExpirationDateTime(self): + """ + Public method to get the date and time the fair use delay will expire. + + @return fair use delay expiration date and time + @rtype QDateTime + """ + return self.__nextRequestNoSoonerThan + + @classmethod + def getThreatMessage(cls, threatType): + """ + Class method to get a warning message for the given threat type. + + @param threatType threat type to get the message for + @type str + @return threat message + @rtype str + """ + threatType = threatType.lower() + if threatType == "malware": + msg = QCoreApplication.translate( + "SafeBrowsingAPI", + "<h3>Malware Warning</h3>" + "<p>The web site you are about to visit may try to install" + " harmful programs on your computer in order to steal or" + " destroy your data.</p>") + elif threatType == "social_engineering": + msg = QCoreApplication.translate( + "SafeBrowsingAPI", + "<h3>Phishing Warning</h3>" + "<p>The web site you are about to visit may try to trick you" + " into doing something dangerous online, such as revealing" + " passwords or personal information, usually through a fake" + " website.</p>") + elif threatType == "unwanted_software": + msg = QCoreApplication.translate( + "SafeBrowsingAPI", + "<h3>Unwanted Software Warning</h3>" + "<p>The software you are about to download may negatively" + " affect your browsing or computing experience.</p>") + elif threatType == "potentially_harmful_application": + msg = QCoreApplication.translate( + "SafeBrowsingAPI", + "<h3>Potentially Harmful Application</h3>" + "<p>The web site you are about to visit may try to trick you" + " into installing applications, that may negatively affect" + " your browsing experience.</p>") + else: + # unknow threat + msg = QCoreApplication.translate( + "SafeBrowsingAPI", + "<h3>Unknown Threat Warning</h3>" + "<p>The web site you are about to visit was found in the Safe" + " Browsing Database but was not classified yet.</p>") + + return msg + + @classmethod + def getThreatType(cls, threatType): + """ + Class method to get a display string for a given threat type. + + @param threatType threat type to get display string for + @type str + @return display string + @rtype str + """ + threatType = threatType.lower() + if threatType == "malware": + displayString = QCoreApplication.translate( + "SafeBrowsingAPI", "Malware") + elif threatType == "social_engineering": + displayString = QCoreApplication.translate( + "SafeBrowsingAPI", "Phishing") + elif threatType == "unwanted_software": + displayString = QCoreApplication.translate( + "SafeBrowsingAPI", "Unwanted Software") + elif threatType == "potentially_harmful_application": + displayString = QCoreApplication.translate( + "SafeBrowsingAPI", "Harmful Application") + elif threatType == "malcious_binary": + displayString = QCoreApplication.translate( + "SafeBrowsingAPI", "Malicious Binary") + else: + displayString = QCoreApplication.translate( + "SafeBrowsingAPI", "Unknown Threat") + + return displayString + + @classmethod + def getPlatformString(cls, platformType): + """ + Class method to get the platform string for a given platform type. + + @param platformType platform type as defined in the v4 API + @type str + @return platform string + @rtype str + """ + platformStrings = { + "WINDOWS": "Windows", + "LINUX": "Linux", + "ANDROID": "Android", + "OSX": "macOS", + "IOS": "iOS", + "CHROME": "Chrome OS", + } + if platformType in platformStrings: + return platformStrings[platformType] + + if platformType == "ANY_PLATFORM": + return QCoreApplication.translate( + "SafeBrowsingAPI", "any defined platform") + elif platformType == "ALL_PLATFORMS": + return QCoreApplication.translate( + "SafeBrowsingAPI", "all defined platforms") + else: + return QCoreApplication.translate( + "SafeBrowsingAPI", "unknown platform") + + @classmethod + def getThreatEntryString(cls, threatEntry): + """ + Class method to get the threat entry string. + + @param threatEntry threat entry type as defined in the v4 API + @type str + @return threat entry string + @rtype str + """ + if threatEntry == "URL": + return "URL" + elif threatEntry == "EXECUTABLE": + return QCoreApplication.translate( + "SafeBrowsingAPI", "executable program") + else: + return QCoreApplication.translate( + "SafeBrowsingAPI", "unknown type") + + @classmethod + def getPlatformTypes(cls, platform): + """ + Class method to get the platform types for a given platform. + + @param platform platform string + @type str (one of 'linux', 'windows', 'macos') + @return list of platform types as defined in the v4 API for the + given platform + @rtype list of str + @exception ValueError raised to indicate an invalid platform string + """ + platform = platform.lower() + + platformTypes = ["ANY_PLATFORM", "ALL_PLATFORMS"] + if platform == "linux": + platformTypes.append("LINUX") + elif platform == "windows": + platformTypes.append("WINDOWS") + elif platform == "macos": + platformTypes.append("OSX") + else: + raise ValueError("Unsupported platform") + + return platformTypes
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/WebBrowser/SafeBrowsing/SafeBrowsingCache.py Fri Aug 04 18:41:50 2017 +0200 @@ -0,0 +1,770 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing a cache for Google Safe Browsing. +""" + +# +# Some part of this code were ported from gglsbl.storage and adapted +# to QtSql. +# +# https://github.com/afilipovich/gglsbl +# + +from __future__ import unicode_literals, division + +import os + +from PyQt5.QtCore import QObject, QByteArray, QCryptographicHash, \ + QCoreApplication, QEventLoop +from PyQt5.QtSql import QSql, QSqlDatabase, QSqlQuery + +from .SafeBrowsingUtilities import toHex + + +class ThreatList(object): + """ + Class implementing the threat list info. + """ + def __init__(self, threatType, platformType, threatEntryType): + """ + Constructor + + @param threatType threat type + @type str + @param platformType platform type + @type str + @param threatEntryType threat entry type + @type str + """ + self.threatType = threatType + self.platformType = platformType + self.threatEntryType = threatEntryType + + @classmethod + def fromApiEntry(cls, entry): + """ + Class method to instantiate a threat list given a threat list entry + dictionary. + + @param entry threat list entry dictionary + @type dict + @return instantiated object + @rtype ThreatList + """ + return cls(entry['threatType'], entry['platformType'], + entry['threatEntryType']) + + def asTuple(self): + """ + Public method to convert the object to a tuple. + + @return tuple containing the threat list info + @rtype tuple of (str, str, str) + """ + return (self.threatType, self.platformType, self.threatEntryType) + + def __repr__(self): + """ + Special method to generate a printable representation. + + @return printable representation + @rtype str + """ + return '/'.join(self.asTuple()) + + +class HashPrefixList(object): + """ + Class implementing a container for threat list data. + """ + def __init__(self, prefixLength, rawHashes): + """ + Constructor + + @param prefixLength length of each hash prefix + @type int + @param rawHashes raw hash prefixes of given length concatenated and + sorted in lexicographical order + @type str + """ + self.__prefixLength = prefixLength + self.__rawHashes = rawHashes + + def __len__(self): + """ + Special method to calculate the number of entries. + + @return length + @rtype int + """ + return len(self.__rawHashes) // self.__prefixLength + + def __iter__(self): + """ + Special method to iterate over the raw hashes. + + @return iterator object + @rtype iterator + """ + n = self.__prefixLength + return (self.__rawHashes[index:index + n] + for index in range(0, len(self.__rawHashes), n) + ) + + +class SafeBrowsingCache(QObject): + """ + Class implementing a cache for Google Safe Browsing. + """ + create_threat_list_stmt = """ + CREATE TABLE threat_list + (threat_type character varying(128) NOT NULL, + platform_type character varying(128) NOT NULL, + threat_entry_type character varying(128) NOT NULL, + client_state character varying(42), + timestamp timestamp without time zone DEFAULT current_timestamp, + PRIMARY KEY (threat_type, platform_type, threat_entry_type) + ) + """ + drop_threat_list_stmt = """DROP TABLE IF EXISTS threat_list""" + + create_full_hashes_stmt = """ + CREATE TABLE full_hash + (value BLOB NOT NULL, + threat_type character varying(128) NOT NULL, + platform_type character varying(128) NOT NULL, + threat_entry_type character varying(128) NOT NULL, + downloaded_at timestamp without time zone DEFAULT current_timestamp, + expires_at timestamp without time zone + NOT NULL DEFAULT current_timestamp, + malware_threat_type varchar(32), + PRIMARY KEY (value, threat_type, platform_type, threat_entry_type) + ) + """ + drop_full_hashes_stmt = """DROP TABLE IF EXISTS full_hash""" + + create_hash_prefix_stmt = """ + CREATE TABLE hash_prefix + (value BLOB NOT NULL, + cue character varying(4) NOT NULL, + threat_type character varying(128) NOT NULL, + platform_type character varying(128) NOT NULL, + threat_entry_type character varying(128) NOT NULL, + timestamp timestamp without time zone DEFAULT current_timestamp, + negative_expires_at timestamp without time zone + NOT NULL DEFAULT current_timestamp, + PRIMARY KEY (value, threat_type, platform_type, threat_entry_type), + FOREIGN KEY(threat_type, platform_type, threat_entry_type) + REFERENCES threat_list(threat_type, platform_type, threat_entry_type) + ON DELETE CASCADE + ) + """ + drop_hash_prefix_stmt = """DROP TABLE IF EXISTS hash_prefix""" + + create_full_hash_cue_idx = """ + CREATE INDEX idx_hash_prefix_cue ON hash_prefix (cue) + """ + drop_full_hash_cue_idx = """DROP INDEX IF EXISTS idx_hash_prefix_cue""" + + create_full_hash_expires_idx = """ + CREATE INDEX idx_full_hash_expires_at ON full_hash (expires_at) + """ + drop_full_hash_expires_idx = """ + DROP INDEX IF EXISTS idx_full_hash_expires_at + """ + + create_full_hash_value_idx = """ + CREATE INDEX idx_full_hash_value ON full_hash (value) + """ + drop_full_hash_value_idx = """DROP INDEX IF EXISTS idx_full_hash_value""" + + def __init__(self, dbPath, parent=None): + """ + Constructor + + @param dbPath path to store the cache DB into + @type str + @param parent reference to the parent object + @type QObject + """ + super(SafeBrowsingCache, self).__init__(parent) + + self.__connectionName = "SafeBrowsingCache" + + if not os.path.exists(dbPath): + os.makedirs(dbPath) + + self.__dbFileName = os.path.join(dbPath, "SafeBrowsingCache.db") + preparationNeeded = not os.path.exists(self.__dbFileName) + + self.__openCacheDb() + if preparationNeeded: + self.prepareCacheDb() + + def close(self): + """ + Public method to close the database. + """ + if QSqlDatabase.database(self.__connectionName).isOpen(): + QSqlDatabase.database(self.__connectionName).close() + QSqlDatabase.removeDatabase(self.__language) + + def __openCacheDb(self): + """ + Private method to open the cache database. + + @return flag indicating the open state + @rtype bool + """ + db = QSqlDatabase.database(self.__connectionName, False) + if not db.isValid(): + # the database connection is a new one + db = QSqlDatabase.addDatabase("QSQLITE", self.__connectionName) + db.setDatabaseName(self.__dbFileName) + opened = db.open() + if not opened: + QSqlDatabase.removeDatabase(self.__connectionName) + else: + opened = True + return opened + + def prepareCacheDb(self): + """ + Public method to prepare the cache database. + """ + db = QSqlDatabase.database(self.__connectionName) + db.transaction() + try: + query = QSqlQuery(db) + # step 1: drop old tables + query.exec_(self.drop_threat_list_stmt) + query.exec_(self.drop_full_hashes_stmt) + query.exec_(self.drop_hash_prefix_stmt) + # step 2: drop old indices + query.exec_(self.drop_full_hash_cue_idx) + query.exec_(self.drop_full_hash_expires_idx) + query.exec_(self.drop_full_hash_value_idx) + # step 3: create tables + query.exec_(self.create_threat_list_stmt) + query.exec_(self.create_full_hashes_stmt) + query.exec_(self.create_hash_prefix_stmt) + # step 4: create indices + query.exec_(self.create_full_hash_cue_idx) + query.exec_(self.create_full_hash_expires_idx) + query.exec_(self.create_full_hash_value_idx) + finally: + del query + db.commit() + + def lookupFullHashes(self, hashValues): + """ + Public method to get a list of threat lists and expiration flag + for the given hashes if a hash is blacklisted. + + @param hashValues list of hash values to look up + @type list of bytes + @return list of tuples containing the threat list info and the + expiration flag + @rtype list of tuple of (ThreatList, bool) + """ + queryStr = """ + SELECT threat_type, platform_type, threat_entry_type, + expires_at < current_timestamp AS has_expired + FROM full_hash WHERE value IN ({0}) + """ + output = [] + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + query = QSqlQuery(db) + query.prepare( + queryStr.format(",".join(["?"] * len(hashValues)))) + for hashValue in hashValues: + query.addBindValue(QByteArray(hashValue), + QSql.In | QSql.Binary) + + query.exec_() + + while query.next(): + threatType = query.value(0) + platformType = query.value(1) + threatEntryType = query.value(2) + hasExpired = bool(query.value(3)) + threatList = ThreatList(threatType, platformType, + threatEntryType) + output.append((threatList, hasExpired)) + del query + finally: + db.commit() + + return output + + def lookupHashPrefix(self, prefixes): + """ + Public method to look up hash prefixes in the local cache. + + @param prefixes list of hash prefixes to look up + @type list of bytes + @return list of tuples containing the threat list, full hash and + negative cache expiration flag + @rtype list of tuple of (ThreatList, bytes, bool) + """ + queryStr = """ + SELECT value,threat_type,platform_type,threat_entry_type, + negative_expires_at < current_timestamp AS negative_cache_expired + FROM hash_prefix WHERE cue IN ({0}) + """ + output = [] + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + query = QSqlQuery(db) + query.prepare( + queryStr.format(",".join(["?"] * len(prefixes)))) + for prefix in prefixes: + query.addBindValue(prefix) + + query.exec_() + + while query.next(): + fullHash = bytes(query.value(0)) + threatType = query.value(1) + platformType = query.value(2) + threatEntryType = query.value(3) + negativeCacheExpired = bool(query.value(4)) + threatList = ThreatList(threatType, platformType, + threatEntryType) + output.append((threatList, fullHash, negativeCacheExpired)) + del query + finally: + db.commit() + + return output + + def storeFullHash(self, threatList, hashValue, cacheDuration, + malwareThreatType): + """ + Public method to store full hash data in the cache database. + + @param threatList threat list info object + @type ThreatList + @param hashValue hash to be stored + @type bytes + @param cacheDuration duration the data should remain in the cache + @type int or float + @param malwareThreatType threat type of the malware + @type str + """ + insertQueryStr = """ + INSERT OR IGNORE INTO full_hash + (value, threat_type, platform_type, threat_entry_type, + malware_threat_type, downloaded_at) + VALUES + (?, ?, ?, ?, ?, current_timestamp) + """ + updateQueryStr = """ + UPDATE full_hash SET + expires_at=datetime(current_timestamp, '+{0} SECONDS') + WHERE value=? AND threat_type=? AND platform_type=? AND + threat_entry_type=? + """.format(int(cacheDuration)) + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + query = QSqlQuery(db) + query.prepare(insertQueryStr) + query.addBindValue(QByteArray(hashValue), + QSql.In | QSql.Binary) + query.addBindValue(threatList.threatType) + query.addBindValue(threatList.platformType) + query.addBindValue(threatList.threatEntryType) + query.addBindValue(malwareThreatType) + query.exec_() + del query + + query = QSqlQuery(db) + query.prepare(updateQueryStr) + query.addBindValue(QByteArray(hashValue), + QSql.In | QSql.Binary) + query.addBindValue(threatList.threatType) + query.addBindValue(threatList.platformType) + query.addBindValue(threatList.threatEntryType) + query.exec_() + del query + finally: + db.commit() + + def deleteHashPrefixList(self, threatList): + """ + Public method to delete hash prefixes for a given threat list. + + @param threatList threat list info object + @type ThreatList + """ + queryStr = """ + DELETE FROM hash_prefix + WHERE threat_type=? AND platform_type=? AND threat_entry_type=? + """ + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + query = QSqlQuery(db) + query.prepare(queryStr) + query.addBindValue(threatList.threatType) + query.addBindValue(threatList.platformType) + query.addBindValue(threatList.threatEntryType) + query.exec_() + del query + finally: + db.commit() + + def cleanupFullHashes(self, keepExpiredFor=43200): + """ + Public method to clean up full hash entries expired more than the + given time. + + @param keepExpiredFor time period in seconds of entries to be expired + @type int or float + """ + queryStr = """ + DELETE FROM full_hash + WHERE expires_at=datetime(current_timestamp, '{0} SECONDS') + """.format(int(keepExpiredFor)) + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + query = QSqlQuery(db) + query.prepare(queryStr) + query.exec_() + del query + finally: + db.commit() + + def updateHashPrefixExpiration(self, threatList, hashPrefix, + negativeCacheDuration): + """ + Public method to update the hash prefix expiration time. + + @param threatList threat list info object + @type ThreatList + @param hashPrefix hash prefix + @type bytes + @param negativeCacheDuration time in seconds the entry should remain + in the cache + @type int or float + """ + queryStr = """ + UPDATE hash_prefix + SET negative_expires_at=datetime(current_timestamp, '+{0} SECONDS') + WHERE value=? AND threat_type=? AND platform_type=? AND + threat_entry_type=? + """.format(int(negativeCacheDuration)) + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + query = QSqlQuery(db) + query.prepare(queryStr) + query.addBindValue(QByteArray(hashPrefix), + QSql.In | QSql.Binary) + query.addBindValue(threatList.threatType) + query.addBindValue(threatList.platformType) + query.addBindValue(threatList.threatEntryType) + query.exec_() + del query + finally: + db.commit() + + def getThreatLists(self): + """ + Public method to get the available threat lists. + + @return list of available threat lists + @rtype list of tuples of (ThreatList, str) + """ + queryStr = """ + SELECT threat_type,platform_type,threat_entry_type,client_state + FROM threat_list + """ + output = [] + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + query = QSqlQuery(db) + query.prepare(queryStr) + + query.exec_() + + while query.next(): + threatType = query.value(0) + platformType = query.value(1) + threatEntryType = query.value(2) + clientState = query.value(3) + threatList = ThreatList(threatType, platformType, + threatEntryType) + output.append((threatList, clientState)) + del query + finally: + db.commit() + + return output + + def addThreatList(self, threatList): + """ + Public method to add a threat list to the cache. + + @param threatList threat list to be added + @type ThreatList + """ + queryStr = """ + INSERT OR IGNORE INTO threat_list + (threat_type, platform_type, threat_entry_type, timestamp) + VALUES (?, ?, ?, current_timestamp) + """ + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + query = QSqlQuery(db) + query.prepare(queryStr) + query.addBindValue(threatList.threatType) + query.addBindValue(threatList.platformType) + query.addBindValue(threatList.threatEntryType) + query.exec_() + del query + finally: + db.commit() + + def deleteThreatList(self, threatList): + """ + Public method to delete a threat list from the cache. + + @param threatList threat list to be deleted + @type ThreatList + """ + queryStr = """ + DELETE FROM threat_list + WHERE threat_type=? AND platform_type=? AND threat_entry_type=? + """ + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + query = QSqlQuery(db) + query.prepare(queryStr) + query.addBindValue(threatList.threatType) + query.addBindValue(threatList.platformType) + query.addBindValue(threatList.threatEntryType) + query.exec_() + del query + finally: + db.commit() + + def updateThreatListClientState(self, threatList, clientState): + """ + Public method to update the client state of a threat list. + + @param threatList threat list to update the client state for + @type ThreatList + @param clientState new client state + @type str + """ + queryStr = """ + UPDATE threat_list SET timestamp=current_timestamp, client_state=? + WHERE threat_type=? AND platform_type=? AND threat_entry_type=? + """ + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + query = QSqlQuery(db) + query.prepare(queryStr) + query.addBindValue(clientState) + query.addBindValue(threatList.threatType) + query.addBindValue(threatList.platformType) + query.addBindValue(threatList.threatEntryType) + query.exec_() + del query + finally: + db.commit() + + def hashPrefixListChecksum(self, threatList): + """ + Public method to calculate the SHA256 checksum for an alphabetically + sorted concatenated list of hash prefixes. + + @param threatList threat list to calculate checksum for + @type ThreatList + @return SHA256 checksum + @rtype bytes + """ + queryStr = """ + SELECT value FROM hash_prefix + WHERE threat_type=? AND platform_type=? AND threat_entry_type=? + ORDER BY value + """ + checksum = None + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + sha256Hash = QCryptographicHash(QCryptographicHash.Sha256) + try: + query = QSqlQuery(db) + query.prepare(queryStr) + query.addBindValue(threatList.threatType) + query.addBindValue(threatList.platformType) + query.addBindValue(threatList.threatEntryType) + + query.exec_() + + while query.next(): + sha256Hash.addData(query.value(0)) + QCoreApplication.processEvents(QEventLoop.AllEvents, 200) + del query + finally: + db.commit() + + checksum = bytes(sha256Hash.result()) + + return checksum + + def populateHashPrefixList(self, threatList, prefixes): + """ + Public method to populate the hash prefixes for a threat list. + + @param threatList threat list of the hash prefixes + @type ThreatList + @param prefixes list of hash prefixes to be inserted + @type HashPrefixList + """ + queryStr = """ + INSERT INTO hash_prefix + (value, cue, threat_type, platform_type, threat_entry_type, + timestamp) + VALUES (?, ?, ?, ?, ?, current_timestamp) + """ + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + for prefix in prefixes: + query = QSqlQuery(db) + query.prepare(queryStr) + query.addBindValue(QByteArray(prefix), + QSql.In | QSql.Binary) + query.addBindValue(toHex(prefix[:4])) + query.addBindValue(threatList.threatType) + query.addBindValue(threatList.platformType) + query.addBindValue(threatList.threatEntryType) + query.exec_() + del query + finally: + db.commit() + + def getHashPrefixValuesToRemove(self, threatList, indexes): + """ + Public method to get the hash prefix values to be removed from the + cache. + + @param threatList threat list to remove prefixes from + @type ThreatList + @param indexes list of indexes of prefixes to be removed + @type list of int + @return list of hash prefixes to be removed + @rtype list of bytes + """ + queryStr = """ + SELECT value FROM hash_prefix + WHERE threat_type=? AND platform_type=? AND threat_entry_type=? + ORDER BY value + """ + indexes = set(indexes) + output = [] + + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + query = QSqlQuery(db) + query.prepare(queryStr) + query.addBindValue(threatList.threatType) + query.addBindValue(threatList.platformType) + query.addBindValue(threatList.threatEntryType) + + query.exec_() + + index = 0 + while query.next(): + if index in indexes: + prefix = bytes(query.value(0)) + output.append(prefix) + index += 1 + del query + finally: + db.commit() + + return output + + def removeHashPrefixIndices(self, threatList, indexes): + """ + Public method to remove hash prefixes from the cache. + + @param threatList threat list to delete hash prefixes of + @type ThreatList + @param indexes list of indexes of prefixes to be removed + @type list of int + """ + queryStr = """ + DELETE FROM hash_prefix + WHERE threat_type=? AND platform_type=? AND + threat_entry_type=? AND value IN ({0}) + """ + batchSize = 40 + + prefixesToRemove = self.getHashPrefixValuesToRemove( + threatList, indexes) + if prefixesToRemove: + db = QSqlDatabase.database(self.__connectionName) + if db.isOpen(): + db.transaction() + try: + for index in range(0, len(prefixesToRemove), batchSize): + removeBatch = \ + prefixesToRemove[index:(index + batchSize)] + + query = QSqlQuery(db) + query.prepare( + queryStr.format(",".join(["?"] * len(removeBatch))) + ) + query.addBindValue(threatList.threatType) + query.addBindValue(threatList.platformType) + query.addBindValue(threatList.threatEntryType) + for prefix in removeBatch: + query.addBindValue(QByteArray(prefix), + QSql.In | QSql.Binary) + query.exec_() + del query + finally: + db.commit()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/WebBrowser/SafeBrowsing/SafeBrowsingDialog.py Fri Aug 04 18:41:50 2017 +0200 @@ -0,0 +1,302 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing a dialog to configure safe browsing support. +""" + +from __future__ import unicode_literals + +from PyQt5.QtCore import pyqtSlot, Qt, QUrl +from PyQt5.QtWidgets import QDialog, QDialogButtonBox, QAbstractButton, \ + QApplication + +from E5Gui import E5MessageBox + +from .Ui_SafeBrowsingDialog import Ui_SafeBrowsingDialog + +import UI.PixmapCache +import Preferences + + +class SafeBrowsingDialog(QDialog, Ui_SafeBrowsingDialog): + """ + Class implementing a dialog to configure safe browsing support. + """ + def __init__(self, manager, parent=None): + """ + Constructor + + @param manager reference to the safe browsing manager + @type SafeBrowsingManager + @param parent reference to the parent widget + @type QWidget + """ + super(SafeBrowsingDialog, self).__init__(parent) + self.setupUi(self) + self.setWindowFlags(Qt.Window) + + self.__manager = manager + self.__manager.progressMessage.connect(self.__setProgressMessage) + self.__manager.progress.connect(self.__setProgress) + + self.iconLabel.setPixmap( + UI.PixmapCache.getPixmap("safeBrowsing48.png")) + + self.__gsbHelpDialog = None + + self.__enabled = Preferences.getWebBrowser("SafeBrowsingEnabled") + self.__apiKey = Preferences.getWebBrowser("SafeBrowsingApiKey") + self.__filterPlatform = Preferences.getWebBrowser( + "SafeBrowsingFilterPlatform") + + self.buttonBox.setFocus() + + msh = self.minimumSizeHint() + self.resize(max(self.width(), msh.width()), msh.height()) + + def show(self): + """ + Public slot to show the dialog. + """ + self.gsbGroupBox.setChecked(self.__enabled) + self.gsbApiKeyEdit.setText(self.__apiKey) + self.gsbFilterPlatformCheckBox.setChecked(self.__filterPlatform) + + self.__updateCacheButtons() + + super(SafeBrowsingDialog, self).show() + + @pyqtSlot() + def on_gsbHelpButton_clicked(self): + """ + Private slot to show some help text "How to create a safe + browsing API key.". + """ + if self.__gsbHelpDialog is None: + from E5Gui.E5SimpleHelpDialog import E5SimpleHelpDialog + from . import SafeBrowsingHelp + + helpStr = SafeBrowsingHelp() + self.__gsbHelpDialog = E5SimpleHelpDialog( + title=self.tr("Google Safe Browsing API Help"), + helpStr=helpStr, parent=self) + + self.__gsbHelpDialog.show() + + @pyqtSlot(QAbstractButton) + def on_buttonBox_clicked(self, button): + """ + Private slot called by a button of the button box clicked. + + @param button button that was clicked (QAbstractButton) + """ + if button == self.buttonBox.button(QDialogButtonBox.Close): + self.close() + + @pyqtSlot() + def __save(self): + """ + Private slot to save the configuration. + + @return flag indicating success + @rtype bool + """ + self.__enabled = self.gsbGroupBox.isChecked() + self.__apiKey = self.gsbApiKeyEdit.text() + self.__filterPlatform = self.gsbFilterPlatformCheckBox.isChecked() + + Preferences.setWebBrowser("SafeBrowsingEnabled", self.__enabled) + Preferences.setWebBrowser("SafeBrowsingApiKey", self.__apiKey) + Preferences.setWebBrowser("SafeBrowsingFilterPlatform", + self.__filterPlatform) + + self.__manager.configurationChanged() + + self.__updateCacheButtons() + + return True + + def closeEvent(self, evt): + """ + Protected method to handle close events. + + @param evt reference to the close event + @type QCloseEvent + """ + if self.__okToClose(): + evt.accept() + else: + evt.ignore() + + def __isModified(self): + """ + Private method to check, if the dialog contains modified data. + + @return flag indicating the presence of modified data + @rtype bool + """ + return ( + self.__enabled != self.gsbGroupBox.isChecked() or + self.__apiKey != self.gsbApiKeyEdit.text() or + self.__filterPlatform != self.gsbFilterPlatformCheckBox.isChecked() + ) + + def __okToClose(self): + """ + Private method to check, if it is safe to close the dialog. + + @return flag indicating safe to close + @rtype bool + """ + QApplication.restoreOverrideCursor() + if self.__isModified(): + res = E5MessageBox.okToClearData( + self, + self.tr("Safe Browsing Management"), + self.tr("""The dialog contains unsaved changes."""), + self.__save) + if not res: + return False + return True + + def __updateCacheButtons(self): + """ + Private method to set enabled state of the cache buttons. + """ + enable = self.__enabled and bool(self.__apiKey) + + self.updateCacheButton.setEnabled(enable) + self.clearCacheButton.setEnabled(enable) + + @pyqtSlot() + def on_updateCacheButton_clicked(self): + """ + Private slot to update the local cache database. + """ + E5MessageBox.information( + self, + self.tr("Update Safe Browsing Cache"), + self.tr("""Updating the Safe Browsing cache might be a lengthy""" + """ operation. Please be patient!""")) + + QApplication.setOverrideCursor(Qt.WaitCursor) + ok, error = self.__manager.updateHashPrefixCache() + self.__resetProgress() + QApplication.restoreOverrideCursor() + if not ok: + if error: + E5MessageBox.critical( + self, + self.tr("Update Safe Browsing Cache"), + self.tr("""<p>Updating the Safe Browsing cache failed.""" + """</p><p>Reason: {0}</p>""").format(error)) + else: + E5MessageBox.critical( + self, + self.tr("Update Safe Browsing Cache"), + self.tr("""<p>Updating the Safe Browsing cache failed.""" + """</p>""")) + + @pyqtSlot() + def on_clearCacheButton_clicked(self): + """ + Private slot to clear the local cache database. + """ + res = E5MessageBox.yesNo( + self, + self.tr("Clear Safe Browsing Cache"), + self.tr("""Do you really want to clear the Safe Browsing cache?""" + """ Re-populating it might take some time.""")) + if res: + QApplication.setOverrideCursor(Qt.WaitCursor) + self.__manager.fullCacheCleanup() + QApplication.restoreOverrideCursor() + + @pyqtSlot(str, int) + def __setProgressMessage(self, message, maximum): + """ + Private slot to set the progress message and the maximum value. + + @param message progress message to be set + @type str + @param maximum maximum value to be set + @type int + """ + self.progressLabel.setText(message) + self.progressBar.setMaximum(maximum) + self.progressBar.setValue(0) + + @pyqtSlot(int) + def __setProgress(self, value): + """ + Private slot to set the progress value. + + @param value progress value to be set + @type int + """ + self.progressBar.setValue(value) + + def __resetProgress(self): + """ + Private method to reset the progress info. + """ + self.progressLabel.clear() + self.progressBar.setMaximum(100) + self.progressBar.setValue(0) + + @pyqtSlot(str) + def on_urlEdit_textChanged(self, text): + """ + Private slot to handle changes of the entered URL text. + + @param text entered URL text + @type str + """ + url = QUrl.fromUserInput(text) + enable = ( + url.isValid() and + bool(url.scheme()) and + url.scheme() not in self.__manager.getIgnoreSchemes() + ) + self.urlCheckButton.setEnabled(enable) + + @pyqtSlot() + def on_urlCheckButton_clicked(self): + """ + Private slot to check the entered URL. + """ + # Malicious URL for testing: + # http://malware.testing.google.test/testing/malware/* + # http://ianfette.org + # + urlStr = self.urlEdit.text() + url = QUrl.fromUserInput(urlStr) + threatLists = self.__manager.lookupUrl(url) + + if threatLists: + threatMessages = self.__manager.getThreatMessages(threatLists) + E5MessageBox.warning( + self, + self.tr("Check URL"), + self.tr("<p>The URL <b>{0}</b> was found in the Safe Browsing" + " Database.</p>{1}").format(urlStr, + "".join(threatMessages)) + ) + else: + E5MessageBox.information( + self, + self.tr("Check URL"), + self.tr("<p>The URL <b>{0}</b> was not found in the Safe" + " Browsing Database and may be considered safe.</p>") + .format(urlStr) + ) + + @pyqtSlot() + def on_saveButton_clicked(self): + """ + Private slot to save the configuration data. + """ + self.__save()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/WebBrowser/SafeBrowsing/SafeBrowsingDialog.ui Fri Aug 04 18:41:50 2017 +0200 @@ -0,0 +1,239 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ui version="4.0"> + <class>SafeBrowsingDialog</class> + <widget class="QDialog" name="SafeBrowsingDialog"> + <property name="geometry"> + <rect> + <x>0</x> + <y>0</y> + <width>650</width> + <height>461</height> + </rect> + </property> + <property name="windowTitle"> + <string>Safe Browsing Management</string> + </property> + <property name="sizeGripEnabled"> + <bool>true</bool> + </property> + <layout class="QVBoxLayout" name="verticalLayout_2"> + <item> + <widget class="QGroupBox" name="gsbGroupBox"> + <property name="toolTip"> + <string>Select to enable the Google sage browsing support</string> + </property> + <property name="title"> + <string>Enable Google Safe Browsing</string> + </property> + <property name="checkable"> + <bool>true</bool> + </property> + <layout class="QGridLayout" name="gridLayout"> + <item row="0" column="0" rowspan="2"> + <widget class="QLabel" name="iconLabel"> + <property name="minimumSize"> + <size> + <width>48</width> + <height>48</height> + </size> + </property> + <property name="text"> + <string notr="true">Icon</string> + </property> + </widget> + </item> + <item row="0" column="1" colspan="2"> + <widget class="QCheckBox" name="gsbFilterPlatformCheckBox"> + <property name="toolTip"> + <string>Select to check against the current platform only</string> + </property> + <property name="text"> + <string>Adjust to current platform</string> + </property> + </widget> + </item> + <item row="1" column="1"> + <widget class="QLabel" name="label_14"> + <property name="text"> + <string>API Key:</string> + </property> + </widget> + </item> + <item row="1" column="2"> + <widget class="QLineEdit" name="gsbApiKeyEdit"> + <property name="toolTip"> + <string>Enter the Google Safe Browsing API key</string> + </property> + </widget> + </item> + <item row="2" column="1" colspan="2"> + <widget class="QPushButton" name="gsbHelpButton"> + <property name="toolTip"> + <string>Press to get some help about obtaining the API key</string> + </property> + <property name="text"> + <string>Google Safe Browsing API Help</string> + </property> + </widget> + </item> + </layout> + </widget> + </item> + <item> + <layout class="QHBoxLayout" name="horizontalLayout_2"> + <item> + <spacer name="horizontalSpacer"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + <item> + <widget class="QPushButton" name="saveButton"> + <property name="toolTip"> + <string>Press to save the current configuration settings</string> + </property> + <property name="text"> + <string>Save Configuration</string> + </property> + </widget> + </item> + <item> + <spacer name="horizontalSpacer_2"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + </layout> + </item> + <item> + <widget class="QGroupBox" name="groupBox"> + <property name="title"> + <string>Manage Local Cache</string> + </property> + <layout class="QVBoxLayout" name="verticalLayout"> + <item> + <layout class="QHBoxLayout" name="horizontalLayout"> + <item> + <widget class="QPushButton" name="updateCacheButton"> + <property name="toolTip"> + <string>Press to update the local cache database</string> + </property> + <property name="text"> + <string>Update Cache</string> + </property> + </widget> + </item> + <item> + <widget class="QPushButton" name="clearCacheButton"> + <property name="toolTip"> + <string>Press to clear the local cache database</string> + </property> + <property name="text"> + <string>Clear Cache</string> + </property> + </widget> + </item> + </layout> + </item> + <item> + <widget class="QLabel" name="progressLabel"> + <property name="text"> + <string notr="true"/> + </property> + <property name="wordWrap"> + <bool>true</bool> + </property> + </widget> + </item> + <item> + <widget class="QProgressBar" name="progressBar"> + <property name="value"> + <number>0</number> + </property> + <property name="format"> + <string>%v/%m</string> + </property> + </widget> + </item> + </layout> + </widget> + </item> + <item> + <widget class="QGroupBox" name="groupBox_2"> + <property name="title"> + <string>URL Check</string> + </property> + <layout class="QGridLayout" name="gridLayout_2"> + <item row="0" column="0"> + <widget class="QLabel" name="label"> + <property name="text"> + <string>URL:</string> + </property> + </widget> + </item> + <item row="0" column="1"> + <widget class="E5ClearableLineEdit" name="urlEdit"/> + </item> + <item row="1" column="0" colspan="2"> + <widget class="QPushButton" name="urlCheckButton"> + <property name="enabled"> + <bool>false</bool> + </property> + <property name="toolTip"> + <string>Press to check the entered URL</string> + </property> + <property name="text"> + <string>Check URL</string> + </property> + </widget> + </item> + </layout> + </widget> + </item> + <item> + <widget class="QDialogButtonBox" name="buttonBox"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="standardButtons"> + <set>QDialogButtonBox::Close</set> + </property> + </widget> + </item> + </layout> + </widget> + <customwidgets> + <customwidget> + <class>E5ClearableLineEdit</class> + <extends>QLineEdit</extends> + <header>E5Gui/E5LineEdit.h</header> + </customwidget> + </customwidgets> + <tabstops> + <tabstop>gsbGroupBox</tabstop> + <tabstop>gsbFilterPlatformCheckBox</tabstop> + <tabstop>gsbApiKeyEdit</tabstop> + <tabstop>gsbHelpButton</tabstop> + <tabstop>saveButton</tabstop> + <tabstop>updateCacheButton</tabstop> + <tabstop>clearCacheButton</tabstop> + <tabstop>urlEdit</tabstop> + <tabstop>urlCheckButton</tabstop> + </tabstops> + <resources/> + <connections/> +</ui>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/WebBrowser/SafeBrowsing/SafeBrowsingInfoWidget.py Fri Aug 04 18:41:50 2017 +0200 @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing a widget to show some threat information. +""" + +from __future__ import unicode_literals + +from PyQt5.QtCore import Qt, QPoint +from PyQt5.QtWidgets import QMenu, QLabel, QHBoxLayout, QSizePolicy + +import UI.PixmapCache + + +class SafeBrowsingInfoWidget(QMenu): + """ + Class implementing a widget to show some threat information. + """ + def __init__(self, info, parent=None): + """ + Constructor + + @param info information string to be shown + @type str + @param parent reference to the parent widget + @type QWidget + """ + super(SafeBrowsingInfoWidget, self).__init__(parent) + + self.setMinimumWidth(500) + + layout = QHBoxLayout(self) + + iconLabel = QLabel(self) + iconLabel.setPixmap(UI.PixmapCache.getPixmap("safeBrowsing48.png")) + layout.addWidget(iconLabel, 0, Qt.AlignTop) + + infoLabel = QLabel(self) + infoLabel.setWordWrap(True) + infoLabel.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding) + infoLabel.setText(info) + layout.addWidget(infoLabel, 0, Qt.AlignTop) + + def showAt(self, pos): + """ + Public method to show the widget. + + @param pos position to show at + @type QPoint + """ + self.adjustSize() + xpos = pos.x() - self.width() + if xpos < 0: + xpos = 10 + p = QPoint(xpos, pos.y() + 10) + self.move(p) + self.show()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/WebBrowser/SafeBrowsing/SafeBrowsingLabel.py Fri Aug 04 18:41:50 2017 +0200 @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing the label to show some SSL info. +""" + +from __future__ import unicode_literals + +from PyQt5.QtCore import Qt, pyqtSlot, pyqtSignal, QPoint +from PyQt5.QtWidgets import QLabel + + +class SafeBrowsingLabel(QLabel): + """ + Class implementing a label to show some Safe Browsing info. + + @signal clicked(pos) emitted to indicate a click of the label (QPoint) + """ + clicked = pyqtSignal(QPoint) + + nokStyle = "QLabel { color : white; background-color : red; }" + + def __init__(self, parent=None): + """ + Constructor + + @param parent reference to the parent widget (QWidget) + """ + super(SafeBrowsingLabel, self).__init__(parent) + + self.setFocusPolicy(Qt.NoFocus) + self.setCursor(Qt.PointingHandCursor) + + self.setStyleSheet(SafeBrowsingLabel.nokStyle) + + self.__threatType = "" + self.__threatMessages = "" + + self.__deafultText = self.tr("Malicious Site") + self.__updateLabel() + + def mouseReleaseEvent(self, evt): + """ + Protected method to handle mouse release events. + + @param evt reference to the mouse event (QMouseEvent) + """ + if evt.button() == Qt.LeftButton: + self.clicked.emit(evt.globalPos()) + else: + super(SafeBrowsingLabel, self).mouseReleaseEvent(evt) + + def mouseDoubleClickEvent(self, evt): + """ + Protected method to handle mouse double click events. + + @param evt reference to the mouse event (QMouseEvent) + """ + if evt.button() == Qt.LeftButton: + self.clicked.emit(evt.globalPos()) + else: + super(SafeBrowsingLabel, self).mouseDoubleClickEvent(evt) + + @pyqtSlot() + def __updateLabel(self): + """ + Private slot to update the label text. + """ + if self.__threatType: + self.setText(self.__threatType) + else: + self.setText(self.__deafultText) + + @pyqtSlot(str, str) + def setThreatInfo(self, threatType, threatMessages): + """ + Public slot to set threat information. + + @param threatType threat type + @type str + @param threatMessages more verbose info about detected threats + @type str + """ + self.__threatType = threatType + self.__threatMessages = threatMessages + + self.__updateLabel() + + def getThreatInfo(self): + """ + Public method to get the threat info text. + + @return threat info text + @rtype str + """ + return self.__threatMessages
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/WebBrowser/SafeBrowsing/SafeBrowsingManager.py Fri Aug 04 18:41:50 2017 +0200 @@ -0,0 +1,485 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing the interface for Google Safe Browsing. +""" + +# +# Some part of this code were ported from gglsbl.client and adapted +# to Qt. +# +# https://github.com/afilipovich/gglsbl +# + +from __future__ import unicode_literals + +import os +import base64 + +from PyQt5.QtCore import pyqtSignal, QObject, QCoreApplication, QUrl + +import Preferences +import Utilities + +from .SafeBrowsingAPIClient import SafeBrowsingAPIClient +from .SafeBrowsingCache import SafeBrowsingCache, ThreatList, HashPrefixList +from .SafeBrowsingUrl import SafeBrowsingUrl +from .SafeBrowsingUtilities import toHex + + +class SafeBrowsingManager(QObject): + """ + Class implementing the interface for Google Safe Browsing. + + @signal progressMessage(message,maximum) emitted to give a message for the + action about to be performed and the maximum value + @signal progress(current) emitted to signal the current progress + """ + progressMessage = pyqtSignal(str, int) + progress = pyqtSignal(int) + + def __init__(self): + """ + Constructor + """ + super(SafeBrowsingManager, self).__init__() + + self.__apiKey = Preferences.getWebBrowser("SafeBrowsingApiKey") + if self.__apiKey: + self.__apiClient = SafeBrowsingAPIClient(self.__apiKey, + parent=self) + # TODO: switch these after debugging is finished +## self.__apiClient = SafeBrowsingAPIClient(self.__apiKey, +## parent=self, +## fairUse=False) + else: + self.__apiClient = None + + self.__enabled = ( + Preferences.getWebBrowser("SafeBrowsingEnabled") and + bool(self.__apiKey)) + + gsbCachePath = os.path.join( + Utilities.getConfigDir(), "web_browser", "safe_browsing") + self.__cache = SafeBrowsingCache(gsbCachePath, self) + + self.__gsbDialog = None + self.__setPlatforms() + + def configurationChanged(self): + """ + Public method to handle changes of the settings. + """ + apiKey = Preferences.getWebBrowser("SafeBrowsingApiKey") + if apiKey != self.__apiKey: + self.__apiKey = apiKey + if self.__apiKey: + if self.__apiClient: + self.__apiClient.setApiKey(self.__apiKey) + else: + self.__apiClient = SafeBrowsingAPIClient(self.__apiKey, + parent=self) + # TODO: switch these after debugging is finished +## self.__apiClient = SafeBrowsingAPIClient(self.__apiKey, +## parent=self, +## fairUse=False) + + self.__enabled = ( + Preferences.getWebBrowser("SafeBrowsingEnabled") and + bool(self.__apiKey)) + + self.__setPlatforms() + + def __setPlatforms(self): + """ + Private method to set the platforms to be checked against. + """ + self.__platforms = None + if Preferences.getWebBrowser("SafeBrowsingFilterPlatform"): + if Utilities.isWindowsPlatform(): + platform = "windows" + elif Utilities.isMacPlatform(): + platform = "macos" + else: + # treat all other platforms like linux + platform = "linux" + self.__platforms = SafeBrowsingAPIClient.getPlatformTypes(platform) + + def isEnabled(self): + """ + Public method to check, if safe browsing is enabled. + + @return flag indicating the enabled state + @rtype bool + """ + return self.__enabled + + def close(self): + """ + Public method to close the safe browsing interface. + """ + self.__cache.close() + + def fairUseDelayExpired(self): + """ + Public method to check, if the fair use wait period has expired. + + @return flag indicating expiration + @rtype bool + """ + return self.__enabled and self.__apiClient.fairUseDelayExpired() + + def updateHashPrefixCache(self): + """ + Public method to load or update the locally cached threat lists. + + @return flag indicating success and an error message + @rtype tuple of (bool, str) + """ + if not self.__enabled: + return False, self.tr("Safe Browsing is disabled.") + + if not self.__apiClient.fairUseDelayExpired(): + return False, \ + self.tr("The fair use wait period has not expired yet." + "Expiration will be at {0}.").format( + self.__apiClient.getFairUseDelayExpirationDateTime() + .toString("yyyy-MM-dd, HH:mm:ss")) + + # step 1: remove expired hashes + self.__cache.cleanupFullHashes() + + # step 2: update threat lists + threatListsForRemove = {} + for threatList, clientState in self.__cache.getThreatLists(): + threatListsForRemove[repr(threatList)] = threatList + threatLists = self.__apiClient.getThreatLists() + maximum = len(threatLists) + current = 0 + self.progressMessage.emit(self.tr("Updating threat lists"), maximum) + for entry in threatLists: + current += 1 + self.progress.emit(current) + QCoreApplication.processEvents() + threatList = ThreatList.fromApiEntry(entry) + if self.__platforms is None or \ + threatList.platformType in self.__platforms: + self.__cache.addThreatList(threatList) + key = repr(threatList) + if key in threatListsForRemove: + del threatListsForRemove[key] + maximum = len(threatListsForRemove.values()) + current = 0 + self.progressMessage.emit(self.tr("Deleting obsolete threat lists"), + maximum) + for threatList in threatListsForRemove.values(): + current += 1 + self.progress.emit(current) + QCoreApplication.processEvents() + self.__cache.deleteHashPrefixList(threatList) + self.__cache.deleteThreatList(threatList) + del threatListsForRemove + + # step 3: update threats + threatLists = self.__cache.getThreatLists() + clientStates = {} + for threatList, clientState in threatLists: + clientStates[threatList.asTuple()] = clientState + threatsUpdateResponses = \ + self.__apiClient.getThreatsUpdate(clientStates) + maximum = len(threatsUpdateResponses) + current = 0 + self.progressMessage.emit(self.tr("Updating hash prefixes"), maximum) + for response in threatsUpdateResponses: + current += 1 + self.progress.emit(current) + QCoreApplication.processEvents() + responseThreatList = ThreatList.fromApiEntry(response) + if response["responseType"] == "FULL_UPDATE": + self.__cache.deleteHashPrefixList(responseThreatList) + for removal in response.get("removals", []): + self.__cache.removeHashPrefixIndices( + responseThreatList, removal["rawIndices"]["indices"]) + QCoreApplication.processEvents() + for addition in response.get("additions", []): + hashPrefixList = HashPrefixList( + addition["rawHashes"]["prefixSize"], + base64.b64decode(addition["rawHashes"]["rawHashes"])) + self.__cache.populateHashPrefixList(responseThreatList, + hashPrefixList) + QCoreApplication.processEvents() + expectedChecksum = base64.b64decode(response["checksum"]["sha256"]) + if self.__verifyThreatListChecksum(responseThreatList, + expectedChecksum): + self.__cache.updateThreatListClientState( + responseThreatList, response["newClientState"]) + else: + return False, \ + self.tr("Local cache checksum does not match the server." + " Consider cleaning the cache. Threat update has" + " been aborted.") + + return True, "" + + def __verifyThreatListChecksum(self, threatList, remoteChecksum): + """ + Private method to verify the local checksum of a threat list with the + checksum of the safe browsing server. + + @param threatList threat list to calculate checksum for + @type ThreatList + @param remoteChecksum SHA256 checksum as reported by the Google server + @type bytes + @return flag indicating equality + @rtype bool + """ + localChecksum = self.__cache.hashPrefixListChecksum(threatList) + return remoteChecksum == localChecksum + + def fullCacheCleanup(self): + """ + Public method to clean up the cache completely. + """ + self.__cache.prepareCacheDb() + + def showSafeBrowsingDialog(self): + """ + Public slot to show the safe browsing management dialog. + """ + if self.__gsbDialog is None: + from WebBrowser.WebBrowserWindow import WebBrowserWindow + from .SafeBrowsingDialog import SafeBrowsingDialog + self.__gsbDialog = SafeBrowsingDialog( + self, parent=WebBrowserWindow.mainWindow()) + + self.__gsbDialog.show() + + def lookupUrl(self, url): + """ + Public method to lookup an URL. + + @param url URL to be checked + @type str or QUrl + @return list of threat lists the URL was found in + @rtype list of ThreatList + @exception ValueError raised for an invalid URL + """ + if self.__enabled: + if isinstance(url, QUrl): + urlStr = url.toString().strip() + else: + urlStr = url.strip() + + if not urlStr: + raise ValueError("Empty URL given.") + + urlHashes = SafeBrowsingUrl(urlStr).hashes() + listNames = self.__lookupHashes(urlHashes) + if listNames: + return listNames + + return None + + def __lookupHashes(self, fullHashes): + """ + Private method to lookup the given hashes. + + @param fullHashes list of hashes to lookup + @type list of bytes + @return names of threat lists hashes were found in + @rtype list of ThreatList + """ + fullHashes = list(fullHashes) + cues = [toHex(fh[:4]) for fh in fullHashes] + result = [] + + matchingPrefixes = {} + matchingFullHashes = set() + isPotentialThreat = False + # Lookup hash prefixes which match full URL hash + for threatList, hashPrefix, negativeCacheExpired in \ + self.__cache.lookupHashPrefix(cues): + for fullHash in fullHashes: + if fullHash.startswith(hashPrefix): + isPotentialThreat = True + # consider hash prefix negative cache as expired if it + # is expired in at least one threat list + matchingPrefixes[hashPrefix] = matchingPrefixes.get( + hashPrefix, False) or negativeCacheExpired + matchingFullHashes.add(fullHash) + + # if none matches, url hash is clear + if not isPotentialThreat: + return [] + + # if there is non-expired full hash, URL is blacklisted + matchingExpiredThreatLists = set() + for threatList, hasExpired in self.__cache.lookupFullHashes( + matchingFullHashes): + if hasExpired: + matchingExpiredThreatLists.add(threatList) + else: + result.append(threatList) + if result: + return result + + # If there are no matching expired full hash entries and negative + # cache is still current for all prefixes, consider it safe. + if len(matchingExpiredThreatLists) == 0 and \ + sum(map(int, matchingPrefixes.values())) == 0: + return [] + + # Now it can be assumed that there are expired matching full hash + # entries and/or cache prefix entries with expired negative cache. + # Both require full hash synchronization. + self.__syncFullHashes(matchingPrefixes.keys()) + + # Now repeat full hash lookup + for threatList, hasExpired in self.__cache.lookupFullHashes( + matchingFullHashes): + if not hasExpired: + result.append(threatList) + + return result + + def __syncFullHashes(self, hashPrefixes): + """ + Private method to download full hashes matching given prefixes. + + This also updates the cache expiration timestamps. + + @param hashPrefixes list of hash prefixes to get full hashes for + @type list of bytes + """ + threatLists = self.__cache.getThreatLists() + clientStates = {} + for threatList, clientState in threatLists: + clientStates[threatList.asTuple()] = clientState + + fullHashResponses = self.__apiClient.getFullHashes( + hashPrefixes, clientStates) + + # update negative cache for each hash prefix + # store full hash with positive cache bumped up + for match in fullHashResponses["matches"]: + threatList = ThreatList.fromApiEntry(match) + hashValue = base64.b64decode(match["threat"]["hash"]) + cacheDuration = int(match["cacheDuration"].rstrip("s")) + malwareThreatType = None + for metadata in match["threatEntryMetadata"].get("entries", []): + key = base64.b64decode(metadata["key"]) + value = base64.b64decode(metadata["value"]) + if key == b"malware_threat_type": + malwareThreatType = value + if not isinstance(malwareThreatType, str): + malwareThreatType = malwareThreatType.decode() + self.__cache.storeFullHash(threatList, hashValue, cacheDuration, + malwareThreatType) + + negativeCacheDuration = int( + fullHashResponses["negativeCacheDuration"].rstrip("s")) + for prefixValue in hashPrefixes: + for threatList, clientState in threatLists: + self.__cache.updateHashPrefixExpiration( + threatList, prefixValue, negativeCacheDuration) + + @classmethod + def getIgnoreSchemes(cls): + """ + Class method to get the schemes not to be checked. + + @return list of schemes to be ignored + @rtype list of str + """ + return [ + "about", + "eric", + "qrc", + "qthelp", + "chrome", + "abp", + "file", + ] + + def getThreatMessage(self, threatType): + """ + Public method to get a warning message for the given threat type. + + @param threatType threat type to get the message for + @type str + @return threat message + @rtype str + """ + if self.__apiClient: + msg = self.__apiClient.getThreatMessage(threatType) + else: + msg = "" + + return msg + + def getThreatMessages(self, threatLists): + """ + Public method to get threat messages for the given threats. + + @param threatLists list of threat lists to get a message for + @type list of ThreatList + @return list of threat messages, one per unique threat type + @rtype list of str + """ + threatTypes = set() + for threatList in threatLists: + threatTypes.add(threatList.threatType) + + messages = [] + if self.__apiClient: + for threatType in sorted(threatTypes): + msg = self.__apiClient.getThreatMessage(threatType) + messages.append(msg) + + return messages + + def getThreatType(self, threatList): + """ + Public method to get a display string for a given threat type. + + @param threatList threat list to get display string for + @type str + @return display string + @rtype str + """ + displayString = "" + if self.__apiClient: + displayString = self.__apiClient.getThreatType( + threatList.threatType) + return displayString + + def getPlatformString(self, platformType): + """ + Public method to get the platform string for a given platform type. + + @param platformType platform type as defined in the v4 API + @type str + @return platform string + @rtype str + """ + if self.__apiClient: + return self.__apiClient.getPlatformString(platformType) + else: + return "" + + def getThreatEntryString(self, threatEntry): + """ + Public method to get the threat entry string. + + @param threatEntry threat entry type as defined in the v4 API + @type str + @return threat entry string + @rtype str + """ + if self.__apiClient: + return self.__apiClient.getThreatEntryString(threatEntry) + else: + return ""
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/WebBrowser/SafeBrowsing/SafeBrowsingUrl.py Fri Aug 04 18:41:50 2017 +0200 @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing an URL representation suitable for Google Safe Browsing. +""" + +from __future__ import unicode_literals + +try: + import urlparse # Py2 + import urllib # Py2 +except ImportError: + import urllib.parse as urllib + from urllib import parse as urlparse + +import re +import posixpath +import socket +import struct +import hashlib + +import Preferences + + +class SafeBrowsingUrl(object): + """ + Class implementing an URL representation suitable for Google Safe Browsing. + """ + # + # Modeled after the URL class of the gglsbl package. + # https://github.com/afilipovich/gglsbl + # + def __init__(self, url): + """ + Constructor + + @param url URL to be embedded + @type str + """ + self.__url = url + + def hashes(self): + """ + Public method to get the hashes of all possible permutations of the URL + in canonical form. + + @return generator for the URL hashes + @rtype generator of bytes + """ + for variant in self.permutations(self.canonical()): + urlHash = self.digest(variant) + yield urlHash + + def canonical(self): + """ + Public method to convert the URL to the canonical form. + + @return canonical form of the URL + @rtype str + """ + def fullUnescape(u): + """ + Method to recursively unescape an URL. + + @param u URL string to unescape + @type str + @return unescaped URL string + @rtype str + """ + uu = urllib.unquote(u) + if uu == u: + return uu + else: + return fullUnescape(uu) + + def quote(s): + """ + Method to quote a string. + + @param string to be quoted + @type str + @return quoted string + @rtype str + """ + safeChars = '!"$&\'()*+,-./:;<=>?@[\\]^_`{|}~' + return urllib.quote(s, safe=safeChars) + + url = self.__url.strip() + url = url.replace('\n', '').replace('\r', '').replace('\t', '') + url = url.split('#', 1)[0] + if url.startswith('//'): + url = Preferences.getWebBrowser("DefaultScheme")[:-3] + url + if len(url.split('://')) <= 1: + url = Preferences.getWebBrowser("DefaultScheme") + url + url = quote(fullUnescape(url)) + urlParts = urlparse.urlsplit(url) + if not urlParts[0]: + url = Preferences.getWebBrowser("DefaultScheme") + url + urlParts = urlparse.urlsplit(url) + protocol = urlParts.scheme + host = fullUnescape(urlParts.hostname) + path = fullUnescape(urlParts.path) + query = urlParts.query + if not query and '?' not in url: + query = None + if not path: + path = '/' + path = posixpath.normpath(path).replace('//', '/') + if path[-1] != '/': + path += '/' + port = urlParts.port + host = host.strip('.') + host = re.sub(r'\.+', '.', host).lower() + if host.isdigit(): + try: + host = socket.inet_ntoa(struct.pack("!I", int(host))) + except Exception: + pass + if host.startswith('0x') and '.' not in host: + try: + host = socket.inet_ntoa(struct.pack("!I", int(host, 16))) + except Exception: + pass + quotedPath = quote(path) + quotedHost = quote(host) + if port is not None: + quotedHost = '{0}:{1}'.format(quotedHost, port) + canonicalUrl = '{0}://{1}{2}'.format(protocol, quotedHost, quotedPath) + if query is not None: + canonicalUrl = '{0}?{1}'.format(canonicalUrl, query) + return canonicalUrl + + @staticmethod + def permutations(url): + """ + Static method to determine all permutations of host name and path + which can be applied to blacklisted URLs. + + @param url URL string to be permuted + @type str + @return generator of permuted URL strings + @type generator of str + """ + def hostPermutations(host): + """ + Method to generate the permutations of the host name. + + @param host host name + @type str + @return generator of permuted host names + @rtype generator of str + """ + if re.match(r'\d+\.\d+\.\d+\.\d+', host): + yield host + return + parts = host.split('.') + partsLen = min(len(parts), 5) + if partsLen > 4: + yield host + for i in range(partsLen - 1): + yield '.'.join(parts[i - partsLen:]) + + def pathPermutations(path): + """ + Method to generate the permutations of the path. + + @param path path to be processed + @type str + @return generator of permuted paths + @rtype generator of str + """ + yield path + query = None + if '?' in path: + path, query = path.split('?', 1) + if query is not None: + yield path + pathParts = path.split('/')[0:-1] + curPath = '' + for i in range(min(4, len(pathParts))): + curPath = curPath + pathParts[i] + '/' + yield curPath + + protocol, addressStr = urllib.splittype(url) + host, path = urllib.splithost(addressStr) + user, host = urllib.splituser(host) + host, port = urllib.splitport(host) + host = host.strip('/') + seenPermutations = set() + for h in hostPermutations(host): + for p in pathPermutations(path): + u = '{0}{1}'.format(h, p) + if u not in seenPermutations: + yield u + seenPermutations.add(u) + + @staticmethod + def digest(url): + """ + Static method to calculate the SHA256 digest of an URL string. + + @param url URL string + @type str + @return SHA256 digest of the URL string + @rtype bytes + """ + return hashlib.sha256(url.encode('utf-8')).digest()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/WebBrowser/SafeBrowsing/SafeBrowsingUtilities.py Fri Aug 04 18:41:50 2017 +0200 @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing some utilities for Google Safe Browsing. +""" + +from __future__ import unicode_literals + +import sys + +if sys.version_info < (3, 0): + def toHex(value): + """ + Public method to convert a bytes array to a hex string. + + @param value value to be converted + @type bytes + @return hex string + @rtype str + """ + return value.encode("hex") +else: + def toHex(value): + """ + Public method to convert a bytes array to a hex string. + + @param value value to be converted + @type bytes + @return hex string + @rtype str + """ + return value.hex()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/WebBrowser/SafeBrowsing/__init__.py Fri Aug 04 18:41:50 2017 +0200 @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Package implementing the safe browsing functionality. +""" + +from __future__ import unicode_literals + + +def SafeBrowsingHelp(): + """ + Module function to get some help about how to enable the Google Mail + OAuth2 service. + + @return help text + @rtype str + """ + return ( + "<h2>Steps to get a Google Safe Browsing API key</h2>" + "<p>In order to use Google Safe Browsing you need a Google Account," + " a Google Developer Console project, and an API key. You also need" + " to activate the Safe Browsing APIs for use with your project.</p>" + "<ol>" + "<li>Google Account<br/>You need a Google Account in order to create" + " a project. If you don't already have an account, sign up at" + " <a href='https://accounts.google.com/SignUp'>Create your Google" + " Account</a>.</li>" + "<li>Developer Console Project<br/>You need a Google Developer Console" + " project in order to create an API key. If you don't already have a" + " project, see" + " <a href='https://support.google.com/cloud/answer/6251787?hl=en'>" + "Create, shut down, and restore projects</a>.</li>" + "<li>API Key<br/>You need an API key to access the Safe Browsing APIs." + " An API key authenticates you as an API user and allows you to" + " interact with the APIs. To set up an API key, see " + "<a href='https://support.google.com/cloud/answer/6158862" + "?hl=en&ref_topic=6262490'>Setting up API keys</a>. Your new API key" + " appears in a table. Copy and paste this key into the line edit above" + " the button used to show this help.</li>" + "<li>Activate the API<br/>Finally, you need to activate the Safe" + " Browsing API for use with your project. To learn how to do this, see" + " <a href='https://support.google.com/cloud/answer/6158841?hl=en'>" + "Activate and deactivate APIs</a>. The API to enable is referred to as" + " "Google Safe Browsing API".</li>" + "</ol>" + )
--- a/WebBrowser/UrlBar/UrlBar.py Fri Aug 04 18:29:30 2017 +0200 +++ b/WebBrowser/UrlBar/UrlBar.py Fri Aug 04 18:41:50 2017 +0200 @@ -13,7 +13,7 @@ except NameError: pass -from PyQt5.QtCore import pyqtSlot, Qt, QPointF, QUrl, QDateTime, QTimer +from PyQt5.QtCore import pyqtSlot, Qt, QPointF, QUrl, QDateTime, QTimer, QPoint from PyQt5.QtGui import QColor, QPalette, QLinearGradient, QIcon from PyQt5.QtWidgets import QDialog, QApplication from PyQt5.QtWebEngineWidgets import QWebEnginePage @@ -23,7 +23,10 @@ from WebBrowser.WebBrowserWindow import WebBrowserWindow +from WebBrowser.SafeBrowsing.SafeBrowsingLabel import SafeBrowsingLabel + from .FavIconLabel import FavIconLabel + import UI.PixmapCache import Preferences @@ -51,6 +54,10 @@ self.__bmInactiveIcon = QIcon( self.__bmActiveIcon.pixmap(16, 16, QIcon.Disabled)) + self.__safeBrowsingLabel = SafeBrowsingLabel(self) + self.addWidget(self.__safeBrowsingLabel, E5LineEdit.LeftSide) + self.__safeBrowsingLabel.setVisible(False) + self.__favicon = FavIconLabel(self) self.addWidget(self.__favicon, E5LineEdit.LeftSide) @@ -68,6 +75,7 @@ self.addWidget(self.__clearButton, E5LineEdit.RightSide) self.__clearButton.setVisible(False) + self.__safeBrowsingLabel.clicked.connect(self.__showThreatInfo) self.__bookmarkButton.clicked.connect(self.__showBookmarkInfo) self.__rssButton.clicked.connect(self.__rssClicked) self.__clearButton.clicked.connect(self.clear) @@ -95,6 +103,9 @@ self.__browser.loadProgress.connect(self.update) self.__browser.loadFinished.connect(self.__loadFinished) self.__browser.loadStarted.connect(self.__loadStarted) + + self.__browser.safeBrowsingBad.connect( + self.__safeBrowsingLabel.setThreatInfo) def browser(self): """ @@ -156,6 +167,10 @@ self.__checkBookmark() self.__bookmarkButton.setVisible(True) + self.__browserUrlChanged(self.__browser.url()) + self.__safeBrowsingLabel.setVisible( + not self.__browser.getSafeBrowsingStatus()) + if ok: QTimer.singleShot(0, self.__setRssButton) @@ -220,16 +235,19 @@ if self.__browser is not None: p = self.palette() progress = self.__browser.progress() + + if not self.__browser.getSafeBrowsingStatus(): + # malicious web site + backgroundColor = QColor(170, 0, 0) + foregroundColor = QColor(Qt.white) + elif self.__browser.url().scheme() == "https": + backgroundColor = Preferences.getWebBrowser( + "SaveUrlColor") + if progress == 0 or progress == 100: - if self.__browser.url().scheme() == "https": - backgroundColor = Preferences.getWebBrowser( - "SaveUrlColor") p.setBrush(QPalette.Base, backgroundColor) p.setBrush(QPalette.Text, foregroundColor) else: - if self.__browser.url().scheme() == "https": - backgroundColor = Preferences.getWebBrowser( - "SaveUrlColor") highlight = QApplication.palette().color(QPalette.Highlight) r = (highlight.red() + 2 * backgroundColor.red()) // 3 g = (highlight.green() + 2 * backgroundColor.green()) // 3 @@ -379,3 +397,18 @@ feeds = self.__browser.getRSS() dlg = FeedsDialog(feeds, self.__browser) dlg.exec_() + + @pyqtSlot(QPoint) + def __showThreatInfo(self, pos): + """ + Private slot to show the threat info widget. + + @param pos position to show the info at + @type QPoint + """ + threatInfo = self.__safeBrowsingLabel.getThreatInfo() + if threatInfo: + from WebBrowser.SafeBrowsing.SafeBrowsingInfoWidget import \ + SafeBrowsingInfoWidget + widget = SafeBrowsingInfoWidget(threatInfo, self.__browser) + widget.showAt(pos)
--- a/WebBrowser/WebBrowserPage.py Fri Aug 04 18:29:30 2017 +0200 +++ b/WebBrowser/WebBrowserPage.py Fri Aug 04 18:41:50 2017 +0200 @@ -14,12 +14,15 @@ except NameError: pass -from PyQt5.QtCore import pyqtSlot, QUrl, QTimer, QEventLoop, QPoint, QPointF +from PyQt5.QtCore import pyqtSlot, pyqtSignal, QUrl, QTimer, QEventLoop, \ + QPoint, QPointF from PyQt5.QtGui import QDesktopServices from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineSettings, \ QWebEngineScript from PyQt5.QtWebChannel import QWebChannel +from E5Gui import E5MessageBox + from WebBrowser.WebBrowserWindow import WebBrowserWindow from .JavaScript.ExternalJsObject import ExternalJsObject @@ -33,6 +36,11 @@ class WebBrowserPage(QWebEnginePage): """ Class implementing an enhanced web page. + + @signal safeBrowsingAbort() emitted to indicate an abort due to a safe + browsing event + @signal safeBrowsingBad(threatType, threatMessages) emitted to indicate a + malicious web site as determined by safe browsing """ if qVersionTuple() >= (5, 7, 0): # SafeJsWorld = QWebEngineScript.ApplicationWorld @@ -40,6 +48,9 @@ else: SafeJsWorld = QWebEngineScript.MainWorld + safeBrowsingAbort = pyqtSignal() + safeBrowsingBad = pyqtSignal(str, str) + def __init__(self, parent=None): """ Constructor @@ -65,6 +76,7 @@ self.urlChanged.connect(self.__urlChanged) self.__printer = None + self.__badSite = False def acceptNavigationRequest(self, url, type_, isMainFrame): """ @@ -96,6 +108,33 @@ WebBrowserWindow.greaseMonkeyManager().downloadScript(url) return False + # Safe Browsing + self.__badSite = False + if url.scheme() not in \ + WebBrowserWindow.safeBrowsingManager().getIgnoreSchemes(): + threatLists = WebBrowserWindow.safeBrowsingManager().lookupUrl(url) + if threatLists: + threatMessages = WebBrowserWindow.safeBrowsingManager()\ + .getThreatMessages(threatLists) + res = E5MessageBox.warning( + WebBrowserWindow.getWindow(), + self.tr("Suspicuous URL detected"), + self.tr("<p>The URL <b>{0}</b> was found in the Safe" + " Browsing database.</p>{1}").format( + url.toString(), "".join(threatMessages)), + E5MessageBox.StandardButtons( + E5MessageBox.Abort | + E5MessageBox.Ignore), + E5MessageBox.Abort) + if res == E5MessageBox.Abort: + self.safeBrowsingAbort.emit() + return False + + self.__badSite = True + threatType = WebBrowserWindow.safeBrowsingManager()\ + .getThreatType(threatLists[0]) + self.safeBrowsingBad.emit(threatType, "".join(threatMessages)) + return QWebEnginePage.acceptNavigationRequest(self, url, type_, isMainFrame) @@ -374,6 +413,19 @@ self.view().mainWindow().javascriptConsole().javaScriptConsoleMessage( level, message, lineNumber, sourceId) + ########################################################################### + ## Methods below implement safe browsing related functions + ########################################################################### + + def getSafeBrowsingStatus(self): + """ + Public method to get the safe browsing status of the current page. + + @return flag indicating a safe site + @rtype bool + """ + return not self.__badSite + ################################################## ## Methods below implement compatibility functions ##################################################
--- a/WebBrowser/WebBrowserView.py Fri Aug 04 18:29:30 2017 +0200 +++ b/WebBrowser/WebBrowserView.py Fri Aug 04 18:41:50 2017 +0200 @@ -53,6 +53,10 @@ @signal search(QUrl) emitted, when a search is requested @signal zoomValueChanged(int) emitted to signal a change of the zoom value @signal faviconChanged() emitted to signal a changed web site icon + @signal safeBrowsingAbort() emitted to indicate an abort due to a safe + browsing event + @signal safeBrowsingBad(threatType, threatMessages) emitted to indicate a + malicious web site as determined by safe browsing """ sourceChanged = pyqtSignal(QUrl) forwardAvailable = pyqtSignal(bool) @@ -61,6 +65,8 @@ search = pyqtSignal(QUrl) zoomValueChanged = pyqtSignal(int) faviconChanged = pyqtSignal() + safeBrowsingAbort = pyqtSignal() + safeBrowsingBad = pyqtSignal(str, str) ZoomLevels = [ 30, 40, 50, 67, 80, 90, @@ -85,8 +91,8 @@ self.__speedDial = WebBrowserWindow.speedDial() - self.__page = WebBrowserPage(self) - self.setPage(self.__page) + self.__page = None + self.__createNewPage() self.__mw = mainWindow self.__isLoading = False @@ -137,6 +143,16 @@ self.grabGesture(Qt.PinchGesture) + def __createNewPage(self): + """ + Private method to create a new page object. + """ + self.__page = WebBrowserPage(self) + self.setPage(self.__page) + + self.__page.safeBrowsingAbort.connect(self.safeBrowsingAbort) + self.__page.safeBrowsingBad.connect(self.safeBrowsingBad) + def __setRwhvqt(self): """ Private slot to set widget that receives input events. @@ -1579,8 +1595,7 @@ @type QWebEnginePage.RenderProcessTerminationStatus """ self.page().deleteLater() - self.__page = WebBrowserPage(self) - self.setPage(self.__page) + self.__createNewPage() html = readAllFileContents(":/html/tabCrashPage.html") html = html.replace("@IMAGE@", pixmapToDataUrl( @@ -2049,3 +2064,19 @@ QUrl.fromUserInput(urlStr)) return title, urlStr, icon + + ########################################################################### + ## Methods below implement safe browsing related functions + ########################################################################### + + def getSafeBrowsingStatus(self): + """ + Public method to get the safe browsing status of the current page. + + @return flag indicating a safe site + @rtype bool + """ + if self.__page: + return self.__page.getSafeBrowsingStatus() + else: + return True
--- a/WebBrowser/WebBrowserWindow.py Fri Aug 04 18:29:30 2017 +0200 +++ b/WebBrowser/WebBrowserWindow.py Fri Aug 04 18:41:50 2017 +0200 @@ -108,6 +108,7 @@ _autoScroller = None _tabManager = None _sessionManager = None + _safeBrowsingManager = None _performingStartup = True _performingShutdown = False @@ -1757,6 +1758,23 @@ self.__showCertificateErrorsDialog) self.__actions.append(self.certificateErrorsAct) + self.safeBrowsingAct = E5Action( + self.tr('Manage Safe Browsing'), + UI.PixmapCache.getIcon("safeBrowsing.png"), + self.tr('Manage Safe Browsing...'), 0, 0, self, + 'webbrowser_manage_safe_browsing') + self.safeBrowsingAct.setStatusTip(self.tr( + 'Configure Safe Browsing and manage local cache')) + self.safeBrowsingAct.setWhatsThis(self.tr( + """<b>Manage Safe Browsing</b>""" + """<p>This opens a dialog to configure Safe Browsing and""" + """ to manage the local cache.</p>""" + )) + if not self.__initShortcutsOnly: + self.safeBrowsingAct.triggered.connect( + self.__showSafeBrowsingDialog) + self.__actions.append(self.safeBrowsingAct) + self.showDownloadManagerAct = E5Action( self.tr('Downloads'), self.tr('Downloads'), @@ -2084,6 +2102,8 @@ menu.addSeparator() menu.addAction(self.adblockAct) menu.addSeparator() + menu.addAction(self.safeBrowsingAct) + menu.addSeparator() self.__settingsMenu = menu self.__settingsMenu.aboutToShow.connect( self.__aboutToShowSettingsMenu) @@ -2205,6 +2225,8 @@ menu.addSeparator() menu.addAction(self.adblockAct) menu.addSeparator() + menu.addAction(self.safeBrowsingAct) + menu.addSeparator() menu.addMenu(self.__userAgentMenu) menu.addAction(self.userAgentManagerAct) menu.addSeparator() @@ -2946,6 +2968,8 @@ self.networkManager().shutdown() + self.safeBrowsingManager().close() + for browser in WebBrowserWindow.BrowserWindows: if browser != self: browser.close() @@ -4833,3 +4857,27 @@ Private slot to show the session manager dialog. """ self.sessionManager().showSessionManagerDialog() + + ########################################################## + ## Methods below implement safe browsing related functions + ########################################################## + + @classmethod + def safeBrowsingManager(cls): + """ + Class method to get a reference to the safe browsing interface. + + @return reference to the safe browsing manager + @rtype SafeBrowsingManager + """ + if cls._safeBrowsingManager is None: + from .SafeBrowsing.SafeBrowsingManager import SafeBrowsingManager + cls._safeBrowsingManager = SafeBrowsingManager() + + return cls._safeBrowsingManager + + def __showSafeBrowsingDialog(self): + """ + Private slot to show the safe browsing management dialog. + """ + self.safeBrowsingManager().showSafeBrowsingDialog()
--- a/eric6.e4p Fri Aug 04 18:29:30 2017 +0200 +++ b/eric6.e4p Fri Aug 04 18:41:50 2017 +0200 @@ -1404,6 +1404,15 @@ <Source>WebBrowser/QtHelp/QtHelpDocumentationSelectionDialog.py</Source> <Source>WebBrowser/QtHelp/QtHelpFiltersDialog.py</Source> <Source>WebBrowser/QtHelp/__init__.py</Source> + <Source>WebBrowser/SafeBrowsing/SafeBrowsingAPIClient.py</Source> + <Source>WebBrowser/SafeBrowsing/SafeBrowsingCache.py</Source> + <Source>WebBrowser/SafeBrowsing/SafeBrowsingDialog.py</Source> + <Source>WebBrowser/SafeBrowsing/SafeBrowsingInfoWidget.py</Source> + <Source>WebBrowser/SafeBrowsing/SafeBrowsingLabel.py</Source> + <Source>WebBrowser/SafeBrowsing/SafeBrowsingManager.py</Source> + <Source>WebBrowser/SafeBrowsing/SafeBrowsingUrl.py</Source> + <Source>WebBrowser/SafeBrowsing/SafeBrowsingUtilities.py</Source> + <Source>WebBrowser/SafeBrowsing/__init__.py</Source> <Source>WebBrowser/SearchWidget.py</Source> <Source>WebBrowser/Session/SessionManager.py</Source> <Source>WebBrowser/Session/SessionManagerDialog.py</Source> @@ -1933,6 +1942,7 @@ <Form>WebBrowser/QtHelp/QtHelpDocumentationDialog.ui</Form> <Form>WebBrowser/QtHelp/QtHelpDocumentationSelectionDialog.ui</Form> <Form>WebBrowser/QtHelp/QtHelpFiltersDialog.ui</Form> + <Form>WebBrowser/SafeBrowsing/SafeBrowsingDialog.ui</Form> <Form>WebBrowser/SearchWidget.ui</Form> <Form>WebBrowser/Session/SessionManagerDialog.ui</Form> <Form>WebBrowser/SiteInfo/SiteInfoDialog.ui</Form>
