Mercurial Repositories > eric / file comparison

comparison: eric6/WebBrowser/SafeBrowsing/SafeBrowsingManager.py

eric6/WebBrowser/SafeBrowsing/SafeBrowsingManager.py

changeset 6942
2602857055c5
parent 6645
ad476851d7e0
child 7192
a22eee00b052
equal deleted inserted replaced
6941:f99d60d6b59b 6942:2602857055c5
1 # -*- coding: utf-8 -*-
2
3 # Copyright (c) 2017 - 2019 Detlev Offenbach <detlev@die-offenbachs.de>
4 #
5
6 """
7 Module implementing the interface for Google Safe Browsing.
8 """
9
10 #
11 # Some part of this code were ported from gglsbl.client and adapted
12 # to Qt.
13 #
14 # https://github.com/afilipovich/gglsbl
15 #
16
17 from __future__ import unicode_literals
18
19 import os
20 import base64
21
22 from PyQt5.QtCore import pyqtSignal, pyqtSlot, QObject, QCoreApplication, \
23 QUrl, QDateTime, QTimer
24
25 import Preferences
26 import Utilities
27
28 import UI.PixmapCache
29
30 from .SafeBrowsingAPIClient import SafeBrowsingAPIClient
31 from .SafeBrowsingCache import SafeBrowsingCache
32 from .SafeBrowsingThreatList import ThreatList, HashPrefixList
33 from .SafeBrowsingUrl import SafeBrowsingUrl
34 from .SafeBrowsingUtilities import toHex
35
36
37 class SafeBrowsingManager(QObject):
38 """
39 Class implementing the interface for Google Safe Browsing.
40
41 @signal progressMessage(message,maximum) emitted to give a message for the
42 action about to be performed and the maximum value
43 @signal progress(current) emitted to signal the current progress
44 """
45 progressMessage = pyqtSignal(str, int)
46 progress = pyqtSignal(int)
47
48 enabled = (
49 Preferences.getWebBrowser("SafeBrowsingEnabled") and
50 bool(Preferences.getWebBrowser("SafeBrowsingApiKey"))
51 )
52
53 def __init__(self):
54 """
55 Constructor
56 """
57 super(SafeBrowsingManager, self).__init__()
58
59 self.__apiKey = Preferences.getWebBrowser("SafeBrowsingApiKey")
60 if self.__apiKey:
61 self.__apiClient = SafeBrowsingAPIClient(self.__apiKey,
62 parent=self)
63 else:
64 self.__apiClient = None
65
66 gsbCachePath = os.path.join(
67 Utilities.getConfigDir(), "web_browser", "safe_browsing")
68 self.__cache = SafeBrowsingCache(gsbCachePath, self)
69
70 self.__gsbDialog = None
71 self.__setPlatforms()
72 self.__setLookupMethod()
73
74 self.__updatingThreatLists = False
75 self.__threatListsUpdateTimer = QTimer(self)
76 self.__threatListsUpdateTimer.setSingleShot(True)
77 self.__threatListsUpdateTimer.timeout.connect(
78 self.__threatListsUpdateTimerTimeout)
79 self.__setAutoUpdateThreatLists()
80
81 def configurationChanged(self):
82 """
83 Public method to handle changes of the settings.
84 """
85 apiKey = Preferences.getWebBrowser("SafeBrowsingApiKey")
86 if apiKey != self.__apiKey:
87 self.__apiKey = apiKey
88 if self.__apiKey:
89 if self.__apiClient:
90 self.__apiClient.setApiKey(self.__apiKey)
91 else:
92 self.__apiClient = SafeBrowsingAPIClient(self.__apiKey,
93 parent=self)
94
95 SafeBrowsingManager.enabled = (
96 Preferences.getWebBrowser("SafeBrowsingEnabled") and
97 bool(self.__apiKey))
98
99 self.__setPlatforms()
100 self.__setLookupMethod()
101 self.__setAutoUpdateThreatLists()
102
103 def __setPlatforms(self):
104 """
105 Private method to set the platforms to be checked against.
106 """
107 self.__platforms = None
108 if Preferences.getWebBrowser("SafeBrowsingFilterPlatform"):
109 if Utilities.isWindowsPlatform():
110 platform = "windows"
111 elif Utilities.isMacPlatform():
112 platform = "macos"
113 else:
114 # treat all other platforms like linux
115 platform = "linux"
116 self.__platforms = SafeBrowsingAPIClient.getPlatformTypes(platform)
117
118 def __setLookupMethod(self):
119 """
120 Private method to set the lookup method (Update API or Lookup API).
121 """
122 self.__useLookupApi = Preferences.getWebBrowser(
123 "SafeBrowsingUseLookupApi")
124
125 @classmethod
126 def isEnabled(cls):
127 """
128 Class method to check, if safe browsing is enabled.
129
130 @return flag indicating the enabled state
131 @rtype bool
132 """
133 return cls.enabled
134
135 def close(self):
136 """
137 Public method to close the safe browsing interface.
138 """
139 self.__cache.close()
140
141 def fairUseDelayExpired(self):
142 """
143 Public method to check, if the fair use wait period has expired.
144
145 @return flag indicating expiration
146 @rtype bool
147 """
148 return self.isEnabled() and self.__apiClient.fairUseDelayExpired()
149
150 def __showNotificationMessage(self, message, timeout=5):
151 """
152 Private method to show some message in a notification widget.
153
154 If desktop notifications have been disabled, the message will
155 be shown in the status bar of the main window (either the main
156 web browser window or the eric main window)
157
158 @param message message to be shown
159 @type str
160 @param timeout amount of time in seconds the message should be shown
161 (0 = indefinitely)
162 @type int
163 """
164 from WebBrowser.WebBrowserWindow import WebBrowserWindow
165
166 if WebBrowserWindow.notificationsEnabled():
167 WebBrowserWindow.showNotification(
168 UI.PixmapCache.getPixmap("safeBrowsing48.png"),
169 self.tr("Google Safe Browsing"),
170 message,
171 timeout=timeout,
172 )
173 else:
174 statusBar = WebBrowserWindow.globalStatusBar()
175 if statusBar is not None:
176 statusBar.showMessage(message, timeout * 1000)
177
178 def __setAutoUpdateThreatLists(self):
179 """
180 Private method to set auto update for the threat lists.
181 """
182 autoUpdateEnabled = \
183 Preferences.getWebBrowser("SafeBrowsingAutoUpdate") and \
184 not Preferences.getWebBrowser("SafeBrowsingUseLookupApi")
185 if autoUpdateEnabled and self.isEnabled():
186 nextUpdateDateTime = Preferences.getWebBrowser(
187 "SafeBrowsingUpdateDateTime")
188 if nextUpdateDateTime.isValid():
189 interval = \
190 QDateTime.currentDateTime().secsTo(nextUpdateDateTime) + 2
191 # 2 seconds extra wait time; interval in milliseconds
192
193 if interval < 5:
194 interval = 5
195 # minimum 5 seconds interval
196 else:
197 interval = 5
198 # just wait 5 seconds
199 self.__threatListsUpdateTimer.start(interval * 1000)
200 else:
201 if self.__threatListsUpdateTimer.isActive():
202 self.__threatListsUpdateTimer.stop()
203
204 @pyqtSlot()
205 def __threatListsUpdateTimerTimeout(self):
206 """
207 Private slot to perform the auto update of the threat lists.
208 """
209 ok = False
210 if self.isEnabled():
211 self.__showNotificationMessage(
212 self.tr("Updating threat lists..."), 0)
213 ok = self.updateHashPrefixCache()[0]
214 if ok:
215 self.__showNotificationMessage(
216 self.tr("Updating threat lists done"))
217 else:
218 self.__showNotificationMessage(
219 self.tr("Updating threat lists failed"))
220
221 if ok:
222 nextUpdateDateTime = \
223 self.__apiClient.getFairUseDelayExpirationDateTime()
224 Preferences.setWebBrowser("SafeBrowsingUpdateDateTime",
225 nextUpdateDateTime)
226 self.__threatListsUpdateTimer.start(
227 (QDateTime.currentDateTime().secsTo(nextUpdateDateTime) + 2) *
228 1000)
229 # 2 seconds extra wait time; interval in milliseconds
230 else:
231 Preferences.setWebBrowser("SafeBrowsingUpdateDateTime",
232 QDateTime())
233
234 def updateHashPrefixCache(self):
235 """
236 Public method to load or update the locally cached threat lists.
237
238 @return flag indicating success and an error message
239 @rtype tuple of (bool, str)
240 """
241 if not self.isEnabled():
242 return False, self.tr("Safe Browsing is disabled.")
243
244 if not self.__apiClient.fairUseDelayExpired():
245 return False, \
246 self.tr("The fair use wait period has not expired yet."
247 "Expiration will be at {0}.").format(
248 self.__apiClient.getFairUseDelayExpirationDateTime()
249 .toString("yyyy-MM-dd, HH:mm:ss"))
250
251 self.__updatingThreatLists = True
252 ok = True
253 errorMessage = ""
254
255 # step 1: remove expired hashes
256 self.__cache.cleanupFullHashes()
257 QCoreApplication.processEvents()
258
259 # step 2: update threat lists
260 threatListsForRemove = {}
261 for threatList, _clientState in self.__cache.getThreatLists():
262 threatListsForRemove[repr(threatList)] = threatList
263 threatLists, error = self.__apiClient.getThreatLists()
264 if error:
265 return False, error
266
267 maximum = len(threatLists)
268 current = 0
269 self.progressMessage.emit(self.tr("Updating threat lists"), maximum)
270 for entry in threatLists:
271 current += 1
272 self.progress.emit(current)
273 QCoreApplication.processEvents()
274 threatList = ThreatList.fromApiEntry(entry)
275 if self.__platforms is None or \
276 threatList.platformType in self.__platforms:
277 self.__cache.addThreatList(threatList)
278 key = repr(threatList)
279 if key in threatListsForRemove:
280 del threatListsForRemove[key]
281 maximum = len(threatListsForRemove.values())
282 current = 0
283 self.progressMessage.emit(self.tr("Deleting obsolete threat lists"),
284 maximum)
285 for threatList in threatListsForRemove.values():
286 current += 1
287 self.progress.emit(current)
288 QCoreApplication.processEvents()
289 self.__cache.deleteHashPrefixList(threatList)
290 self.__cache.deleteThreatList(threatList)
291 del threatListsForRemove
292
293 # step 3: update threats
294 threatLists = self.__cache.getThreatLists()
295 clientStates = {}
296 for threatList, clientState in threatLists:
297 clientStates[threatList.asTuple()] = clientState
298 threatsUpdateResponses, error = \
299 self.__apiClient.getThreatsUpdate(clientStates)
300 if error:
301 return False, error
302
303 maximum = len(threatsUpdateResponses)
304 current = 0
305 self.progressMessage.emit(self.tr("Updating hash prefixes"), maximum)
306 for response in threatsUpdateResponses:
307 current += 1
308 self.progress.emit(current)
309 QCoreApplication.processEvents()
310 responseThreatList = ThreatList.fromApiEntry(response)
311 if response["responseType"] == "FULL_UPDATE":
312 self.__cache.deleteHashPrefixList(responseThreatList)
313 for removal in response.get("removals", []):
314 self.__cache.removeHashPrefixIndices(
315 responseThreatList, removal["rawIndices"]["indices"])
316 QCoreApplication.processEvents()
317 for addition in response.get("additions", []):
318 hashPrefixList = HashPrefixList(
319 addition["rawHashes"]["prefixSize"],
320 base64.b64decode(addition["rawHashes"]["rawHashes"]))
321 self.__cache.populateHashPrefixList(responseThreatList,
322 hashPrefixList)
323 QCoreApplication.processEvents()
324 expectedChecksum = base64.b64decode(response["checksum"]["sha256"])
325 if self.__verifyThreatListChecksum(responseThreatList,
326 expectedChecksum):
327 self.__cache.updateThreatListClientState(
328 responseThreatList, response["newClientState"])
329 else:
330 ok = False
331 errorMessage = self.tr(
332 "Local cache checksum does not match the server. Consider"
333 " cleaning the cache. Threat update has been aborted.")
334
335 self.__updatingThreatLists = False
336
337 return ok, errorMessage
338
339 def isUpdatingThreatLists(self):
340 """
341 Public method to check, if we are in the process of updating the
342 threat lists.
343
344 @return flag indicating an update process is active
345 @rtype bool
346 """
347 return self.__updatingThreatLists
348
349 def __verifyThreatListChecksum(self, threatList, remoteChecksum):
350 """
351 Private method to verify the local checksum of a threat list with the
352 checksum of the safe browsing server.
353
354 @param threatList threat list to calculate checksum for
355 @type ThreatList
356 @param remoteChecksum SHA256 checksum as reported by the Google server
357 @type bytes
358 @return flag indicating equality
359 @rtype bool
360 """
361 localChecksum = self.__cache.hashPrefixListChecksum(threatList)
362 return remoteChecksum == localChecksum
363
364 def fullCacheCleanup(self):
365 """
366 Public method to clean up the cache completely.
367 """
368 self.__cache.prepareCacheDb()
369
370 def showSafeBrowsingDialog(self):
371 """
372 Public slot to show the safe browsing management dialog.
373 """
374 if self.__gsbDialog is None:
375 from WebBrowser.WebBrowserWindow import WebBrowserWindow
376 from .SafeBrowsingDialog import SafeBrowsingDialog
377 self.__gsbDialog = SafeBrowsingDialog(
378 self, parent=WebBrowserWindow.mainWindow())
379
380 self.__gsbDialog.show()
381
382 def lookupUrl(self, url):
383 """
384 Public method to lookup an URL.
385
386 @param url URL to be checked
387 @type str or QUrl
388 @return tuple containing the list of threat lists the URL was found in
389 and an error message
390 @rtype tuple of (list of ThreatList, str)
391 @exception ValueError raised for an invalid URL
392 """
393 if self.isEnabled():
394 if self.__useLookupApi:
395 if isinstance(url, str):
396 url = QUrl(url.strip())
397
398 if url.isEmpty():
399 raise ValueError("Empty URL given.")
400
401 listNames, error = \
402 self.__apiClient.lookupUrl(url, self.__platforms)
403 return listNames, error
404 else:
405 if isinstance(url, QUrl):
406 urlStr = url.toString().strip()
407 else:
408 urlStr = url.strip()
409
410 if not urlStr:
411 raise ValueError("Empty URL given.")
412
413 urlHashes = SafeBrowsingUrl(urlStr).hashes()
414 listNames = self.__lookupHashes(urlHashes)
415
416 return listNames, ""
417
418 return None, ""
419
420 def __lookupHashes(self, fullHashes):
421 """
422 Private method to lookup the given hashes.
423
424 @param fullHashes list of hashes to lookup
425 @type list of bytes
426 @return names of threat lists hashes were found in
427 @rtype list of ThreatList
428 """
429 fullHashes = list(fullHashes)
430 cues = [toHex(fh[:4]) for fh in fullHashes]
431 result = []
432
433 matchingPrefixes = {}
434 matchingFullHashes = set()
435 isPotentialThreat = False
436 # Lookup hash prefixes which match full URL hash
437 for _threatList, hashPrefix, negativeCacheExpired in \
438 self.__cache.lookupHashPrefix(cues):
439 for fullHash in fullHashes:
440 if fullHash.startswith(hashPrefix):
441 isPotentialThreat = True
442 # consider hash prefix negative cache as expired if it
443 # is expired in at least one threat list
444 matchingPrefixes[hashPrefix] = matchingPrefixes.get(
445 hashPrefix, False) or negativeCacheExpired
446 matchingFullHashes.add(fullHash)
447
448 # if none matches, url hash is clear
449 if not isPotentialThreat:
450 return []
451
452 # if there is non-expired full hash, URL is blacklisted
453 matchingExpiredThreatLists = set()
454 for threatList, hasExpired in self.__cache.lookupFullHashes(
455 matchingFullHashes):
456 if hasExpired:
457 matchingExpiredThreatLists.add(threatList)
458 else:
459 result.append(threatList)
460 if result:
461 return result
462
463 # If there are no matching expired full hash entries and negative
464 # cache is still current for all prefixes, consider it safe.
465 if len(matchingExpiredThreatLists) == 0 and \
466 sum(map(int, matchingPrefixes.values())) == 0:
467 return []
468
469 # Now it can be assumed that there are expired matching full hash
470 # entries and/or cache prefix entries with expired negative cache.
471 # Both require full hash synchronization.
472 self.__syncFullHashes(matchingPrefixes.keys())
473
474 # Now repeat full hash lookup
475 for threatList, hasExpired in self.__cache.lookupFullHashes(
476 matchingFullHashes):
477 if not hasExpired:
478 result.append(threatList)
479
480 return result
481
482 def __syncFullHashes(self, hashPrefixes):
483 """
484 Private method to download full hashes matching given prefixes.
485
486 This also updates the cache expiration timestamps.
487
488 @param hashPrefixes list of hash prefixes to get full hashes for
489 @type list of bytes
490 """
491 threatLists = self.__cache.getThreatLists()
492 clientStates = {}
493 for threatList, clientState in threatLists:
494 clientStates[threatList.asTuple()] = clientState
495
496 fullHashResponses = self.__apiClient.getFullHashes(
497 hashPrefixes, clientStates)
498
499 # update negative cache for each hash prefix
500 # store full hash with positive cache bumped up
501 for match in fullHashResponses["matches"]:
502 threatList = ThreatList.fromApiEntry(match)
503 hashValue = base64.b64decode(match["threat"]["hash"])
504 cacheDuration = int(match["cacheDuration"].rstrip("s"))
505 malwareThreatType = None
506 for metadata in match["threatEntryMetadata"].get("entries", []):
507 key = base64.b64decode(metadata["key"])
508 value = base64.b64decode(metadata["value"])
509 if key == b"malware_threat_type":
510 malwareThreatType = value
511 if not isinstance(malwareThreatType, str):
512 malwareThreatType = malwareThreatType.decode()
513 self.__cache.storeFullHash(threatList, hashValue, cacheDuration,
514 malwareThreatType)
515
516 negativeCacheDuration = int(
517 fullHashResponses["negativeCacheDuration"].rstrip("s"))
518 for prefixValue in hashPrefixes:
519 for threatList, _clientState in threatLists:
520 self.__cache.updateHashPrefixExpiration(
521 threatList, prefixValue, negativeCacheDuration)
522
523 @classmethod
524 def getIgnoreSchemes(cls):
525 """
526 Class method to get the schemes not to be checked.
527
528 @return list of schemes to be ignored
529 @rtype list of str
530 """
531 return [
532 "about",
533 "eric",
534 "qrc",
535 "qthelp",
536 "chrome",
537 "abp",
538 "file",
539 ]
540
541 def getThreatMessage(self, threatType):
542 """
543 Public method to get a warning message for the given threat type.
544
545 @param threatType threat type to get the message for
546 @type str
547 @return threat message
548 @rtype str
549 """
550 if self.__apiClient:
551 msg = self.__apiClient.getThreatMessage(threatType)
552 else:
553 msg = ""
554
555 return msg
556
557 def getThreatMessages(self, threatLists):
558 """
559 Public method to get threat messages for the given threats.
560
561 @param threatLists list of threat lists to get a message for
562 @type list of ThreatList
563 @return list of threat messages, one per unique threat type
564 @rtype list of str
565 """
566 threatTypes = set()
567 for threatList in threatLists:
568 threatTypes.add(threatList.threatType)
569
570 messages = []
571 if self.__apiClient:
572 for threatType in sorted(threatTypes):
573 msg = self.__apiClient.getThreatMessage(threatType)
574 messages.append(msg)
575
576 return messages
577
578 def getThreatType(self, threatList):
579 """
580 Public method to get a display string for a given threat type.
581
582 @param threatList threat list to get display string for
583 @type str
584 @return display string
585 @rtype str
586 """
587 displayString = ""
588 if self.__apiClient:
589 displayString = self.__apiClient.getThreatType(
590 threatList.threatType)
591 return displayString
592
593 def getPlatformString(self, platformType):
594 """
595 Public method to get the platform string for a given platform type.
596
597 @param platformType platform type as defined in the v4 API
598 @type str
599 @return platform string
600 @rtype str
601 """
602 if self.__apiClient:
603 return self.__apiClient.getPlatformString(platformType)
604 else:
605 return ""
606
607 def getThreatEntryString(self, threatEntry):
608 """
609 Public method to get the threat entry string.
610
611 @param threatEntry threat entry type as defined in the v4 API
612 @type str
613 @return threat entry string
614 @rtype str
615 """
616 if self.__apiClient:
617 return self.__apiClient.getThreatEntryString(threatEntry)
618 else:
619 return ""

Mercurial Repository: eric

eric ide

mercurial