Mercurial Repositories > eric / file comparison

comparison: eric7/WebBrowser/SafeBrowsing/SafeBrowsingManager.py

eric7/WebBrowser/SafeBrowsing/SafeBrowsingManager.py

branch
eric7
changeset 8312
800c432b34c8
parent 8265
0090cfa83159
child 8318
962bce857696
equal deleted inserted replaced
8311:4e8b98454baa 8312:800c432b34c8
1 # -*- coding: utf-8 -*-
2
3 # Copyright (c) 2017 - 2021 Detlev Offenbach <detlev@die-offenbachs.de>
4 #
5
6 """
7 Module implementing the interface for Google Safe Browsing.
8 """
9
10 #
11 # Some part of this code were ported from gglsbl.client and adapted
12 # to Qt.
13 #
14 # https://github.com/afilipovich/gglsbl
15 #
16
17 import os
18 import base64
19
20 from PyQt5.QtCore import (
21 pyqtSignal, pyqtSlot, QObject, QCoreApplication, QUrl, QDateTime, QTimer
22 )
23
24 import Preferences
25 import Utilities
26
27 import UI.PixmapCache
28 from UI.NotificationWidget import NotificationTypes
29
30 from .SafeBrowsingAPIClient import SafeBrowsingAPIClient
31 from .SafeBrowsingCache import SafeBrowsingCache
32 from .SafeBrowsingThreatList import ThreatList, HashPrefixList
33 from .SafeBrowsingUrl import SafeBrowsingUrl
34
35
36 class SafeBrowsingManager(QObject):
37 """
38 Class implementing the interface for Google Safe Browsing.
39
40 @signal progressMessage(message,maximum) emitted to give a message for the
41 action about to be performed and the maximum value
42 @signal progress(current) emitted to signal the current progress
43 """
44 progressMessage = pyqtSignal(str, int)
45 progress = pyqtSignal(int)
46
47 enabled = (
48 Preferences.getWebBrowser("SafeBrowsingEnabled") and
49 bool(Preferences.getWebBrowser("SafeBrowsingApiKey"))
50 )
51
52 def __init__(self):
53 """
54 Constructor
55 """
56 super().__init__()
57
58 self.__apiKey = Preferences.getWebBrowser("SafeBrowsingApiKey")
59 if self.__apiKey:
60 self.__apiClient = SafeBrowsingAPIClient(self.__apiKey,
61 parent=self)
62 else:
63 self.__apiClient = None
64
65 gsbCachePath = os.path.join(
66 Utilities.getConfigDir(), "web_browser", "safe_browsing")
67 self.__cache = SafeBrowsingCache(gsbCachePath, self)
68
69 self.__gsbDialog = None
70 self.__setPlatforms()
71 self.__setLookupMethod()
72
73 self.__updatingThreatLists = False
74 self.__threatListsUpdateTimer = QTimer(self)
75 self.__threatListsUpdateTimer.setSingleShot(True)
76 self.__threatListsUpdateTimer.timeout.connect(
77 self.__threatListsUpdateTimerTimeout)
78 self.__setAutoUpdateThreatLists()
79
80 def configurationChanged(self):
81 """
82 Public method to handle changes of the settings.
83 """
84 apiKey = Preferences.getWebBrowser("SafeBrowsingApiKey")
85 if apiKey != self.__apiKey:
86 self.__apiKey = apiKey
87 if self.__apiKey:
88 if self.__apiClient:
89 self.__apiClient.setApiKey(self.__apiKey)
90 else:
91 self.__apiClient = SafeBrowsingAPIClient(self.__apiKey,
92 parent=self)
93
94 SafeBrowsingManager.enabled = (
95 Preferences.getWebBrowser("SafeBrowsingEnabled") and
96 bool(self.__apiKey))
97
98 self.__setPlatforms()
99 self.__setLookupMethod()
100 self.__setAutoUpdateThreatLists()
101
102 def __setPlatforms(self):
103 """
104 Private method to set the platforms to be checked against.
105 """
106 self.__platforms = None
107 if Preferences.getWebBrowser("SafeBrowsingFilterPlatform"):
108 if Utilities.isWindowsPlatform():
109 platform = "windows"
110 elif Utilities.isMacPlatform():
111 platform = "macos"
112 else:
113 # treat all other platforms like linux
114 platform = "linux"
115 self.__platforms = SafeBrowsingAPIClient.getPlatformTypes(platform)
116
117 def __setLookupMethod(self):
118 """
119 Private method to set the lookup method (Update API or Lookup API).
120 """
121 self.__useLookupApi = Preferences.getWebBrowser(
122 "SafeBrowsingUseLookupApi")
123
124 @classmethod
125 def isEnabled(cls):
126 """
127 Class method to check, if safe browsing is enabled.
128
129 @return flag indicating the enabled state
130 @rtype bool
131 """
132 return cls.enabled
133
134 def close(self):
135 """
136 Public method to close the safe browsing interface.
137 """
138 self.__cache.close()
139
140 def fairUseDelayExpired(self):
141 """
142 Public method to check, if the fair use wait period has expired.
143
144 @return flag indicating expiration
145 @rtype bool
146 """
147 return self.isEnabled() and self.__apiClient.fairUseDelayExpired()
148
149 def __showNotificationMessage(self, message, timeout=5):
150 """
151 Private method to show some message in a notification widget.
152
153 @param message message to be shown
154 @type str
155 @param timeout amount of time in seconds the message should be shown
156 (0 = indefinitely)
157 @type int
158 """
159 from WebBrowser.WebBrowserWindow import WebBrowserWindow
160
161 kind = (
162 NotificationTypes.CRITICAL
163 if timeout == 0 else
164 NotificationTypes.INFORMATION
165 )
166
167 WebBrowserWindow.showNotification(
168 UI.PixmapCache.getPixmap("safeBrowsing48"),
169 self.tr("Google Safe Browsing"),
170 message,
171 kind=kind,
172 timeout=timeout,
173 )
174
175 def __setAutoUpdateThreatLists(self):
176 """
177 Private method to set auto update for the threat lists.
178 """
179 autoUpdateEnabled = (
180 Preferences.getWebBrowser("SafeBrowsingAutoUpdate") and
181 not Preferences.getWebBrowser("SafeBrowsingUseLookupApi")
182 )
183 if autoUpdateEnabled and self.isEnabled():
184 nextUpdateDateTime = Preferences.getWebBrowser(
185 "SafeBrowsingUpdateDateTime")
186 if nextUpdateDateTime.isValid():
187 interval = (
188 QDateTime.currentDateTime().secsTo(nextUpdateDateTime) + 2
189 # 2 seconds extra wait time; interval in milliseconds
190 )
191
192 if interval < 5:
193 interval = 5
194 # minimum 5 seconds interval
195 else:
196 interval = 5
197 # just wait 5 seconds
198 self.__threatListsUpdateTimer.start(interval * 1000)
199 else:
200 if self.__threatListsUpdateTimer.isActive():
201 self.__threatListsUpdateTimer.stop()
202
203 @pyqtSlot()
204 def __threatListsUpdateTimerTimeout(self):
205 """
206 Private slot to perform the auto update of the threat lists.
207 """
208 ok = False
209 if self.isEnabled():
210 self.__showNotificationMessage(
211 self.tr("Updating threat lists..."), 0)
212 ok = self.updateHashPrefixCache()[0]
213 if ok:
214 self.__showNotificationMessage(
215 self.tr("Updating threat lists done."))
216 else:
217 self.__showNotificationMessage(
218 self.tr("Updating threat lists failed."),
219 timeout=0)
220
221 if ok:
222 nextUpdateDateTime = (
223 self.__apiClient.getFairUseDelayExpirationDateTime()
224 )
225 Preferences.setWebBrowser("SafeBrowsingUpdateDateTime",
226 nextUpdateDateTime)
227 self.__threatListsUpdateTimer.start(
228 (QDateTime.currentDateTime().secsTo(nextUpdateDateTime) + 2) *
229 1000)
230 # 2 seconds extra wait time; interval in milliseconds
231 else:
232 Preferences.setWebBrowser("SafeBrowsingUpdateDateTime",
233 QDateTime())
234
235 def updateHashPrefixCache(self):
236 """
237 Public method to load or update the locally cached threat lists.
238
239 @return flag indicating success and an error message
240 @rtype tuple of (bool, str)
241 """
242 if not self.isEnabled():
243 return False, self.tr("Safe Browsing is disabled.")
244
245 if not self.__apiClient.fairUseDelayExpired():
246 return (
247 False,
248 self.tr("The fair use wait period has not expired yet."
249 "Expiration will be at {0}.").format(
250 self.__apiClient.getFairUseDelayExpirationDateTime()
251 .toString("yyyy-MM-dd, HH:mm:ss"))
252 )
253
254 self.__updatingThreatLists = True
255 ok = True
256 errorMessage = ""
257
258 # step 1: remove expired hashes
259 self.__cache.cleanupFullHashes()
260 QCoreApplication.processEvents()
261
262 # step 2: update threat lists
263 threatListsForRemove = {}
264 for threatList, _clientState in self.__cache.getThreatLists():
265 threatListsForRemove[repr(threatList)] = threatList
266 threatLists, error = self.__apiClient.getThreatLists()
267 if error:
268 return False, error
269
270 maximum = len(threatLists)
271 self.progressMessage.emit(self.tr("Updating threat lists"), maximum)
272 for current, entry in enumerate(threatLists, start=1):
273 self.progress.emit(current)
274 QCoreApplication.processEvents()
275 threatList = ThreatList.fromApiEntry(entry)
276 if (
277 self.__platforms is None or
278 threatList.platformType in self.__platforms
279 ):
280 self.__cache.addThreatList(threatList)
281 key = repr(threatList)
282 if key in threatListsForRemove:
283 del threatListsForRemove[key]
284
285 maximum = len(threatListsForRemove.values())
286 self.progressMessage.emit(self.tr("Deleting obsolete threat lists"),
287 maximum)
288 for current, threatList in enumerate(
289 threatListsForRemove.values(), start=1
290 ):
291 self.progress.emit(current)
292 QCoreApplication.processEvents()
293 self.__cache.deleteHashPrefixList(threatList)
294 self.__cache.deleteThreatList(threatList)
295 del threatListsForRemove
296
297 # step 3: update threats
298 threatLists = self.__cache.getThreatLists()
299 clientStates = {}
300 for threatList, clientState in threatLists:
301 clientStates[threatList.asTuple()] = clientState
302 threatsUpdateResponses, error = self.__apiClient.getThreatsUpdate(
303 clientStates)
304 if error:
305 return False, error
306
307 maximum = len(threatsUpdateResponses)
308 self.progressMessage.emit(self.tr("Updating hash prefixes"), maximum)
309 for current, response in enumerate(threatsUpdateResponses, start=1):
310 self.progress.emit(current)
311 QCoreApplication.processEvents()
312 responseThreatList = ThreatList.fromApiEntry(response)
313 if response["responseType"] == "FULL_UPDATE":
314 self.__cache.deleteHashPrefixList(responseThreatList)
315 for removal in response.get("removals", []):
316 self.__cache.removeHashPrefixIndices(
317 responseThreatList, removal["rawIndices"]["indices"])
318 QCoreApplication.processEvents()
319 for addition in response.get("additions", []):
320 hashPrefixList = HashPrefixList(
321 addition["rawHashes"]["prefixSize"],
322 base64.b64decode(addition["rawHashes"]["rawHashes"]))
323 self.__cache.populateHashPrefixList(responseThreatList,
324 hashPrefixList)
325 QCoreApplication.processEvents()
326 expectedChecksum = base64.b64decode(response["checksum"]["sha256"])
327 if self.__verifyThreatListChecksum(responseThreatList,
328 expectedChecksum):
329 self.__cache.updateThreatListClientState(
330 responseThreatList, response["newClientState"])
331 else:
332 ok = False
333 errorMessage = self.tr(
334 "Local cache checksum does not match the server. Consider"
335 " cleaning the cache. Threat update has been aborted.")
336
337 self.__updatingThreatLists = False
338
339 return ok, errorMessage
340
341 def isUpdatingThreatLists(self):
342 """
343 Public method to check, if we are in the process of updating the
344 threat lists.
345
346 @return flag indicating an update process is active
347 @rtype bool
348 """
349 return self.__updatingThreatLists
350
351 def __verifyThreatListChecksum(self, threatList, remoteChecksum):
352 """
353 Private method to verify the local checksum of a threat list with the
354 checksum of the safe browsing server.
355
356 @param threatList threat list to calculate checksum for
357 @type ThreatList
358 @param remoteChecksum SHA256 checksum as reported by the Google server
359 @type bytes
360 @return flag indicating equality
361 @rtype bool
362 """
363 localChecksum = self.__cache.hashPrefixListChecksum(threatList)
364 return remoteChecksum == localChecksum
365
366 def fullCacheCleanup(self):
367 """
368 Public method to clean up the cache completely.
369 """
370 self.__cache.prepareCacheDb()
371
372 def showSafeBrowsingDialog(self):
373 """
374 Public slot to show the safe browsing management dialog.
375 """
376 if self.__gsbDialog is None:
377 from WebBrowser.WebBrowserWindow import WebBrowserWindow
378 from .SafeBrowsingDialog import SafeBrowsingDialog
379 self.__gsbDialog = SafeBrowsingDialog(
380 self, parent=WebBrowserWindow.mainWindow())
381
382 self.__gsbDialog.show()
383
384 def lookupUrl(self, url):
385 """
386 Public method to lookup an URL.
387
388 @param url URL to be checked
389 @type str or QUrl
390 @return tuple containing the list of threat lists the URL was found in
391 and an error message
392 @rtype tuple of (list of ThreatList, str)
393 @exception ValueError raised for an invalid URL
394 """
395 if self.isEnabled():
396 if self.__useLookupApi:
397 if isinstance(url, str):
398 url = QUrl(url.strip())
399
400 if url.isEmpty():
401 raise ValueError("Empty URL given.")
402
403 listNames, error = self.__apiClient.lookupUrl(
404 url, self.__platforms)
405 return listNames, error
406 else:
407 if isinstance(url, QUrl):
408 urlStr = url.toString().strip()
409 else:
410 urlStr = url.strip()
411
412 if not urlStr:
413 raise ValueError("Empty URL given.")
414
415 urlHashes = SafeBrowsingUrl(urlStr).hashes()
416 listNames = self.__lookupHashes(urlHashes)
417
418 return listNames, ""
419
420 return None, ""
421
422 def __lookupHashes(self, fullHashes):
423 """
424 Private method to lookup the given hashes.
425
426 @param fullHashes list of hashes to lookup
427 @type list of bytes
428 @return names of threat lists hashes were found in
429 @rtype list of ThreatList
430 """
431 fullHashes = list(fullHashes)
432 cues = [fh[:4].hex() for fh in fullHashes]
433 result = []
434
435 matchingPrefixes = {}
436 matchingFullHashes = set()
437 isPotentialThreat = False
438 # Lookup hash prefixes which match full URL hash
439 for _threatList, hashPrefix, negativeCacheExpired in (
440 self.__cache.lookupHashPrefix(cues)
441 ):
442 for fullHash in fullHashes:
443 if fullHash.startswith(hashPrefix):
444 isPotentialThreat = True
445 # consider hash prefix negative cache as expired if it
446 # is expired in at least one threat list
447 matchingPrefixes[hashPrefix] = matchingPrefixes.get(
448 hashPrefix, False) or negativeCacheExpired
449 matchingFullHashes.add(fullHash)
450
451 # if none matches, url hash is clear
452 if not isPotentialThreat:
453 return []
454
455 # if there is non-expired full hash, URL is blacklisted
456 matchingExpiredThreatLists = set()
457 for threatList, hasExpired in self.__cache.lookupFullHashes(
458 matchingFullHashes):
459 if hasExpired:
460 matchingExpiredThreatLists.add(threatList)
461 else:
462 result.append(threatList)
463 if result:
464 return result
465
466 # If there are no matching expired full hash entries and negative
467 # cache is still current for all prefixes, consider it safe.
468 if (
469 len(matchingExpiredThreatLists) == 0 and
470 sum(map(int, matchingPrefixes.values())) == 0
471 ):
472 return []
473
474 # Now it can be assumed that there are expired matching full hash
475 # entries and/or cache prefix entries with expired negative cache.
476 # Both require full hash synchronization.
477 self.__syncFullHashes(matchingPrefixes.keys())
478
479 # Now repeat full hash lookup
480 for threatList, hasExpired in self.__cache.lookupFullHashes(
481 matchingFullHashes):
482 if not hasExpired:
483 result.append(threatList)
484
485 return result
486
487 def __syncFullHashes(self, hashPrefixes):
488 """
489 Private method to download full hashes matching given prefixes.
490
491 This also updates the cache expiration timestamps.
492
493 @param hashPrefixes list of hash prefixes to get full hashes for
494 @type list of bytes
495 """
496 threatLists = self.__cache.getThreatLists()
497 clientStates = {}
498 for threatList, clientState in threatLists:
499 clientStates[threatList.asTuple()] = clientState
500
501 fullHashResponses = self.__apiClient.getFullHashes(
502 hashPrefixes, clientStates)
503
504 # update negative cache for each hash prefix
505 # store full hash with positive cache bumped up
506 for match in fullHashResponses["matches"]:
507 threatList = ThreatList.fromApiEntry(match)
508 hashValue = base64.b64decode(match["threat"]["hash"])
509 cacheDuration = int(match["cacheDuration"].rstrip("s"))
510 malwareThreatType = None
511 for metadata in match["threatEntryMetadata"].get("entries", []):
512 key = base64.b64decode(metadata["key"])
513 value = base64.b64decode(metadata["value"])
514 if key == b"malware_threat_type":
515 malwareThreatType = value
516 if not isinstance(malwareThreatType, str):
517 malwareThreatType = malwareThreatType.decode()
518 self.__cache.storeFullHash(threatList, hashValue, cacheDuration,
519 malwareThreatType)
520
521 negativeCacheDuration = int(
522 fullHashResponses["negativeCacheDuration"].rstrip("s"))
523 for prefixValue in hashPrefixes:
524 for threatList, _clientState in threatLists:
525 self.__cache.updateHashPrefixExpiration(
526 threatList, prefixValue, negativeCacheDuration)
527
528 @classmethod
529 def getIgnoreSchemes(cls):
530 """
531 Class method to get the schemes not to be checked.
532
533 @return list of schemes to be ignored
534 @rtype list of str
535 """
536 return [
537 "about",
538 "eric",
539 "qrc",
540 "qthelp",
541 "chrome",
542 "abp",
543 "file",
544 ]
545
546 def getThreatMessage(self, threatType):
547 """
548 Public method to get a warning message for the given threat type.
549
550 @param threatType threat type to get the message for
551 @type str
552 @return threat message
553 @rtype str
554 """
555 msg = (
556 self.__apiClient.getThreatMessage(threatType)
557 if self.__apiClient else
558 ""
559 )
560
561 return msg
562
563 def getThreatMessages(self, threatLists):
564 """
565 Public method to get threat messages for the given threats.
566
567 @param threatLists list of threat lists to get a message for
568 @type list of ThreatList
569 @return list of threat messages, one per unique threat type
570 @rtype list of str
571 """
572 threatTypes = set()
573 for threatList in threatLists:
574 threatTypes.add(threatList.threatType)
575
576 messages = []
577 if self.__apiClient:
578 for threatType in sorted(threatTypes):
579 msg = self.__apiClient.getThreatMessage(threatType)
580 messages.append(msg)
581
582 return messages
583
584 def getThreatType(self, threatList):
585 """
586 Public method to get a display string for a given threat type.
587
588 @param threatList threat list to get display string for
589 @type str
590 @return display string
591 @rtype str
592 """
593 displayString = ""
594 if self.__apiClient:
595 displayString = self.__apiClient.getThreatType(
596 threatList.threatType)
597 return displayString
598
599 def getPlatformString(self, platformType):
600 """
601 Public method to get the platform string for a given platform type.
602
603 @param platformType platform type as defined in the v4 API
604 @type str
605 @return platform string
606 @rtype str
607 """
608 if self.__apiClient:
609 return self.__apiClient.getPlatformString(platformType)
610 else:
611 return ""
612
613 def getThreatEntryString(self, threatEntry):
614 """
615 Public method to get the threat entry string.
616
617 @param threatEntry threat entry type as defined in the v4 API
618 @type str
619 @return threat entry string
620 @rtype str
621 """
622 if self.__apiClient:
623 return self.__apiClient.getThreatEntryString(threatEntry)
624 else:
625 return ""

Mercurial Repository: eric

eric ide

mercurial