--- a/src/eric7/WebBrowser/SafeBrowsing/SafeBrowsingCache.py Wed Jul 13 11:16:20 2022 +0200
+++ b/src/eric7/WebBrowser/SafeBrowsing/SafeBrowsingCache.py Wed Jul 13 14:55:47 2022 +0200
@@ -17,7 +17,11 @@
import os
from PyQt6.QtCore import (
- QObject, QByteArray, QCryptographicHash, QCoreApplication, QEventLoop
+ QObject,
+ QByteArray,
+ QCryptographicHash,
+ QCoreApplication,
+ QEventLoop,
)
from PyQt6.QtSql import QSql, QSqlDatabase, QSqlQuery
@@ -28,6 +32,7 @@
"""
Class implementing a cache for Google Safe Browsing.
"""
+
create_threat_list_stmt = """
CREATE TABLE threat_list
(threat_type character varying(128) NOT NULL,
@@ -39,7 +44,7 @@
)
"""
drop_threat_list_stmt = """DROP TABLE IF EXISTS threat_list"""
-
+
create_full_hashes_stmt = """
CREATE TABLE full_hash
(value BLOB NOT NULL,
@@ -54,7 +59,7 @@
)
"""
drop_full_hashes_stmt = """DROP TABLE IF EXISTS full_hash"""
-
+
create_hash_prefix_stmt = """
CREATE TABLE hash_prefix
(value BLOB NOT NULL,
@@ -72,49 +77,49 @@
)
"""
drop_hash_prefix_stmt = """DROP TABLE IF EXISTS hash_prefix"""
-
+
create_full_hash_cue_idx = """
CREATE INDEX idx_hash_prefix_cue ON hash_prefix (cue)
"""
drop_full_hash_cue_idx = """DROP INDEX IF EXISTS idx_hash_prefix_cue"""
-
+
create_full_hash_expires_idx = """
CREATE INDEX idx_full_hash_expires_at ON full_hash (expires_at)
"""
drop_full_hash_expires_idx = """
DROP INDEX IF EXISTS idx_full_hash_expires_at
"""
-
+
create_full_hash_value_idx = """
CREATE INDEX idx_full_hash_value ON full_hash (value)
"""
drop_full_hash_value_idx = """DROP INDEX IF EXISTS idx_full_hash_value"""
-
+
maxProcessEventsTime = 500
-
+
def __init__(self, dbPath, parent=None):
"""
Constructor
-
+
@param dbPath path to store the cache DB into
@type str
@param parent reference to the parent object
@type QObject
"""
super().__init__(parent)
-
+
self.__connectionName = "SafeBrowsingCache"
-
+
if not os.path.exists(dbPath):
os.makedirs(dbPath)
-
+
self.__dbFileName = os.path.join(dbPath, "SafeBrowsingCache.db")
preparationNeeded = not os.path.exists(self.__dbFileName)
-
+
self.__openCacheDb()
if preparationNeeded:
self.prepareCacheDb()
-
+
def close(self):
"""
Public method to close the database.
@@ -122,11 +127,11 @@
if QSqlDatabase.database(self.__connectionName).isOpen():
QSqlDatabase.database(self.__connectionName).close()
QSqlDatabase.removeDatabase(self.__connectionName)
-
+
def __openCacheDb(self):
"""
Private method to open the cache database.
-
+
@return flag indicating the open state
@rtype bool
"""
@@ -141,7 +146,7 @@
else:
opened = True
return opened
-
+
def prepareCacheDb(self):
"""
Public method to prepare the cache database.
@@ -169,12 +174,12 @@
finally:
del query
db.commit()
-
+
def lookupFullHashes(self, hashValues):
"""
Public method to get a list of threat lists and expiration flag
for the given hashes if a hash is blacklisted.
-
+
@param hashValues list of hash values to look up
@type list of bytes
@return list of tuples containing the threat list info and the
@@ -187,42 +192,42 @@
FROM full_hash WHERE value IN ({0})
"""
output = []
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
try:
query = QSqlQuery(db)
- query.prepare(
- queryStr.format(",".join(["?"] * len(hashValues))))
+ query.prepare(queryStr.format(",".join(["?"] * len(hashValues))))
for hashValue in hashValues:
query.addBindValue(
QByteArray(hashValue),
- QSql.ParamTypeFlag.In | QSql.ParamTypeFlag.Binary)
-
+ QSql.ParamTypeFlag.In | QSql.ParamTypeFlag.Binary,
+ )
+
query.exec()
-
- while query.next(): # __IGNORE_WARNING_M523__
+
+ while query.next(): # __IGNORE_WARNING_M523__
threatType = query.value(0)
platformType = query.value(1)
threatEntryType = query.value(2)
hasExpired = bool(query.value(3))
- threatList = ThreatList(threatType, platformType,
- threatEntryType)
+ threatList = ThreatList(threatType, platformType, threatEntryType)
output.append((threatList, hasExpired))
QCoreApplication.processEvents(
QEventLoop.ProcessEventsFlag.AllEvents,
- self.maxProcessEventsTime)
+ self.maxProcessEventsTime,
+ )
del query
finally:
db.commit()
-
+
return output
-
+
def lookupHashPrefix(self, prefixes):
"""
Public method to look up hash prefixes in the local cache.
-
+
@param prefixes list of hash prefixes to look up
@type list of bytes
@return list of tuples containing the threat list, full hash and
@@ -235,42 +240,40 @@
FROM hash_prefix WHERE cue IN ({0})
"""
output = []
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
try:
query = QSqlQuery(db)
- query.prepare(
- queryStr.format(",".join(["?"] * len(prefixes))))
+ query.prepare(queryStr.format(",".join(["?"] * len(prefixes))))
for prefix in prefixes:
query.addBindValue(prefix)
-
+
query.exec()
-
- while query.next(): # __IGNORE_WARNING_M523__
+
+ while query.next(): # __IGNORE_WARNING_M523__
fullHash = bytes(query.value(0))
threatType = query.value(1)
platformType = query.value(2)
threatEntryType = query.value(3)
negativeCacheExpired = bool(query.value(4))
- threatList = ThreatList(threatType, platformType,
- threatEntryType)
+ threatList = ThreatList(threatType, platformType, threatEntryType)
output.append((threatList, fullHash, negativeCacheExpired))
QCoreApplication.processEvents(
QEventLoop.ProcessEventsFlag.AllEvents,
- self.maxProcessEventsTime)
+ self.maxProcessEventsTime,
+ )
del query
finally:
db.commit()
-
+
return output
-
- def storeFullHash(self, threatList, hashValue, cacheDuration,
- malwareThreatType):
+
+ def storeFullHash(self, threatList, hashValue, cacheDuration, malwareThreatType):
"""
Public method to store full hash data in the cache database.
-
+
@param threatList threat list info object
@type ThreatList
@param hashValue hash to be stored
@@ -287,15 +290,15 @@
VALUES
(?, ?, ?, ?, ?, current_timestamp)
"""
- updateQueryStr = (
- """
+ updateQueryStr = """
UPDATE full_hash SET
expires_at=datetime(current_timestamp, '+{0} SECONDS')
WHERE value=? AND threat_type=? AND platform_type=? AND
threat_entry_type=?
- """.format(int(cacheDuration))
+ """.format(
+ int(cacheDuration)
)
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
@@ -304,19 +307,21 @@
query.prepare(insertQueryStr)
query.addBindValue(
QByteArray(hashValue),
- QSql.ParamTypeFlag.In | QSql.ParamTypeFlag.Binary)
+ QSql.ParamTypeFlag.In | QSql.ParamTypeFlag.Binary,
+ )
query.addBindValue(threatList.threatType)
query.addBindValue(threatList.platformType)
query.addBindValue(threatList.threatEntryType)
query.addBindValue(malwareThreatType)
query.exec()
del query
-
+
query = QSqlQuery(db)
query.prepare(updateQueryStr)
query.addBindValue(
QByteArray(hashValue),
- QSql.ParamTypeFlag.In | QSql.ParamTypeFlag.Binary)
+ QSql.ParamTypeFlag.In | QSql.ParamTypeFlag.Binary,
+ )
query.addBindValue(threatList.threatType)
query.addBindValue(threatList.platformType)
query.addBindValue(threatList.threatEntryType)
@@ -324,11 +329,11 @@
del query
finally:
db.commit()
-
+
def deleteHashPrefixList(self, threatList):
"""
Public method to delete hash prefixes for a given threat list.
-
+
@param threatList threat list info object
@type ThreatList
"""
@@ -336,7 +341,7 @@
DELETE FROM hash_prefix
WHERE threat_type=? AND platform_type=? AND threat_entry_type=?
"""
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
@@ -350,22 +355,22 @@
del query
finally:
db.commit()
-
+
def cleanupFullHashes(self, keepExpiredFor=43200):
"""
Public method to clean up full hash entries expired more than the
given time.
-
+
@param keepExpiredFor time period in seconds of entries to be expired
@type int or float
"""
- queryStr = (
- """
+ queryStr = """
DELETE FROM full_hash
WHERE expires_at=datetime(current_timestamp, '{0} SECONDS')
- """.format(int(keepExpiredFor))
+ """.format(
+ int(keepExpiredFor)
)
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
@@ -376,12 +381,11 @@
del query
finally:
db.commit()
-
- def updateHashPrefixExpiration(self, threatList, hashPrefix,
- negativeCacheDuration):
+
+ def updateHashPrefixExpiration(self, threatList, hashPrefix, negativeCacheDuration):
"""
Public method to update the hash prefix expiration time.
-
+
@param threatList threat list info object
@type ThreatList
@param hashPrefix hash prefix
@@ -390,15 +394,15 @@
in the cache
@type int or float
"""
- queryStr = (
- """
+ queryStr = """
UPDATE hash_prefix
SET negative_expires_at=datetime(current_timestamp, '+{0} SECONDS')
WHERE value=? AND threat_type=? AND platform_type=? AND
threat_entry_type=?
- """.format(int(negativeCacheDuration))
+ """.format(
+ int(negativeCacheDuration)
)
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
@@ -407,7 +411,8 @@
query.prepare(queryStr)
query.addBindValue(
QByteArray(hashPrefix),
- QSql.ParamTypeFlag.In | QSql.ParamTypeFlag.Binary)
+ QSql.ParamTypeFlag.In | QSql.ParamTypeFlag.Binary,
+ )
query.addBindValue(threatList.threatType)
query.addBindValue(threatList.platformType)
query.addBindValue(threatList.threatEntryType)
@@ -415,11 +420,11 @@
del query
finally:
db.commit()
-
+
def getThreatLists(self):
"""
Public method to get the available threat lists.
-
+
@return list of available threat lists
@rtype list of tuples of (ThreatList, str)
"""
@@ -428,37 +433,37 @@
FROM threat_list
"""
output = []
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
try:
query = QSqlQuery(db)
query.prepare(queryStr)
-
+
query.exec()
-
- while query.next(): # __IGNORE_WARNING_M523__
+
+ while query.next(): # __IGNORE_WARNING_M523__
threatType = query.value(0)
platformType = query.value(1)
threatEntryType = query.value(2)
clientState = query.value(3)
- threatList = ThreatList(threatType, platformType,
- threatEntryType)
+ threatList = ThreatList(threatType, platformType, threatEntryType)
output.append((threatList, clientState))
QCoreApplication.processEvents(
QEventLoop.ProcessEventsFlag.AllEvents,
- self.maxProcessEventsTime)
+ self.maxProcessEventsTime,
+ )
del query
finally:
db.commit()
-
+
return output
-
+
def addThreatList(self, threatList):
"""
Public method to add a threat list to the cache.
-
+
@param threatList threat list to be added
@type ThreatList
"""
@@ -467,7 +472,7 @@
(threat_type, platform_type, threat_entry_type, timestamp)
VALUES (?, ?, ?, current_timestamp)
"""
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
@@ -481,11 +486,11 @@
del query
finally:
db.commit()
-
+
def deleteThreatList(self, threatList):
"""
Public method to delete a threat list from the cache.
-
+
@param threatList threat list to be deleted
@type ThreatList
"""
@@ -493,7 +498,7 @@
DELETE FROM threat_list
WHERE threat_type=? AND platform_type=? AND threat_entry_type=?
"""
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
@@ -507,11 +512,11 @@
del query
finally:
db.commit()
-
+
def updateThreatListClientState(self, threatList, clientState):
"""
Public method to update the client state of a threat list.
-
+
@param threatList threat list to update the client state for
@type ThreatList
@param clientState new client state
@@ -521,7 +526,7 @@
UPDATE threat_list SET timestamp=current_timestamp, client_state=?
WHERE threat_type=? AND platform_type=? AND threat_entry_type=?
"""
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
@@ -536,12 +541,12 @@
del query
finally:
db.commit()
-
+
def hashPrefixListChecksum(self, threatList):
"""
Public method to calculate the SHA256 checksum for an alphabetically
sorted concatenated list of hash prefixes.
-
+
@param threatList threat list to calculate checksum for
@type ThreatList
@return SHA256 checksum
@@ -553,38 +558,38 @@
ORDER BY value
"""
checksum = None
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
- sha256Hash = QCryptographicHash(
- QCryptographicHash.Algorithm.Sha256)
+ sha256Hash = QCryptographicHash(QCryptographicHash.Algorithm.Sha256)
try:
query = QSqlQuery(db)
query.prepare(queryStr)
query.addBindValue(threatList.threatType)
query.addBindValue(threatList.platformType)
query.addBindValue(threatList.threatEntryType)
-
+
query.exec()
-
- while query.next(): # __IGNORE_WARNING_M523__
+
+ while query.next(): # __IGNORE_WARNING_M523__
sha256Hash.addData(query.value(0))
QCoreApplication.processEvents(
QEventLoop.ProcessEventsFlag.AllEvents,
- self.maxProcessEventsTime)
+ self.maxProcessEventsTime,
+ )
del query
finally:
db.commit()
-
+
checksum = bytes(sha256Hash.result())
-
+
return checksum
-
+
def populateHashPrefixList(self, threatList, prefixes):
"""
Public method to populate the hash prefixes for a threat list.
-
+
@param threatList threat list of the hash prefixes
@type ThreatList
@param prefixes list of hash prefixes to be inserted
@@ -596,7 +601,7 @@
timestamp)
VALUES (?, ?, ?, ?, ?, current_timestamp)
"""
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
@@ -606,7 +611,8 @@
query.prepare(queryStr)
query.addBindValue(
QByteArray(prefix),
- QSql.ParamTypeFlag.In | QSql.ParamTypeFlag.Binary)
+ QSql.ParamTypeFlag.In | QSql.ParamTypeFlag.Binary,
+ )
query.addBindValue(prefix[:4].hex())
query.addBindValue(threatList.threatType)
query.addBindValue(threatList.platformType)
@@ -615,15 +621,16 @@
del query
QCoreApplication.processEvents(
QEventLoop.ProcessEventsFlag.AllEvents,
- self.maxProcessEventsTime)
+ self.maxProcessEventsTime,
+ )
finally:
db.commit()
-
+
def getHashPrefixValuesToRemove(self, threatList, indexes):
"""
Public method to get the hash prefix values to be removed from the
cache.
-
+
@param threatList threat list to remove prefixes from
@type ThreatList
@param indexes list of indexes of prefixes to be removed
@@ -638,7 +645,7 @@
"""
indexes = set(indexes)
output = []
-
+
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
@@ -648,54 +655,50 @@
query.addBindValue(threatList.threatType)
query.addBindValue(threatList.platformType)
query.addBindValue(threatList.threatEntryType)
-
+
query.exec()
-
+
index = 0
- while query.next(): # __IGNORE_WARNING_M523__
+ while query.next(): # __IGNORE_WARNING_M523__
if index in indexes:
prefix = bytes(query.value(0))
output.append(prefix)
index += 1
QCoreApplication.processEvents(
QEventLoop.ProcessEventsFlag.AllEvents,
- self.maxProcessEventsTime)
+ self.maxProcessEventsTime,
+ )
del query
finally:
db.commit()
-
+
return output
-
+
def removeHashPrefixIndices(self, threatList, indexes):
"""
Public method to remove hash prefixes from the cache.
-
+
@param threatList threat list to delete hash prefixes of
@type ThreatList
@param indexes list of indexes of prefixes to be removed
@type list of int
"""
- queryStr = (
- """
+ queryStr = """
DELETE FROM hash_prefix
WHERE threat_type=? AND platform_type=? AND
threat_entry_type=? AND value IN ({0})
"""
- )
batchSize = 40
-
- prefixesToRemove = self.getHashPrefixValuesToRemove(
- threatList, indexes)
+
+ prefixesToRemove = self.getHashPrefixValuesToRemove(threatList, indexes)
if prefixesToRemove:
db = QSqlDatabase.database(self.__connectionName)
if db.isOpen():
db.transaction()
try:
for index in range(0, len(prefixesToRemove), batchSize):
- removeBatch = prefixesToRemove[
- index:(index + batchSize)
- ]
-
+ removeBatch = prefixesToRemove[index : (index + batchSize)]
+
query = QSqlQuery(db)
query.prepare(
queryStr.format(",".join(["?"] * len(removeBatch)))
@@ -706,15 +709,17 @@
for prefix in removeBatch:
query.addBindValue(
QByteArray(prefix),
- QSql.ParamTypeFlag.In |
- QSql.ParamTypeFlag.Binary)
+ QSql.ParamTypeFlag.In | QSql.ParamTypeFlag.Binary,
+ )
query.exec()
del query
QCoreApplication.processEvents(
QEventLoop.ProcessEventsFlag.AllEvents,
- self.maxProcessEventsTime)
+ self.maxProcessEventsTime,
+ )
finally:
db.commit()
+
#
# eflag: noqa = S608
