Mercurial Repositories > eric / file comparison
comparison: WebBrowser/SafeBrowsing/SafeBrowsingCache.py
WebBrowser/SafeBrowsing/SafeBrowsingCache.py
- branch
- safe_browsing
- changeset 5817
- a5f6c9128500
- child 5818
- cae9956be67e
equal
deleted
inserted
replaced
| 5816:93c74269d59e | 5817:a5f6c9128500 |
|---|---|
| 1 # -*- coding: utf-8 -*- | |
| 2 | |
| 3 # Copyright (c) 2017 Detlev Offenbach <detlev@die-offenbachs.de> | |
| 4 # | |
| 5 | |
| 6 """ | |
| 7 Module implementing a cache for Google Safe Browsing. | |
| 8 """ | |
| 9 | |
| 10 # | |
| 11 # Some part of this code were ported from gglsbl.storage and adapted | |
| 12 # to QtSql. | |
| 13 # | |
| 14 # https://github.com/afilipovich/gglsbl | |
| 15 # | |
| 16 | |
| 17 from __future__ import unicode_literals | |
| 18 | |
| 19 import os | |
| 20 | |
| 21 from PyQt5.QtCore import QObject | |
| 22 from PyQt5.QtSql import QSqlDatabase, QSqlQuery | |
| 23 | |
| 24 | |
| 25 class ThreatList(object): | |
| 26 """ | |
| 27 Class implementing the threat list info. | |
| 28 """ | |
| 29 def __init__(self, threatType, platformType, threatEntryType): | |
| 30 """ | |
| 31 Constructor | |
| 32 | |
| 33 @param threatType threat type | |
| 34 @type str | |
| 35 @param platformType platform type | |
| 36 @type str | |
| 37 @param threatEntryType threat entry type | |
| 38 @type str | |
| 39 """ | |
| 40 self.__threatType = threatType | |
| 41 self.__platformType = platformType | |
| 42 self.__threatEntryType = threatEntryType | |
| 43 | |
| 44 @classmethod | |
| 45 def fromApiEntry(cls, entry): | |
| 46 """ | |
| 47 Class method to instantiate a threat list given a threat list entry | |
| 48 dictionary. | |
| 49 | |
| 50 @param entry threat list entry dictionary | |
| 51 @type dict | |
| 52 @return instantiated object | |
| 53 @rtype ThreatList | |
| 54 """ | |
| 55 return cls(entry['threatType'], entry['platformType'], | |
| 56 entry['threatEntryType']) | |
| 57 | |
| 58 def asTuple(self): | |
| 59 """ | |
| 60 Public method to convert the object to a tuple. | |
| 61 | |
| 62 @return tuple containing the threat list info | |
| 63 @rtype tuple of (str, str, str) | |
| 64 """ | |
| 65 return (self.__threatType, self.platformType, self.__threatEntryType) | |
| 66 | |
| 67 def __repr__(self): | |
| 68 """ | |
| 69 Special method to generate a printable representation. | |
| 70 | |
| 71 @return printable representation | |
| 72 @rtype str | |
| 73 """ | |
| 74 return '/'.join(self.asTuple()) | |
| 75 | |
| 76 | |
| 77 class SafeBrowsingCache(QObject): | |
| 78 """ | |
| 79 Class implementing a cache for Google Safe Browsing. | |
| 80 """ | |
| 81 | |
| 82 create_threat_list_stmt = """ | |
| 83 CREATE TABLE threat_list | |
| 84 (threat_type character varying(128) NOT NULL, | |
| 85 platform_type character varying(128) NOT NULL, | |
| 86 threat_entry_type character varying(128) NOT NULL, | |
| 87 client_state character varying(42), | |
| 88 timestamp timestamp without time zone DEFAULT current_timestamp, | |
| 89 PRIMARY KEY (threat_type, platform_type, threat_entry_type) | |
| 90 ) | |
| 91 """ | |
| 92 drop_threat_list_stmt = """DROP TABLE IF EXISTS threat_list""" | |
| 93 | |
| 94 create_full_hashes_stmt = """ | |
| 95 CREATE TABLE full_hash | |
| 96 (value BLOB NOT NULL, | |
| 97 threat_type character varying(128) NOT NULL, | |
| 98 platform_type character varying(128) NOT NULL, | |
| 99 threat_entry_type character varying(128) NOT NULL, | |
| 100 downloaded_at timestamp without time zone DEFAULT current_timestamp, | |
| 101 expires_at timestamp without time zone | |
| 102 NOT NULL DEFAULT current_timestamp, | |
| 103 malware_threat_type varchar(32), | |
| 104 PRIMARY KEY (value, threat_type, platform_type, threat_entry_type) | |
| 105 ) | |
| 106 """ | |
| 107 drop_full_hashes_stmt = """DROP TABLE IF EXISTS full_hash""" | |
| 108 | |
| 109 create_hash_prefix_stmt = """ | |
| 110 CREATE TABLE hash_prefix | |
| 111 (value BLOB NOT NULL, | |
| 112 cue character varying(4) NOT NULL, | |
| 113 threat_type character varying(128) NOT NULL, | |
| 114 platform_type character varying(128) NOT NULL, | |
| 115 threat_entry_type character varying(128) NOT NULL, | |
| 116 timestamp timestamp without time zone DEFAULT current_timestamp, | |
| 117 negative_expires_at timestamp without time zone | |
| 118 NOT NULL DEFAULT current_timestamp, | |
| 119 PRIMARY KEY (value, threat_type, platform_type, threat_entry_type), | |
| 120 FOREIGN KEY(threat_type, platform_type, threat_entry_type) | |
| 121 REFERENCES threat_list(threat_type, platform_type, threat_entry_type) | |
| 122 ON DELETE CASCADE | |
| 123 ) | |
| 124 """ | |
| 125 drop_hash_prefix_stmt = """DROP TABLE IF EXISTS hash_prefix""" | |
| 126 | |
| 127 create_full_hash_cue_idx = """ | |
| 128 CREATE INDEX idx_hash_prefix_cue ON hash_prefix (cue) | |
| 129 """ | |
| 130 drop_full_hash_cue_idx = """DROP INDEX IF EXISTS idx_hash_prefix_cue""" | |
| 131 | |
| 132 create_full_hash_expires_idx = """ | |
| 133 CREATE INDEX idx_full_hash_expires_at ON full_hash (expires_at) | |
| 134 """ | |
| 135 drop_full_hash_expires_idx = """ | |
| 136 DROP INDEX IF EXISTS idx_full_hash_expires_at | |
| 137 """ | |
| 138 | |
| 139 create_full_hash_value_idx = """ | |
| 140 CREATE INDEX idx_full_hash_value ON full_hash (value) | |
| 141 """ | |
| 142 drop_full_hash_value_idx = """DROP INDEX IF EXISTS idx_full_hash_value""" | |
| 143 | |
| 144 def __init__(self, dbPath, parent=None): | |
| 145 """ | |
| 146 Constructor | |
| 147 | |
| 148 @param dbPath path to store the cache DB into | |
| 149 @type str | |
| 150 @param parent reference to the parent object | |
| 151 @type QObject | |
| 152 """ | |
| 153 super(SafeBrowsingCache, self).__init__(parent) | |
| 154 | |
| 155 self.__connectionName = "SafeBrowsingCache" | |
| 156 | |
| 157 if not os.path.exists(dbPath): | |
| 158 os.makedirs(dbPath) | |
| 159 | |
| 160 self.__dbFileName = os.path.join(dbPath, "SafeBrowsingCache.db") | |
| 161 preparationNeeded = not os.path.exists(self.__dbFileName) | |
| 162 | |
| 163 self.__openCacheDb() | |
| 164 if preparationNeeded: | |
| 165 self.__prepareCacheDb() | |
| 166 | |
| 167 def __openCacheDb(self): | |
| 168 """ | |
| 169 Private method to open the cache database. | |
| 170 | |
| 171 @return flag indicating the open state | |
| 172 @rtype bool | |
| 173 """ | |
| 174 db = QSqlDatabase.database(self.__connectionName, False) | |
| 175 if not db.isValid(): | |
| 176 # the database connection is a new one | |
| 177 db = QSqlDatabase.addDatabase("QSQLITE", self.__connectionName) | |
| 178 db.setDatabaseName(self.__dbFileName) | |
| 179 opened = db.open() | |
| 180 if not opened: | |
| 181 QSqlDatabase.removeDatabase(self.__connectionName) | |
| 182 else: | |
| 183 opened = True | |
| 184 return opened | |
| 185 | |
| 186 def __prepareCacheDb(self): | |
| 187 """ | |
| 188 Private method to prepare the cache database. | |
| 189 """ | |
| 190 db = QSqlDatabase.database(self.__connectionName) | |
| 191 db.transaction() | |
| 192 try: | |
| 193 query = QSqlQuery(db) | |
| 194 # step 1: drop old tables | |
| 195 query.exec_(self.drop_threat_list_stmt) | |
| 196 query.exec_(self.drop_full_hashes_stmt) | |
| 197 query.exec_(self.drop_hash_prefix_stmt) | |
| 198 # step 2: drop old indices | |
| 199 query.exec_(self.drop_full_hash_cue_idx) | |
| 200 query.exec_(self.drop_full_hash_expires_idx) | |
| 201 query.exec_(self.drop_full_hash_value_idx) | |
| 202 # step 3: create tables | |
| 203 query.exec_(self.create_threat_list_stmt) | |
| 204 query.exec_(self.create_full_hashes_stmt) | |
| 205 query.exec_(self.create_hash_prefix_stmt) | |
| 206 # step 4: create indices | |
| 207 query.exec_(self.create_full_hash_cue_idx) | |
| 208 query.exec_(self.create_full_hash_expires_idx) | |
| 209 query.exec_(self.create_full_hash_value_idx) | |
| 210 finally: | |
| 211 del query | |
| 212 db.commit() | |
| 213 | |
| 214 def lookupFullHashes(self, hashValues): | |
| 215 """ | |
| 216 Public method to get a list of threat lists and expiration flag | |
| 217 for the given hashes if a hash is blacklisted. | |
| 218 | |
| 219 @param hashValues list of hash values to look up | |
| 220 @type list of bytes | |
| 221 @return list of tuples containing the threat list info and the | |
| 222 expiration flag | |
| 223 @rtype list of tuple of (ThreatList, bool) | |
| 224 """ | |
| 225 queryStr = """ | |
| 226 SELECT threat_type, platform_type, threat_entry_type, | |
| 227 expires_at < current_timestamp AS has_expired | |
| 228 FROM full_hash WHERE value IN ({}) | |
| 229 """ | |
| 230 output = [] | |
| 231 | |
| 232 db = QSqlDatabase.database(self.__connectionName) | |
| 233 if db.isOpen(): | |
| 234 db.transaction() | |
| 235 try: | |
| 236 query = QSqlQuery(db) | |
| 237 query.prepare( | |
| 238 queryStr.format(",".join(["?" * len(hashValues)]))) | |
| 239 for hashValue in hashValues: | |
| 240 query.addBindValue(hashValue) | |
| 241 | |
| 242 query.exec_() | |
| 243 | |
| 244 while query.next(): | |
| 245 threatType = query.value(0) | |
| 246 platformType = query.value(1) | |
| 247 threatEntryType = query.value(2) | |
| 248 hasExpired = query.value(3) # TODO: check if bool | |
| 249 threatList = ThreatList(threatType, platformType, | |
| 250 threatEntryType) | |
| 251 output.append((threatList, hasExpired)) | |
| 252 del query | |
| 253 finally: | |
| 254 db.commit() | |
| 255 | |
| 256 return output | |
| 257 | |
| 258 def lookupHashPrefix(self, prefixes): | |
| 259 """ | |
| 260 Public method to look up hash prefixes in the local cache. | |
| 261 | |
| 262 @param prefix hash prefix to look up | |
| 263 @type list of bytes | |
| 264 @return list of tuples containing the threat list, full hash and | |
| 265 negative cache expiration flag | |
| 266 @rtype list of tuple of (ThreatList, bytes, bool) | |
| 267 """ | |
| 268 queryStr = """ | |
| 269 SELECT value,threat_type,platform_type,threat_entry_type, | |
| 270 negative_expires_at < current_timestamp AS negative_cache_expired | |
| 271 FROM hash_prefix WHERE cue IN ({}) | |
| 272 """ | |
| 273 output = [] | |
| 274 | |
| 275 db = QSqlDatabase.database(self.__connectionName) | |
| 276 if db.isOpen(): | |
| 277 db.transaction() | |
| 278 try: | |
| 279 query = QSqlQuery(db) | |
| 280 query.prepare( | |
| 281 queryStr.format(",".join(["?" * len(prefixes)]))) | |
| 282 for prefix in prefixes: | |
| 283 query.addBindValue(prefix) | |
| 284 | |
| 285 query.exec_() | |
| 286 | |
| 287 while query.next(): | |
| 288 fullHash = bytes(query.value(0)) | |
| 289 threatType = query.value(1) | |
| 290 platformType = query.value(2) | |
| 291 threatEntryType = query.value(3) | |
| 292 negativeCacheExpired = query.value(4) | |
| 293 threatList = ThreatList(threatType, platformType, | |
| 294 threatEntryType) | |
| 295 output.append((threatList, fullHash, negativeCacheExpired)) | |
| 296 del query | |
| 297 finally: | |
| 298 db.commit() | |
| 299 | |
| 300 return output |
