Mercurial Repositories > eric / file comparison
comparison: WebBrowser/SafeBrowsing/SafeBrowsingCache.py
WebBrowser/SafeBrowsing/SafeBrowsingCache.py
- branch
- safe_browsing
- changeset 5820
- b610cb5b501a
- parent 5819
- 69fa45e95673
- child 5829
- d3448873ced3
equal
deleted
inserted
replaced
| 5819:69fa45e95673 | 5820:b610cb5b501a |
|---|---|
| 12 # to QtSql. | 12 # to QtSql. |
| 13 # | 13 # |
| 14 # https://github.com/afilipovich/gglsbl | 14 # https://github.com/afilipovich/gglsbl |
| 15 # | 15 # |
| 16 | 16 |
| 17 from __future__ import unicode_literals | 17 from __future__ import unicode_literals, division |
| 18 | 18 |
| 19 import os | 19 import os |
| 20 import hashlib | 20 |
| 21 | 21 from PyQt5.QtCore import QObject, QByteArray, QCryptographicHash |
| 22 from PyQt5.QtCore import QObject | |
| 23 from PyQt5.QtSql import QSql, QSqlDatabase, QSqlQuery | 22 from PyQt5.QtSql import QSql, QSqlDatabase, QSqlQuery |
| 24 | 23 |
| 25 from .SafeBrowsingUtilities import toHex | 24 from .SafeBrowsingUtilities import toHex |
| 26 | 25 |
| 27 | 26 |
| 73 | 72 |
| 74 @return printable representation | 73 @return printable representation |
| 75 @rtype str | 74 @rtype str |
| 76 """ | 75 """ |
| 77 return '/'.join(self.asTuple()) | 76 return '/'.join(self.asTuple()) |
| 77 | |
| 78 | |
| 79 class HashPrefixList(object): | |
| 80 """ | |
| 81 Class implementing a container for threat list data. | |
| 82 """ | |
| 83 def __init__(self, prefixLength, rawHashes): | |
| 84 """ | |
| 85 Constructor | |
| 86 | |
| 87 @param prefixLength length of each hash prefix | |
| 88 @type int | |
| 89 @param rawHashes raw hash prefixes of given length concatenated and | |
| 90 sorted in lexicographical order | |
| 91 @type str | |
| 92 """ | |
| 93 self.__prefixLength = prefixLength | |
| 94 self.__rawHashes = rawHashes | |
| 95 | |
| 96 def __len__(self): | |
| 97 """ | |
| 98 Special method to calculate the number of entries. | |
| 99 | |
| 100 @return length | |
| 101 @rtype int | |
| 102 """ | |
| 103 return len(self.__rawHashes) // self.__prefixLength | |
| 104 | |
| 105 def __iter__(self): | |
| 106 """ | |
| 107 Special method to iterate over the raw hashes. | |
| 108 | |
| 109 @return iterator object | |
| 110 @rtype iterator | |
| 111 """ | |
| 112 n = self.__prefixLength | |
| 113 return (self.__rawHashes[index:index + n] | |
| 114 for index in range(0, len(self.__rawHashes), n) | |
| 115 ) | |
| 78 | 116 |
| 79 | 117 |
| 80 class SafeBrowsingCache(QObject): | 118 class SafeBrowsingCache(QObject): |
| 81 """ | 119 """ |
| 82 Class implementing a cache for Google Safe Browsing. | 120 Class implementing a cache for Google Safe Browsing. |
| 245 try: | 283 try: |
| 246 query = QSqlQuery(db) | 284 query = QSqlQuery(db) |
| 247 query.prepare( | 285 query.prepare( |
| 248 queryStr.format(",".join(["?" * len(hashValues)]))) | 286 queryStr.format(",".join(["?" * len(hashValues)]))) |
| 249 for hashValue in hashValues: | 287 for hashValue in hashValues: |
| 250 query.addBindValue(hashValue, QSql.In | QSql.Binary) | 288 query.addBindValue(QByteArray(hashValue), |
| 289 QSql.In | QSql.Binary) | |
| 251 | 290 |
| 252 query.exec_() | 291 query.exec_() |
| 253 | 292 |
| 254 while query.next(): | 293 while query.next(): |
| 255 threatType = query.value(0) | 294 threatType = query.value(0) |
| 297 while query.next(): | 336 while query.next(): |
| 298 fullHash = bytes(query.value(0)) | 337 fullHash = bytes(query.value(0)) |
| 299 threatType = query.value(1) | 338 threatType = query.value(1) |
| 300 platformType = query.value(2) | 339 platformType = query.value(2) |
| 301 threatEntryType = query.value(3) | 340 threatEntryType = query.value(3) |
| 302 negativeCacheExpired = query.value(4) | 341 negativeCacheExpired = query.value(4) # TODO: check if bool |
| 303 threatList = ThreatList(threatType, platformType, | 342 threatList = ThreatList(threatType, platformType, |
| 304 threatEntryType) | 343 threatEntryType) |
| 305 output.append((threatList, fullHash, negativeCacheExpired)) | 344 output.append((threatList, fullHash, negativeCacheExpired)) |
| 306 del query | 345 del query |
| 307 finally: | 346 finally: |
| 341 if db.isOpen(): | 380 if db.isOpen(): |
| 342 db.transaction() | 381 db.transaction() |
| 343 try: | 382 try: |
| 344 query = QSqlQuery(db) | 383 query = QSqlQuery(db) |
| 345 query.prepare(insertQueryStr) | 384 query.prepare(insertQueryStr) |
| 346 query.addBindValue(hashValue, QSql.In | QSql.Binary) | 385 query.addBindValue(QByteArray(hashValue), |
| 386 QSql.In | QSql.Binary) | |
| 347 query.addBindValue(threatList.threatType) | 387 query.addBindValue(threatList.threatType) |
| 348 query.addBindValue(threatList.platformType) | 388 query.addBindValue(threatList.platformType) |
| 349 query.addBindValue(threatList.threatEntryType) | 389 query.addBindValue(threatList.threatEntryType) |
| 350 query.addBindValue(malwareThreatType) | 390 query.addBindValue(malwareThreatType) |
| 351 query.exec_() | 391 query.exec_() |
| 352 del query | 392 del query |
| 353 | 393 |
| 354 query = QSqlQuery(db) | 394 query = QSqlQuery(db) |
| 355 query.prepare(updateQueryStr) | 395 query.prepare(updateQueryStr) |
| 356 query.addBindValue(hashValue, QSql.In | QSql.Binary) | 396 query.addBindValue(QByteArray(hashValue), |
| 397 QSql.In | QSql.Binary) | |
| 357 query.addBindValue(threatList.threatType) | 398 query.addBindValue(threatList.threatType) |
| 358 query.addBindValue(threatList.platformType) | 399 query.addBindValue(threatList.platformType) |
| 359 query.addBindValue(threatList.threatEntryType) | 400 query.addBindValue(threatList.threatEntryType) |
| 360 query.exec_() | 401 query.exec_() |
| 361 del query | 402 del query |
| 436 if db.isOpen(): | 477 if db.isOpen(): |
| 437 db.transaction() | 478 db.transaction() |
| 438 try: | 479 try: |
| 439 query = QSqlQuery(db) | 480 query = QSqlQuery(db) |
| 440 query.prepare(queryStr) | 481 query.prepare(queryStr) |
| 441 query.addBindValue(hashPrefix, QSql.In | QSql.Binary) | 482 query.addBindValue(QByteArray(hashPrefix), |
| 483 QSql.In | QSql.Binary) | |
| 442 query.addBindValue(threatList.threatType) | 484 query.addBindValue(threatList.threatType) |
| 443 query.addBindValue(threatList.platformType) | 485 query.addBindValue(threatList.platformType) |
| 444 query.addBindValue(threatList.threatEntryType) | 486 query.addBindValue(threatList.threatEntryType) |
| 445 query.exec_() | 487 query.exec_() |
| 446 del query | 488 del query |
| 512 | 554 |
| 513 def deleteThreatList(self, threatList): | 555 def deleteThreatList(self, threatList): |
| 514 """ | 556 """ |
| 515 Public method to delete a threat list from the cache. | 557 Public method to delete a threat list from the cache. |
| 516 | 558 |
| 517 @param threatlist threat list to be deleted | 559 @param threatList threat list to be deleted |
| 518 @type ThreatList | 560 @type ThreatList |
| 519 """ | 561 """ |
| 520 queryStr = """ | 562 queryStr = """ |
| 521 DELETE FROM threat_list | 563 DELETE FROM threat_list |
| 522 WHERE threat_type=? AND platform_type=? AND threat_entry_type=? | 564 WHERE threat_type=? AND platform_type=? AND threat_entry_type=? |
| 583 checksum = None | 625 checksum = None |
| 584 | 626 |
| 585 db = QSqlDatabase.database(self.__connectionName) | 627 db = QSqlDatabase.database(self.__connectionName) |
| 586 if db.isOpen(): | 628 if db.isOpen(): |
| 587 db.transaction() | 629 db.transaction() |
| 588 allHashes = b"" | 630 hash = QCryptographicHash(QCryptographicHash.Sha256) |
| 589 try: | 631 try: |
| 590 query = QSqlQuery(db) | 632 query = QSqlQuery(db) |
| 591 query.prepare(queryStr) | 633 query.prepare(queryStr) |
| 592 query.addBindValue(threatList.threatType) | 634 query.addBindValue(threatList.threatType) |
| 593 query.addBindValue(threatList.platformType) | 635 query.addBindValue(threatList.platformType) |
| 594 query.addBindValue(threatList.threatEntryType) | 636 query.addBindValue(threatList.threatEntryType) |
| 595 | 637 |
| 596 query.exec_() | 638 query.exec_() |
| 597 | 639 |
| 598 while query.next(): | 640 while query.next(): |
| 599 allHashes += bytes(query.value(0)) | 641 hash.addData(query.value(0)) |
| 600 del query | 642 del query |
| 601 finally: | 643 finally: |
| 602 db.commit() | 644 db.commit() |
| 603 | 645 |
| 604 checksum = hashlib.sha256(allHashes).digest() | 646 checksum = bytes(hash.result()) |
| 605 | 647 |
| 606 return checksum | 648 return checksum |
| 607 | 649 |
| 608 def populateHashPrefixList(self, threatList, prefixes): | 650 def populateHashPrefixList(self, threatList, prefixes): |
| 609 """ | 651 """ |
| 610 Public method to populate the hash prefixes for a threat list. | 652 Public method to populate the hash prefixes for a threat list. |
| 611 | 653 |
| 612 @param threatList threat list of the hash prefixes | 654 @param threatList threat list of the hash prefixes |
| 613 @type ThreatList | 655 @type ThreatList |
| 614 @param prefixes hash prefixes to be inserted | 656 @param prefixes list of hash prefixes to be inserted |
| 615 @type bytes | 657 @type HashPrefixList |
| 616 """ | 658 """ |
| 617 queryStr = """ | 659 queryStr = """ |
| 618 INSERT INTO hash_prefix | 660 INSERT INTO hash_prefix |
| 619 (value, cue, threat_type, platform_type, threat_entry_type, | 661 (value, cue, threat_type, platform_type, threat_entry_type, |
| 620 timestamp) | 662 timestamp) |
| 626 db.transaction() | 668 db.transaction() |
| 627 try: | 669 try: |
| 628 for prefix in prefixes: | 670 for prefix in prefixes: |
| 629 query = QSqlQuery(db) | 671 query = QSqlQuery(db) |
| 630 query.prepare(queryStr) | 672 query.prepare(queryStr) |
| 631 query.addBindValue(prefix, QSql.In | QSql.Binary) | 673 query.addBindValue(QByteArray(prefix), |
| 674 QSql.In | QSql.Binary) | |
| 632 query.addBindValue(toHex(prefix[:4])) | 675 query.addBindValue(toHex(prefix[:4])) |
| 633 query.addBindValue(threatList.threatType) | 676 query.addBindValue(threatList.threatType) |
| 634 query.addBindValue(threatList.platformType) | 677 query.addBindValue(threatList.platformType) |
| 635 query.addBindValue(threatList.threatEntryType) | 678 query.addBindValue(threatList.threatEntryType) |
| 636 query.exec_() | 679 query.exec_() |
| 715 ) | 758 ) |
| 716 query.addBindValue(threatList.threatType) | 759 query.addBindValue(threatList.threatType) |
| 717 query.addBindValue(threatList.platformType) | 760 query.addBindValue(threatList.platformType) |
| 718 query.addBindValue(threatList.threatEntryType) | 761 query.addBindValue(threatList.threatEntryType) |
| 719 for prefix in removeBatch: | 762 for prefix in removeBatch: |
| 720 query.addBindValue(prefix, QSql.In | QSql.Binary) | 763 query.addBindValue(QByteArray(prefix), |
| 764 QSql.In | QSql.Binary) | |
| 721 query.exec_() | 765 query.exec_() |
| 722 del query | 766 del query |
| 723 finally: | 767 finally: |
| 724 db.commit() | 768 db.commit() |
