action.py 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114
  1. import sys
  2. sys.path.append('/filters-maker')
  3. import os
  4. import crawler
  5. import maker
  6. import duplicat
  7. incoming = "/reprwiki/Private-build/ucate/domains.txt"
  8. excluded = "/repros/Resources/excluded.txt"
  9. rpz_locat = "/reprwiki/Private-build/ucate/rpz.txt"
  10. hosts_locat = "/reprwiki/Private-build/ucate/hosts.txt"
  11. abp_locat = "/reprwiki/Private-build/ucate/adblock.txt"
  12. unb_locat = "/reprwiki/Private-build/ucate/unbound.conf"
  13. dnq_locat = "/reprwiki/Private-build/ucate/dnsmasq.conf"
  14. Version = "UCATE"
  15. os.makedirs('/reprwiki/Private-build/ucate',exist_ok=True)
  16. UCATE_SOURCE = [
  17. 'https://badmojr.github.io/1Hosts/Pro/rpz.txt',
  18. 'https://github.com/minoplhy/filters/releases/download/filters-build/Adguard-dns_rpz.txt',
  19. 'https://github.com/minoplhy/filters/releases/download/filters-build/stevenblack-f_rpz.txt',
  20. 'https://github.com/minoplhy/filters/releases/download/filters-build/someonewhocares_rpz.txt',
  21. 'https://github.com/minoplhy/filters/releases/download/filters-build/hostsVN-all_rpz.txt',
  22. 'https://github.com/minoplhy/filters/raw/main/Resources/blocked.txt',
  23. 'https://block.energized.pro/basic/formats/filter',
  24. 'https://hblock.molinero.dev/hosts_unbound.conf',
  25. 'https://github.com/minoplhy/filters/releases/download/filters-build/Adguard-Base_rpz.txt',
  26. 'https://github.com/minoplhy/filters/releases/download/filters-build/Adguard-Mobile_rpz.txt',
  27. 'https://github.com/yous/YousList/raw/master/youslist.txt',
  28. 'https://github.com/uBlockOrigin/uAssets/raw/master/filters/filters.txt',
  29. 'https://github.com/uBlockOrigin/uAssets/raw/master/filters/filters-2020.txt',
  30. 'https://github.com/uBlockOrigin/uAssets/raw/master/filters/filters-2021.txt',
  31. 'https://github.com/uBlockOrigin/uAssets/raw/master/filters/filters-2022.txt'
  32. ]
  33. crawler.clear_old_files(incoming)
  34. crawler.download_group_filters(UCATE_SOURCE ,incoming)
  35. crawler.filtering(incoming)
  36. crawler.filteringcon(incoming)
  37. crawler.killingdup(incoming)
  38. crawler.IP_URL_FILTERING(incoming)
  39. crawler.excluded(excluded, incoming)
  40. crawler.blankremover(incoming)
  41. crawler.sort(incoming)
  42. duplicat.add_file(incoming, "/reprwiki/Private-build/ruleset.txt")
  43. UCATEline = maker.linecounter(incoming)
  44. maker.RPZBlocklist(excluded, incoming, rpz_locat ,Version)
  45. maker.HOSTBlocklist(excluded, incoming, hosts_locat ,Version)
  46. maker.ABPBlocklist(excluded, incoming, abp_locat ,Version)
  47. maker.UNBOUNDBlocklist(excluded, incoming, unb_locat ,Version)
  48. maker.DNSMASQBlocklist(excluded, incoming, dnq_locat ,Version)
  49. maker.DOMAINBlocklist(excluded, incoming ,Version)
  50. incoming = "/reprwiki/Private-build/veneto/domains.txt"
  51. excluded = "/repros/Resources/excluded.txt"
  52. rpz_locat = "/reprwiki/Private-build/veneto/rpz.txt"
  53. hosts_locat = "/reprwiki/Private-build/veneto/hosts.txt"
  54. abp_locat = "/reprwiki/Private-build/veneto/adblock.txt"
  55. unb_locat = "/reprwiki/Private-build/veneto/unbound.conf"
  56. dnq_locat = "/reprwiki/Private-build/veneto/dnsmasq.conf"
  57. Version = "VENETO"
  58. os.makedirs('/reprwiki/Private-build/veneto',exist_ok=True)
  59. VENETO_SOURCE = [
  60. 'https://blokada.org/mirror/v5/exodusprivacy/standard/hosts.txt',
  61. 'https://github.com/crazy-max/WindowsSpyBlocker/raw/master/data/hosts/spy.txt',
  62. 'https://energized.pro/extensions/xtreme/formats/rpz.txt',
  63. 'https://phishing.army/download/phishing_army_blocklist_extended.txt',
  64. 'https://github.com/minoplhy/filters/releases/download/filters-build/Adguard-Tracking_rpz.txt',
  65. 'https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt',
  66. 'https://github.com/DandelionSprout/adfilt/raw/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareDomains.txt',
  67. 'https://github.com/minoplhy/filters/releases/download/filters-build/Adguard-cname-tracker_rpz.txt',
  68. 'https://github.com/minoplhy/filters/releases/download/filters-build/Adguard-cname-original_rpz.txt',
  69. 'https://urlhaus.abuse.ch/downloads/rpz/',
  70. 'https://github.com/uBlockOrigin/uAssets/raw/master/filters/privacy.txt',
  71. 'https://github.com/mitchellkrogza/Phishing.Database/raw/master/phishing-domains-ACTIVE.txt'
  72. ]
  73. crawler.clear_old_files(incoming)
  74. crawler.download_group_filters(VENETO_SOURCE ,incoming)
  75. crawler.filtering(incoming)
  76. crawler.filteringcon(incoming)
  77. crawler.killingdup(incoming)
  78. crawler.IP_URL_FILTERING(incoming)
  79. crawler.excluded(excluded, incoming)
  80. crawler.blankremover(incoming)
  81. crawler.sort(incoming)
  82. duplicat.check_n_kill_dupes("/reprwiki/Private-build/ruleset.txt", incoming)
  83. duplicat.add_file(incoming, "/reprwiki/Private-build/ruleset.txt")
  84. VENETOline = maker.linecounter(incoming)
  85. maker.RPZBlocklist(excluded, incoming, rpz_locat ,Version)
  86. maker.HOSTBlocklist(excluded, incoming, hosts_locat ,Version)
  87. maker.ABPBlocklist(excluded, incoming, abp_locat ,Version)
  88. maker.UNBOUNDBlocklist(excluded, incoming, unb_locat ,Version)
  89. maker.DNSMASQBlocklist(excluded, incoming, dnq_locat ,Version)
  90. maker.DOMAINBlocklist(excluded, incoming ,Version)
  91. excluded = "/repros/Resources/excluded.txt"
  92. os.makedirs('/reprwiki/Private-build/Allowlist',exist_ok=True)
  93. Version = "Allowlist"
  94. rpz_locat = "/reprwiki/Private-build/Allowlist/rpz.txt"
  95. abp_locat = "/reprwiki/Private-build/Allowlist/adblock.txt"
  96. domains_locat = "/reprwiki/Private-build/Allowlist/domains.txt"
  97. ALLOWLISTline = maker.linecounter(excluded)
  98. maker.RPZAllowlist(excluded, rpz_locat ,Version)
  99. maker.ABPAllowlist(excluded, abp_locat ,Version)
  100. maker.DOMAINAllowlist(excluded ,domains_locat ,Version)
  101. import version
  102. het = "/repros/version.md"
  103. addition = "\n# Rule Counter \n ### UCATE : " + str(UCATEline) +"\n\n ### Veneto : " + str(VENETOline) + "\n\n ### Allowlist : " + str(ALLOWLISTline)
  104. version.build(het, addition)