endlessh_scoreboard.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487
  1. #!/usr/bin/env python3
  2. ## Written by demure (demuredemeanor)
  3. ## This tool makes generates a static html page for showing an endlessh scoreboard.
  4. ## This program depends on ... to work
  5. ## TODO: Add flag to exclude IPs
  6. ## TODO: Add flag to disable GeoIp and sleep
  7. ## TODO: Add flag to change log location
  8. ## TODO: Add flag to change output file
  9. ## TODO: Use config file
  10. ## TODO: Sort default output, maybe by Org
  11. ## NOTE: best point to apply is the html table string
  12. ### Imports ### {{{
  13. ## Imports
  14. import sys ## Need for argv
  15. import os ## For file open
  16. import datetime ## For log line time and math
  17. import re ## For RegEx, <3
  18. import time ## For sleep between geoip lookups
  19. import inspect ## For cleaning up multi line strings
  20. import hashlib ## For hiding the IPs
  21. import requests ## For curl type requests
  22. ### End Imports ### }}}
  23. ### f_open_log_file ### {{{
  24. def f_open_log_file():
  25. Log_Location = '/var/log/' ## Declare log path
  26. Log_Name = 'endlessh.log' ## Declare log name
  27. ## Open file
  28. for filename in os.listdir(Log_Location):
  29. if filename == Log_Name:
  30. fh = open(os.path.join(Log_Location, Log_Name), "r")
  31. FilterArray = f_parse_log(fh) ## Parse log and store
  32. # FilterArray.sort() ## Sort array to make next part easier
  33. fh.close() ## Clean up and close file
  34. return(FilterArray) ## Return FilterArray
  35. ### End f_open_log_file ### }}}
  36. ### f_parse_log ### {{{
  37. def f_parse_log(fh):
  38. ## Declare variables
  39. CutOff = int(datetime.datetime.utcnow().strftime("%s")) - int(datetime.timedelta(days=7).total_seconds())
  40. LineParse = []
  41. LineHost = ""
  42. LineTime = ""
  43. FilterArray = []
  44. for x in fh: ## Iterate over file
  45. LineParse = x.split() ## Split line into array
  46. if LineParse[1] == "CLOSE": ## Only process CLOSE lines
  47. ## Convert line timestamp to epoch
  48. LineTime = int(datetime.datetime.strptime(LineParse[0], '%Y-%m-%dT%H:%M:%S.%fZ').strftime("%s"))
  49. if LineTime > CutOff: ## Discard lines that are too old
  50. if re.match('host=::ffff:', LineParse[2]) is not None:
  51. LineHost = re.sub('host=::ffff:', "", LineParse[2]) ## Clean up IPv4 host
  52. else:
  53. LineHost = re.sub('host=', "", LineParse[2]) ## Clean up IPv6 host
  54. LineTime = re.sub('time=', "", LineParse[5]) ## Clean up time
  55. FilterArray.append((LineHost, LineTime)) ## Add to Array
  56. return(FilterArray) ## Return FilterArray
  57. ### End f_parse_log ###}}}
  58. ### f_dict_computation ### {{{
  59. def f_dict_computation(FilterArray):
  60. ## Declare variables
  61. HostDict = dict()
  62. Host = ""
  63. HostTime = ""
  64. GeoLookup = ""
  65. for x in FilterArray:
  66. Host = x[0]
  67. HostTime = float(x[1])
  68. if Host in HostDict: ## Test if Host already in HostDict
  69. HostDict[Host]["count"] += 1
  70. if HostTime < HostDict[Host]["min"]: ## Replace min if new HostTime is less
  71. HostDict[Host]["min"] = HostTime
  72. if HostTime > HostDict[Host]["max"]: ## Replace max if new HostTime is more
  73. HostDict[Host]["max"] = HostTime
  74. HostDict[Host]["sum"] += HostTime ## Add current HostTime to sum
  75. HostDict[Host]["ave"] = HostDict[Host]["sum"] / HostDict[Host]["count"]
  76. else: ## Initialize if not already in HostDict
  77. HostDict[Host] = dict()
  78. HostDict[Host]["count"] = 1
  79. HostDict[Host]["min"] = HostTime
  80. HostDict[Host]["max"] = HostTime
  81. HostDict[Host]["sum"] = HostTime
  82. HostDict[Host]["ave"] = HostTime / 1
  83. ### GeoIp lookup ### {{{
  84. ## Get newline output, splitline into array
  85. ## NOTE: Delay must be enabled if real lookup enabled!
  86. # time.sleep(0.5) ## Add delay to not exceed lookup limit
  87. time.sleep(1) ## Add delay to not exceed lookup limit
  88. GeoLookup = requests.get('http://ip-api.com/line/' + Host + '?fields=status,message,country,regionName,org').text.splitlines()
  89. # GeoLookup = ['success', 'DEBUG_Country', 'DEBUG_State', 'DEBUG_Org'] ## DEBUG pass
  90. # GeoLookup = ['fail', 'Reserved'] ## DEBUG fail
  91. ## Check for lack of response
  92. if not bool(GeoLookup) == False:
  93. ## Check for successful lookup
  94. if GeoLookup[0] == 'success':
  95. HostDict[Host]["country"] = GeoLookup[1]
  96. HostDict[Host]["region"] = GeoLookup[2]
  97. HostDict[Host]["org"] = GeoLookup[3]
  98. else:
  99. ## If lookup had issues, set null
  100. HostDict[Host]["country"] = "Error2"
  101. HostDict[Host]["region"] = ""
  102. HostDict[Host]["org"] = ""
  103. else:
  104. ## If lookup had issues, set null
  105. HostDict[Host]["country"] = "Error1"
  106. HostDict[Host]["region"] = ""
  107. HostDict[Host]["org"] = ""
  108. ### End GeoIp lookup ### }}}
  109. return(HostDict)
  110. ### End f_dict_computation ### }}}
  111. ### f_dict_total ### {{{
  112. def f_dict_total(HostDict):
  113. ## Declare variables
  114. HighDict = dict()
  115. ServerDict = dict()
  116. Uniq = 0
  117. ## Process for HighDict
  118. for Host in HostDict:
  119. Time = HostDict[Host]["max"]
  120. Country = HostDict[Host]["country"]
  121. Count = HostDict[Host]["count"]
  122. Ave = HostDict[Host]["ave"]
  123. Sum = HostDict[Host]["sum"]
  124. if 'count' not in HighDict: ## Initialize if not set
  125. HighDict["count"] = dict(Count=Count,Host=Host,Country=Country)
  126. HighDict["max"] = dict(Time=Time,Host=Host,Country=Country)
  127. HighDict["ave"] = dict(Time=0,Host="None Qualified",Country="") ## First run not qualified to win
  128. HighDict["sum"] = dict(Time=0,Host="None Qualified",Country="") ## First run not qualified to win
  129. else:
  130. if Count > HighDict["count"]["Count"]: ## Update count if greater
  131. HighDict["count"] = dict(Count=Count,Host=Host,Country=Country)
  132. if Time > HighDict["max"]["Time"]: ## Update max time if greater
  133. HighDict["max"] = dict(Time=Time,Host=Host,Country=Country)
  134. if Ave > HighDict["ave"]["Time"] and Count >= 3: ## Update ave time if greater, and count three or more
  135. HighDict["ave"] = dict(Time=Ave,Host=Host,Country=Country)
  136. if Sum > HighDict["sum"]["Time"] and Count >= 2: ## Update total time if greater, and count two or more
  137. HighDict["sum"] = dict(Time=Sum,Host=Host,Country=Country)
  138. ## Process for ServerDict
  139. for Host in HostDict:
  140. Count = HostDict[Host]["count"]
  141. Sum = HostDict[Host]["sum"]
  142. if 'count' not in ServerDict: ## Initialize if not set
  143. ServerDict["count"] = Count
  144. ServerDict["sum"] = Sum
  145. ServerDict["uniq"] = 1
  146. else:
  147. ServerDict["count"] += Count
  148. ServerDict["sum"] += Sum
  149. ServerDict["uniq"] += 1
  150. return(HighDict,ServerDict)
  151. ### End f_dict_total ### }}}
  152. ### f_gen_html ### {{{
  153. def f_gen_html(HighDict,HostDict,ServerDict):
  154. ## Declare variables
  155. s_main_table = ""
  156. ### s_html_top ### {{{
  157. s_html_top = inspect.cleandoc("""
  158. <html>
  159. <center>
  160. <body>
  161. <style> table {border-spacing: 0; border: 1px solid black; font-family: monospace;} th {cursor: pointer;} th, td {border: 1px solid black; border-collapse: collapse; padding: 2px;}</style>
  162. <h2>Tar Pit Score Board</h2>
  163. <h3>Info</h3>
  164. <p>Here is a little Score Board of <strike>ssh attempts</strike> players from the last seven days.<BR>
  165. To <i>"keep things fair"</i>, players who try too many times are given a time out.</p>
  166. """)
  167. ### End s_html_top ### }}}
  168. ### s_top_table ### {{{
  169. s_top_table = """
  170. <h3>Top Players</h3>
  171. <table id="topTable">
  172. <tr><th>Category</th><th>Value</th><th>Host</th><th>Country</th></tr>
  173. <tr><td>Most Attempts</td><td align=right>{Count}</td><td>{CHost}</td><td>{CCountry}</td></tr>
  174. <tr><td>Longest Conn</td><td align=right>{LTime}</td><td>{LHost}</td><td>{LCountry}</td></tr>
  175. <tr><td>Highest Ave*</td><td align=right>{ATime}</td><td>{AHost}</td><td>{ACountry}</td></tr>
  176. <tr><td>Highest Total*</td><td align=right>{TTime}</td><td>{THost}</td><td>{TCountry}</td></tr>
  177. </table>
  178. <p><small>* Top Average requires three connections, <BR>and Top Total two connections to qualify.</small></p>
  179. """.format(Count=HighDict["count"]["Count"],
  180. CHost=HighDict["count"]["Host"],
  181. CCountry=HighDict["count"]["Country"],
  182. LTime=f_mins(HighDict["max"]["Time"]),
  183. LHost=HighDict["max"]["Host"],
  184. LCountry=HighDict["max"]["Country"],
  185. ATime=f_mins(HighDict["ave"]["Time"]),
  186. AHost=HighDict["ave"]["Host"],
  187. ACountry=HighDict["ave"]["Country"],
  188. TTime=f_mins(HighDict["sum"]["Time"]),
  189. THost=HighDict["sum"]["Host"],
  190. TCountry=HighDict["sum"]["Country"])
  191. ### End s_top_table ### }}}
  192. ### s_main_table_pre ### {{{
  193. s_main_table_pre = """
  194. <h3>Player List</h3>
  195. <p>In the past seven days there have been {TPlayers} players, and {TAttempts} connections.<BR>
  196. The average connection time is {TAve} minutes,<BR>
  197. with a total accumulated time of {TTime} minutes.<BR>
  198. <small>All <b>Conn</b>ection lengths are in minutes.<BR>
  199. Number columns may be sorted by clicking on their header.</small></p>
  200. <table id="mainTable">
  201. <thead>
  202. <tr><th>Host</th><th>Country</th><th>Region</th><th>Org</th><th>Attempts</th><th>Shortest Conn</th><th>Longest Conn</th><th>Average Conn</th><th>Total Conns</th></tr>
  203. </thead>
  204. <tbody>""".format(TAttempts=ServerDict["count"],
  205. TPlayers=ServerDict["uniq"],
  206. TAve=f_mins(ServerDict["sum"] / ServerDict["count"]),
  207. TTime=f_mins(ServerDict["sum"]))
  208. ### End s_main_table_pre ### }}}
  209. ### s_main_table ### {{{
  210. for Host in HostDict:
  211. ## Make easy to read in code without added newlines
  212. s_main_table_temp = (
  213. "<tr>"
  214. "<td>{Host}</td>"
  215. "<td>{Country}</td>"
  216. "<td>{Region}</td>"
  217. "<td>{Org}</td>"
  218. "<td align=right>{Count}</td>"
  219. "<td align=right>{Min}</td>"
  220. "<td align=right>{Max}</td>"
  221. "<td align=right>{Ave:.3f}</td>"
  222. "<td align=right>{Sum:.3f}</td>"
  223. "</tr>"
  224. ).format(Host=Host,
  225. Country=HostDict[Host]["country"],
  226. Region=HostDict[Host]["region"],
  227. Org=HostDict[Host]["org"],
  228. Count=HostDict[Host]["count"],
  229. Min=f_mins(HostDict[Host]["min"]),
  230. Max=f_mins(HostDict[Host]["max"]),
  231. Ave=f_mins(HostDict[Host]["ave"]),
  232. Sum=f_mins(HostDict[Host]["sum"]))
  233. s_main_table = """{0}
  234. {1}""".format(s_main_table, s_main_table_temp)
  235. ### End s_main_table ### }}}
  236. ### s_main_table_post ### {{{
  237. ## This was broken off of s_html_bottom so that the .format didn't error
  238. ## No inspect.cleandoc as it would collapse to beginning of line
  239. s_main_table_post = """
  240. </tbody>
  241. </table>
  242. <p>This page is brought to you by a python <a href="https://notabug.org/demure/scripts/src/master/endlessh_scoreboard.py">script</a>,<BR>
  243. which parses an <a href="https://github.com/skeeto/endlessh">endlessh</a> log file.</p>
  244. <p><b>Last updated {Date}</b></p>
  245. """.format(Date=datetime.datetime.utcnow().strftime("%a %d %b %Y %H:%M:%S UTC"))
  246. ### End s_main_table_post ### }}}
  247. ### s_html_bottom ### {{{
  248. s_html_bottom = inspect.cleandoc("""
  249. <!-- from https://developer.mozilla.org/en-US/docs/Web/HTML/Element/table -->
  250. <script>
  251. for (let table of document.querySelectorAll('#mainTable')) {
  252. for (let th of table.tHead.rows[0].cells) {
  253. th.onclick = function(){
  254. const tBody = table.tBodies[0];
  255. const rows = tBody.rows;
  256. for (let tr of rows) {
  257. Array.prototype.slice.call(rows)
  258. .sort(function(tr1, tr2){
  259. const cellIndex = th.cellIndex;
  260. return tr1.cells[cellIndex].textContent.localeCompare(tr2.cells[cellIndex].textContent);
  261. })
  262. .forEach(function(tr){
  263. this.appendChild(this.removeChild(tr));
  264. }, tBody);
  265. }
  266. }
  267. }
  268. }
  269. </script>
  270. </body>
  271. </center>
  272. </html>
  273. """)
  274. ### End s_html_bottom ### }}}
  275. return(s_html_top,s_top_table,s_main_table_pre,s_main_table,s_main_table_post,s_html_bottom)
  276. ### End f_gen_html ### }}}
  277. ### f_write_out_file ### {{{
  278. def f_write_out_file(s_output_html):
  279. #Out_Path = '/var/www/pit/index.html' ## Set path to output
  280. Out_Path = '/usr/share/nginx/pit/index.html' ## Set path to output
  281. OutFile = open(Out_Path, "w") ## Open file for writing
  282. for String in s_output_html: ## Iterate over strings
  283. OutFile.write(String) ## Write each string in order
  284. OutFile.close() ## Close file
  285. ### End f_write_out_file ### }}}
  286. ### f_hash ### {{{
  287. ## Make the Hashing a simple function call
  288. def f_hash(Host):
  289. ## Has, and salt with repeatability
  290. return(hashlib.sha512(os.uname()[1].encode('utf-8') + Host.encode('utf-8')).hexdigest()[0:16])
  291. ### End f_hash ### }}}
  292. ### f_mins ### {{{
  293. ## Convert and round seconds to minutes
  294. def f_mins(Time):
  295. return(round((Time / 60), 3))
  296. ### End f_mins ### }}}
  297. ## DEBUG
  298. ### f_debug_print ### {{{
  299. def f_debug_print(Mode,HostDict,HighDict,ServerDict):
  300. if Mode == 0 or Mode == 1:
  301. for Host in HostDict:
  302. ## Print host matches
  303. print(("Host: {Host}, "
  304. "Country: {Country}, "
  305. "Region: {Region}, "
  306. "Org: {Org}, "
  307. "Count: {Count}, "
  308. "Min: {Min}, "
  309. "Max: {Max}, "
  310. "Ave: {Ave:.3f}, "
  311. "Sum: {Sum:.3f}"
  312. ).format(Host=Host,
  313. Country=HostDict[Host]["country"],
  314. Region=HostDict[Host]["region"],
  315. Org=HostDict[Host]["org"],
  316. Count=HostDict[Host]["count"],
  317. Min=f_mins(HostDict[Host]["min"]),
  318. Max=f_mins(HostDict[Host]["max"]),
  319. Ave=f_mins(HostDict[Host]["ave"]),
  320. Sum=f_mins(HostDict[Host]["sum"])))
  321. if Mode == 0 or Mode == 2:
  322. ## Print High Scores
  323. print(("Most Attempts: {Count}, "
  324. "Host: {Host}, "
  325. "Country: {Country}"
  326. ).format(Count=HighDict["count"]["Count"],
  327. Host=HighDict["count"]["Host"],
  328. Country=HighDict["count"]["Country"]))
  329. print(("Longest Conn: {Time}, "
  330. "Host: {Host}, "
  331. "Country: {Country}"
  332. ).format(Time=f_mins(HighDict["max"]["Time"]),
  333. Host=HighDict["max"]["Host"],
  334. Country=HighDict["max"]["Country"]))
  335. print(("Highest Average: {Time}, "
  336. "Host: {Host}, "
  337. "Country: {Country}"
  338. ).format(Time=f_mins(HighDict["ave"]["Time"]),
  339. Host=HighDict["ave"]["Host"],
  340. Country=HighDict["ave"]["Country"]))
  341. print(("Highest Total: {Time}, "
  342. "Host: {Host}, "
  343. "Country: {Country}"
  344. ).format(Time=f_mins(HighDict["sum"]["Time"]),
  345. Host=HighDict["sum"]["Host"],
  346. Country=HighDict["sum"]["Country"]))
  347. if Mode == 0 or Mode == 3:
  348. print(("Total Attempts: {TAttempts}, "
  349. "Total Players: {TPlayers}, "
  350. "Total Ave Conn: {TAve}, "
  351. "Total Conn Time: {TTime}"
  352. ).format(TAttempts=ServerDict["count"],
  353. TPlayers=ServerDict["uniq"],
  354. TAve=f_mins(ServerDict["sum"] / ServerDict["count"]),
  355. TTime=f_mins(ServerDict["sum"])))
  356. ### f_debug_print ### }}}
  357. ### Arguments ### {{{
  358. ## Parse the command line arguments
  359. def parse_argv(argv):
  360. arg1=argv[1]
  361. ## Single augment
  362. if len(argv) == 2:
  363. ## Option help
  364. if arg1 == "-h" or arg1 == "--help" or arg1 == "help":
  365. msg_help()
  366. return
  367. ## Option version
  368. if arg1 == "-V" or arg1 == "--version" or arg1 == "version":
  369. msg_version()
  370. return
  371. ## If they get here, the didn't match; print help.
  372. print("["+arg1+"] invalid use, or invalid option!")
  373. msg_help()
  374. return
  375. ## Two Augments
  376. elif len(argv) > 2:
  377. arg2=argv[2]
  378. ## Option mute
  379. if arg1 == "-e" or arg1 == "--exclude" or arg1 == "exclude":
  380. if re.match("^(\d+)$", arg2):
  381. mute_toggle(arg2)
  382. else:
  383. print("["+arg2+"] is not a valid sink input!")
  384. msg_help()
  385. return
  386. return
  387. ### End Arguments ### }}}
  388. ## Print help message
  389. def msg_help():
  390. help_msg = inspect.cleandoc("""
  391. Usage: endlessh_scoreboard.py [OPTION...]
  392. When run without OPTION, will DO A THING
  393. When run with BLAH, WILL DO OTHER THING
  394. -e, --exclude Exclude IP [ip1] [ip2] [ip3]...
  395. -V, --version Display version information
  396. -h, --help Display this help message
  397. """)
  398. print(help_msg)
  399. quit()
  400. return
  401. ## Print version number message
  402. def msg_version():
  403. version_msg="endlessh_scoreboard version: 0.3.2"
  404. print(version_msg)
  405. quit()
  406. return
  407. ## Main
  408. def main(argv):
  409. ## Process input, if given
  410. if len(argv) > 1:
  411. parse_argv(argv)
  412. FilterArray = f_open_log_file()
  413. HostDict = f_dict_computation(FilterArray)
  414. HighDict,ServerDict = f_dict_total(HostDict)
  415. s_output_html=f_gen_html(HighDict,HostDict,ServerDict)
  416. f_write_out_file(s_output_html)
  417. # f_debug_print(3,HostDict,HighDict,ServerDict)
  418. if __name__ == '__main__':
  419. main(sys.argv)