noball/backend/score-updater.py

#!/usr/bin/env python3

import json
import os
import re
import shutil
import sys
import threading
import time

import bs4
import requests


seriesRequestComplete = False
majorSeries = set()
majorMatches = {}
minorMatches = set()
backupFileName = os.path.join(os.path.dirname(sys.argv[0]), "scores.json")
fileName = os.path.join(os.path.dirname(sys.argv[0]), "../noball/scores.json")
matchData = {}
rssContent = None

def check():
    global seriesRequestComplete
    try:
        websiteContent = requests.get("http://www.espncricinfo.com/?edition-view=espncricinfo-en-ww").text
    finally:
        seriesRequestComplete = True
        
    soup = bs4.BeautifulSoup(websiteContent, 'html.parser')
    majorSeries_list = []
    for link in soup.find_all("a", attrs={
            "class": "quicklinks_list__link",
            "name": lambda name: name.startswith('&lpos=cricket:keyseries:') and not name.endswith(':app')
    }):
        try:
            majorSeries_list.append(re.match('/series/_/id/([0-9]+)/', link['href']).groups()[0])
        except AttributeError:
            pass
    majorSeries_list = majorSeries_list[:-3 + len(majorSeries_list) // 2]
    global majorSeries
    majorSeries = set(majorSeries_list)

    global majorMatches
    global minorMatches
    for match, series in majorMatches.items():
        if series not in majorSeries:
            minorMatches.add(match)
            majorMatches.pop(match)

    rssContent_local = bs4.BeautifulSoup(requests.get("http://static.espncricinfo.com/rss/livescores.xml").text, 'html.parser')
    matchUrlList = set()
    for match in rssContent_local.find_all('item'):
        url = match.guid.get_text().strip()
        matchUrlList.add(url)
        match.link['href'] = url
    global rssContent
    rssContent = rssContent_local

    curMatches = set(majorMatches.keys()).union(minorMatches)
    
    for match in matchUrlList - curMatches:
        try:
            seriesId, matchId = getIds(match)
        except:
            minorMatches.add(match)
            continue
        
        if seriesId in majorSeries:
            majorMatches[match] = seriesId
            index = max(list(matchData.keys()) + [ -1 ]) + 1
            matchData[index] = { 'url': match, 'status': rssContent.find('link', href=match).parent.title.get_text() }
            threading.Thread(target=startUpdater, args=(match, matchId, seriesId, index)).start()
        else:
            minorMatches.add(match)
            
    for match in curMatches - matchUrlList:
        try:
            majorMatches.pop(match)
        except KeyError:
            minorMatches.remove(match)
            
    with open(backupFileName, 'w') as _file:
        json.dump(list(matchData.values()), _file)
    shutil.copy2(backupFileName, fileName)
    os.remove(backupFileName)

def getIds(match):
    return re.search("/series/([0-9]+)/[^/]+/([0-9]+)", requests.get(match).url).groups()

def startUpdater(matchUrl, matchId, seriesId, key):
    page = 1
    while True:
        try:
            matchSummary = rssContent.find('link', href=matchUrl).parent.title.get_text()
            
            if re.search(" [*]( |$)", matchSummary) and not re.search(" [&] ( |$)", matchSummary):
                apiLink = "http://site.api.espn.com/apis/site/v2/sports/cricket/{}/playbyplay".format(seriesId)
                
                innings = 1
                ballByBallData = [ [] ]
                
                cgiParams = { "contentorigin": "espn",
                              "lang": "en",
                              "event": matchId,
                              "page": page
                }


                commentaryData = requests.get(apiLink, cgiParams).json()['commentary']
                pageCount = commentaryData['pageCount']
                if pageCount > 1:
                    page = pageCount
                    cgiParams['page'] = page
                    commentaryData = requests.get(apiLink, cgiParams).json()['commentary']
        
                lastBall = next(ball for ball in reversed(commentaryData['items']) if 'overs' in ball['over'])
                    
                innings = lastBall['innings']
                runs = innings['runs']
                wickets = innings['wickets']
                target = innings['target']
                over = lastBall['over']
                limit = over['limit']
                batsman = lastBall['batsman']
                otherBatsman = lastBall['otherBatsman']
                bowler = lastBall['bowler']
    
                score = "{}{} ({}{} ov{}) ({}{} {}*, {} {}/{})".format(runs, "/{}".format(wickets) if wickets < 10 else "", over['overs'], "/{:.0f}".format(limit) if limit > 0 else "", ", target {}".format(target) if target > 0 else "", "{} {}*, ".format(batsman['athlete']['displayName'], batsman['totalRuns']) if not lastBall['dismissal']['dismissal'] else "", otherBatsman['athlete']['displayName'], otherBatsman['totalRuns'], bowler['athlete']['displayName'], bowler['wickets'], bowler['conceded'])

                ballsRemaining = innings['remainingBalls']
                battingTeam = lastBall['team']['displayName']
                bowlingTeam = bowler['team']['displayName']

                result = None
                if target > 0:
                    remaining = " (with {} remaining)".format("{} ov".format(innings['remainingOvers']) if ballsRemaining > 90 else "{} balls".format(ballsRemaining)) if limit > 0 else ""
                    if runs >= target:
                        result = "{} won by {} wickets{}".format(battingTeam, 10 - wickets, remaining)
                    elif ballsRemaining == 0 or wickets == 10:
                        if runs == target - 1:
                            result = "Match tied"
                        else:
                            result = "{} won by {} runs".format(bowlingTeam, target - runs - 1)
                    else:
                        result = "{} need {} runs to win{}".format(battingTeam, target - runs, remaining)

                matchSummary = re.sub("[^ ]+ +[*]", score.replace('\\', '\\\\'), matchSummary)
                if result:
                    matchSummary += " - " + result
                
            matchData[key]['status'] = matchSummary
        except Exception as e:
            print("{} - {}: {}".format(time.time(), type(e), e))
        finally:
            time.sleep(10)
            if matchUrl not in majorMatches:
                matchData.pop(key)
                break

while True:
    checkThread = threading.Thread(target=check, args=[])
    checkThread.start()
    while not seriesRequestComplete:
        time.sleep(.01)
    seriesRequestComplete = False
    nextThreadStart = time.time() + 20
    checkThread.join()
    time.sleep(max(0, nextThreadStart - time.time()))