xrn-tools/scripts/compile_documentation.py

#!/usr/bin/env python3

import json
import os
import sys
import re
from datetime import date

#type_file=xoronosconfidential ; legal_info="Confidential document"
#type_file=xoronosinternal ; legal_info="Internal document"
type_file="xoronosopen" ; legal_info="Public document"
template = "/usr/share/pandoc/data/templates/xoronos.tex"
texmfhome = ".local/xoronos/texmf/"

# read the file list
with open('./backend/filelist.json', 'r') as file:
    jsondata = file.read()
filelists_json = json.loads( jsondata )

script_dir = os.popen("pwd").read().replace("\n", "")
project_dir = os.path.dirname(script_dir) 

cache_dir = project_dir + "/cache"
build_dir = project_dir + "/build"
doc_dir = build_dir + "/doc"
html_dir = doc_dir + "/html"
pdf_dir = doc_dir + "/pdf"
svg_dir = html_dir + "/svg"
out_dir = cache_dir + "/libdoc"
resdir = project_dir + "/archives/resources/miscellaneus/"

# extract archive
if not os.path.exists(resdir):
    cmd = "cd ../archives ; tar -xzf resources.tar.gz"
    os.system(cmd)

# create directories
cmd = ""
if not os.path.exists(cache_dir):
    cmd = cmd + "; mkdir " + cache_dir 

if not os.path.exists(out_dir):
    cmd = cmd + "; mkdir " + out_dir 

if not os.path.exists(build_dir):
    cmd = cmd + "; mkdir " + build_dir 

if not os.path.exists(html_dir):
    cmd = cmd + "; mkdir " + html_dir

if not os.path.exists(pdf_dir):
    cmd = cmd + "; mkdir " + pdf_dir

if not os.path.exists(svg_dir):
    cmd = cmd + "; mkdir " + svg_dir + "/ ;"

os.system(cmd[2:])

# clean content of directories
cmd = ""
cmd = cmd + "; rm -rf " + out_dir + "/*"
cmd = cmd + "; rm -rf " + svg_dir + "/*"
cmd = cmd + "; rm -rf " + html_dir + "/*.html"
cmd = cmd + "; rm -rf " + pdf_dir + "/*"

os.system(cmd[2:])


# get all the files to analyze
all_files_doc = []
all_files_src = []
for file in filelists_json["lib-function-docs"]:
    if os.path.exists( project_dir + "/" + file ) == True :
        all_files_doc.append( project_dir + "/" + file )
    path_file_src = "/".join(file.split("/")[:-2]) + "/src/" + file.split("/")[-1]
    path_file_src =  path_file_src[:-3] + ".c" 
    if os.path.exists( project_dir + "/" + path_file_src ) == True :
        all_files_src.append( project_dir + "/" + path_file_src )

# construct a keyword database
functions_keywords = ""
functions_returns = ""
for file in all_files_doc :
    cmd = "cat " + file + " | awk '/int .*\(/||/void .*\(/{print $0}' | sed 's/(.*$//' | tr -d '*' | awk '//{print $2}'"
    outcmd = os.popen( cmd ).read()
    functions_keywords = functions_keywords + "\n" + outcmd
    if "xrn_common" in file :
        cmd = "cat " + file + " | grep define | grep '(' | awk '//{print $2 }'"
        outcmd = os.popen( cmd ).read()
        functions_returns = functions_returns + "\n" + outcmd

functions_keywords = "\n".join([s for s in functions_keywords.split("\n") if s])
functions_returns = "\n".join([s for s in functions_returns.split("\n") if s])
list_functions = functions_keywords.split("\n")
list_return = functions_returns.split("\n")

list_return_msgs = [] 
# construct the error warning messages
for file in all_files_doc :
    if "xrn_common" in file :
        for returnstr in list_return :
            cmd = "cat " + file + " | grep " + returnstr +"_MSG  |  sed 's/^.* \"//' | tr -d \"\\\"\" | tr -d '\n' "
            outcmd = os.popen( cmd ).read()
            list_return_msgs.append( dict( code = returnstr , msg = outcmd) )

# get library version
lib_version = ""
for file in filelists_json["libh"]:
    if "xrn_settings" in file :
        cmd = "cat " + project_dir + "/"+ file + " | awk '/XRN_ALGORITHM_VERSION/ {print $3}' | head -n 1 "
        algorithm_version = os.popen( cmd ).read()
        cmd = "cat " + project_dir + "/"+ file + " | awk '/XRN_IO_VERSION/ {print $3}' | head -n 1 "
        ioversion = os.popen( cmd ).read()
        lib_version = algorithm_version[:-1] + "." + ioversion[:-1]
        break

#file, function, list of ret, list of calls
database = []
# read the src files and compile a list of functions and keywords 
firstentry = 0
for file in all_files_src :
    filep = open(file, 'r')
    lines = filep.readlines()
    filep.close()
    for line in lines :
        # get function name
        if ((None != re.search("^int .*\(", line)) or (None != re.search("^void .*\(", line))) :
            funcstr = line
            funcstr = funcstr.replace('*','') 
            funcstr = funcstr.split(" ")[1]
            funcstr = funcstr.split("(")[0]
            found = 0
            for function in list_functions:
                if (( found == 0 ) and ( function == funcstr )):
                     database = database + [ dict(srcfile = file, func = function, lst_ret = [], lst_call = [])]
                     firstentry = 1
                     found = 1
            if found == 0 :
                 print ( "error in line: "+line + "\nfile: " + file )
                 exit(-1)
        # get return values
        for return_value in list_return :
            if return_value in line :
                 if not ( return_value in database[-1]['lst_ret'] ):
                     database[-1]['lst_ret'].append(return_value)

        # get function calls
        for call_funct in list_functions :
            if (( call_funct in line ) and ( firstentry == 1 )):
                 if (not "strcmp" in line ):
                     if not database[-1]['func'] in line :
                          database[-1]['lst_call'].append(call_funct)
                          # remove duplicates
                          database[-1]['lst_call'] = list(dict.fromkeys(database[-1]['lst_call']))

# print info
def print_autocomplete ( name , file ):
    for entry in database :
        if name == entry['func'] :
            string = "\n**Return values:**\n\n"
            for ret in entry['lst_ret'] :
                if 'XERR_' in ret :
                    string = string + "> **error:** " + ret + " $\\rightarrow$ "
                elif 'XWAR_' in ret :
                    string = string + "> **warning:** " + ret + " $\\rightarrow$ "
                else :
                    string = string + "> **standard:** " + ret + " $\\rightarrow$ "
                for code_dic in list_return_msgs :
                    if ret == code_dic['code'] :
                        string = string + code_dic['msg'] + ".\\\n"
            if ( string != "\n**Return values:**\n\n") :
                string = string[:-2] + "\n\n"
                file.write(string)

            string = "\n**Function calls:**\n\n"
            already_called = []
            for call in entry['lst_call'] :
                for entry2 in database :
                    if ( call == entry2['func'] ) and ( not ( call in already_called ) ) :
                        already_called.append(call) 
                        string = string + "> **" + os.path.basename(entry2['srcfile'])[:-2] + ".md:** " + call + "\\\n" 
                    #if ( call == entry2['func'] ) :
            if ( string != "\n**Function calls:**\n\n") :
                string = string[:-2] + "\n\n"
                file.write(string)
            break 

for file in all_files_doc :
    print( "file " + file)
    title = os.path.basename(file).replace("_", " ")[:-3]
    filep = open(file, 'r')
    lines = filep.readlines()
    filep.close()
    fileo = open(out_dir + "/" +os.path.basename(file), 'w') 

    # dump header
    today = date.today()
    fileo.write("---\n")
    fileo.write("title: \"" + title + "\""+ "\n")
    fileo.write("author: []\n")
    fileo.write("legalinfo: \""+ legal_info+"\""+ "\n")
    fileo.write("page-background: \""+ os.path.expanduser('~') + "/" + texmfhome + "tex/" + type_file + ".pdf"+"\""+ "\n")
    fileo.write("date: \""+today.strftime("%d-%m-%Y")+"\""+ "\n")
    fileo.write("lang: en"+"\n")
    fileo.write("titlepage: true,"+"\n")
    fileo.write("titlepage-background: \"" + os.path.expanduser('~') + "/" +texmfhome + "tex/xoronostitlepage.pdf" + "\""+ "\n")
    fileo.write("titlepage-text-color: \"000000\"" + "\n")
    fileo.write("titlepage-rule-color: \"360049\"" + "\n")
    fileo.write("titlepage-rule-height: 0" + "\n")
    fileo.write("..." + "\n")
    fileo.write("\\vspace*{2cm}" + "\n")
    fileo.write("\\centerline{" + "\n")
    fileo.write("\\Large" + "\n")
    fileo.write("\\textbf{" + "\n")
    fileo.write("Xoron Library Version " + lib_version + "\n")
    fileo.write("}" + "\n")
    fileo.write("}" + "\n")
    fileo.write("\\vspace*{2cm}" + "\n")

    # dump lines
    for line in lines :
        # check for autocomplete
        if "docautocomplete" in line :
            maxlen = 0
            for function in list_functions :
                if function in line :
                    if maxlen < len ( function ) :
                         best_match = function
                         maxlen = len ( function ) 
            if ( maxlen != 0 ) :
                print_autocomplete ( best_match, fileo )
        else :
            fileo.write(line)
    fileo.close()

    # dump pdf
    cmd = "pandoc " + out_dir + "/" +os.path.basename(file) + " -o "
    cmd = cmd + pdf_dir + "/" + os.path.basename(file)[:-3] + ".pdf --from markdown --template "
    cmd = cmd + " " + template + " --listings --toc "
    os.system(cmd)

    name6 = out_dir + "/" + os.path.basename(file)[:-3] + "6.html"
    name5 = out_dir + "/" + os.path.basename(file)[:-3] + "5.html"
    name4 = out_dir + "/" + os.path.basename(file)[:-3] + "4.html"
    name3 = out_dir + "/" + os.path.basename(file)[:-3] + "3.html"
    name2 = out_dir + "/" + os.path.basename(file)[:-3] + "2.html"
    name1 = out_dir + "/" + os.path.basename(file)[:-3] + "1.html"
    name = html_dir + "/" + os.path.basename(file)[:-3] + ".html"
    cmd = "pandoc --standalone " + out_dir + "/" +os.path.basename(file) + " --from markdown --mathjax --toc "
    cmd = cmd + " -o " + name6
    os.system(cmd)

    # copy css into html
    cmd = "line=$(grep -n '/style' " + name6 + " | cut -d ':' -f 1); "
    cmd = cmd + "{ head -n $(($line-1)) " + name6 + " ; cat " + resdir + "/xoronos.css; tail -n +$line " + name6 + " ; } > " + name5 + " ;"
    os.system(cmd)

    # copy fonts into html
    cmd = "line=$(grep -n '<body>' " + name5 + " | cut -d ':' -f 1); "
    cmd = cmd + "{ head -n $(($line-1)) " + name5 + " ; cat " + resdir + "/xoronos_font.html; tail -n +$line " + name5 + " ; } > " + name4 + " ;"
    os.system(cmd)

    # fix formatting
    cmd = "cat " + name4 + " | sed 's/<\\/nav>/<\/nav>\\n<div>/g; s/<\\/body>/<\\/div>\\n<\\/body>/g' > " + name3
    os.system(cmd)

    # add java script into html
    cmd = "line4=$( grep -n '/body>' " + name3 + " | cut -d ':' -f 1 ); "
    cmd = cmd + "{ head -n $(($line4-1)) " + name3 + "; cat " + resdir + "/xoronos.js; tail -n +$line4 " + name3 + "; } > " + name2 + ";"
    os.system(cmd)

    # add menu collapsable
    cmd = "cat " + name2 + " | sed '/<li>.*.<ul>/s/li><a/li><a class=\"caret\"/; /<li>.*.<ul>/s/<ul/<ul class=\"nested\"/; ' > " + name1 + ";"
    os.system(cmd)

    # fix hdots
    cmd = "cat " + name1 + " | sed \"s/\\\hdots/.../g; s/\\././; s/cline[{][^}]*[}]/\\\\\ \\\hline/ \" > " + name
    os.system(cmd)

    # remove temporary files
    cmd = "rm " + name6 + " ; rm " + name5 + " ; rm " + name4 + " ; rm " + name3 + " ; rm " + name2 + " ; rm " + name1
    os.system(cmd)