xrn-tools/scripts/compile_code.py

#!/usr/bin/env python3

import json
import os
import sys
import random

#####################
# Variables to edit #
#####################

# commands options
commandlist_complete = ["tests-128", "xrnconv-cli", "tests-256", "tests-dbg-128", "tests-dbg-256", "examples-256", "examples-128", "parser-dbg" ,"parser", "parser-128", "parser-256", "libstatic-128", "libstatic-256", "libdynamic-128", "libdynamic-256", "swigpy-128" , "swigpy-256", "lexer-only", "xrnarchive-cli", "auxiliary-tools", "testapp", "xmessage", "xtransfer" ]
if ( len(sys.argv) > 1 ):
    commandlist = sys.argv[1:]
    for command in commandlist :
        if (( command in commandlist_complete ) == False ):
            print ( "command " + command + " not in: " + str(commandlist_complete))
            exit()
else:
    commandlist = commandlist_complete # to compile everithing
    #commandlist = ["libdynamic-256", "libdynamic-128", "parser"]

production_code = 1
platform = "gnu-linux" # alternatives: "gnu-linux" "windows"
force_indentation = 1 # does not indent the original code
enable_clang = 0 # enable clang code analysis
enable_clang_tidy = 0 # enable clang-tidy code analysis
enable_cppcheck = 0 # enable cppcheck code analysis
skip_configuration_prompt = 1 # skip the post compilation program
regenerate_shared_lib = 1 # regenerate shared library when command list contains "libdynamic-256", "libdynamic-128", "parser"

cmd="python3-config --includes"
pythondir_inc = os.popen( cmd ).read()

#######################
# Do not modify below #
#######################

if (( len(sys.argv) == 1 ) and ( production_code == 1 )):
    force_indentation = 0
    enable_clang = 0
    enable_clang_tidy = 0 
    enable_cppcheck = 0
    skip_configuration_prompt = 0
    regenerate_shared_lib = 1
    commandlist = ["xrnconv-cli", "xrnarchive-cli", "auxiliary-tools", "libdynamic-256", "libdynamic-128", "libstatic-256", "libstatic-128", "parser", "swigpy-128", "swigpy-256", "lexer-only" ]

#############################
# Copy and extract function #
#############################

def mkdir_cp_exctract_lib_addcppheader( directory_str ) : 

    # copy code in the cache directory

    dst_dir = project_dir + directory_str
    cmd = "mkdir " + dst_dir + " ;  cp -r " + original_code_dir + " " + dst_dir 
    os.system( cmd )

    # add visibility attributes to backend headers
    cmd = "du -a " + dst_dir + " | cut -f 2 | grep _backend | grep '\.h' | xargs -I + sh -c \"./backend/add_visibility_attributes.sh + +.bkp ; rm + ; mv +.bkp + ;\""
    os.system( cmd )


    # extract header files from documentation

    cmd = "du -a " + dst_dir + " | cut -f 2 | awk '/\.md/ {print $0}' | xargs -I + sh -c \"./backend/doc2lib.sh + ; \""
    os.system( cmd )

    # add cpp headers

    cmd = "du -a " + dst_dir  + " | cut -f 2 | awk '/\.h/{print $0}' | xargs -I + sh -c \"cat ../archives/resources/miscellaneus/cpp/head.txt + ../archives/resources/miscellaneus/cpp/tail.txt > +.bkp ; rm + ; mv +.bkp + ;\""
    os.system( cmd )


############################
# Dump compilation command #
############################

def dump_compilation ( compiler, outputtype, targetname, all_flags, codeconf_flags, all_files, flatdumpdir, first_stage ) :

    cppchecksuppressions = " --suppress=uninitStructMember:test_io_crypto.c  --suppress=objectIndex "
    all_file_str = str(all_files).replace('\'','').replace('[','').replace(']','').replace(',','')

    if ( outputtype == "debug" ):
        compilation_str = compiler + str(all_flags) + " -g -O0 -fno-omit-frame-pointer " + all_file_str  + " -o " + targetname + " 2>gcc_"+targetname+".log ; "

    if ( outputtype == "debug-dynamic-lib" ):
        compilation_str = compiler + str(all_flags) + " -g -O0 -fno-omit-frame-pointer " + all_file_str  + " -ldl -o " + targetname + " 2>gcc_"+targetname+".log ; "

    elif ( outputtype == "executable" ):
        compilation_str = compiler + str(all_flags) + " -O3 " + all_file_str + " -o " + targetname + " 2>gcc_"+targetname+".log ; "

    elif ( outputtype == "executable-dynamic-lib" ):
        compilation_str = compiler + str(all_flags) + " -O3 " + all_file_str + " -ldl -o " + targetname + " 2>gcc_"+targetname+".log ; "

    elif ( outputtype == "libdynamic" ) :
        compilation_str = compiler + str(all_flags) + " -O3 -fPIC -shared -o " + targetname + " " + all_file_str + " 2>gcc_"+targetname+".log ; "

    elif ( outputtype == "libstatic" ):
        compilation_str = compiler + str(all_flags) + " -O3 -fPIC -c " + all_file_str + " 2>gcc_"+targetname+".log ; ar rc -o " + targetname + " " + flatdumpdir + "/*.o ; "

    elif (( outputtype == "swigpy-128" ) or ( outputtype == "swigpy-256" )) :
        compilation_str = compiler + str(all_flags) + " -O3 -fPIC -shared -o " + targetname + " " + all_file_str + " " + pythondir_inc + " 2>gcc_"+targetname+".log ; "

    if ((( outputtype == "libdynamic" ) or ( outputtype == "libstatic" ) or ( outputtype == "debug" ) or ( outputtype == "debug-dynamic-lib" ) or ( outputtype == "executable" ) or ( outputtype == "executable-dynamic-lib" )) and ( first_stage == 0 )):
        if ( enable_clang == 1 ) :
            compilation_str = compilation_str + " clang --analyze " + str(all_flags) + " ./*.c ./*.h 2>clang_"+targetname+".log ; "
        if ( enable_cppcheck == 1 ) :
            compilation_str = compilation_str + " cppcheck " + str(codeconf_flags) + cppchecksuppressions + " ./*.c ./*.h 1>cppcheck_stdout_"+targetname+".log 2>cppcheck_stderr_"+targetname+".log ; "
        if ( enable_clang_tidy == 1 ) :
            compilation_str = compilation_str + " clang-tidy ./*.c -- " + str(codeconf_flags) + " 1>clang_tidy_"+targetname+".log 2>clang_tidy_summary_" +targetname + ".log ; " 

    with open( flatdumpdir + "/compile_"+targetname+".sh", 'w') as file:
        file.write(compilation_str)
    file.close()
    os.system("chmod 777 " + flatdumpdir + "/compile_"+targetname+".sh")

########################
# Compilation routine ##
########################

def compile ( platform, targetname, outputtype, codeconf_flags, filelists ) : 

    cachedirrel = "/cache/" + targetname 
    builddir = project_dir +"/build" 
    mkdir_cp_exctract_lib_addcppheader( cachedirrel ) 
    cachedirabs = project_dir + cachedirrel

    # configure flags
    if platform == "gnu-linux" :
        all_flags = " -D XRN_GNU_LINUX=1 " + code_cstd_flags + code_cleaner_flags + code_pthread_flags + stack_protection_flags + fortify_source_flags + warnings_flags + git_flags + codeconf_flags
        compiler = " gcc "
    else :
        all_flags = " -D XRN_GNU_LINUX=0 " + windows_flags + windows_end_flags + code_cstd_flags + code_cleaner_flags + code_pthread_flags + stack_protection_flags + fortify_source_flags + warnings_flags + git_flags + codeconf_flags
        compiler = " x86_64-w64-mingw32-gcc "

    # configure swig files
    if ( outputtype == "swigpy-128" ):
        cmd = "du -a " + cachedirabs + " | cut -f 2 | awk ' /\.i/ {print $0}' | grep python | xargs -I  + sh -c \"./backend/create_interface_python.sh + " + builddir + "/py 128; \""
        os.system(cmd)
    elif ( outputtype == "swigpy-256" ):
        cmd = "du -a " + cachedirabs + " | cut -f 2 | awk ' /\.i/ {print $0}' | grep python | xargs -I  + sh -c \"./backend/create_interface_python.sh + " + builddir + "/py 256; \""
        os.system(cmd)

    if ( targetname == "xrnconf" ):
         first_stage = 1
    else :
         first_stage = 0

    # configure files
    flatdumpdir = cachedirabs + "/flatdump"

    all_files = []
    all_files_flat = []

    for filelist in filelists:
        for file in filelists_json[filelist]:
            if os.path.exists( cachedirabs + "/" + file ) == True :
                all_files.append( cachedirabs + "/" + file )
                all_files_flat.append( os.path.basename(file) )

    # do a flat dump of all the source files
    os.system( "mkdir " + flatdumpdir )
    for file in all_files:
        os.system("cp " + file + " " + flatdumpdir )

    # if the run configuration has been run (include those files in the compilation
    if ( os.path.exists( project_dir + "/cache/headers" )) :
        all_files_flat.append( "default_permutation_settings.h" )
        all_files_flat.append( "default_encoding_settings.h" )
        os.system( "cp " + project_dir + "/cache/headers/* " + flatdumpdir )

    # dump compilation commands
    dump_compilation ( compiler, outputtype, targetname, all_flags, codeconf_flags, all_files_flat, flatdumpdir, first_stage )

    corerefdir = builddir + "/coderef/" +  targetname
    cmd = "mkdir "  + corerefdir + "; cp -r " + flatdumpdir + " " + corerefdir 
    os.system( cmd )

    print("compile " + targetname )
    os.system("cd " + flatdumpdir + ";" + " ./compile_"+targetname+".sh ")

    # copy the results in the build directory

    if (( outputtype == "executable" ) or ( outputtype == "executable-dynamic-lib" ) or ( outputtype == "debug" )or ( outputtype == "debug-dynamic-lib" )):
        os.system("cp " + flatdumpdir + "/" +targetname + " " + project_dir + "/build/bin/")
        os.system("chmod 777 " + project_dir + "/build/bin/" +targetname )
    elif (( outputtype == "libdynamic" ) or ( outputtype == "libstatic" )) :
        os.system("cp " + flatdumpdir + "/" +targetname + " " + project_dir + "/build/sharedlib/")
    elif (( outputtype == "swigpy-128" ) or ( outputtype == "swigpy-256" )) :
        os.system("cp " + flatdumpdir + "/" +targetname + " " + project_dir + "/build/py/")

    os.system("cp " + flatdumpdir + "/gcc_" +targetname + ".log " + project_dir + "/build/reports/compilationlogs ")
    outcmd = os.popen("cat " + flatdumpdir + "/gcc_" + targetname + ".log " ).read()
    print("################## gcc stderr  ##################")
    print(outcmd)

    if ( first_stage == 0 ) :
        if ( enable_clang == 1 ) :
            os.system("cp " + flatdumpdir + "/clang_" +targetname + ".log " + project_dir + "/build/reports/compilationlogs")
        if ( enable_clang_tidy == 1 ) :
            os.system("cp " + flatdumpdir + "/clang_tidy_" +targetname + ".log " + project_dir + "/build/reports/compilationlogs")
        if ( enable_cppcheck == 1 ) :
            os.system("cp " + flatdumpdir + "/cppcheck_stdout_" +targetname + ".log " + project_dir + "/build/reports/compilationlogs")
            os.system("cp " + flatdumpdir + "/cppcheck_stderr_" +targetname + ".log " + project_dir + "/build/reports/compilationlogs")

        if (( enable_clang == 1 ) and (targetname != "xrnconf")) :
            outcmd = os.popen("cat " + flatdumpdir + "/clang_" + targetname + ".log " ).read()
            print("################# clang stderr  #################")
            print(outcmd)
        if (( enable_clang_tidy == 1 ) and (targetname != "xrnconf")) :
            outcmd = os.popen("cat " + flatdumpdir + "/clang_tidy_" + targetname + ".log " ).read()
            print("############### clang-tidy stderr ###############")
            print(outcmd)
        if (( enable_cppcheck == 1 ) and (targetname != "xrnconf")) :
            outcmd = os.popen("cat " + flatdumpdir + "/cppcheck_stderr_" + targetname + ".log " ).read()
            print("############### cppcheck stderr  ################")
            print(outcmd)

############################
# Compilation App routine ##
############################

def compile_app ( targetname, filelists ):

    builddir = project_dir +"/build/py" 
    #outfile = builddir + "/" + targetname + ".py"
    outfile = builddir + "/" + targetname

    license_file_path = "../archives/resources/miscellaneus/license.txt"
    with open( license_file_path, 'r') as f:
        license_file = f.read().replace("/*","'''").replace("*/","'''")

    with open(outfile, 'w') as f:
        f.write( "#!/usr/bin/env python3\n\n" )

    with open(outfile, 'a') as f:
        f.write( license_file )

    # configure files

    all_files = []
    all_files_flat = []
    all_files_json = []
    all_files_py = []
    all_files_behaviour = []
    all_files_converters = []

    import_str = ""

    for filelist in filelists:
        for file in filelists_json[filelist]:
            all_files.append( file )
            all_files_flat.append( os.path.basename(file) )
            if len(file) > len(".json") :
                if file[-(len(".json")):] == ".json" :
                    all_files_json.append(file)

            if len(file) > len("_behaviour.py") :
                if file[-(len("_behaviour.py")):] == "_behaviour.py" :
                    all_files_behaviour.append(file)

            if len(file) > len("_converters.py") :
                if file[-(len("_converters.py")):] == "_converters.py" :
                    all_files_converters.append(file)

            if file[-(len(".py")):] == ".py" :
                cmd = "cat " + project_dir + "/" + file + " | awk ' /import / { print $0 } ' " 
                import_str = import_str + os.popen(cmd).read()
                all_files_py.append(file)

    # check behaviour json files
    if ( len(all_files_json) != len(all_files_behaviour) ):
        print ("not matching behaviour json found")
        exit(0)

    uniq_import = []
    for row in import_str.split("\n") : 
        if row not in uniq_import :
            uniq_import = uniq_import + [ row ]
    uniq_import = uniq_import + ["\n\n"]

    with open(outfile, 'a') as f:
        f.write('\n'.join( uniq_import ).replace("\n\n","\n"))

    first = 0
    for jfile in all_files_json :
        found = 0
        for pyfile in all_files_behaviour :
            if ( jfile[:-(len(".json"))] == pyfile[:-(len("_behaviour.py"))] ): 
                found = 1
                cmd = "./backend/check_json_functions.sh " + project_dir + "/" + jfile + " " + project_dir + "/" + pyfile
                ret = os.popen(cmd).read().replace("\n", "")
                if ret == "fail" :
                    print ("non coherent functions")
                    print (cmd)
                    exit(0)
        if found == 0 :
            print ("behaviour file not found")
            exit(0)
        else :
            # put json files into
            cmd = "cat " + project_dir + "/" + jfile + " | sed 's/$/\\\\/' | sed '1s/^/def get_" + os.path.basename(jfile)[:-(len(".json"))] + "_layout ():\\n    return ++ / ; $s/$/\\n++/ ' | sed \"s/++/'/\" >> " + outfile
            #print(cmd)
            os.system(cmd)

    # concatenate all the python files
    for pyfile in all_files_py :
         cmd = "cat " + project_dir + "/" + pyfile + " | grep -v 'import ' >> " + outfile
         os.system(cmd)

    cmd = "chmod 777 " + outfile
    os.system(cmd)

# get project directory

script_dir = os.popen("pwd").read().replace("\n", "")
project_dir = os.path.dirname(script_dir) 
original_code_dir = project_dir + "/code"
reports_dir = project_dir + "/build/reports"

# remove and recreate directories
if regenerate_shared_lib == 1 :
    cmd = "rm -rf ../build ../cache; mkdir ../build ../build/coderef ../build/reports ../build/reports/compilationlogs ../build/py ../build/sharedlib ../build/bin ../build/doc ../build/doc/pdf ../build/doc/html ../build/doc/html/svg ../cache "
else :
    if (( os.path.isfile( "../build/sharedlib/xrnlib-128.so" ) == False ) or \
        ( os.path.isfile( "../build/sharedlib/xrnlib-128.so" ) == False )):
        print("shared libraries need to be generate if regenerate_shared_lib == 0 ")
        exit(-1)
    cmd = ""
    if os.path.exists( "../build" ) == False :
        cmd = "mkdir ../build ; "
    cmd = cmd + "rm -rf ../build/bin ../build/coderef ../build/doc ../build/py ../build/reports ../cache; mkdir ../build/coderef ../build/reports ../build/reports/compilationlogs ../build/py ../build/bin ../build/doc ../build/doc/pdf ../build/doc/html ../build/doc/html/svg ../cache "
os.system( cmd )

# decide output setting files

permutation_header_file="default_permutation_settings.h"
encoder_header_file="default_encoding_settings.h"

# read the file list
with open('./backend/filelist.json', 'r') as file:
    jsondata = file.read()
filelists_json = json.loads( jsondata )


# define windows flags

windows_flags = " -Wl,--stack -Wl,8454272 "
windows_compiler = " x86_64-w64-mingw32-gcc "
windows_end_flags =  " -lbcrypt "

# get git variables

git_version = os.popen("git rev-list --count HEAD").read().replace("\n", "")
git_dirty_flags = os.popen("git describe --dirty --always --tags | awk '{if (/dirty/){ print 1} else {print 0}}'").read().replace("\n", "")
git_version_str = os.popen("git describe --dirty --always --tags").read().replace("\n", "")

# define compilation option

#code_cstd_flags = " -ansi -pedantic " # use the c89 flags
#code_cstd_flags = " -std=c11 " # use the c11 flags
code_cstd_flags = " "
code_cleaner_flags = " -fdata-sections -ffunction-sections -Wl,--gc-sections " # optimize out not used functions (printing debugging ect)
#code_optimization_flags = " -O3 " # maximum level of optimization
code_pthread_flags = " -pthread " # compile with threads
stack_protection_flags = " -fstack-protector-all " # add stack protection to all the functions (performance penalty)
fortify_source_flags = " -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3 " # add compiler checks for buffer overflows
warnings_flags = " -Wall -Wconversion -Wformat -Wformat-security -Wnonnull " # add wornings during compilation
git_flags = " -D GIT_VERSION="+ git_version + " -D XRN_GIT_VERSION_STR="+ "\""+git_version_str+"\""+ " -D XRN_GIT_DIRTY_FLAG="+ "\\\"" + git_dirty_flags+ "\\\" "
codeconf_flags = " -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=256 " # code configuration flags

#######################
# perform code checks #
#######################

# check errors codes
cmd =  "du " + project_dir + " -a | cut -f 2 | grep '\/lib\/' | grep '\.c' | xargs cat | grep -v '\/\/' |  grep XPRINT_ERR | awk '// { print $2 }' | tr -d ',' "
outcmd = os.popen(cmd).read().split("\n")[:-1]
outcmdint = [int(i) for i in outcmd]
available_codes = []
error_in_codes = 0
error_code = 0
for i in range(0, len(outcmdint)):
    for j in range(i+1, len(outcmdint)):
        if ( outcmdint[i] == outcmdint[j] ):
            error_in_codes = 1
            error_code = outcmdint[i]
    seen = 0
    for j in range(0, len(outcmdint)):
        if ( outcmdint[j] == i ):
            seen = 1
    if ( seen == 0 ):
        if ( production_code == 0 ) :
            available_codes = available_codes + [i]
if ( production_code == 0 ) :
    available_codes = available_codes + [len(outcmdint)+1]
    if production_code == 0 :
        print("available error codes: " + str(available_codes))
if error_in_codes == 1 :
    print("error in the error codes: " + str( error_code ))
    exit(0)

#check for TODO HANDLER and CHECK comments in source files
keywords = ["TODO","HANDLER","CHECK"]
for keyword in keywords :
    cmd = "grep -Rl '\/\/.*"+keyword+"' " + original_code_dir
    outcmd = os.popen(cmd).read() 
    if ( outcmd.isupper() or outcmd.islower()) :
        print("\nfiles with "+keyword+"s comments")
        print( os.popen(cmd).read())

#check for not safe functions and suggest replacements
keywords     = ["strcpy",   "strcat",  "gets",  "sprintf",  "strlen" , "strcmp" , "atoi",   "atol",   "atoll",   "atof"  , "free"]
replacements = ["snprintf", "strncat", "fgets", "snprintf", "strnlen", "strncmp", "strtol", "strtol", "strtoll", "strtof", "XRN_FREE"]

for i in range(0, len(keywords)):
    keyword = keywords[i]
    replacement = replacements[i]
    if ( keyword == "free" ): # only for the library
        cmd = "grep -Rl '"+keyword+"(' " + original_code_dir + " | grep -v app | grep -v xrn_common_backend | grep -v python "
        outcmd = os.popen(cmd).read() 
    else :
        cmd = "grep -Rl '\/\/.*"+keyword+"' " + original_code_dir + " | grep -v examples | grep -v tests "
        outcmd = os.popen(cmd).read() 
    if ( outcmd.isupper() or outcmd.islower()) :
        print("\nfiles with "+keyword+"() function detected, cosider " + replacement + "() as replacement")
        print( os.popen(cmd).read())

#check if line continuation in comments 
cmd = "du -a " + project_dir + " | cut -f 2 | awk '/\.c/ || /\.h/ || /\.cpp/ {print $0}' | xargs -I + sh -c \"cat + | awk '/\/\/.*\\\\\\\\$/ {print \$0}'\""
outcmd = os.popen(cmd).read() 
if ( outcmd.isupper() or outcmd.islower()) :
    print("found wrong comment termination: " + outcmd)
    exit()

########################
# change original code #
########################

# indent code
if ( force_indentation == 1 ) :
    # do it twice to avoid oscillations in files due to the indentation
    for i in range(2) :
        indentcmd = " indent "
        indentopt = " -kr -l110 -lc110 -bad -bbb -br -bli1 -ce -cli4 -saf -sai -saw -prs -sbi4 -nut -bfde "

        cmd =       "du -a " + original_code_dir + " | cut -f 2 | awk '/\.c/ || /\.h/ || /\.cpp/ {print $0}' | grep -v 'xrn_lib_interposer_dll.h' | grep -v 'xrn_lib_interposer.h' | grep -v 'xrnlib.h' | grep -v 'steg_data.h' | xargs -I + sh -c \"" + indentcmd + " + " + indentopt + " \";\n"
        cmd = cmd + "du -a " + original_code_dir + " | cut -f 2 | awk '/\.c~/ || /\.h~/ || /\.cpp~/ {print $0}' | xargs -I + sh -c 'rm -fr +';\n"
        cmd = cmd + "du -a " + original_code_dir + " | cut -f 2 | awk '/\.c/ || /\.h/ || /\.cpp/ {print $0}' | grep -v 'xrn_lib_interposer_dll.h' | grep -v 'xrn_lib_interposer.h' | grep -v 'xrnlib.h' | grep -v 'steg_data.h' | xargs -I + sh -c \"./backend/remove_double_lines.sh + > +.bkp ; rm + ; mv +.bkp + \""

        os.system( cmd )

# add license

cmd = "grep -rL \"Software License\" " + original_code_dir + " | awk '/\.c/ || /\.h/ || /\.cpp/ {print $0}' | grep -v 'xrn_lib_interposer_dll.h' | grep -v 'xrn_lib_interposer.h' | grep -v 'xrnlib.h' | xargs -I + sh -c \"cat ../archives/resources/miscellaneus/license.txt + > +.bkp ; rm + ; mv +.bkp + \" ; "
os.system( cmd )

#############################################
# check what operation need to be performed #
#############################################

# done to skip the first compilation stage if only documents need to be compiled
first_stage_compilation = 0
parser_with_lexer = 0
for command in commandlist :
    if (( command == "tests-128" ) or ( command == "tests-256" ) or \
        ( command == "tests-dbg-128" ) or ( command == "tests-dbg-256" ) or \
        ( command == "examples-128" ) or ( command == "examples-256" ) or \
        ( command == "parser-128" ) or ( command == "parser-256" ) or \
        ( command == "swigpy-128" ) or ( command == "swigpy-256" ) or \
        ( command == "libstatic-128" ) or ( command == "libstatic-256" ) or \
        (( command == "libdynamic-128" ) and regenerate_shared_lib ) or \
        (( command == "libdynamic-256" ) and regenerate_shared_lib ) \
        ):
        first_stage_compilation = 1
    if ( command == "parser" ) :
        parser_with_lexer = 1

#########################################
# perform lexer and autocomplete checks #
#########################################

if (( parser_with_lexer == 1 ) and ( production_code == 0)):
    # check lexcheck.h
    cmd = "cd ./backend/lexer ; ./check_lexcheck.sh"
    errors = os.popen( cmd ).read()
    if ( errors != "" ) :
       print("error in lexcheck.h\n")
       print("trace " + errors + "\n")
       exit()
    # autogenerate xrnlib-cli-autocomplete.sh
    cmd = "cd ./backend ; ./generate_autocomplete.sh " + "../../cache/autocomplete_tmp.sh 2>/dev/null ; diff ../../cache/autocomplete_tmp.sh ./xrnlib-cli-autocomplete.sh " 
    errors = os.popen( cmd ).read()
    if ( errors != "" ) :
       print("autocomplete out of date\ncommand run: " + cmd + "\nto fix it run\n./install.sh; source ./source.sh; cd backend; ./generate_autocomplete.sh; cd .. ;")
       print("remember to push changes into repo")
    if os.path.exists( "../cache/autocomplete_tmp.sh" ) == True :
       os.system("rm ../cache/autocomplete_tmp.sh")


###################################
# perform lexer and parser checks #
###################################

if ( parser_with_lexer == 1 ) :
   cmd = "cd ./backend/lexer ; ./parser_options_functions_check.sh"
   errors = os.popen( cmd ).read()
   if ( "correct" not in errors ) :
      print("error in parser_options.h or parser_functions.h\n")
      print("inconsistent options/functions compared to lexcheck.h\n")
      print(errors)
      exit()

#######################################
# perform the first stage compilation #
#######################################

if ( first_stage_compilation == 1 ) :
    targetname = "xrnconf"
    codeconf_flags = " -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=1 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=256 "

    if ( skip_configuration_prompt == 1 ) :
        codeconf_flags = codeconf_flags + " -D XRN_SKIP_CONFIGURATION_PROMPT=1 "
    else :
        codeconf_flags = codeconf_flags + " -D XRN_SKIP_CONFIGURATION_PROMPT=0 "

    filelists = ["libh" ,"libc" ,"confgenh" ,"confgenc" , "mainc", "main_xconfig_ch" ]
    outputtype = "executable"
    compile ( platform, targetname, outputtype, codeconf_flags, filelists ) 

    # compile
    cacheheaders = project_dir + "/cache/headers"
    permutation_header_file_path = cacheheaders+"/"+permutation_header_file
    encoder_header_file_path = cacheheaders+"/"+encoder_header_file
    os.system( "rm -rf " + project_dir + "/cache/* ; mkdir " + cacheheaders + " ; " + project_dir + "/build/bin/"+targetname + " " + permutation_header_file_path + " " + encoder_header_file_path )

    # check if it produced outputs
    if ( os.path.exists(permutation_header_file_path) and os.path.exists( encoder_header_file_path )):
        if ( os.path.getsize(permutation_header_file_path) <= 0 ) or ( os.path.getsize(encoder_header_file_path) <= 0 ) :
            exit()
        elif (( commandlist[0] != "examples-256" ) and ( skip_configuration_prompt == 1 )):

            ###############################
            # check if swig is up to date #
            ###############################

            flatdumpdir = project_dir + "/build/coderef/" + targetname + "/flatdump"

            # check swig compatibility
            swigheaderfile = flatdumpdir + "/swigheader.h"

            # regenerate the swigpy header
            cmd = "cat " + permutation_header_file_path + " "
            cmd = cmd + encoder_header_file_path + " "
            cmd = cmd + flatdumpdir + "/xrn_settings.h "
            cmd = cmd + flatdumpdir + "/xrn_common.h "
            cmd = cmd + flatdumpdir + "/xrn_arithmetic.h "
            cmd = cmd + flatdumpdir + "/xrn_encoder.h "
            cmd = cmd + flatdumpdir + "/xrn_core.h "
            cmd = cmd + flatdumpdir + "/xrn_miners.h "
            cmd = cmd + flatdumpdir + "/xrn_crypto.h "
            cmd = cmd + " > " + swigheaderfile
            os.system(cmd)
            cmd = "diff -E -Z -b -w -B " + swigheaderfile + " " + "../" + filelists_json['swigpyh'][0] 
            errors = os.popen( cmd ).read()
            if ( errors != "" ) :
                print("../" + filelists_json['swigpyh'][0] + " out of date")
                print(errors)
                print("to fix the issue, you can run:\n./compile_code.py examples-256; cd backend; ./create_header_python_interface.sh ; cd .. ")
                exit(-1)
            else :
                cmd = "rm " + swigheaderfile

    else :
        exit()

#############################
# perform interposer checks #
#############################


def check_interposer ( targetname ):
    if (( parser_with_lexer == 1 ) and ( commandlist[0] != "examples-256" )) :
        flatdump_path = "../../build/coderef/" + targetname +"/flatdump/"
        # autogenerate interposer headers
        cmd = "cd ./backend ; python3 autogen_interposer.py ../../cache/xrn_lib_interposer.h ../../cache/xrn_lib_interposer_dll.h " + flatdump_path + " ;"
        os.system( cmd )
        cmd0 = "diff ../cache/xrn_lib_interposer.h ../code/xrnlib/lib/xrn_interfaces/c/inc/xrn_lib_interposer.h " 
        errors1 = os.popen( cmd0 ).read()
        cmd1 = "diff ../cache/xrn_lib_interposer_dll.h ../code/xrnlib/app/parser/inc/xrn_lib_interposer_dll.h " 
        errors2 = os.popen( cmd1 ).read()
        if (( errors1 != "" ) or ( errors2 != "" )):
           print("interposer headers out of date\ncommand run: " + cmd0 + "\ncommand run: " + cmd1 + "\nto fix it run\n./compile_code.py examples-256 ; cd backend; python3 autogen_interposer.py; cd .. ;")
           print("remember to push changes into repo")
        #if os.path.exists( "../cache/xrn_lib_interposer.h" ) == True :
        #   os.system("rm ../cache/xrn_lib_interposer.h")
        #if os.path.exists( "../cache/xrn_lib_interposer_dll.h" ) == True :
        #   os.system("rm ../cache/xrn_lib_interposer_dll.h")

##########################
# generate configuration #
##########################

interposer_check_done = 0

for command in commandlist :
    perform_final_compilation = 1
    if ( command == "lexer-only" ) :
        targetname = "xrnlib-lexer-cli"
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=0 -D XCOMPILELEXERONLY=1 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=128 -D XRN_DEBUG=1 "
        filelists = ["lexerc" ,"lexerh" , "mainc" ]
        outputtype = "executable"

    elif ( command == "tests-dbg-128" ) :
        targetname = "tests-dbg-128"
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=1 -D XRUNCONFGEN=0 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=128 -D XRN_DEBUG=1 "
        filelists = ["libh" ,"libc" ,"libc-dev" ,"testc" ,"testh" , "mainc", "main_tests_ch" ]
        outputtype = "debug"

    elif ( command == "tests-dbg-256" ) :
        targetname = "tests-dbg-256"
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=1 -D XRUNCONFGEN=0 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=256  -D XRN_DEBUG=1 "
        filelists = ["libh" ,"libc" ,"libc-dev" ,"testc" ,"testh" , "mainc", "main_tests_ch" ]
        outputtype = "debug"

    elif ( command == "tests-128" ) :
        targetname = "tests-128"
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=1 -D XRUNCONFGEN=0 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=128 "
        filelists = ["libh" ,"libc" ,"libc-dev" ,"testc" ,"testh" , "mainc", "main_tests_ch" ]
        outputtype = "executable"

    elif ( command == "tests-256" ) :
        targetname = "tests-256"
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=1 -D XRUNCONFGEN=0 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=256 "
        filelists = ["libh" ,"libc" ,"libc-dev" ,"testc" ,"testh" , "mainc", "main_tests_ch" ]
        outputtype = "executable"

    elif ( command == "examples-256" ) :
        targetname = "examples-256"
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=1 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=256 "
        filelists = ["libh" ,"libc" ,"libc-dev" ,"examplesc" ,"examplesh" , "mainc", "main_examples_ch" ]
        outputtype = "executable"

    elif ( command == "examples-128" ) :
        targetname = "examples-128"
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=1 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=128 "
        filelists = ["libh" ,"libc" ,"libc-dev" ,"examplesc" ,"examplesh" , "mainc", "main_examples_ch" ]
        outputtype = "executable"

    elif ( command == "parser-128" ) :
        targetname = "xrnlib-128-cli"
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=1 -D XNBITS=128 "
        filelists = ["libh" ,"libc" ,"parserc" ,"parserh" , "lexerc", "lexerh", "mainc", "main_parser_ch" ]
        outputtype = "executable"

    elif ( command == "parser-256" ) :
        targetname = "xrnlib-256-cli"
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=1 -D XNBITS=256 "
        filelists = ["libh" ,"libc" ,"parserc" ,"parserh" , "lexerc", "lexerh", "mainc", "main_parser_ch" ]
        outputtype = "executable"

    elif ( command == "parser" ) :
        targetname = "xrnlib-cli"
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=0 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=1 -D XNBITS=256 "
        filelists = [ "parserc" ,"parserh" , "lexerc", "lexerh", "mainc", "main_parser_ch" ]
        outputtype = "executable-dynamic-lib"

    elif ( command == "parser-dbg" ) :
        targetname = "xrnlib-cli"
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=0 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=0 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=1 -D XNBITS=256 "
        filelists = [ "parserc" ,"parserh" , "lexerc", "lexerh", "mainc", "main_parser_ch" ]
        outputtype = "debug-dynamic-lib"

    elif ( command == "libstatic-128" ) :
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=1 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=128 "
        outputtype = "libstatic"
        targetname = "xrnlib-128.a"
        filelists = ["libh" ,"libc", "libc-dev"]

    elif ( command == "libstatic-256" ) :
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=1 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=256 "
        outputtype = "libstatic"
        targetname = "xrnlib-256.a"
        filelists = ["libh" ,"libc", "libc-dev"]

    elif (( command == "libdynamic-128" ) and regenerate_shared_lib ) :
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=1 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=128 "
        filelists = ["libh" ,"libc","libc-dev"]
        targetname = "xrnlib-128.so"
        outputtype = "libdynamic"

    elif (( command == "libdynamic-256" ) and regenerate_shared_lib ) :
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=1 -D SWIGPYTHONEN=0 -D XCOMPILEPARSER=0 -D XNBITS=256 "
        outputtype = "libdynamic"
        filelists = ["libh" ,"libc","libc-dev"]
        targetname = "xrnlib-256.so"

    elif ( command == "swigpy-128" ) :
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=1 -D SWIGPYTHONEN=1 -D XCOMPILEPARSER=0 -D XNBITS=128 "
        outputtype = "swigpy-128"
        targetname = "_xrnlib128.so"
        filelists = ["libh" ,"libc" ,"libc-dev" ,"swigpyc128", "swigpyh" ]

    elif ( command == "swigpy-256" ) :
        codeconf_flags = " -D XRN_MONOLITIC_EXECUTABLE=1 -D XCOMPILELEXERONLY=0 -D XCOMPILEEXAMPLES=0 -D XCOMPILETESTS=0 -D XRUNCONFGEN=0 -D XSKIPMAIN=1 -D SWIGPYTHONEN=1 -D XCOMPILEPARSER=0 -D XNBITS=256 "
        outputtype = "swigpy-256"
        targetname = "_xrnlib256.so"
        filelists = ["libh" ,"libc" ,"libc-dev" ,"swigpyc256", "swigpyh" ]

    elif ( command == "xrnarchive-cli" ) :
        perform_final_compilation = 0
        os.system("cd backend; ./compile_xrnarchive.sh")

    elif ( command == "xrnconv-cli" ) :
        perform_final_compilation = 0
        os.system("cd backend; ./compile_xrnconv.sh")

    elif ( command == "auxiliary-tools" ) :
        perform_final_compilation = 0
        os.system("cp ../code/auxiliary_tools/*-cli ../build/bin ")

    elif ( command == "testapp" ) :
        perform_final_compilation = 0
        filelists = ["gui_backend", "gui_frontend", "testapp" ] 
        targetname = "testapp"
        compile_app ( targetname, filelists )

    elif ( command == "xmessage" ) :
        perform_final_compilation = 0
        filelists = ["gui_backend", "gui_frontend", "xmessage" ]
        targetname = "xmessage_app"
        compile_app ( targetname, filelists )

    elif ( command == "xtransfer" ) :
        perform_final_compilation = 0
        filelists = ["gui_backend", "gui_frontend", "xtransfer" ]
        targetname = "xtransfer"
        compile_app ( targetname, filelists )

    else :
        continue

    if ( perform_final_compilation == 1 ) :
        compile ( platform, targetname, outputtype, codeconf_flags, filelists ) 
        if ( interposer_check_done == 0 ) :
            check_interposer (  targetname  )
            interposer_check_done = 1


#clean cache directory
# os.system( "rm -rf " + project_dir + "/cache/* ")