spectacle/tools/spec2spectacle

#!/usr/bin/python -tt
# vim: ai ts=4 sts=4 et sw=4

#    Copyright (c) 2009 Intel Corporation
#
#    This program is free software; you can redistribute it and/or modify it
#    under the terms of the GNU General Public License as published by the Free
#    Software Foundation; version 2 of the License
#
#    This program is distributed in the hope that it will be useful, but
#    WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
#    or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
#    for more details.
#
#    You should have received a copy of the GNU General Public License along
#    with this program; if not, write to the Free Software Foundation, Inc., 59
#    Temple Place - Suite 330, Boston, MA 02111-1307, USA.

"""Overview of spec2spectacle
"""

import os
import sys
import re
import glob
import optparse

# spectacle modules
from spectacle.convertor import *
from spectacle.dumper import *
from spectacle import logger

class SpecError(Exception):
    def __ini__(self, cur_state, cur_pkg, cur_line):
        self.cur_state = cur_state
        self.cur_pkg = cur_pkg
        self.cur_line = cur_line

    def __repr__(self):
        return self.cur_state + self.cur_pkg + self.cur_line

class SpecFormatError(SpecError):
    pass

class SpecUnknowLineError(SpecError):
    pass

class SpecUnknowHeaderError(SpecError):
    pass

HEADERS = ( 'package',
            'description',
            'prep',
            'build',
            'install',
            'clean',
            'check',
            'preun',
            'pre',
            'postun',
            'post',
            'files',
            'changelog' )
SINGLES = ( 'Summary',
            'Name',
            'Version',
            'Release',
            'Epoch',
            'URL',
            'Url',
            'Group',
            'BuildArch',
            'AutoReq',
            'AutoProv',
            'AutoReqProv',
            'Autoreq',
            'Autoprov',
            'Autoreqprov',
            'ExclusiveArch',
            'Prefix',
            'License' )
REQUIRES = ('BuildRequires',
            'Requires',
            'Requires(post)',
            'Requires(postun)',
            'Requires(pre)',
            'PreRequires', 'PreReq', 'Prereq', # alias in old spec
            'Requires(preun)',
            'Provides',
            'Obsoletes',
            'Conflicts',
            'BuildConflicts',
           )
SKIPS   = ( 'BuildRoot',)

# must have keys for 'main' package
MUSTHAVE = {'Release': '1',
           }

# state definition of parser
(
    ST_DEFINE,
    ST_MAIN,
    ST_INLINE,
    ST_SUBPKG,
)   = range(4)

class SpecConvertor(Convertor):
    """ Convertor for SpecBuild ini files """

    def __init__(self):
        sb_cv_table = {
                'BuildRequires': 'PkgBR',
                'pre': 'install-pre',
                'description': 'Description',
                'Requires(post)': 'RequiresPost',
                'Requires(postun)': 'RequiresPostUn',
                'Requires(pre)': 'RequiresPre',
                'PreRequires': 'RequiresPre',
                'PreReq': 'RequiresPre',
                'Prereq': 'RequiresPre',
                'Requires(preun)': 'RequiresPreUn',
                'Url': 'URL',
                'Autoreq': 'AutoReq',
                'Autoprov': 'AutoProv',
                'Autoreqprov': 'AutoReqProv',
                }
        Convertor.__init__(self, sb_cv_table)

class SpecParser(object):
    """ Parser of SPEC file of rpm package """

    def __init__(self, replace_macros, builder_parsing, include_files):
        # runtime variables
        self.items = {}
        self.table = {}
        self.cur_pkg = 'main'

        self.include_files = include_files

        self.builder_parsing = builder_parsing
        self._Builder = None
        self._Configure = None

        self.replace_macros = replace_macros
        self.macros = []

    def _switch_subpkg(self, subpkg, create=False):

        # whether '-n subpkg'?
        wholename = False
        ls = subpkg.split()
        if '-n' in ls:
            try:
                subpkg = ls[ls.index('-n')+1]
                wholename = True
            except IndexError:
                raise SpecFormatError(subpkg)
        else:
            subpkg = ls[0]

        subpkg_new = 'SubPackages' not in self.items or subpkg not in self.items['SubPackages']


        if subpkg_new and not create:
            logger.warning('<spec2spectacle> un-declared subpkg %s found in spec' % subpkg)
            return None

        if subpkg_new:
            if 'SubPackages' not in self.items:
                self.items['SubPackages'] = {}
            if subpkg not in self.items['SubPackages']:
                self.items['SubPackages'][subpkg] = {}
            if wholename:
                self.items['SubPackages'][subpkg]['AsWholeName'] = True

        # switch
        self.cur_pkg = subpkg
        return self.items['SubPackages'][subpkg]

    def _do_package(self, items, pkg, h, v):
        # skip
        pass

    def _do_prep(self, items, pkg, h, v):
        logger.info('<spec2spectacle> the following is the content of PREP in original spec, please compare them with the generated carefully: \n%s' % v)

    def _do_build(self, items, pkg, h, v):
        """ to handle build script:
            trying to find out the most of the generic cases
        """

        def _save_raw_in_post(items, lines):
            # make sure no 'make' generated auto
            items['Configure'] = 'none'
            items['Builder'] = 'none'
            # put all script to 'post install'
            items['extra']['PostMakeExtras'] = lines

        ### Sub START
        lines = v.splitlines()
        if lines[0].startswith('-'):
            lines.pop(0)

        if not self.builder_parsing:
            _save_raw_in_post(items, lines)
            return

        # parts of build script
        (PRE, POST_CFG, POST_BUILD) = range(3)
        parts = { 'pre': [], 'post': [] }
        cur_part = PRE

        cont_line = False
        for line in lines:
            if cont_line:
                whole_line = whole_line + ' ' + line.strip()
            else:
                if not line.strip() or line.startswith('#'):
                    # empty line or comment line, skip
                    continue

                whole_line = line.strip()

            if line[-1:] == '\\':
                cont_line = True
                whole_line = whole_line[:-1].strip()
                continue
            else:
                cont_line = False

            found_cfgr = False
            found_bldr = False

            # find configure in current line
            for cfgr in ('configure', 'reconfigure', 'autogen'):
                pieces = whole_line.split()
                if cfgr in pieces or '%'+cfgr in pieces:
                    found_cfgr = True
                    self._Configure = cfgr
                    cfgr_line = whole_line[len(cfgr)+1:].strip()
                    break

            # find builder in current line
            if re.match('^(%\{__)?make', whole_line):
                found_bldr = True
                self._Builder = 'make'
            elif re.search('python\W+setup.py\W+build', whole_line):
                found_bldr = True
                self._Builder = 'python'

            if cur_part == PRE:
                if found_cfgr:
                    cur_part = POST_CFG
                elif found_bldr:
                    cur_part = POST_BUILD
                else:
                    parts['pre'].append(whole_line)

            elif cur_part == POST_CFG:
                if found_cfgr:
                    # more 'configr', wrong
                    cur_part = PRE
                    break
                elif found_bldr:
                    cur_part = POST_BUILD
                else:
                    # another line(s) between configr and buildr, wrong
                    # or no supported buildr found, wrong
                    cur_part = PRE
                    break

            elif cur_part == POST_BUILD:
                parts['post'].append(whole_line)

        if cur_part == PRE:
            # means match failed
            _save_raw_in_post(items, lines)
        else:
            if self._Configure:
                items['Configure'] = self._Configure
                # parse the configure options
                opts = map(lambda s: '--'+s,
                             [opt for opt in map(str.strip, cfgr_line.split('--')) if opt])
                if opts:
                    items['ConfigOptions'] = opts
            else:
                # no configure found, only need builder
                items['Configure'] = 'none'

            if self._Builder:
                items['Builder'] = self._Builder

            if parts['pre']:
                items['extra']['PreMakeExtras'] = parts['pre']
            if parts['post']:
                items['extra']['PostMakeExtras'] = parts['post']

    def _do_install(self, items, pkg, h, v):
        """ to handle install script:
            trying to find out the most of the generic cases
        """

        lines = v.splitlines()
        if lines[0].startswith('-'):
            lines.pop(0)

        # try to search %find_lang
        filter_lines = []
        for line in lines:
            if line.startswith('%find_lang'):
                m = re.compile('^%find_lang\s+(.*)\s*').match(line)
                if m:
                    parts = m.group(1).split()
                    if '||' in parts:
                        parts = parts[:parts.index('||')]
                    items['LocaleName'] = parts.pop(0)
                    if parts:
                        items['LocaleOptions'] = parts
                    continue

            filter_lines.append(line)
        lines = filter_lines

        parts = { 'pre': [], 'post': [] }

        if self._Builder == 'make':
            re_installer = re.compile('make[_ \t]*install')
        elif self._Builder == 'python':
            re_installer = re.compile('python\W+setup.py\W+install')

        re_cleanup = re.compile('(rm|\%\{__rm\})\W+-rf\W+(\$RPM_BUILD_ROOT|\%\{buildroot\})')

        if self._Builder:
            # have found 'Builder' in build scripts

            found_insaller = False
            for line in lines:
                if not found_insaller:
                    if re_installer.search(line):
                        found_insaller = True
                    elif re_cleanup.search(line):
                        # skip cleanup line
                        pass
                    else:
                        parts['pre'].append(line)
                else:
                    parts['post'].append(line)

            if found_insaller:
                if parts['pre']:
                    items['extra']['PreMakeInstallExtras'] = parts['pre']
                if parts['post']:
                    items['extra']['PostMakeInstallExtras'] = parts['post']

                return

        # false safe case
        items['extra']['PostMakeInstallExtras'] = lines

    def _do_clean(self, items, pkg, h, v):
        # skip
        pass

    def _do_check(self, items, pkg, h, v):
        items['extra']['check'] = v.strip()
        items['Check'] = True

    def _parse_prog_in_opt(self, header):
        ls = header.split()
        if '-p' in ls:
            try:
                return ls[ls.index('-p')+1]
            except IndexError:
                raise SpecFormatError(h)
        else:
            return ''

    def _do_extra_scripts(self, items, h, v):
        section = h.split()[0][1:]
        items['extra'][section] = v.strip().splitlines()
        inline_prog = self._parse_prog_in_opt(h)
        if inline_prog:
            items['extra'][section].insert(0, inline_prog)

    def _do_pre(self, items, pkg, h, v):
        self._do_extra_scripts(items, h, v)

    def _do_preun(self, items, pkg, h, v):
        self._do_extra_scripts(items, h, v)

    def _do_post(self, items, pkg, h, v):
        self._do_extra_scripts(items, h, v)

    def _do_postun(self, items, pkg, h, v):
        self._do_extra_scripts(items, h, v)

    def _do_changelog(self, items, pkg, h, v):
        logger.warning('<spec2spectacle> Please move changelog in %changelog to *.changes file.')

    def _remove_attrs(self, files):
        # try to remove duplicate '%defattr' in files list
        dup = '%defattr(-,root,root,-)'
        dup2 = '%defattr(-,root,root)'
        if dup in files:
            files.remove(dup)
        if dup2 in files:
            files.remove(dup2)

    def _do_files(self, items, pkg, h, v):
        files = map(str.strip, v.strip().splitlines())

        if files:
            # skip option line
            if files[0].startswith('-'):
                files.pop(0)

            self._remove_attrs(files)

        if self.include_files:
            items['Files'] = files
        else:
            items['extra']['Files'] = files

    def _do_description(self, items, pkg, h, v):
        items['Description'] = v.strip()

    def read(self, filename):
        """ read in all recognized directives and headers """

        comment = re.compile('^#.*')
        directive = re.compile('^([\w()]+):[ \t]*(.*)')
        define_re = re.compile('^%(define|global)\s+(\w+)\s+(.*)')
        header_re = re.compile('^%(' + '|'.join(HEADERS) + ')\s*(.*)')

        state = ST_DEFINE
        items = self.items
        for line in file(filename):

            if state == ST_DEFINE:
                line = line.strip()
                if not line or comment.match(line):
                    # skip comment line and empty line
                    continue

                m = define_re.match(line)
                if m:
                    if self.replace_macros:
                        self.table[m.group(2)] = m.group(3)
                    else:
                        self.macros.append(line)

                    continue #short-cut
                else:
                    state = ST_MAIN
                    # fall through

            if state == ST_INLINE:
                if header_re.match(line):
                    state = ST_MAIN
                    # fall through
                else:
                    if items:
                        items[cur_block] += line
                    continue

            if state == ST_MAIN:
                line = line.strip()
                if not line or comment.match(line):
                    # skip comment line and empty line
                    continue

                dm = directive.match(line)
                if not dm:
                    hm = header_re.match(line)

                if dm:
                    key = dm.group(1)
                    val = dm.group(2)

                    # special case for Source and Patch
                    if key.startswith('Source'):
                        key = 'Sources'
                    elif key.startswith('Patch'):
                        key = 'Patches'

                    if key not in items:
                        items[key] = [val]
                    else:
                        items[key].append(val)

                elif hm:
                    header = hm.group(1)
                    opt = hm.group(2)
                    if header not in HEADERS:
                        raise SpecUnknowHeaderError(state, self.cur_pkg, header)

                    if header == 'package':
                        if not opt:
                            raise SpecFormatError(line)
                        items = self._switch_subpkg(opt, True)
                    else:
                        # inline sections of other headers
                        state = ST_INLINE
                        if opt and (not opt.startswith('-') or '-n' in opt):
                            # section with sub-pkg specified
                            items = self._switch_subpkg(opt)
                        else:
                            # for 'main' package
                            items = self.items

                        if items:
                            cur_block = header
                            if cur_block not in items:
                                items[cur_block] = line+'\n'
                                """
                                # options in header line as the first line
                                if ' -' in line:
                                    items[cur_block] = line[line.index(' -'):] + '\n'
                                """

                else:
                    # unparsed line
                    known_skips = ('python_sitelib', 'python_sitearch')
                    msg = True
                    for skip in known_skips:
                        if skip in line:
                            msg = False
                            break

                    if msg:
                        logger.warning('<spec2spectacle> un-parsed spec line skipped: %s' % line)

    def cooked_items(self):
        """ return all items, cooked to the input of convertor """
        return self._cook_items('main', self.items)

    def _cook_items(self, pkg_name, items):
        """ helper function to transfer data structure
            <recursive>
        """

        # pattern of macros
        macro_re = re.compile('%{(\w+)}')

        ck_items = {'extra': {}}
        if pkg_name != 'main':
            ck_items['Name'] = pkg_name

        for k, v in items.iteritems():
            if k in SKIPS or k in HEADERS or k == 'SubPackages':
                continue

            if self.table:
                # macro replacing
                nv = []
                for vi in v:
                    while macro_re.search(vi):
                        nvi = vi
                        for m in macro_re.finditer(vi):
                            macro, name = m.group(0, 1)
                            if name in self.table:
                                nvi = nvi.replace(macro, self.table[name])
                        if vi == nvi:
                            break # break to exit 'while' loop
                        vi = nvi

                    # now nvi is the replaced string
                    nv.append(vi)
                v = nv

            if k in SINGLES:
                # special case for Release
                if k == 'Release':
                    m = re.match('(\S+)%{\?dist}', v[0])
                    if m: ck_items[k] = m.group(1)
                    else: ck_items[k] = v[0]
                else:
                    ck_items[k] = v[0]

            elif k in REQUIRES:
                nbr = [] # new 'PkgConfigBR' list
                nv = []
                for vi in v:
                    if 'perl' in vi:
                        nv.append(vi)
                    elif ',' in vi:
                        reqs = []
                        for entry in re.findall('\S+\s+[<>=]+\s+[^,\s]+|[^,\s]+', vi):
                            reqs.append(entry)
                        nv += reqs
                    elif ' ' in vi:
                        reqs = []
                        for entry in re.findall('\S+\s+[<>=]+\s+\S+|\S+', vi):
                            reqs.append(entry)
                        nv += reqs
                    else:
                        nv.append(vi)

                    if 'pkgconfig' in vi and k == 'BuildRequires':
                        for nvi in nv:
                            pkgbr = re.sub(r'pkgconfig\s*\(\s*([^\)]*)\s*\)', r'\1', nvi)
                            if pkgbr != nvi:
                                nbr.append(pkgbr)
                                nv.remove(nvi)

                ck_items[k] = nv
                if nbr:
                    ck_items['PkgConfigBR'] = nbr
            else:
                ck_items[k] = v

        # handle all sectinos with header, IN-ORDER
        for hdr in HEADERS:
            if hdr in items:
                routine = getattr(self, '_do_' + hdr)
                hdr_line, Drop, content = items[hdr].partition('\n')
                routine(ck_items, pkg_name, hdr_line, content)

        if pkg_name != 'main':
            # shortcut for subpkg
            return ck_items

        # handle subpackages
        if 'SubPackages' in items:
            ck_items['SubPackages'] = []
            for sub, sub_items in items['SubPackages'].iteritems():
                ck_items['SubPackages'].append(self._cook_items(sub, sub_items))

        # check must-have keys
        for key, default in MUSTHAVE.iteritems():
            if key not in ck_items:
                ck_items[key] = default

        # check for global macros
        if self.macros:
            ck_items['extra']['macros'] = self.macros

        return ck_items

def parse_options(args):
    import spectacle.__version__

    usage = "Usage: %prog [options] [spec-path]"
    parser = optparse.OptionParser(usage, version=spectacle.__version__.VERSION)

    parser.add_option("-o", "--output", type="string",
                      dest="outfile_path", default=None,
                      help="Path of output yaml file")
    parser.add_option("-r", "--replace-macros", action="store_true",
                      dest="replace_macros", default=False,
                      help="To replace self-defined macros in spec file")
    parser.add_option("",   "--no-builder-parsing", action="store_false",
                      dest="builder_parsing", default=True,
                      help="Do NOT try to parse build/install scripts")
    parser.add_option("-f", "--include-files", action="store_true",
                      dest="include_files", default=False,
                      help="To store files list in YAML file")

    return parser.parse_args()

def check_yaml_file(spec_fpath):
    specDir = os.path.dirname(spec_fpath)
    if not specDir:
        specDir = os.path.curdir

    yaml_s = glob.glob('*.yaml')
    if yaml_s:
        answer = logger.ask(""""*.yaml" file(s) exists in working dir: %s
    Maybe this package has been converted to spectacle enabled one.
    Continue?"""  % ' '.join(yaml_s), False)

        if not answer:
            sys.exit(1)

def check_ini_file(spec_fpath):
    specDir = os.path.dirname(spec_fpath)
    if not specDir:
        specDir = os.path.curdir

    ini_s = glob.glob('*.ini')
    if ini_s:
        answer = logger.ask(""""*.ini" file(s) exists in working dir: %s
    If being spec-builder file(s), please use ini2spectacle to convert.
    Continue?"""  % ' '.join(ini_s), False)

        if not answer:
            sys.exit(1)

def check_spec_file(spec_fpath):
    heads = """# 
# Do not Edit! Generated by:
# spectacle version """
    new_heads = """# 
# Do NOT Edit the Auto-generated Part!
# Generated by: spectacle version """
    fcont = file(spec_fpath).read()
    if fcont.startswith(heads) or fcont.startswith(new_heads):
        logger.error('<spec2spectacle> Input spec file is a spectacle generated one, do NOT convert it again.')

if __name__ == '__main__':
    """ Main Function """

    (options, args) = parse_options(sys.argv[1:])

    if not args:
        # no spec-path specified, search in CWD
        specls = glob.glob('*.spec')
        if not specls:
            logger.error('<spec2spectacle> Cannot find valid spec file in current directory, please specify one.')
        elif len(specls) > 1:
            logger.error('<spec2spectacle> Find multiple spec files in current directory, please specify one.')

        spec_fpath = specls[0]
    else:
        spec_fpath = args[0]

    # Check if YAML file exists
    check_yaml_file(spec_fpath)

    # Check if spec-build's INI file exists
    check_ini_file(spec_fpath)

    # Check if the input file exists
    if not os.path.exists(spec_fpath):
        # input file does not exist
        logger.error("<spec2spectacle> %s: File does not exist" % spec_fpath)

    # Check if spec file is spectacle generated one
    check_spec_file(spec_fpath)

    # check the working path
    if spec_fpath.find('/') != -1 and os.path.dirname(spec_fpath) != os.path.curdir:
        wdir = os.path.dirname(spec_fpath)
        logger.info('<spec2spectacle> Changing to working dir: %s' % wdir)
        os.chdir(wdir)

    spec_fname = os.path.basename(spec_fpath)

    if options.outfile_path:
        out_fpath = options.outfile_path
    else:
        if spec_fname.endswith('.spec'):
            out_fpath = spec_fname[:-4] + 'yaml'
        else:
            out_fpath = spec_fname + '.yaml'

    """Read the input file"""
    spec_parser = SpecParser(replace_macros = options.replace_macros,
                             builder_parsing = options.builder_parsing,
                             include_files = options.include_files
                            )
    try:
        spec_parser.read(spec_fname)
    except SpecFormatError, e:
        logger.warning('<spec2spectacle> Spec syntax error: %s' % str(e))
    except SpecUnknowHeaderError, e:
        logger.warning('<spec2spectacle> Unknown spec header: %s' % str(e))

    convertor = SpecConvertor()

    """Dump them to spectacle file"""
    dumper = SpectacleDumper(format='yaml', opath = out_fpath)
    newspec_fpath = dumper.dump(convertor.convert(spec_parser.cooked_items()))

    logger.info('<spec2spectacle> Yaml file %s created' % out_fpath)
    if newspec_fpath:
        bak_spec_fpath = os.path.join('spec.backup', newspec_fpath)
        logger.info('<spec2spectacle> New spec file %s was generated by new yaml file,' % newspec_fpath)
        logger.info('<spec2spectacle> and orignal spec file was saved as %s' % bak_spec_fpath)