Ignore:
Timestamp:
Jul 24, 2017, 2:12:55 PM (3 years ago)
Author:
Paul Brossier <piem@piem.org>
Branches:
feature/autosink, feature/constantq, feature/pitchshift, feature/pydocstrings, feature/timestretch, master
Children:
1070378
Parents:
482641d (diff), ddea34b (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.
Message:

Merge branch 'master' into yinfast

File:
1 edited

Legend:

Unmodified
Added
Removed
  • python/lib/gen_external.py

    r482641d r9fa0ed1  
    11import distutils.ccompiler
    2 import sys, os, subprocess, glob
     2import sys
     3import os
     4import subprocess
     5import glob
    36
    47header = os.path.join('src', 'aubio.h')
     
    912"""
    1013
    11 skip_objects = [
    12   # already in ext/
    13   'fft',
    14   'pvoc',
    15   'filter',
    16   'filterbank',
    17   # AUBIO_UNSTABLE
    18   'hist',
    19   'parameter',
    20   'scale',
    21   'beattracking',
    22   'resampler',
    23   'peakpicker',
    24   'pitchfcomb',
    25   'pitchmcomb',
    26   'pitchschmitt',
    27   'pitchspecacf',
    28   'pitchyin',
    29   'pitchyinfft',
    30   'pitchyinfast',
    31   'sink',
    32   'sink_apple_audio',
    33   'sink_sndfile',
    34   'sink_wavwrite',
    35   #'mfcc',
    36   'source',
    37   'source_apple_audio',
    38   'source_sndfile',
    39   'source_avcodec',
    40   'source_wavread',
    41   #'sampler',
    42   'audio_unit',
    43   'spectral_whitening',
    44   ]
     14default_skip_objects = [
     15    # already in ext/
     16    'fft',
     17    'pvoc',
     18    'filter',
     19    'filterbank',
     20    # AUBIO_UNSTABLE
     21    'hist',
     22    'parameter',
     23    'scale',
     24    'beattracking',
     25    'resampler',
     26    'peakpicker',
     27    'pitchfcomb',
     28    'pitchmcomb',
     29    'pitchschmitt',
     30    'pitchspecacf',
     31    'pitchyin',
     32    'pitchyinfft',
     33    'pitchyinfast',
     34    'sink',
     35    'sink_apple_audio',
     36    'sink_sndfile',
     37    'sink_wavwrite',
     38    #'mfcc',
     39    'source',
     40    'source_apple_audio',
     41    'source_sndfile',
     42    'source_avcodec',
     43    'source_wavread',
     44    #'sampler',
     45    'audio_unit',
     46    'spectral_whitening',
     47]
     48
    4549
    4650def get_preprocessor():
     
    6165
    6266    cpp_cmd = None
    63     if hasattr(compiler, 'preprocessor'): # for unixccompiler
     67    if hasattr(compiler, 'preprocessor'):  # for unixccompiler
    6468        cpp_cmd = compiler.preprocessor
    65     elif hasattr(compiler, 'compiler'): # for ccompiler
     69    elif hasattr(compiler, 'compiler'):  # for ccompiler
    6670        cpp_cmd = compiler.compiler.split()
    6771        cpp_cmd += ['-E']
    68     elif hasattr(compiler, 'cc'): # for msvccompiler
     72    elif hasattr(compiler, 'cc'):  # for msvccompiler
    6973        cpp_cmd = compiler.cc.split()
    7074        cpp_cmd += ['-E']
     
    7478        cpp_cmd = os.environ.get('CC', 'cc').split()
    7579        cpp_cmd += ['-E']
    76 
     80    cpp_cmd += ['-x', 'c']  # force C language (emcc defaults to c++)
    7781    return cpp_cmd
    7882
    79 def get_cpp_objects(header=header, usedouble=False):
     83
     84def get_c_declarations(header=header, usedouble=False):
     85    ''' return a dense and preprocessed  string of all c declarations implied by aubio.h
     86    '''
    8087    cpp_cmd = get_preprocessor()
    8188
     
    93100    print("Running command: {:s}".format(" ".join(cpp_cmd)))
    94101    proc = subprocess.Popen(cpp_cmd,
    95             stderr=subprocess.PIPE,
    96             stdout=subprocess.PIPE)
     102                            stderr=subprocess.PIPE,
     103                            stdout=subprocess.PIPE)
    97104    assert proc, 'Proc was none'
    98105    cpp_output = proc.stdout.read()
     
    101108        raise Exception("preprocessor output is empty:\n%s" % err_output)
    102109    elif err_output:
    103         print ("Warning: preprocessor produced warnings:\n%s" % err_output)
     110        print("Warning: preprocessor produced warnings:\n%s" % err_output)
    104111    if not isinstance(cpp_output, list):
    105112        cpp_output = [l.strip() for l in cpp_output.decode('utf8').split('\n')]
     
    110117    i = 1
    111118    while 1:
    112         if i >= len(cpp_output): break
    113         if cpp_output[i-1].endswith(',') or cpp_output[i-1].endswith('{') or cpp_output[i].startswith('}'):
    114             cpp_output[i] = cpp_output[i-1] + ' ' + cpp_output[i]
    115             cpp_output.pop(i-1)
     119        if i >= len(cpp_output):
     120            break
     121        if ('{' in cpp_output[i - 1]) and (not '}' in cpp_output[i - 1]) or (not ';' in cpp_output[i - 1]):
     122            cpp_output[i] = cpp_output[i - 1] + ' ' + cpp_output[i]
     123            cpp_output.pop(i - 1)
     124        elif ('}' in cpp_output[i]):
     125            cpp_output[i] = cpp_output[i - 1] + ' ' + cpp_output[i]
     126            cpp_output.pop(i - 1)
    116127        else:
    117128            i += 1
    118129
    119     typedefs = filter(lambda y: y.startswith ('typedef struct _aubio'), cpp_output)
    120 
     130    # clean pointer notations
     131    tmp = []
     132    for l in cpp_output:
     133        tmp += [l.replace(' *', ' * ')]
     134    cpp_output = tmp
     135
     136    return cpp_output
     137
     138
     139def get_cpp_objects_from_c_declarations(c_declarations, skip_objects=None):
     140    if skip_objects == None:
     141        skip_objects = default_skip_objects
     142    typedefs = filter(lambda y: y.startswith('typedef struct _aubio'), c_declarations)
    121143    cpp_objects = [a.split()[3][:-1] for a in typedefs]
    122 
    123     return cpp_output, cpp_objects
    124 
    125 
    126 def analyze_cpp_output(cpp_objects, cpp_output):
     144    cpp_objects_filtered = filter(lambda y: not y[6:-2] in skip_objects, cpp_objects)
     145    return cpp_objects_filtered
     146
     147
     148def get_all_func_names_from_lib(lib, depth=0):
     149    ''' return flat string of all function used in lib
     150    '''
     151    res = []
     152    indent = " " * depth
     153    for k, v in lib.items():
     154        if isinstance(v, dict):
     155            res += get_all_func_names_from_lib(v, depth + 1)
     156        elif isinstance(v, list):
     157            for elem in v:
     158                e = elem.split('(')
     159                if len(e) < 2:
     160                    continue  # not a function
     161                fname_part = e[0].strip().split(' ')
     162                fname = fname_part[-1]
     163                if fname:
     164                    res += [fname]
     165                else:
     166                    raise NameError('gen_lib : weird function: ' + str(e))
     167
     168    return res
     169
     170
     171def generate_lib_from_c_declarations(cpp_objects, c_declarations):
     172    ''' returns a lib from given cpp_object names
     173
     174    a lib is a dict grouping functions by family (onset,pitch...)
     175        each eement is itself a dict of functions grouped by puposes as :
     176        struct, new, del, do, get, set and other
     177    '''
    127178    lib = {}
    128179
    129180    for o in cpp_objects:
    130         if o[:6] != 'aubio_':
    131             continue
    132         shortname = o[6:-2]
    133         if shortname in skip_objects:
    134             continue
     181        shortname = o
     182        if o[:6] == 'aubio_':
     183            shortname = o[6:-2]  # without aubio_ prefix and _t suffix
     184
    135185        lib[shortname] = {'struct': [], 'new': [], 'del': [], 'do': [], 'get': [], 'set': [], 'other': []}
    136186        lib[shortname]['longname'] = o
    137187        lib[shortname]['shortname'] = shortname
    138         for fn in cpp_output:
    139             if o[:-1] in fn:
    140                 #print "found", o[:-1], "in", fn
     188
     189        fullshortname = o[:-2]  # name without _t suffix
     190
     191        for fn in c_declarations:
     192            func_name = fn.split('(')[0].strip().split(' ')[-1]
     193            if func_name.startswith(fullshortname + '_') or func_name.endswith(fullshortname):
     194                # print "found", shortname, "in", fn
    141195                if 'typedef struct ' in fn:
    142196                    lib[shortname]['struct'].append(fn)
     
    152206                    lib[shortname]['set'].append(fn)
    153207                else:
    154                     #print "no idea what to do about", fn
     208                    # print "no idea what to do about", fn
    155209                    lib[shortname]['other'].append(fn)
    156210    return lib
    157211
    158 def print_cpp_output_results(lib, cpp_output):
    159     for fn in cpp_output:
     212
     213def print_c_declarations_results(lib, c_declarations):
     214    for fn in c_declarations:
    160215        found = 0
    161216        for o in lib:
     
    164219                    found = 1
    165220        if found == 0:
    166             print ("missing", fn)
     221            print("missing", fn)
    167222
    168223    for o in lib:
    169224        for family in lib[o]:
    170225            if type(lib[o][family]) == str:
    171                 print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family] ) )
     226                print("{:15s} {:10s} {:s}".format(o, family, lib[o][family]))
    172227            elif len(lib[o][family]) == 1:
    173                 print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family][0] ) )
     228                print("{:15s} {:10s} {:s}".format(o, family, lib[o][family][0]))
    174229            else:
    175                 print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family] ) )
     230                print("{:15s} {:10s} {:s}".format(o, family, lib[o][family]))
    176231
    177232
    178233def generate_external(header=header, output_path=output_path, usedouble=False, overwrite=True):
    179     if not os.path.isdir(output_path): os.mkdir(output_path)
    180     elif not overwrite: return sorted(glob.glob(os.path.join(output_path, '*.c')))
    181 
    182     cpp_output, cpp_objects = get_cpp_objects(header, usedouble=usedouble)
    183 
    184     lib = analyze_cpp_output(cpp_objects, cpp_output)
    185     # print_cpp_output_results(lib, cpp_output)
     234    if not os.path.isdir(output_path):
     235        os.mkdir(output_path)
     236    elif not overwrite:
     237        return sorted(glob.glob(os.path.join(output_path, '*.c')))
     238
     239    c_declarations = get_c_declarations(header, usedouble=usedouble)
     240    cpp_objects = get_cpp_objects_from_c_declarations(c_declarations)
     241
     242    lib = generate_lib_from_c_declarations(cpp_objects, c_declarations)
     243    # print_c_declarations_results(lib, c_declarations)
    186244
    187245    sources_list = []
     
    192250    for o in lib:
    193251        out = source_header
    194         mapped = MappedObject(lib[o], usedouble = usedouble)
     252        mapped = MappedObject(lib[o], usedouble=usedouble)
    195253        out += mapped.gen_code()
    196254        output_file = os.path.join(output_path, 'gen-%s.c' % o)
    197255        with open(output_file, 'w') as f:
    198256            f.write(out)
    199             print ("wrote %s" % output_file )
     257            print("wrote %s" % output_file)
    200258            sources_list.append(output_file)
    201259
     
    209267  return ({pycheck_types});
    210268}}
    211 """.format(pycheck_types = check_types)
     269""".format(pycheck_types=check_types)
    212270
    213271    add_types = "".join(["""
    214272  Py_INCREF (&Py_{name}Type);
    215   PyModule_AddObject(m, "{name}", (PyObject *) & Py_{name}Type);""".format(name = o) for o in lib])
     273  PyModule_AddObject(m, "{name}", (PyObject *) & Py_{name}Type);""".format(name=o) for o in lib])
    216274    out += """
    217275
     
    220278{add_types}
    221279}}
    222 """.format(add_types = add_types)
     280""".format(add_types=add_types)
    223281
    224282    output_file = os.path.join(output_path, 'aubio-generated.c')
    225283    with open(output_file, 'w') as f:
    226284        f.write(out)
    227         print ("wrote %s" % output_file )
     285        print("wrote %s" % output_file)
    228286        sources_list.append(output_file)
    229287
     
    243301int generated_objects ( void );
    244302void add_generated_objects( PyObject *m );
    245 """.format(objlist = objlist)
     303""".format(objlist=objlist)
    246304
    247305    output_file = os.path.join(output_path, 'aubio-generated.h')
    248306    with open(output_file, 'w') as f:
    249307        f.write(out)
    250         print ("wrote %s" % output_file )
     308        print("wrote %s" % output_file)
    251309        # no need to add header to list of sources
    252310
     
    254312
    255313if __name__ == '__main__':
    256     if len(sys.argv) > 1: header = sys.argv[1]
    257     if len(sys.argv) > 2: output_path = sys.argv[2]
     314    if len(sys.argv) > 1:
     315        header = sys.argv[1]
     316    if len(sys.argv) > 2:
     317        output_path = sys.argv[2]
    258318    generate_external(header, output_path)
Note: See TracChangeset for help on using the changeset viewer.