Changeset ddea34b


Ignore:
Timestamp:
Jul 24, 2017, 2:09:57 PM (7 years ago)
Author:
Paul Brossier <piem@piem.org>
Branches:
feature/autosink, feature/cnn, feature/cnn_org, feature/constantq, feature/crepe, feature/crepe_org, feature/pitchshift, feature/pydocstrings, feature/timestretch, fix/ffmpeg5, master
Children:
1070378, 9fa0ed1
Parents:
a1cce65 (diff), caf5fce (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.
Message:

Merge branch 'emscripten'

Files:
3 edited

Legend:

Unmodified
Added
Removed
  • python/lib/gen_external.py

    ra1cce65 rddea34b  
    11import distutils.ccompiler
    2 import sys, os, subprocess, glob
     2import sys
     3import os
     4import subprocess
     5import glob
    36
    47header = os.path.join('src', 'aubio.h')
     
    912"""
    1013
    11 skip_objects = [
    12   # already in ext/
    13   'fft',
    14   'pvoc',
    15   'filter',
    16   'filterbank',
    17   # AUBIO_UNSTABLE
    18   'hist',
    19   'parameter',
    20   'scale',
    21   'beattracking',
    22   'resampler',
    23   'peakpicker',
    24   'pitchfcomb',
    25   'pitchmcomb',
    26   'pitchschmitt',
    27   'pitchspecacf',
    28   'pitchyin',
    29   'pitchyinfft',
    30   'sink',
    31   'sink_apple_audio',
    32   'sink_sndfile',
    33   'sink_wavwrite',
    34   #'mfcc',
    35   'source',
    36   'source_apple_audio',
    37   'source_sndfile',
    38   'source_avcodec',
    39   'source_wavread',
    40   #'sampler',
    41   'audio_unit',
    42   'spectral_whitening',
    43   ]
     14default_skip_objects = [
     15    # already in ext/
     16    'fft',
     17    'pvoc',
     18    'filter',
     19    'filterbank',
     20    # AUBIO_UNSTABLE
     21    'hist',
     22    'parameter',
     23    'scale',
     24    'beattracking',
     25    'resampler',
     26    'peakpicker',
     27    'pitchfcomb',
     28    'pitchmcomb',
     29    'pitchschmitt',
     30    'pitchspecacf',
     31    'pitchyin',
     32    'pitchyinfft',
     33    'sink',
     34    'sink_apple_audio',
     35    'sink_sndfile',
     36    'sink_wavwrite',
     37    #'mfcc',
     38    'source',
     39    'source_apple_audio',
     40    'source_sndfile',
     41    'source_avcodec',
     42    'source_wavread',
     43    #'sampler',
     44    'audio_unit',
     45    'spectral_whitening',
     46]
     47
    4448
    4549def get_preprocessor():
     
    6064
    6165    cpp_cmd = None
    62     if hasattr(compiler, 'preprocessor'): # for unixccompiler
     66    if hasattr(compiler, 'preprocessor'):  # for unixccompiler
    6367        cpp_cmd = compiler.preprocessor
    64     elif hasattr(compiler, 'compiler'): # for ccompiler
     68    elif hasattr(compiler, 'compiler'):  # for ccompiler
    6569        cpp_cmd = compiler.compiler.split()
    6670        cpp_cmd += ['-E']
    67     elif hasattr(compiler, 'cc'): # for msvccompiler
     71    elif hasattr(compiler, 'cc'):  # for msvccompiler
    6872        cpp_cmd = compiler.cc.split()
    6973        cpp_cmd += ['-E']
     
    7377        cpp_cmd = os.environ.get('CC', 'cc').split()
    7478        cpp_cmd += ['-E']
    75 
     79    cpp_cmd += ['-x', 'c']  # force C language (emcc defaults to c++)
    7680    return cpp_cmd
    7781
    78 def get_cpp_objects(header=header, usedouble=False):
     82
     83def get_c_declarations(header=header, usedouble=False):
     84    ''' return a dense and preprocessed  string of all c declarations implied by aubio.h
     85    '''
    7986    cpp_cmd = get_preprocessor()
    8087
     
    9299    print("Running command: {:s}".format(" ".join(cpp_cmd)))
    93100    proc = subprocess.Popen(cpp_cmd,
    94             stderr=subprocess.PIPE,
    95             stdout=subprocess.PIPE)
     101                            stderr=subprocess.PIPE,
     102                            stdout=subprocess.PIPE)
    96103    assert proc, 'Proc was none'
    97104    cpp_output = proc.stdout.read()
     
    100107        raise Exception("preprocessor output is empty:\n%s" % err_output)
    101108    elif err_output:
    102         print ("Warning: preprocessor produced warnings:\n%s" % err_output)
     109        print("Warning: preprocessor produced warnings:\n%s" % err_output)
    103110    if not isinstance(cpp_output, list):
    104111        cpp_output = [l.strip() for l in cpp_output.decode('utf8').split('\n')]
     
    109116    i = 1
    110117    while 1:
    111         if i >= len(cpp_output): break
    112         if cpp_output[i-1].endswith(',') or cpp_output[i-1].endswith('{') or cpp_output[i].startswith('}'):
    113             cpp_output[i] = cpp_output[i-1] + ' ' + cpp_output[i]
    114             cpp_output.pop(i-1)
     118        if i >= len(cpp_output):
     119            break
     120        if ('{' in cpp_output[i - 1]) and (not '}' in cpp_output[i - 1]) or (not ';' in cpp_output[i - 1]):
     121            cpp_output[i] = cpp_output[i - 1] + ' ' + cpp_output[i]
     122            cpp_output.pop(i - 1)
     123        elif ('}' in cpp_output[i]):
     124            cpp_output[i] = cpp_output[i - 1] + ' ' + cpp_output[i]
     125            cpp_output.pop(i - 1)
    115126        else:
    116127            i += 1
    117128
    118     typedefs = filter(lambda y: y.startswith ('typedef struct _aubio'), cpp_output)
    119 
     129    # clean pointer notations
     130    tmp = []
     131    for l in cpp_output:
     132        tmp += [l.replace(' *', ' * ')]
     133    cpp_output = tmp
     134
     135    return cpp_output
     136
     137
     138def get_cpp_objects_from_c_declarations(c_declarations, skip_objects=None):
     139    if skip_objects == None:
     140        skip_objects = default_skip_objects
     141    typedefs = filter(lambda y: y.startswith('typedef struct _aubio'), c_declarations)
    120142    cpp_objects = [a.split()[3][:-1] for a in typedefs]
    121 
    122     return cpp_output, cpp_objects
    123 
    124 
    125 def analyze_cpp_output(cpp_objects, cpp_output):
     143    cpp_objects_filtered = filter(lambda y: not y[6:-2] in skip_objects, cpp_objects)
     144    return cpp_objects_filtered
     145
     146
     147def get_all_func_names_from_lib(lib, depth=0):
     148    ''' return flat string of all function used in lib
     149    '''
     150    res = []
     151    indent = " " * depth
     152    for k, v in lib.items():
     153        if isinstance(v, dict):
     154            res += get_all_func_names_from_lib(v, depth + 1)
     155        elif isinstance(v, list):
     156            for elem in v:
     157                e = elem.split('(')
     158                if len(e) < 2:
     159                    continue  # not a function
     160                fname_part = e[0].strip().split(' ')
     161                fname = fname_part[-1]
     162                if fname:
     163                    res += [fname]
     164                else:
     165                    raise NameError('gen_lib : weird function: ' + str(e))
     166
     167    return res
     168
     169
     170def generate_lib_from_c_declarations(cpp_objects, c_declarations):
     171    ''' returns a lib from given cpp_object names
     172
     173    a lib is a dict grouping functions by family (onset,pitch...)
     174        each eement is itself a dict of functions grouped by puposes as :
     175        struct, new, del, do, get, set and other
     176    '''
    126177    lib = {}
    127178
    128179    for o in cpp_objects:
    129         if o[:6] != 'aubio_':
    130             continue
    131         shortname = o[6:-2]
    132         if shortname in skip_objects:
    133             continue
     180        shortname = o
     181        if o[:6] == 'aubio_':
     182            shortname = o[6:-2]  # without aubio_ prefix and _t suffix
     183
    134184        lib[shortname] = {'struct': [], 'new': [], 'del': [], 'do': [], 'get': [], 'set': [], 'other': []}
    135185        lib[shortname]['longname'] = o
    136186        lib[shortname]['shortname'] = shortname
    137         for fn in cpp_output:
    138             if o[:-1] in fn:
    139                 #print "found", o[:-1], "in", fn
     187
     188        fullshortname = o[:-2]  # name without _t suffix
     189
     190        for fn in c_declarations:
     191            func_name = fn.split('(')[0].strip().split(' ')[-1]
     192            if func_name.startswith(fullshortname + '_') or func_name.endswith(fullshortname):
     193                # print "found", shortname, "in", fn
    140194                if 'typedef struct ' in fn:
    141195                    lib[shortname]['struct'].append(fn)
     
    151205                    lib[shortname]['set'].append(fn)
    152206                else:
    153                     #print "no idea what to do about", fn
     207                    # print "no idea what to do about", fn
    154208                    lib[shortname]['other'].append(fn)
    155209    return lib
    156210
    157 def print_cpp_output_results(lib, cpp_output):
    158     for fn in cpp_output:
     211
     212def print_c_declarations_results(lib, c_declarations):
     213    for fn in c_declarations:
    159214        found = 0
    160215        for o in lib:
     
    163218                    found = 1
    164219        if found == 0:
    165             print ("missing", fn)
     220            print("missing", fn)
    166221
    167222    for o in lib:
    168223        for family in lib[o]:
    169224            if type(lib[o][family]) == str:
    170                 print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family] ) )
     225                print("{:15s} {:10s} {:s}".format(o, family, lib[o][family]))
    171226            elif len(lib[o][family]) == 1:
    172                 print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family][0] ) )
     227                print("{:15s} {:10s} {:s}".format(o, family, lib[o][family][0]))
    173228            else:
    174                 print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family] ) )
     229                print("{:15s} {:10s} {:s}".format(o, family, lib[o][family]))
    175230
    176231
    177232def generate_external(header=header, output_path=output_path, usedouble=False, overwrite=True):
    178     if not os.path.isdir(output_path): os.mkdir(output_path)
    179     elif not overwrite: return sorted(glob.glob(os.path.join(output_path, '*.c')))
    180 
    181     cpp_output, cpp_objects = get_cpp_objects(header, usedouble=usedouble)
    182 
    183     lib = analyze_cpp_output(cpp_objects, cpp_output)
    184     # print_cpp_output_results(lib, cpp_output)
     233    if not os.path.isdir(output_path):
     234        os.mkdir(output_path)
     235    elif not overwrite:
     236        return sorted(glob.glob(os.path.join(output_path, '*.c')))
     237
     238    c_declarations = get_c_declarations(header, usedouble=usedouble)
     239    cpp_objects = get_cpp_objects_from_c_declarations(c_declarations)
     240
     241    lib = generate_lib_from_c_declarations(cpp_objects, c_declarations)
     242    # print_c_declarations_results(lib, c_declarations)
    185243
    186244    sources_list = []
     
    191249    for o in lib:
    192250        out = source_header
    193         mapped = MappedObject(lib[o], usedouble = usedouble)
     251        mapped = MappedObject(lib[o], usedouble=usedouble)
    194252        out += mapped.gen_code()
    195253        output_file = os.path.join(output_path, 'gen-%s.c' % o)
    196254        with open(output_file, 'w') as f:
    197255            f.write(out)
    198             print ("wrote %s" % output_file )
     256            print("wrote %s" % output_file)
    199257            sources_list.append(output_file)
    200258
     
    208266  return ({pycheck_types});
    209267}}
    210 """.format(pycheck_types = check_types)
     268""".format(pycheck_types=check_types)
    211269
    212270    add_types = "".join(["""
    213271  Py_INCREF (&Py_{name}Type);
    214   PyModule_AddObject(m, "{name}", (PyObject *) & Py_{name}Type);""".format(name = o) for o in lib])
     272  PyModule_AddObject(m, "{name}", (PyObject *) & Py_{name}Type);""".format(name=o) for o in lib])
    215273    out += """
    216274
     
    219277{add_types}
    220278}}
    221 """.format(add_types = add_types)
     279""".format(add_types=add_types)
    222280
    223281    output_file = os.path.join(output_path, 'aubio-generated.c')
    224282    with open(output_file, 'w') as f:
    225283        f.write(out)
    226         print ("wrote %s" % output_file )
     284        print("wrote %s" % output_file)
    227285        sources_list.append(output_file)
    228286
     
    242300int generated_objects ( void );
    243301void add_generated_objects( PyObject *m );
    244 """.format(objlist = objlist)
     302""".format(objlist=objlist)
    245303
    246304    output_file = os.path.join(output_path, 'aubio-generated.h')
    247305    with open(output_file, 'w') as f:
    248306        f.write(out)
    249         print ("wrote %s" % output_file )
     307        print("wrote %s" % output_file)
    250308        # no need to add header to list of sources
    251309
     
    253311
    254312if __name__ == '__main__':
    255     if len(sys.argv) > 1: header = sys.argv[1]
    256     if len(sys.argv) > 2: output_path = sys.argv[2]
     313    if len(sys.argv) > 1:
     314        header = sys.argv[1]
     315    if len(sys.argv) > 2:
     316        output_path = sys.argv[2]
    257317    generate_external(header, output_path)
  • src/wscript_build

    ra1cce65 rddea34b  
    2727    build_features = ['cstlib', 'cshlib']
    2828elif ctx.env['DEST_OS'] in ['emscripten']:
    29     build_features = ['cstlib']
     29    build_features = ['cstlib','cshlib']
    3030elif '--static' in ctx.env['LDFLAGS'] or '--static' in ctx.env['LINKFLAGS']:
    3131    # static in cflags, ...
  • wscript

    ra1cce65 rddea34b  
    108108    ctx.load('gnu_dirs')
    109109
     110    target_platform = sys.platform
     111    if ctx.options.target_platform:
     112        target_platform = ctx.options.target_platform
     113
     114
     115    if target_platform=='emscripten':
     116        # need to force spaces between flag -o and path
     117        # inspired from :
     118        # https://github.com/waf-project/waf/blob/master/waflib/extras/c_emscripten.py (#1885)
     119        # (OSX /emscripten 1.37.9)
     120        ctx.env.CC_TGT_F            = ['-c', '-o', '']
     121        ctx.env.CCLNK_TGT_F         = ['-o', '']
    110122    # check for common headers
    111123    ctx.check(header_name='stdlib.h')
     
    118130    ctx.check(header_name='unistd.h', mandatory = False)
    119131
    120     target_platform = sys.platform
    121     if ctx.options.target_platform:
    122         target_platform = ctx.options.target_platform
    123132    ctx.env['DEST_OS'] = target_platform
    124133
     
    133142            ctx.env.prepend_value('CFLAGS', ['-O0'])
    134143        else:
    135             # default to -O2 in release mode
    136             ctx.env.prepend_value('CFLAGS', ['-O2'])
     144            if target_platform == 'emscripten':
     145                # -Oz for small js file generation
     146                ctx.env.prepend_value('CFLAGS', ['-Oz'])
     147            else:
     148                # default to -O2 in release mode
     149                ctx.env.prepend_value('CFLAGS', ['-O2'])
    137150        # enable debug symbols and configure warnings
    138151        ctx.env.prepend_value('CFLAGS', ['-g', '-Wall', '-Wextra'])
     
    216229        import os.path
    217230        ctx.env.CFLAGS += [ '-I' + os.path.join(os.environ['EMSCRIPTEN'], 'system', 'include') ]
    218         ctx.env.CFLAGS += ['-Oz']
     231       
     232        if ctx.options.build_type == "debug":
     233            ctx.env.cshlib_PATTERN = '%s.js'
     234            ctx.env.LINKFLAGS += ['-s','ASSERTIONS=2']
     235            ctx.env.LINKFLAGS += ['-s','SAFE_HEAP=1']
     236            ctx.env.LINKFLAGS += ['-s','ALIASING_FUNCTION_POINTERS=0']
     237            ctx.env.LINKFLAGS += ['-O0']
     238        else:
     239            ctx.env.LINKFLAGS += ['-Oz']
     240            ctx.env.cshlib_PATTERN = '%s.min.js'
     241
     242        # doesnt ship file system support in lib
     243        ctx.env.LINKFLAGS_cshlib += ['-s', 'NO_FILESYSTEM=1']
     244        # put memory file inside generated js files for easier portability
     245        ctx.env.LINKFLAGS += ['--memory-init-file', '0']
    219246        ctx.env.cprogram_PATTERN = "%s.js"
    220         if (ctx.options.enable_atlas != True):
    221             ctx.options.enable_atlas = False
     247        ctx.env.cstlib_PATTERN = '%s.a'
     248
     249        # tell emscripten functions we want to expose
     250        from python.lib.gen_external import get_c_declarations, get_cpp_objects_from_c_declarations, get_all_func_names_from_lib, generate_lib_from_c_declarations
     251        c_decls = get_c_declarations(usedouble=False)  # emscripten can't use double
     252        objects = get_cpp_objects_from_c_declarations(c_decls)
     253        # ensure that aubio structs are exported
     254        objects += ['fvec_t', 'cvec_t', 'fmat_t']
     255        lib = generate_lib_from_c_declarations(objects, c_decls)
     256        exported_funcnames = get_all_func_names_from_lib(lib)
     257        c_mangled_names = ['_' + s for s in exported_funcnames]
     258        ctx.env.LINKFLAGS_cshlib += ['-s', 'EXPORTED_FUNCTIONS=%s' % c_mangled_names]
     259
     260    if (ctx.options.enable_atlas != True):
     261        ctx.options.enable_atlas = False
    222262
    223263    # check support for C99 __VA_ARGS__ macros
Note: See TracChangeset for help on using the changeset viewer.