Changes in / [a1cce65:caf5fce]

Files:

• python/lib/gen_external.py (modified) (14 diffs)
• src/wscript_build (modified) (1 diff)
• wscript (modified) (4 diffs)

python/lib/gen_external.py

@@ -1 +1 @@
 import distutils.ccompiler
-import sys, os, subprocess, glob
+import sys
+import os
+import subprocess
+import glob
 
 header = os.path.join('src', 'aubio.h')
@@ -9 +12 @@
 """
 
-skip_objects = [
-  # already in ext/
-  'fft',
-  'pvoc',
-  'filter',
-  'filterbank',
-  # AUBIO_UNSTABLE
-  'hist',
-  'parameter',
-  'scale',
-  'beattracking',
-  'resampler',
-  'peakpicker',
-  'pitchfcomb',
-  'pitchmcomb',
-  'pitchschmitt',
-  'pitchspecacf',
-  'pitchyin',
-  'pitchyinfft',
-  'sink',
-  'sink_apple_audio',
-  'sink_sndfile',
-  'sink_wavwrite',
-  #'mfcc',
-  'source',
-  'source_apple_audio',
-  'source_sndfile',
-  'source_avcodec',
-  'source_wavread',
-  #'sampler',
-  'audio_unit',
-  'spectral_whitening',
-  ]
+default_skip_objects = [
+    # already in ext/
+    'fft',
+    'pvoc',
+    'filter',
+    'filterbank',
+    # AUBIO_UNSTABLE
+    'hist',
+    'parameter',
+    'scale',
+    'beattracking',
+    'resampler',
+    'peakpicker',
+    'pitchfcomb',
+    'pitchmcomb',
+    'pitchschmitt',
+    'pitchspecacf',
+    'pitchyin',
+    'pitchyinfft',
+    'sink',
+    'sink_apple_audio',
+    'sink_sndfile',
+    'sink_wavwrite',
+    #'mfcc',
+    'source',
+    'source_apple_audio',
+    'source_sndfile',
+    'source_avcodec',
+    'source_wavread',
+    #'sampler',
+    'audio_unit',
+    'spectral_whitening',
+]
+
 
 def get_preprocessor():
@@ -60 +64 @@
 
     cpp_cmd = None
-    if hasattr(compiler, 'preprocessor'): # for unixccompiler
+    if hasattr(compiler, 'preprocessor'):  # for unixccompiler
         cpp_cmd = compiler.preprocessor
-    elif hasattr(compiler, 'compiler'): # for ccompiler
+    elif hasattr(compiler, 'compiler'):  # for ccompiler
         cpp_cmd = compiler.compiler.split()
         cpp_cmd += ['-E']
-    elif hasattr(compiler, 'cc'): # for msvccompiler
+    elif hasattr(compiler, 'cc'):  # for msvccompiler
         cpp_cmd = compiler.cc.split()
         cpp_cmd += ['-E']
@@ -73 +77 @@
         cpp_cmd = os.environ.get('CC', 'cc').split()
         cpp_cmd += ['-E']
-
+    cpp_cmd += ['-x', 'c']  # force C language (emcc defaults to c++)
     return cpp_cmd
 
-def get_cpp_objects(header=header, usedouble=False):
+
+def get_c_declarations(header=header, usedouble=False):
+    ''' return a dense and preprocessed  string of all c declarations implied by aubio.h
+    '''
     cpp_cmd = get_preprocessor()
 
@@ -92 +99 @@
     print("Running command: {:s}".format(" ".join(cpp_cmd)))
     proc = subprocess.Popen(cpp_cmd,
-            stderr=subprocess.PIPE,
-            stdout=subprocess.PIPE)
+                            stderr=subprocess.PIPE,
+                            stdout=subprocess.PIPE)
     assert proc, 'Proc was none'
     cpp_output = proc.stdout.read()
@@ -100 +107 @@
         raise Exception("preprocessor output is empty:\n%s" % err_output)
     elif err_output:
-        print ("Warning: preprocessor produced warnings:\n%s" % err_output)
+        print("Warning: preprocessor produced warnings:\n%s" % err_output)
     if not isinstance(cpp_output, list):
         cpp_output = [l.strip() for l in cpp_output.decode('utf8').split('\n')]
@@ -109 +116 @@
     i = 1
     while 1:
-        if i >= len(cpp_output): break
-        if cpp_output[i-1].endswith(',') or cpp_output[i-1].endswith('{') or cpp_output[i].startswith('}'):
-            cpp_output[i] = cpp_output[i-1] + ' ' + cpp_output[i]
-            cpp_output.pop(i-1)
+        if i >= len(cpp_output):
+            break
+        if ('{' in cpp_output[i - 1]) and (not '}' in cpp_output[i - 1]) or (not ';' in cpp_output[i - 1]):
+            cpp_output[i] = cpp_output[i - 1] + ' ' + cpp_output[i]
+            cpp_output.pop(i - 1)
+        elif ('}' in cpp_output[i]):
+            cpp_output[i] = cpp_output[i - 1] + ' ' + cpp_output[i]
+            cpp_output.pop(i - 1)
         else:
             i += 1
 
-    typedefs = filter(lambda y: y.startswith ('typedef struct _aubio'), cpp_output)
-
+    # clean pointer notations
+    tmp = []
+    for l in cpp_output:
+        tmp += [l.replace(' *', ' * ')]
+    cpp_output = tmp
+
+    return cpp_output
+
+
+def get_cpp_objects_from_c_declarations(c_declarations, skip_objects=None):
+    if skip_objects == None:
+        skip_objects = default_skip_objects
+    typedefs = filter(lambda y: y.startswith('typedef struct _aubio'), c_declarations)
     cpp_objects = [a.split()[3][:-1] for a in typedefs]
-
-    return cpp_output, cpp_objects
-
-
-def analyze_cpp_output(cpp_objects, cpp_output):
+    cpp_objects_filtered = filter(lambda y: not y[6:-2] in skip_objects, cpp_objects)
+    return cpp_objects_filtered
+
+
+def get_all_func_names_from_lib(lib, depth=0):
+    ''' return flat string of all function used in lib
+    '''
+    res = []
+    indent = " " * depth
+    for k, v in lib.items():
+        if isinstance(v, dict):
+            res += get_all_func_names_from_lib(v, depth + 1)
+        elif isinstance(v, list):
+            for elem in v:
+                e = elem.split('(')
+                if len(e) < 2:
+                    continue  # not a function
+                fname_part = e[0].strip().split(' ')
+                fname = fname_part[-1]
+                if fname:
+                    res += [fname]
+                else:
+                    raise NameError('gen_lib : weird function: ' + str(e))
+
+    return res
+
+
+def generate_lib_from_c_declarations(cpp_objects, c_declarations):
+    ''' returns a lib from given cpp_object names
+
+    a lib is a dict grouping functions by family (onset,pitch...)
+        each eement is itself a dict of functions grouped by puposes as : 
+        struct, new, del, do, get, set and other
+    '''
     lib = {}
 
     for o in cpp_objects:
-        if o[:6] != 'aubio_':
-            continue
-        shortname = o[6:-2]
-        if shortname in skip_objects:
-            continue
+        shortname = o
+        if o[:6] == 'aubio_':
+            shortname = o[6:-2]  # without aubio_ prefix and _t suffix
+
         lib[shortname] = {'struct': [], 'new': [], 'del': [], 'do': [], 'get': [], 'set': [], 'other': []}
         lib[shortname]['longname'] = o
         lib[shortname]['shortname'] = shortname
-        for fn in cpp_output:
-            if o[:-1] in fn:
-                #print "found", o[:-1], "in", fn
+
+        fullshortname = o[:-2]  # name without _t suffix
+
+        for fn in c_declarations:
+            func_name = fn.split('(')[0].strip().split(' ')[-1]
+            if func_name.startswith(fullshortname + '_') or func_name.endswith(fullshortname):
+                # print "found", shortname, "in", fn
                 if 'typedef struct ' in fn:
                     lib[shortname]['struct'].append(fn)
@@ -151 +205 @@
                     lib[shortname]['set'].append(fn)
                 else:
-                    #print "no idea what to do about", fn
+                    # print "no idea what to do about", fn
                     lib[shortname]['other'].append(fn)
     return lib
 
-def print_cpp_output_results(lib, cpp_output):
-    for fn in cpp_output:
+
+def print_c_declarations_results(lib, c_declarations):
+    for fn in c_declarations:
         found = 0
         for o in lib:
@@ -163 +218 @@
                     found = 1
         if found == 0:
-            print ("missing", fn)
+            print("missing", fn)
 
     for o in lib:
         for family in lib[o]:
             if type(lib[o][family]) == str:
-                print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family] ) )
+                print("{:15s} {:10s} {:s}".format(o, family, lib[o][family]))
             elif len(lib[o][family]) == 1:
-                print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family][0] ) )
+                print("{:15s} {:10s} {:s}".format(o, family, lib[o][family][0]))
             else:
-                print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family] ) )
+                print("{:15s} {:10s} {:s}".format(o, family, lib[o][family]))
 
 
 def generate_external(header=header, output_path=output_path, usedouble=False, overwrite=True):
-    if not os.path.isdir(output_path): os.mkdir(output_path)
-    elif not overwrite: return sorted(glob.glob(os.path.join(output_path, '*.c')))
-
-    cpp_output, cpp_objects = get_cpp_objects(header, usedouble=usedouble)
-
-    lib = analyze_cpp_output(cpp_objects, cpp_output)
-    # print_cpp_output_results(lib, cpp_output)
+    if not os.path.isdir(output_path):
+        os.mkdir(output_path)
+    elif not overwrite:
+        return sorted(glob.glob(os.path.join(output_path, '*.c')))
+
+    c_declarations = get_c_declarations(header, usedouble=usedouble)
+    cpp_objects = get_cpp_objects_from_c_declarations(c_declarations)
+
+    lib = generate_lib_from_c_declarations(cpp_objects, c_declarations)
+    # print_c_declarations_results(lib, c_declarations)
 
     sources_list = []
@@ -191 +249 @@
     for o in lib:
         out = source_header
-        mapped = MappedObject(lib[o], usedouble = usedouble)
+        mapped = MappedObject(lib[o], usedouble=usedouble)
         out += mapped.gen_code()
         output_file = os.path.join(output_path, 'gen-%s.c' % o)
         with open(output_file, 'w') as f:
             f.write(out)
-            print ("wrote %s" % output_file )
+            print("wrote %s" % output_file)
             sources_list.append(output_file)
 
@@ -208 +266 @@
   return ({pycheck_types});
 }}
-""".format(pycheck_types = check_types)
+""".format(pycheck_types=check_types)
 
     add_types = "".join(["""
   Py_INCREF (&Py_{name}Type);
-  PyModule_AddObject(m, "{name}", (PyObject *) & Py_{name}Type);""".format(name = o) for o in lib])
+  PyModule_AddObject(m, "{name}", (PyObject *) & Py_{name}Type);""".format(name=o) for o in lib])
     out += """
 
@@ -219 +277 @@
 {add_types}
 }}
-""".format(add_types = add_types)
+""".format(add_types=add_types)
 
     output_file = os.path.join(output_path, 'aubio-generated.c')
     with open(output_file, 'w') as f:
         f.write(out)
-        print ("wrote %s" % output_file )
+        print("wrote %s" % output_file)
         sources_list.append(output_file)
 
@@ -242 +300 @@
 int generated_objects ( void );
 void add_generated_objects( PyObject *m );
-""".format(objlist = objlist)
+""".format(objlist=objlist)
 
     output_file = os.path.join(output_path, 'aubio-generated.h')
     with open(output_file, 'w') as f:
         f.write(out)
-        print ("wrote %s" % output_file )
+        print("wrote %s" % output_file)
         # no need to add header to list of sources
 
@@ -253 +311 @@
 
 if __name__ == '__main__':
-    if len(sys.argv) > 1: header = sys.argv[1]
-    if len(sys.argv) > 2: output_path = sys.argv[2]
+    if len(sys.argv) > 1:
+        header = sys.argv[1]
+    if len(sys.argv) > 2:
+        output_path = sys.argv[2]
     generate_external(header, output_path)

src/wscript_build

@@ -27 +27 @@
     build_features = ['cstlib', 'cshlib']
 elif ctx.env['DEST_OS'] in ['emscripten']:
-    build_features = ['cstlib']
+    build_features = ['cstlib','cshlib']
 elif '--static' in ctx.env['LDFLAGS'] or '--static' in ctx.env['LINKFLAGS']:
     # static in cflags, ...

wscript

@@ -108 +108 @@
     ctx.load('gnu_dirs')
 
+    target_platform = sys.platform
+    if ctx.options.target_platform:
+        target_platform = ctx.options.target_platform
+
+
+    if target_platform=='emscripten':
+        # need to force spaces between flag -o and path 
+        # inspired from :
+        # https://github.com/waf-project/waf/blob/master/waflib/extras/c_emscripten.py (#1885)
+        # (OSX /emscripten 1.37.9)
+        ctx.env.CC_TGT_F            = ['-c', '-o', '']
+        ctx.env.CCLNK_TGT_F         = ['-o', '']
     # check for common headers
     ctx.check(header_name='stdlib.h')
@@ -118 +130 @@
     ctx.check(header_name='unistd.h', mandatory = False)
 
-    target_platform = sys.platform
-    if ctx.options.target_platform:
-        target_platform = ctx.options.target_platform
     ctx.env['DEST_OS'] = target_platform
 
@@ -133 +142 @@
             ctx.env.prepend_value('CFLAGS', ['-O0'])
         else:
-            # default to -O2 in release mode
-            ctx.env.prepend_value('CFLAGS', ['-O2'])
+            if target_platform == 'emscripten':
+                # -Oz for small js file generation
+                ctx.env.prepend_value('CFLAGS', ['-Oz'])
+            else:
+                # default to -O2 in release mode
+                ctx.env.prepend_value('CFLAGS', ['-O2'])
         # enable debug symbols and configure warnings
         ctx.env.prepend_value('CFLAGS', ['-g', '-Wall', '-Wextra'])
@@ -216 +229 @@
         import os.path
         ctx.env.CFLAGS += [ '-I' + os.path.join(os.environ['EMSCRIPTEN'], 'system', 'include') ]
-        ctx.env.CFLAGS += ['-Oz']
+        
+        if ctx.options.build_type == "debug":
+            ctx.env.cshlib_PATTERN = '%s.js'
+            ctx.env.LINKFLAGS += ['-s','ASSERTIONS=2']
+            ctx.env.LINKFLAGS += ['-s','SAFE_HEAP=1']
+            ctx.env.LINKFLAGS += ['-s','ALIASING_FUNCTION_POINTERS=0']
+            ctx.env.LINKFLAGS += ['-O0']
+        else:
+            ctx.env.LINKFLAGS += ['-Oz']
+            ctx.env.cshlib_PATTERN = '%s.min.js'
+
+        # doesnt ship file system support in lib
+        ctx.env.LINKFLAGS_cshlib += ['-s', 'NO_FILESYSTEM=1']
+        # put memory file inside generated js files for easier portability
+        ctx.env.LINKFLAGS += ['--memory-init-file', '0']
         ctx.env.cprogram_PATTERN = "%s.js"
-        if (ctx.options.enable_atlas != True):
-            ctx.options.enable_atlas = False
+        ctx.env.cstlib_PATTERN = '%s.a'
+
+        # tell emscripten functions we want to expose
+        from python.lib.gen_external import get_c_declarations, get_cpp_objects_from_c_declarations, get_all_func_names_from_lib, generate_lib_from_c_declarations
+        c_decls = get_c_declarations(usedouble=False)  # emscripten can't use double
+        objects = get_cpp_objects_from_c_declarations(c_decls)
+        # ensure that aubio structs are exported
+        objects += ['fvec_t', 'cvec_t', 'fmat_t']
+        lib = generate_lib_from_c_declarations(objects, c_decls)
+        exported_funcnames = get_all_func_names_from_lib(lib)
+        c_mangled_names = ['_' + s for s in exported_funcnames]
+        ctx.env.LINKFLAGS_cshlib += ['-s', 'EXPORTED_FUNCTIONS=%s' % c_mangled_names]
+
+    if (ctx.options.enable_atlas != True):
+        ctx.options.enable_atlas = False
 
     # check support for C99 __VA_ARGS__ macros