Changeset 9fa0ed1 for python/lib

Timestamp:

Jul 24, 2017, 2:12:55 PM (7 years ago)

Author:

Paul Brossier <piem@piem.org>

Branches:

feature/autosink, feature/cnn, feature/cnn_org, feature/constantq, feature/crepe, feature/crepe_org, feature/pitchshift, feature/pydocstrings, feature/timestretch, fix/ffmpeg5, master

Children:

1070378

Parents:

482641d (diff), ddea34b (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' into yinfast

Location:

python/lib

Files:

• aubio/cmd.py (modified) (1 diff)
• gen_external.py (modified) (14 diffs)

python/lib/aubio/cmd.py

@@ -188 +188 @@
         return samples2samples
     else:
-        raise ValueError('invalid time format %s' % mode)
+        raise ValueError("invalid time format '%s'" % mode)
 
 # definition of processing classes

python/lib/gen_external.py

@@ -1 +1 @@
 import distutils.ccompiler
-import sys, os, subprocess, glob
+import sys
+import os
+import subprocess
+import glob
 
 header = os.path.join('src', 'aubio.h')
@@ -9 +12 @@
 """
 
-skip_objects = [
-  # already in ext/
-  'fft',
-  'pvoc',
-  'filter',
-  'filterbank',
-  # AUBIO_UNSTABLE
-  'hist',
-  'parameter',
-  'scale',
-  'beattracking',
-  'resampler',
-  'peakpicker',
-  'pitchfcomb',
-  'pitchmcomb',
-  'pitchschmitt',
-  'pitchspecacf',
-  'pitchyin',
-  'pitchyinfft',
-  'pitchyinfast',
-  'sink',
-  'sink_apple_audio',
-  'sink_sndfile',
-  'sink_wavwrite',
-  #'mfcc',
-  'source',
-  'source_apple_audio',
-  'source_sndfile',
-  'source_avcodec',
-  'source_wavread',
-  #'sampler',
-  'audio_unit',
-  'spectral_whitening',
-  ]
+default_skip_objects = [
+    # already in ext/
+    'fft',
+    'pvoc',
+    'filter',
+    'filterbank',
+    # AUBIO_UNSTABLE
+    'hist',
+    'parameter',
+    'scale',
+    'beattracking',
+    'resampler',
+    'peakpicker',
+    'pitchfcomb',
+    'pitchmcomb',
+    'pitchschmitt',
+    'pitchspecacf',
+    'pitchyin',
+    'pitchyinfft',
+    'pitchyinfast',
+    'sink',
+    'sink_apple_audio',
+    'sink_sndfile',
+    'sink_wavwrite',
+    #'mfcc',
+    'source',
+    'source_apple_audio',
+    'source_sndfile',
+    'source_avcodec',
+    'source_wavread',
+    #'sampler',
+    'audio_unit',
+    'spectral_whitening',
+]
+
 
 def get_preprocessor():
@@ -61 +65 @@
 
     cpp_cmd = None
-    if hasattr(compiler, 'preprocessor'): # for unixccompiler
+    if hasattr(compiler, 'preprocessor'):  # for unixccompiler
         cpp_cmd = compiler.preprocessor
-    elif hasattr(compiler, 'compiler'): # for ccompiler
+    elif hasattr(compiler, 'compiler'):  # for ccompiler
         cpp_cmd = compiler.compiler.split()
         cpp_cmd += ['-E']
-    elif hasattr(compiler, 'cc'): # for msvccompiler
+    elif hasattr(compiler, 'cc'):  # for msvccompiler
         cpp_cmd = compiler.cc.split()
         cpp_cmd += ['-E']
@@ -74 +78 @@
         cpp_cmd = os.environ.get('CC', 'cc').split()
         cpp_cmd += ['-E']
-
+    cpp_cmd += ['-x', 'c']  # force C language (emcc defaults to c++)
     return cpp_cmd
 
-def get_cpp_objects(header=header, usedouble=False):
+
+def get_c_declarations(header=header, usedouble=False):
+    ''' return a dense and preprocessed  string of all c declarations implied by aubio.h
+    '''
     cpp_cmd = get_preprocessor()
 
@@ -93 +100 @@
     print("Running command: {:s}".format(" ".join(cpp_cmd)))
     proc = subprocess.Popen(cpp_cmd,
-            stderr=subprocess.PIPE,
-            stdout=subprocess.PIPE)
+                            stderr=subprocess.PIPE,
+                            stdout=subprocess.PIPE)
     assert proc, 'Proc was none'
     cpp_output = proc.stdout.read()
@@ -101 +108 @@
         raise Exception("preprocessor output is empty:\n%s" % err_output)
     elif err_output:
-        print ("Warning: preprocessor produced warnings:\n%s" % err_output)
+        print("Warning: preprocessor produced warnings:\n%s" % err_output)
     if not isinstance(cpp_output, list):
         cpp_output = [l.strip() for l in cpp_output.decode('utf8').split('\n')]
@@ -110 +117 @@
     i = 1
     while 1:
-        if i >= len(cpp_output): break
-        if cpp_output[i-1].endswith(',') or cpp_output[i-1].endswith('{') or cpp_output[i].startswith('}'):
-            cpp_output[i] = cpp_output[i-1] + ' ' + cpp_output[i]
-            cpp_output.pop(i-1)
+        if i >= len(cpp_output):
+            break
+        if ('{' in cpp_output[i - 1]) and (not '}' in cpp_output[i - 1]) or (not ';' in cpp_output[i - 1]):
+            cpp_output[i] = cpp_output[i - 1] + ' ' + cpp_output[i]
+            cpp_output.pop(i - 1)
+        elif ('}' in cpp_output[i]):
+            cpp_output[i] = cpp_output[i - 1] + ' ' + cpp_output[i]
+            cpp_output.pop(i - 1)
         else:
             i += 1
 
-    typedefs = filter(lambda y: y.startswith ('typedef struct _aubio'), cpp_output)
-
+    # clean pointer notations
+    tmp = []
+    for l in cpp_output:
+        tmp += [l.replace(' *', ' * ')]
+    cpp_output = tmp
+
+    return cpp_output
+
+
+def get_cpp_objects_from_c_declarations(c_declarations, skip_objects=None):
+    if skip_objects == None:
+        skip_objects = default_skip_objects
+    typedefs = filter(lambda y: y.startswith('typedef struct _aubio'), c_declarations)
     cpp_objects = [a.split()[3][:-1] for a in typedefs]
-
-    return cpp_output, cpp_objects
-
-
-def analyze_cpp_output(cpp_objects, cpp_output):
+    cpp_objects_filtered = filter(lambda y: not y[6:-2] in skip_objects, cpp_objects)
+    return cpp_objects_filtered
+
+
+def get_all_func_names_from_lib(lib, depth=0):
+    ''' return flat string of all function used in lib
+    '''
+    res = []
+    indent = " " * depth
+    for k, v in lib.items():
+        if isinstance(v, dict):
+            res += get_all_func_names_from_lib(v, depth + 1)
+        elif isinstance(v, list):
+            for elem in v:
+                e = elem.split('(')
+                if len(e) < 2:
+                    continue  # not a function
+                fname_part = e[0].strip().split(' ')
+                fname = fname_part[-1]
+                if fname:
+                    res += [fname]
+                else:
+                    raise NameError('gen_lib : weird function: ' + str(e))
+
+    return res
+
+
+def generate_lib_from_c_declarations(cpp_objects, c_declarations):
+    ''' returns a lib from given cpp_object names
+
+    a lib is a dict grouping functions by family (onset,pitch...)
+        each eement is itself a dict of functions grouped by puposes as : 
+        struct, new, del, do, get, set and other
+    '''
     lib = {}
 
     for o in cpp_objects:
-        if o[:6] != 'aubio_':
-            continue
-        shortname = o[6:-2]
-        if shortname in skip_objects:
-            continue
+        shortname = o
+        if o[:6] == 'aubio_':
+            shortname = o[6:-2]  # without aubio_ prefix and _t suffix
+
         lib[shortname] = {'struct': [], 'new': [], 'del': [], 'do': [], 'get': [], 'set': [], 'other': []}
         lib[shortname]['longname'] = o
         lib[shortname]['shortname'] = shortname
-        for fn in cpp_output:
-            if o[:-1] in fn:
-                #print "found", o[:-1], "in", fn
+
+        fullshortname = o[:-2]  # name without _t suffix
+
+        for fn in c_declarations:
+            func_name = fn.split('(')[0].strip().split(' ')[-1]
+            if func_name.startswith(fullshortname + '_') or func_name.endswith(fullshortname):
+                # print "found", shortname, "in", fn
                 if 'typedef struct ' in fn:
                     lib[shortname]['struct'].append(fn)
@@ -152 +206 @@
                     lib[shortname]['set'].append(fn)
                 else:
-                    #print "no idea what to do about", fn
+                    # print "no idea what to do about", fn
                     lib[shortname]['other'].append(fn)
     return lib
 
-def print_cpp_output_results(lib, cpp_output):
-    for fn in cpp_output:
+
+def print_c_declarations_results(lib, c_declarations):
+    for fn in c_declarations:
         found = 0
         for o in lib:
@@ -164 +219 @@
                     found = 1
         if found == 0:
-            print ("missing", fn)
+            print("missing", fn)
 
     for o in lib:
         for family in lib[o]:
             if type(lib[o][family]) == str:
-                print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family] ) )
+                print("{:15s} {:10s} {:s}".format(o, family, lib[o][family]))
             elif len(lib[o][family]) == 1:
-                print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family][0] ) )
+                print("{:15s} {:10s} {:s}".format(o, family, lib[o][family][0]))
             else:
-                print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family] ) )
+                print("{:15s} {:10s} {:s}".format(o, family, lib[o][family]))
 
 
 def generate_external(header=header, output_path=output_path, usedouble=False, overwrite=True):
-    if not os.path.isdir(output_path): os.mkdir(output_path)
-    elif not overwrite: return sorted(glob.glob(os.path.join(output_path, '*.c')))
-
-    cpp_output, cpp_objects = get_cpp_objects(header, usedouble=usedouble)
-
-    lib = analyze_cpp_output(cpp_objects, cpp_output)
-    # print_cpp_output_results(lib, cpp_output)
+    if not os.path.isdir(output_path):
+        os.mkdir(output_path)
+    elif not overwrite:
+        return sorted(glob.glob(os.path.join(output_path, '*.c')))
+
+    c_declarations = get_c_declarations(header, usedouble=usedouble)
+    cpp_objects = get_cpp_objects_from_c_declarations(c_declarations)
+
+    lib = generate_lib_from_c_declarations(cpp_objects, c_declarations)
+    # print_c_declarations_results(lib, c_declarations)
 
     sources_list = []
@@ -192 +250 @@
     for o in lib:
         out = source_header
-        mapped = MappedObject(lib[o], usedouble = usedouble)
+        mapped = MappedObject(lib[o], usedouble=usedouble)
         out += mapped.gen_code()
         output_file = os.path.join(output_path, 'gen-%s.c' % o)
         with open(output_file, 'w') as f:
             f.write(out)
-            print ("wrote %s" % output_file )
+            print("wrote %s" % output_file)
             sources_list.append(output_file)
 
@@ -209 +267 @@
   return ({pycheck_types});
 }}
-""".format(pycheck_types = check_types)
+""".format(pycheck_types=check_types)
 
     add_types = "".join(["""
   Py_INCREF (&Py_{name}Type);
-  PyModule_AddObject(m, "{name}", (PyObject *) & Py_{name}Type);""".format(name = o) for o in lib])
+  PyModule_AddObject(m, "{name}", (PyObject *) & Py_{name}Type);""".format(name=o) for o in lib])
     out += """
 
@@ -220 +278 @@
 {add_types}
 }}
-""".format(add_types = add_types)
+""".format(add_types=add_types)
 
     output_file = os.path.join(output_path, 'aubio-generated.c')
     with open(output_file, 'w') as f:
         f.write(out)
-        print ("wrote %s" % output_file )
+        print("wrote %s" % output_file)
         sources_list.append(output_file)
 
@@ -243 +301 @@
 int generated_objects ( void );
 void add_generated_objects( PyObject *m );
-""".format(objlist = objlist)
+""".format(objlist=objlist)
 
     output_file = os.path.join(output_path, 'aubio-generated.h')
     with open(output_file, 'w') as f:
         f.write(out)
-        print ("wrote %s" % output_file )
+        print("wrote %s" % output_file)
         # no need to add header to list of sources
 
@@ -254 +312 @@
 
 if __name__ == '__main__':
-    if len(sys.argv) > 1: header = sys.argv[1]
-    if len(sys.argv) > 2: output_path = sys.argv[2]
+    if len(sys.argv) > 1:
+        header = sys.argv[1]
+    if len(sys.argv) > 2:
+        output_path = sys.argv[2]
     generate_external(header, output_path)