source: python/lib/gen_external.py @ 415e360

feature/autosinkfeature/cnnfeature/cnn_orgfeature/constantqfeature/crepefeature/crepe_orgfeature/pitchshiftfeature/pydocstringsfeature/timestretchfix/ffmpeg5
Last change on this file since 415e360 was 415e360, checked in by Martin Hermant <martin.hermant@gmail.com>, 7 years ago

gen_external.py : fix wrong in longname in lib generation

  • Property mode set to 100644
File size: 8.9 KB
RevLine 
[19c3d75]1import distutils.ccompiler
2import sys, os, subprocess, glob
[ccb9fb5]3
[89b04e8]4header = os.path.join('src', 'aubio.h')
5output_path = os.path.join('python', 'gen')
[1167631]6
7source_header = """// this file is generated! do not modify
[ccb9fb5]8#include "aubio-types.h"
9"""
10
11skip_objects = [
12  # already in ext/
13  'fft',
14  'pvoc',
15  'filter',
16  'filterbank',
17  # AUBIO_UNSTABLE
18  'hist',
19  'parameter',
20  'scale',
21  'beattracking',
22  'resampler',
23  'peakpicker',
24  'pitchfcomb',
25  'pitchmcomb',
26  'pitchschmitt',
27  'pitchspecacf',
28  'pitchyin',
29  'pitchyinfft',
30  'sink',
31  'sink_apple_audio',
32  'sink_sndfile',
33  'sink_wavwrite',
34  #'mfcc',
35  'source',
36  'source_apple_audio',
37  'source_sndfile',
38  'source_avcodec',
39  'source_wavread',
40  #'sampler',
41  'audio_unit',
[1013c5d]42  'spectral_whitening',
[ccb9fb5]43  ]
44
[19c3d75]45def get_preprocessor():
46    # findout which compiler to use
47    from distutils.sysconfig import customize_compiler
48    compiler_name = distutils.ccompiler.get_default_compiler()
49    compiler = distutils.ccompiler.new_compiler(compiler=compiler_name)
50    try:
51        customize_compiler(compiler)
52    except AttributeError as e:
53        print("Warning: failed customizing compiler ({:s})".format(repr(e)))
54
55    if hasattr(compiler, 'initialize'):
56        try:
57            compiler.initialize()
58        except ValueError as e:
59            print("Warning: failed initializing compiler ({:s})".format(repr(e)))
60
61    cpp_cmd = None
62    if hasattr(compiler, 'preprocessor'): # for unixccompiler
63        cpp_cmd = compiler.preprocessor
64    elif hasattr(compiler, 'compiler'): # for ccompiler
65        cpp_cmd = compiler.compiler.split()
66        cpp_cmd += ['-E']
67    elif hasattr(compiler, 'cc'): # for msvccompiler
68        cpp_cmd = compiler.cc.split()
69        cpp_cmd += ['-E']
70
71    if not cpp_cmd:
72        print("Warning: could not guess preprocessor, using env's CC")
73        cpp_cmd = os.environ.get('CC', 'cc').split()
74        cpp_cmd += ['-E']
[b075ad8]75    cpp_cmd += ['-x', 'c']  # force C language (emcc defaults to c++)
[19c3d75]76    return cpp_cmd
[ccb9fb5]77
[150ec2d]78
79def get_c_declarations(header=header, usedouble=False):
80    ''' return a dense and preprocessed  string of all c declarations implied by aubio.h
81    '''
[19c3d75]82    cpp_cmd = get_preprocessor()
83
84    macros = [('AUBIO_UNSTABLE', 1)]
[3d14829]85    if usedouble:
86        macros += [('HAVE_AUBIO_DOUBLE', 1)]
[19c3d75]87
88    if not os.path.isfile(header):
89        raise Exception("could not find include file " + header)
90
91    includes = [os.path.dirname(header)]
92    cpp_cmd += distutils.ccompiler.gen_preprocess_options(macros, includes)
93    cpp_cmd += [header]
94
95    print("Running command: {:s}".format(" ".join(cpp_cmd)))
96    proc = subprocess.Popen(cpp_cmd,
97            stderr=subprocess.PIPE,
98            stdout=subprocess.PIPE)
99    assert proc, 'Proc was none'
100    cpp_output = proc.stdout.read()
101    err_output = proc.stderr.read()
102    if not cpp_output:
103        raise Exception("preprocessor output is empty:\n%s" % err_output)
104    elif err_output:
105        print ("Warning: preprocessor produced warnings:\n%s" % err_output)
106    if not isinstance(cpp_output, list):
107        cpp_output = [l.strip() for l in cpp_output.decode('utf8').split('\n')]
[ccb9fb5]108
109    cpp_output = filter(lambda y: len(y) > 1, cpp_output)
[19c3d75]110    cpp_output = list(filter(lambda y: not y.startswith('#'), cpp_output))
[ccb9fb5]111
112    i = 1
113    while 1:
[15a43e0]114        if i >= len(cpp_output):break
115        if ('{' in cpp_output[i - 1]) and (not '}' in cpp_output[i - 1]) or (not ';' in cpp_output[i - 1]):
[ccb9fb5]116            cpp_output[i] = cpp_output[i-1] + ' ' + cpp_output[i]
117            cpp_output.pop(i-1)
[15a43e0]118        elif ('}' in cpp_output[i]):
119            cpp_output[i] = cpp_output[i - 1] + ' ' + cpp_output[i]
120            cpp_output.pop(i - 1)
[ccb9fb5]121        else:
122            i += 1
123
[eea6101]124    # clean pointer notations
125    tmp = []
126    for l in cpp_output:
127        tmp+=[ l.replace(' *','* ')]
128    cpp_output = tmp;
129
130
[150ec2d]131    return cpp_output
[ccb9fb5]132
[150ec2d]133def get_cpp_objects_from_c_declarations(c_declarations):
134    typedefs = filter(lambda y: y.startswith ('typedef struct _aubio'), c_declarations)
[ccb9fb5]135    cpp_objects = [a.split()[3][:-1] for a in typedefs]
[150ec2d]136    return cpp_objects
[ccb9fb5]137
[150ec2d]138def analyze_c_declarations(cpp_objects, c_declarations):
[ccb9fb5]139    lib = {}
140
141    for o in cpp_objects:
[5674833]142        shortname = ''
143        if o[:6] == 'aubio_':
144            shortname = o[6:-2]  # without aubio_
145            longname = o[:-2]  # without _t
146        else: # support object not starting with aubio_ (fvec...)
147            shortname = o
148            longname = shortname
149       
[ccb9fb5]150        if shortname in skip_objects:
151            continue
152        lib[shortname] = {'struct': [], 'new': [], 'del': [], 'do': [], 'get': [], 'set': [], 'other': []}
[415e360]153        lib[shortname]['longname'] = longname
[ccb9fb5]154        lib[shortname]['shortname'] = shortname
[5674833]155        valid_funcname_part = ['_'+longname,longname+'_']
156
[150ec2d]157        for fn in c_declarations:
[5674833]158            func_name = fn.split('(')[0].strip().split(' ')[1:]
159            if func_name:
160                func_name = func_name[-1]
161            else:
162                raise NameError('Warning : error while parsing : unexpected line %s' % fn)
163            if any(x in func_name for x in valid_funcname_part):
164                # print "found", shortname, "in", fn
[ccb9fb5]165                if 'typedef struct ' in fn:
166                    lib[shortname]['struct'].append(fn)
167                elif '_do' in fn:
168                    lib[shortname]['do'].append(fn)
169                elif 'new_' in fn:
170                    lib[shortname]['new'].append(fn)
171                elif 'del_' in fn:
172                    lib[shortname]['del'].append(fn)
173                elif '_get_' in fn:
174                    lib[shortname]['get'].append(fn)
175                elif '_set_' in fn:
176                    lib[shortname]['set'].append(fn)
177                else:
178                    #print "no idea what to do about", fn
179                    lib[shortname]['other'].append(fn)
[41fc24f]180    return lib
[ccb9fb5]181
[150ec2d]182def print_c_declarations_results(lib, c_declarations):
183    for fn in c_declarations:
[ccb9fb5]184        found = 0
185        for o in lib:
186            for family in lib[o]:
187                if fn in lib[o][family]:
188                    found = 1
189        if found == 0:
[41fc24f]190            print ("missing", fn)
[ccb9fb5]191
192    for o in lib:
193        for family in lib[o]:
194            if type(lib[o][family]) == str:
195                print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family] ) )
196            elif len(lib[o][family]) == 1:
197                print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family][0] ) )
[a7f398d]198            else:
[41fc24f]199                print ( "{:15s} {:10s} {:s}".format(o, family, lib[o][family] ) )
200
[ccb9fb5]201
[41fc24f]202def generate_external(header=header, output_path=output_path, usedouble=False, overwrite=True):
203    if not os.path.isdir(output_path): os.mkdir(output_path)
[2e40231]204    elif not overwrite: return sorted(glob.glob(os.path.join(output_path, '*.c')))
[41fc24f]205
[150ec2d]206    c_declarations = get_c_declarations(header, usedouble=usedouble)
207    cpp_objects = get_cpp_objects_from_c_declarations(c_declarations)
[41fc24f]208
[150ec2d]209    lib = analyze_c_declarations(cpp_objects, c_declarations)
210    # print_c_declarations_results(lib, c_declarations)
[41fc24f]211
212    sources_list = []
[1167631]213    try:
214        from .gen_code import MappedObject
[19c3d75]215    except (SystemError, ValueError):
[1167631]216        from gen_code import MappedObject
[ccb9fb5]217    for o in lib:
[1167631]218        out = source_header
[fbcee4f]219        mapped = MappedObject(lib[o], usedouble = usedouble)
[ccb9fb5]220        out += mapped.gen_code()
221        output_file = os.path.join(output_path, 'gen-%s.c' % o)
222        with open(output_file, 'w') as f:
223            f.write(out)
224            print ("wrote %s" % output_file )
225            sources_list.append(output_file)
226
[1167631]227    out = source_header
[ccb9fb5]228    out += "#include \"aubio-generated.h\""
229    check_types = "\n     ||  ".join(["PyType_Ready(&Py_%sType) < 0" % o for o in lib])
230    out += """
231
232int generated_types_ready (void)
233{{
234  return ({pycheck_types});
235}}
236""".format(pycheck_types = check_types)
237
238    add_types = "".join(["""
239  Py_INCREF (&Py_{name}Type);
240  PyModule_AddObject(m, "{name}", (PyObject *) & Py_{name}Type);""".format(name = o) for o in lib])
241    out += """
242
243void add_generated_objects ( PyObject *m )
244{{
245{add_types}
246}}
247""".format(add_types = add_types)
248
249    output_file = os.path.join(output_path, 'aubio-generated.c')
250    with open(output_file, 'w') as f:
251        f.write(out)
252        print ("wrote %s" % output_file )
253        sources_list.append(output_file)
[a7f398d]254
[b6230d8]255    objlist = "".join(["extern PyTypeObject Py_%sType;\n" % p for p in lib])
[a89ed31]256    out = """// generated list of objects created with gen_external.py
[ccb9fb5]257
[a89ed31]258#include <Python.h>
259"""
260    if usedouble:
261        out += """
262#ifndef HAVE_AUBIO_DOUBLE
263#define HAVE_AUBIO_DOUBLE 1
264#endif
265"""
266    out += """
[ccb9fb5]267{objlist}
268int generated_objects ( void );
269void add_generated_objects( PyObject *m );
270""".format(objlist = objlist)
271
272    output_file = os.path.join(output_path, 'aubio-generated.h')
273    with open(output_file, 'w') as f:
274        f.write(out)
275        print ("wrote %s" % output_file )
276        # no need to add header to list of sources
277
[ee7e543]278    return sorted(sources_list)
[ccb9fb5]279
280if __name__ == '__main__':
[1167631]281    if len(sys.argv) > 1: header = sys.argv[1]
282    if len(sys.argv) > 2: output_path = sys.argv[2]
283    generate_external(header, output_path)
Note: See TracBrowser for help on using the repository browser.