Changeset 3502119

Timestamp:

Apr 5, 2017, 11:49:57 AM (8 years ago)

Author:

Paul Brossier <piem@piem.org>

Branches:

feature/autosink, feature/cnn, feature/cnn_org, feature/constantq, feature/crepe, feature/crepe_org, feature/pitchshift, feature/pydocstrings, feature/timestretch, fix/ffmpeg5, master, sampler

Children:

c3b1a7d

Parents:

0eca01f (diff), c82859f (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'awhitening'

Files:

• examples/aubioonset.c (modified) (3 diffs)
• examples/aubiotrack.c (modified) (1 diff)
• python/lib/gen_code.py (modified) (1 diff)
• python/lib/gen_external.py (modified) (1 diff)
• python/tests/test_onset.py (modified) (1 diff)
• src/aubio.h (modified) (1 diff)
• src/cvec.c (modified) (1 diff)
• src/cvec.h (modified) (1 diff)
• src/mathutils.c (modified) (1 diff)
• src/mathutils.h (modified) (1 diff)
• src/musicutils.h (modified) (1 diff)
• src/onset/onset.c (modified) (7 diffs)
• src/onset/onset.h (modified) (2 diffs)
• src/onset/peakpicker.c (modified) (1 diff)
• src/spectral/awhitening.c (added)
• src/spectral/awhitening.h (added)
• src/spectral/specdesc.c (modified) (8 diffs)
• src/spectral/specdesc.h (modified) (2 diffs)
• src/synth/wavetable.c (modified) (1 diff)
• tests/src/spectral/test-awhitening.c (added)

examples/aubioonset.c

@@ -44 +44 @@
     aubio_wavetable_stop ( wavetable );
   }
-  if (mix_input)
+  if (mix_input) {
     aubio_wavetable_do (wavetable, ibuf, obuf);
-  else
+  } else {
     aubio_wavetable_do (wavetable, obuf, obuf);
+  }
 }
 
@@ -62 +63 @@
   examples_common_init(argc,argv);
 
-  verbmsg ("using source: %s at %dHz\n", source_uri, samplerate);
-  verbmsg ("onset method: %s, ", onset_method);
-  verbmsg ("buffer_size: %d, ", buffer_size);
-  verbmsg ("hop_size: %d, ", hop_size);
-  verbmsg ("silence: %f, ", silence_threshold);
-  verbmsg ("threshold: %f\n", onset_threshold);
-
   o = new_aubio_onset (onset_method, buffer_size, hop_size, samplerate);
   if (o == NULL) { ret = 1; goto beach; }
@@ -77 +71 @@
   if (onset_minioi != 0.)
     aubio_onset_set_minioi_s (o, onset_minioi);
+
+  verbmsg ("using source: %s at %dHz\n", source_uri, samplerate);
+  verbmsg ("onset method: %s, ", onset_method);
+  verbmsg ("buffer_size: %d, ", buffer_size);
+  verbmsg ("hop_size: %d, ", hop_size);
+  verbmsg ("silence: %f, ", aubio_onset_get_silence(o));
+  verbmsg ("threshold: %f, ", aubio_onset_get_threshold(o));
+  verbmsg ("awhitening: %f, ", aubio_onset_get_awhitening(o));
+  verbmsg ("compression: %f\n", aubio_onset_get_compression(o));
 
   onset = new_fvec (1);

examples/aubiotrack.c

@@ -47 +47 @@
     aubio_wavetable_stop ( wavetable );
   }
-  if (mix_input)
+  if (mix_input) {
     aubio_wavetable_do (wavetable, ibuf, obuf);
-  else
+  } else {
     aubio_wavetable_do (wavetable, obuf, obuf);
+  }
 }
 

python/lib/gen_code.py

@@ -184 +184 @@
     def gen_code(self):
         out = ""
-        out += self.gen_struct()
-        out += self.gen_doc()
-        out += self.gen_new()
-        out += self.gen_init()
-        out += self.gen_del()
-        out += self.gen_do()
-        out += self.gen_memberdef()
-        out += self.gen_set()
-        out += self.gen_get()
-        out += self.gen_methodef()
-        out += self.gen_typeobject()
+        try:
+            out += self.gen_struct()
+            out += self.gen_doc()
+            out += self.gen_new()
+            out += self.gen_init()
+            out += self.gen_del()
+            out += self.gen_do()
+            out += self.gen_memberdef()
+            out += self.gen_set()
+            out += self.gen_get()
+            out += self.gen_methodef()
+            out += self.gen_typeobject()
+        except Exception as e:
+            print ("Failed generating code for", self.shortname)
+            raise
         return out
 

python/lib/gen_external.py

@@ -40 +40 @@
   #'sampler',
   'audio_unit',
+  'spectral_whitening',
   ]
 

python/tests/test_onset.py

@@ -20 +20 @@
 
     def test_get_delay(self):
-        assert_equal (self.o.get_delay(), int(4.3 * self.o.hop_size))
+        self.assertGreater(self.o.get_delay(), 0)
 
     def test_get_delay_s(self):
-        assert_almost_equal (self.o.get_delay_s(), self.o.get_delay() / float(self.samplerate))
+        self.assertGreater(self.o.get_delay_s(), 0.)
 
     def test_get_delay_ms(self):
-        assert_almost_equal (self.o.get_delay_ms(), self.o.get_delay() * 1000. / self.samplerate, 5)
+        self.assertGreater(self.o.get_delay_ms(), 0.)
 
     def test_get_minioi(self):
-        assert_almost_equal (self.o.get_minioi(), 0.02 * self.samplerate)
+        self.assertGreater(self.o.get_minioi(), 0)
 
     def test_get_minioi_s(self):
-        assert_almost_equal (self.o.get_minioi_s(), 0.02)
+        self.assertGreater(self.o.get_minioi_s(), 0.)
 
     def test_get_minioi_ms(self):
-        assert_equal (self.o.get_minioi_ms(), 20.)
+        self.assertGreater(self.o.get_minioi_ms(), 0.)
 
     def test_get_threshold(self):
-        assert_almost_equal (self.o.get_threshold(), 0.3)
+        self.assertGreater(self.o.get_threshold(), 0.)
 
     def test_set_delay(self):

src/aubio.h

@@ -188 +188 @@
 #include "spectral/mfcc.h"
 #include "spectral/specdesc.h"
+#include "spectral/awhitening.h"
 #include "spectral/tss.h"
 #include "pitch/pitch.h"

src/cvec.c

@@ -140 +140 @@
   cvec_phas_zeros(s);
 }
+
+void cvec_logmag(cvec_t *s, smpl_t lambda) {
+  uint_t j;
+  for (j=0; j< s->length; j++) {
+    s->norm[j] = LOG(lambda * s->norm[j] + 1);
+  }
+}

src/cvec.h

@@ -231 +231 @@
 void cvec_zeros(cvec_t *s);
 
+/** take logarithmic magnitude
+
+  \param s input cvec to compress
+  \param lambda value to use for normalisation
+
+  \f$ S_k = log( \lambda * S_k + 1 ) \f$
+
+*/
+void cvec_logmag(cvec_t *s, smpl_t lambda);
+
 #ifdef __cplusplus
 }

src/mathutils.c

@@ -290 +290 @@
 }
 
+void fvec_push(fvec_t *in, smpl_t new_elem) {
+  uint_t i;
+  for (i = 0; i < in->length - 1; i++) {
+    in->data[i] = in->data[i + 1];
+  }
+  in->data[in->length - 1] = new_elem;
+}
+
+void fvec_clamp(fvec_t *in, smpl_t absmax) {
+  uint_t i;
+  for (i = 0; i < in->length; i++) {
+    if (in->data[i] > 0 && in->data[i] > ABS(absmax)) {
+      in->data[i] = absmax;
+    } else if (in->data[i] < 0 && in->data[i] < -ABS(absmax)) {
+      in->data[i] = -absmax;
+    }
+  }
+}
+
 smpl_t
 aubio_level_lin (const fvec_t * f)

src/mathutils.h

@@ -117 +117 @@
 */
 void fvec_ishift (fvec_t * v);
+
+/** push a new element to the end of a vector, erasing the first element and
+ * sliding all others
+
+  \param in vector to push to
+  \param new_elem new_element to add at the end of the vector
+
+  In numpy words, this is equivalent to: in = np.concatenate([in, [new_elem]])[1:]
+
+*/
+void fvec_push(fvec_t *in, smpl_t new_elem);
 
 /** compute the sum of all elements of a vector

src/musicutils.h

@@ -157 +157 @@
 smpl_t aubio_level_detection (const fvec_t * v, smpl_t threshold);
 
+/** clamp the values of a vector within the range [-abs(max), abs(max)]
+
+  \param in vector to clamp
+  \param absmax maximum value over which input vector elements should be clamped
+
+*/
+void fvec_clamp(fvec_t *in, smpl_t absmax);
+
 #ifdef __cplusplus
 }

src/onset/onset.c

@@ -24 +24 @@
 #include "spectral/specdesc.h"
 #include "spectral/phasevoc.h"
+#include "spectral/awhitening.h"
 #include "onset/peakpicker.h"
 #include "mathutils.h"
 #include "onset/onset.h"
+
+void aubio_onset_default_parameters (aubio_onset_t *o, const char_t * method);
 
 /** structure to store object state */
@@ -43 +46 @@
   uint_t total_frames;          /**< total number of frames processed since the beginning */
   uint_t last_onset;            /**< last detected onset location, in frames */
+
+  uint_t apply_compression;
+  smpl_t lambda_compression;
+  uint_t apply_awhitening;      /**< apply adaptive spectral whitening */
+  aubio_spectral_whitening_t *spectral_whitening;
 };
 
@@ -50 +58 @@
   smpl_t isonset = 0;
   aubio_pvoc_do (o->pv,input, o->fftgrain);
+  /*
+  if (apply_filtering) {
+  }
+  */
+  if (o->apply_awhitening) {
+    aubio_spectral_whitening_do(o->spectral_whitening, o->fftgrain);
+  }
+  if (o->apply_compression) {
+    cvec_logmag(o->fftgrain, o->lambda_compression);
+  }
   aubio_specdesc_do (o->od, o->fftgrain, o->desc);
   aubio_peakpicker_do(o->pp, o->desc, onset);
@@ -58 +76 @@
       isonset  = 0;
     } else {
+      // we have an onset
       uint_t new_onset = o->total_frames + (uint_t)ROUND(isonset * o->hop_size);
+      // check if last onset time was more than minioi ago
       if (o->last_onset + o->minioi < new_onset) {
-        //AUBIO_DBG ("accepted detection, marking as onset\n");
-        o->last_onset = new_onset;
+        // start of file: make sure (new_onset - delay) >= 0
+        if (o->last_onset > 0 && o->delay > new_onset) {
+          isonset = 0;
+        } else {
+          //AUBIO_DBG ("accepted detection, marking as onset\n");
+          o->last_onset = MAX(o->delay, new_onset);
+        }
       } else {
         //AUBIO_DBG ("doubled onset, not marking as onset\n");
@@ -100 +125 @@
 }
 
+uint_t aubio_onset_set_awhitening (aubio_onset_t *o, uint_t enable)
+{
+  o->apply_awhitening = enable == 1 ? 1 : 0;
+  return AUBIO_OK;
+}
+
+smpl_t aubio_onset_get_awhitening (aubio_onset_t *o)
+{
+  return o->apply_awhitening;
+}
+
+uint_t aubio_onset_set_compression (aubio_onset_t *o, smpl_t lambda)
+{
+  if (lambda < 0.) {
+    return AUBIO_FAIL;
+  }
+  o->lambda_compression = lambda;
+  o->apply_compression = (o->lambda_compression > 0.) ? 1 : 0;
+  return AUBIO_OK;
+}
+
+smpl_t aubio_onset_get_compression (aubio_onset_t *o)
+{
+  return o->apply_compression ? o->lambda_compression : 0;
+}
+
 uint_t aubio_onset_set_silence(aubio_onset_t * o, smpl_t silence) {
   o->silence = silence;
@@ -209 +260 @@
   o->fftgrain = new_cvec(buf_size);
   o->desc = new_fvec(1);
-
-  /* set some default parameter */
-  aubio_onset_set_threshold (o, 0.3);
-  aubio_onset_set_delay(o, 4.3 * hop_size);
-  aubio_onset_set_minioi_ms(o, 20.);
-  aubio_onset_set_silence(o, -70.);
+  o->spectral_whitening = new_aubio_spectral_whitening(buf_size, hop_size, samplerate);
 
   /* initialize internal variables */
-  o->last_onset = 0;
-  o->total_frames = 0;
+  aubio_onset_set_default_parameters (o, onset_mode);
+
+  aubio_onset_reset(o);
   return o;
 
@@ -229 +276 @@
 }
 
+void aubio_onset_reset (aubio_onset_t *o) {
+  o->last_onset = 0;
+  o->total_frames = 0;
+}
+
+uint_t aubio_onset_set_default_parameters (aubio_onset_t * o, const char_t * onset_mode)
+{
+  uint_t ret = AUBIO_OK;
+  /* set some default parameter */
+  aubio_onset_set_threshold (o, 0.3);
+  aubio_onset_set_delay (o, 4.3 * o->hop_size);
+  aubio_onset_set_minioi_ms (o, 50.);
+  aubio_onset_set_silence (o, -70.);
+  // disable spectral whitening
+  aubio_onset_set_awhitening (o, 0);
+  // disable logarithmic magnitude
+  aubio_onset_set_compression (o, 0.);
+
+  /* method specific optimisations */
+  if (strcmp (onset_mode, "energy") == 0) {
+  } else if (strcmp (onset_mode, "hfc") == 0 || strcmp (onset_mode, "default") == 0) {
+    aubio_onset_set_threshold (o, 0.058);
+    aubio_onset_set_compression (o, 1.);
+  } else if (strcmp (onset_mode, "complexdomain") == 0
+             || strcmp (onset_mode, "complex") == 0) {
+    aubio_onset_set_delay (o, 4.6 * o->hop_size);
+    aubio_onset_set_threshold (o, 0.15);
+    aubio_onset_set_awhitening(o, 1);
+    aubio_onset_set_compression (o, 1.);
+  } else if (strcmp (onset_mode, "phase") == 0) {
+    o->apply_compression = 0;
+    aubio_onset_set_awhitening (o, 0);
+  } else if (strcmp (onset_mode, "mkl") == 0) {
+    aubio_onset_set_threshold (o, 0.05);
+    aubio_onset_set_awhitening(o, 1);
+    aubio_onset_set_compression (o, 0.02);
+  } else if (strcmp (onset_mode, "kl") == 0) {
+    aubio_onset_set_threshold (o, 0.35);
+    aubio_onset_set_awhitening(o, 1);
+    aubio_onset_set_compression (o, 0.02);
+  } else if (strcmp (onset_mode, "specflux") == 0) {
+    aubio_onset_set_threshold (o, 0.18);
+    aubio_onset_set_awhitening(o, 1);
+    aubio_spectral_whitening_set_relax_time(o->spectral_whitening, 100);
+    aubio_spectral_whitening_set_floor(o->spectral_whitening, 1.);
+    aubio_onset_set_compression (o, 10.);
+  } else if (strcmp (onset_mode, "specdiff") == 0) {
+  } else if (strcmp (onset_mode, "old_default") == 0) {
+    // used to reproduce results obtained with the previous version
+    aubio_onset_set_threshold (o, 0.3);
+    aubio_onset_set_minioi_ms (o, 20.);
+    aubio_onset_set_compression (o, 0.);
+  } else {
+    AUBIO_WRN("onset: unknown spectral descriptor type %s, "
+               "using default parameters.\n", onset_mode);
+    ret = AUBIO_FAIL;
+  }
+  return ret;
+}
+
 void del_aubio_onset (aubio_onset_t *o)
 {
+  del_aubio_spectral_whitening(o->spectral_whitening);
   del_aubio_specdesc(o->od);
   del_aubio_peakpicker(o->pp);

src/onset/onset.h

@@ -118 +118 @@
 smpl_t aubio_onset_get_last_ms (const aubio_onset_t *o);
 
+/** set onset detection adaptive whitening
+
+  \param o onset detection object as returned by new_aubio_onset()
+  \param enable 1 to enable, 0 to disable
+
+  \return 0 if successful, 1 otherwise
+
+*/
+uint_t aubio_onset_set_awhitening(aubio_onset_t * o, uint_t enable);
+
+/** get onset detection adaptive whitening
+
+  \param o onset detection object as returned by new_aubio_onset()
+
+  \return 1 if enabled, 0 otherwise
+
+*/
+smpl_t aubio_onset_get_awhitening(aubio_onset_t * o);
+
+/** set or disable log compression
+
+  \param o onset detection object as returned by new_aubio_onset()
+  \param lambda logarithmic compression factor, 0 to disable
+
+  \return 0 if successful, 1 otherwise
+
+ */
+uint_t aubio_onset_set_compression(aubio_onset_t *o, smpl_t lambda);
+
+/** get onset detection log compression
+
+  \param o onset detection object as returned by new_aubio_onset()
+
+  \returns 0 if disabled, compression factor otherwise
+
+ */
+smpl_t aubio_onset_get_compression(aubio_onset_t *o);
+
 /** set onset detection silence threshold
 
@@ -274 +312 @@
 */
 smpl_t aubio_onset_get_threshold(const aubio_onset_t * o);
+
+/** set default parameters
+
+  \param o onset detection object as returned by new_aubio_onset()
+  \param onset_mode detection mode to adjust
+
+  This function is called at the end of new_aubio_onset().
+
+ */
+uint_t aubio_onset_set_default_parameters (aubio_onset_t * o, const char_t * onset_mode);
+
+/** reset onset detection
+
+  \param o onset detection object as returned by new_aubio_onset()
+
+  Reset current time and last onset to 0.
+
+  This function is called at the end of new_aubio_onset().
+
+ */
+void aubio_onset_reset(aubio_onset_t * o);
 
 /** delete onset detection object

src/onset/peakpicker.c

@@ -93 +93 @@
   fvec_t *scratch = p->scratch;
   smpl_t mean = 0., median = 0.;
-  uint_t length = p->win_post + p->win_pre + 1;
   uint_t j = 0;
 
-  /* store onset in onset_keep */
-  /* shift all elements but last, then write last */
-  for (j = 0; j < length - 1; j++) {
-    onset_keep->data[j] = onset_keep->data[j + 1];
-    onset_proc->data[j] = onset_keep->data[j];
-  }
-  onset_keep->data[length - 1] = onset->data[0];
-  onset_proc->data[length - 1] = onset->data[0];
+  /* push new novelty to the end */
+  fvec_push(onset_keep, onset->data[0]);
+  /* store a copy */
+  fvec_copy(onset_keep, onset_proc);
 
-  /* filter onset_proc */
-  /** \bug filtfilt calculated post+pre times, should be only once !? */
+  /* filter this copy */
   aubio_filter_do_filtfilt (p->biquad, onset_proc, scratch);
 
   /* calculate mean and median for onset_proc */
   mean = fvec_mean (onset_proc);
-  /* copy to scratch */
-  for (j = 0; j < length; j++)
-    scratch->data[j] = onset_proc->data[j];
+
+  /* copy to scratch and compute its median */
+  fvec_copy(onset_proc, scratch);
   median = p->thresholdfn (scratch);
 

src/spectral/specdesc.c

@@ -31 +31 @@
 void aubio_specdesc_complex(aubio_specdesc_t *o, const cvec_t * fftgrain, fvec_t * onset);
 void aubio_specdesc_phase(aubio_specdesc_t *o, const cvec_t * fftgrain, fvec_t * onset);
+void aubio_specdesc_wphase(aubio_specdesc_t *o, const cvec_t * fftgrain, fvec_t * onset);
 void aubio_specdesc_specdiff(aubio_specdesc_t *o, const cvec_t * fftgrain, fvec_t * onset);
 void aubio_specdesc_kl(aubio_specdesc_t *o, const cvec_t * fftgrain, fvec_t * onset);
@@ -58 +59 @@
         aubio_onset_complex,        /**< complex domain */        
         aubio_onset_phase,          /**< phase fast */            
+        aubio_onset_wphase,         /**< weighted phase */
         aubio_onset_kl,             /**< Kullback Liebler */
         aubio_onset_mkl,            /**< modified Kullback Liebler */
@@ -158 +160 @@
   onset->data[0] = aubio_hist_mean(o->histog);  
   //onset->data[0] = fvec_mean(o->dev1);
+}
+
+/* weighted phase */
+void
+aubio_specdesc_wphase(aubio_specdesc_t *o,
+    const cvec_t *fftgrain, fvec_t *onset) {
+  uint_t i;
+  aubio_specdesc_phase(o, fftgrain, onset);
+  for (i = 0; i < fftgrain->length; i++) {
+    o->dev1->data[i] *= fftgrain->norm[i];
+  }
+  /* apply o->histogram */
+  aubio_hist_dyn_notnull(o->histog,o->dev1);
+  /* weight it */
+  aubio_hist_weight(o->histog);
+  /* its mean is the result */
+  onset->data[0] = aubio_hist_mean(o->histog);
 }
 
@@ -251 +270 @@
   else if (strcmp (onset_mode, "phase") == 0)
       onset_type = aubio_onset_phase;
+  else if (strcmp (onset_mode, "wphase") == 0)
+      onset_type = aubio_onset_wphase;
   else if (strcmp (onset_mode, "mkl") == 0)
       onset_type = aubio_onset_mkl;
@@ -271 +292 @@
   else if (strcmp (onset_mode, "rolloff") == 0)
       onset_type = aubio_specmethod_rolloff;
+  else if (strcmp (onset_mode, "old_default") == 0)
+      onset_type = aubio_onset_default;
   else if (strcmp (onset_mode, "default") == 0)
       onset_type = aubio_onset_default;
@@ -292 +315 @@
       break;
     case aubio_onset_phase:
+    case aubio_onset_wphase:
       o->dev1   = new_fvec(rsize);
       o->theta1 = new_fvec(rsize);
@@ -326 +350 @@
       o->funcpointer = aubio_specdesc_phase;
       break;
+    case aubio_onset_wphase:
+      o->funcpointer = aubio_specdesc_wphase;
+      break;
     case aubio_onset_specdiff:
       o->funcpointer = aubio_specdesc_specdiff;
@@ -379 +406 @@
       break;
     case aubio_onset_phase:
+    case aubio_onset_wphase:
       del_fvec(o->dev1);
       del_fvec(o->theta1);

src/spectral/specdesc.h

@@ -60 +60 @@
   Hong-Kong, 2003.
 
+  \b \p wphase : Weighted Phase Deviation onset detection function
+
+  S. Dixon. Onset detection revisited. In Proceedings of the 9th International
+  Conference on Digital Audio Ef- fects (DAFx) , pages 133–137, 2006.
+
+  http://www.eecs.qmul.ac.uk/~simond/pub/2006/dafx.pdf
+
   \b \p specdiff : Spectral difference method onset detection function
 
@@ -175 +182 @@
   The parameter \p method is a string that can be any of:
 
-    - `energy`, `hfc`, `complex`, `phase`, `specdiff`, `kl`, `mkl`, `specflux`
-    - `centroid`, `spread`, `skewness`, `kurtosis`, `slope`, `decrease`, `rolloff`
+    - onset novelty functions: `complex`, `energy`, `hfc`, `kl`, `mkl`,
+    `phase`, `specdiff`, `specflux`, `wphase`,
+
+    - spectral descriptors: `centroid`, `decrease`, `kurtosis`, `rolloff`,
+    `skewness`, `slope`, `spread`.
 
 */

src/synth/wavetable.c

@@ -104 +104 @@
       output->data[i] += input->data[i];
     }
+    fvec_clamp(output, 1.);
   }
 }