[4cc9fe5] | 1 | #! /usr/bin/python |
---|
| 2 | |
---|
| 3 | from aubio.bench.node import * |
---|
[4f4a8a4] | 4 | from aubio.tasks import * |
---|
[75139a9] | 5 | |
---|
[e968939] | 6 | |
---|
| 7 | |
---|
| 8 | |
---|
| 9 | def mmean(l): |
---|
| 10 | return sum(l)/float(len(l)) |
---|
| 11 | |
---|
| 12 | def stdev(l): |
---|
| 13 | smean = 0 |
---|
| 14 | lmean = mmean(l) |
---|
| 15 | for i in l: |
---|
| 16 | smean += (i-lmean)**2 |
---|
| 17 | smean *= 1. / len(l) |
---|
| 18 | return smean**.5 |
---|
| 19 | |
---|
[336cf77] | 20 | class benchonset(bench): |
---|
[e968939] | 21 | |
---|
[c912c67] | 22 | """ list of values to store per file """ |
---|
[e968939] | 23 | valuenames = ['orig','missed','Tm','expc','bad','Td'] |
---|
[c912c67] | 24 | """ list of lists to store per file """ |
---|
[e968939] | 25 | valuelists = ['l','labs'] |
---|
[c912c67] | 26 | """ list of values to print per dir """ |
---|
| 27 | printnames = [ 'mode', 'thres', 'dist', 'prec', 'recl', |
---|
| 28 | 'Ttrue', 'Tfp', 'Tfn', 'Tm', 'Td', |
---|
| 29 | 'aTtrue', 'aTfp', 'aTfn', 'aTm', 'aTd', |
---|
| 30 | 'mean', 'smean', 'amean', 'samean'] |
---|
| 31 | |
---|
| 32 | """ per dir """ |
---|
| 33 | formats = {'mode': "%12s" , 'thres': "%5.4s", |
---|
| 34 | 'dist': "%5.4s", 'prec': "%5.4s", 'recl': "%5.4s", |
---|
| 35 | 'Ttrue': "%5.4s", 'Tfp': "%5.4s", 'Tfn': "%5.4s", |
---|
| 36 | 'Tm': "%5.4s", 'Td': "%5.4s", |
---|
| 37 | 'aTtrue':"%5.4s", 'aTfp': "%5.4s", 'aTfn': "%5.4s", |
---|
| 38 | 'aTm': "%5.4s", 'aTd': "%5.4s", |
---|
| 39 | 'mean': "%5.40s", 'smean': "%5.40s", |
---|
| 40 | 'amean': "%5.40s", 'samean': "%5.40s"} |
---|
[4f4a8a4] | 41 | |
---|
[e968939] | 42 | def dir_eval(self): |
---|
[c912c67] | 43 | """ evaluate statistical data over the directory """ |
---|
| 44 | totaltrue = sum(self.v['expc'])-sum(self.v['bad'])-sum(self.v['Td']) |
---|
| 45 | totalfp = sum(self.v['bad'])+sum(self.v['Td']) |
---|
| 46 | totalfn = sum(self.v['missed'])+sum(self.v['Tm']) |
---|
[e968939] | 47 | self.P = 100*float(totaltrue)/max(totaltrue + totalfp,1) |
---|
| 48 | self.R = 100*float(totaltrue)/max(totaltrue + totalfn,1) |
---|
| 49 | if self.R < 0: self.R = 0 |
---|
| 50 | self.F = 2.* self.P*self.R / max(float(self.P+self.R),1) |
---|
| 51 | N = float(len(self.reslist)) |
---|
| 52 | self.v['mode'] = self.params.onsetmode |
---|
[c912c67] | 53 | self.v['thres'] = self.params.threshold |
---|
[e968939] | 54 | self.v['thres'] = "%2.3f" % self.params.threshold |
---|
| 55 | self.v['dist'] = "%2.3f" % self.F |
---|
| 56 | self.v['prec'] = "%2.3f" % self.P |
---|
| 57 | self.v['recl'] = "%2.3f" % self.R |
---|
| 58 | self.v['Ttrue'] = totaltrue |
---|
| 59 | self.v['Tfp'] = totalfp |
---|
| 60 | self.v['Tfn'] = totalfn |
---|
| 61 | self.v['aTtrue'] = totaltrue/N |
---|
| 62 | self.v['aTfp'] = totalfp/N |
---|
| 63 | self.v['aTfn'] = totalfn/N |
---|
[c912c67] | 64 | self.v['aTm'] = sum(self.v['Tm'])/N |
---|
| 65 | self.v['aTd'] = sum(self.v['Td'])/N |
---|
[d998190] | 66 | self.v['Tm'] = sum(self.v['Tm']) |
---|
| 67 | self.v['Td'] = sum(self.v['Td']) |
---|
[e968939] | 68 | self.v['mean'] = mmean(self.v['l']) |
---|
| 69 | self.v['smean'] = stdev(self.v['l']) |
---|
| 70 | self.v['amean'] = mmean(self.v['labs']) |
---|
| 71 | self.v['samean'] = stdev(self.v['labs']) |
---|
[75139a9] | 72 | |
---|
| 73 | def run_bench(self,modes=['dual'],thresholds=[0.5]): |
---|
| 74 | self.modes = modes |
---|
| 75 | self.thresholds = thresholds |
---|
[e968939] | 76 | self.pretty_titles() |
---|
[75139a9] | 77 | for mode in self.modes: |
---|
[4f4a8a4] | 78 | self.params.onsetmode = mode |
---|
[75139a9] | 79 | for threshold in self.thresholds: |
---|
| 80 | self.params.threshold = threshold |
---|
[4f4a8a4] | 81 | self.dir_exec() |
---|
| 82 | self.dir_eval() |
---|
[e968939] | 83 | self.pretty_print() |
---|
| 84 | #print self.v |
---|
| 85 | |
---|
[75139a9] | 86 | def auto_learn(self,modes=['dual'],thresholds=[0.1,1.5]): |
---|
| 87 | """ simple dichotomia like algorithm to optimise threshold """ |
---|
| 88 | self.modes = modes |
---|
[e968939] | 89 | self.pretty_titles() |
---|
[75139a9] | 90 | for mode in self.modes: |
---|
[c912c67] | 91 | steps = 11 |
---|
[75139a9] | 92 | lesst = thresholds[0] |
---|
| 93 | topt = thresholds[1] |
---|
[4f4a8a4] | 94 | self.params.onsetmode = mode |
---|
[75139a9] | 95 | |
---|
| 96 | self.params.threshold = topt |
---|
[4f4a8a4] | 97 | self.dir_exec() |
---|
| 98 | self.dir_eval() |
---|
[e968939] | 99 | self.pretty_print() |
---|
[75139a9] | 100 | topF = self.F |
---|
| 101 | |
---|
| 102 | self.params.threshold = lesst |
---|
[4f4a8a4] | 103 | self.dir_exec() |
---|
| 104 | self.dir_eval() |
---|
[e968939] | 105 | self.pretty_print() |
---|
[75139a9] | 106 | lessF = self.F |
---|
| 107 | |
---|
| 108 | for i in range(steps): |
---|
| 109 | self.params.threshold = ( lesst + topt ) * .5 |
---|
[4f4a8a4] | 110 | self.dir_exec() |
---|
| 111 | self.dir_eval() |
---|
[e968939] | 112 | self.pretty_print() |
---|
[75139a9] | 113 | if self.F == 100.0 or self.F == topF: |
---|
| 114 | print "assuming we converged, stopping" |
---|
| 115 | break |
---|
| 116 | #elif abs(self.F - topF) < 0.01 : |
---|
| 117 | # print "done converging" |
---|
| 118 | # break |
---|
| 119 | if topF < self.F: |
---|
| 120 | #lessF = topF |
---|
| 121 | #lesst = topt |
---|
| 122 | topF = self.F |
---|
| 123 | topt = self.params.threshold |
---|
| 124 | elif lessF < self.F: |
---|
| 125 | lessF = self.F |
---|
| 126 | lesst = self.params.threshold |
---|
| 127 | if topt == lesst: |
---|
| 128 | lesst /= 2. |
---|
| 129 | |
---|
[e968939] | 130 | def auto_learn2(self,modes=['dual'],thresholds=[0.00001,1.0]): |
---|
[af445db] | 131 | """ simple dichotomia like algorithm to optimise threshold """ |
---|
| 132 | self.modes = modes |
---|
[e968939] | 133 | self.pretty_titles([]) |
---|
[af445db] | 134 | for mode in self.modes: |
---|
| 135 | steps = 10 |
---|
[e968939] | 136 | step = 0.4 |
---|
[af445db] | 137 | self.params.onsetmode = mode |
---|
[e968939] | 138 | self.params.threshold = thresholds[0] |
---|
| 139 | cur = 0 |
---|
[af445db] | 140 | |
---|
| 141 | for i in range(steps): |
---|
| 142 | self.dir_exec() |
---|
| 143 | self.dir_eval() |
---|
[e968939] | 144 | self.pretty_print() |
---|
| 145 | new = self.P |
---|
| 146 | if self.R == 0.0: |
---|
| 147 | #print "Found maximum, highering" |
---|
| 148 | step /= 2. |
---|
| 149 | self.params.threshold -= step |
---|
| 150 | elif new == 100.0: |
---|
| 151 | #print "Found maximum, highering" |
---|
| 152 | step *= .99 |
---|
| 153 | self.params.threshold += step |
---|
| 154 | elif cur > new: |
---|
| 155 | #print "lower" |
---|
| 156 | step /= 2. |
---|
| 157 | self.params.threshold -= step |
---|
| 158 | elif cur < new: |
---|
| 159 | #print "higher" |
---|
| 160 | step *= .99 |
---|
| 161 | self.params.threshold += step |
---|
| 162 | else: |
---|
| 163 | print "Assuming we converged" |
---|
[af445db] | 164 | break |
---|
[e968939] | 165 | cur = new |
---|
| 166 | |
---|
[75139a9] | 167 | |
---|
[4f4a8a4] | 168 | if __name__ == "__main__": |
---|
| 169 | import sys |
---|
| 170 | if len(sys.argv) > 1: datapath = sys.argv[1] |
---|
| 171 | else: print "ERR: a path is required"; sys.exit(1) |
---|
| 172 | modes = ['complex', 'energy', 'phase', 'specdiff', 'kl', 'mkl', 'dual'] |
---|
[c912c67] | 173 | #modes = [ 'mkl' ] |
---|
[e968939] | 174 | thresholds = [ 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2] |
---|
[4f4a8a4] | 175 | #thresholds = [1.5] |
---|
| 176 | |
---|
| 177 | #datapath = "%s%s" % (DATADIR,'/onset/DB/*/') |
---|
| 178 | respath = '/var/tmp/DB-testings' |
---|
| 179 | |
---|
| 180 | benchonset = benchonset(datapath,respath,checkres=True,checkanno=True) |
---|
| 181 | benchonset.params = taskparams() |
---|
| 182 | benchonset.task = taskonset |
---|
[e968939] | 183 | benchonset.valuesdict = {} |
---|
[4f4a8a4] | 184 | |
---|
| 185 | try: |
---|
[e968939] | 186 | #benchonset.auto_learn2(modes=modes) |
---|
| 187 | benchonset.run_bench(modes=modes,thresholds=thresholds) |
---|
[4f4a8a4] | 188 | except KeyboardInterrupt: |
---|
| 189 | sys.exit(1) |
---|