summaryrefslogtreecommitdiff
path: root/NT/src/nodes_audio_analysis.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'NT/src/nodes_audio_analysis.cpp')
-rw-r--r--NT/src/nodes_audio_analysis.cpp475
1 files changed, 475 insertions, 0 deletions
diff --git a/NT/src/nodes_audio_analysis.cpp b/NT/src/nodes_audio_analysis.cpp
new file mode 100644
index 0000000..1890b8c
--- /dev/null
+++ b/NT/src/nodes_audio_analysis.cpp
@@ -0,0 +1,475 @@
+#include "nodes_audio_analysis.h"
+
+namespace Rotor{
+ bool Audio_thumbnailer::init(int _channels,int _bits,int _samples,int _rate) {
+ //base_audio_processor::init(_channels,_bits,_samples);
+ channels=_channels;
+ bits=_bits;
+ samples=_samples;
+ samples_per_column=samples/width;
+ offset=0x1<<(bits-1); //signed audio
+ scale=1.0/offset;
+
+ out_sample=0; //sample in whole track
+ sample=0;
+ samples=0;
+ accum=0.0;
+ return true;
+ }
+ int Audio_thumbnailer::process_frame(uint8_t *_data,int samples_in_frame){
+ //begin by processing remaining samples
+ //samples per column could be larger than a frame! (probably is)
+ //but all we are doing is averaging
+ int bytes=(bits>>3);
+ int stride=channels*bytes;
+ int in_sample=0;
+ while (in_sample<samples_in_frame) {
+ //continue the column
+ while (sample<samples_per_column&&in_sample<samples_in_frame) {
+ //accumulate samples for this column until we run out of samples
+ for (int i=0;i<channels;i++) {
+ unsigned int this_val=0;
+ for (int j=0;j<bytes;j++) {
+ this_val+=_data[(in_sample*stride)+(i*bytes)+j]<<(j*8);
+ }
+ //convert from integer data format - i.e s16p - to audio signal in -1..1 range
+ //presume 16 bits for now...
+ double val=((double)((int16_t)this_val))*scale;
+ accum+=val*val;
+ samples++;
+ }
+ in_sample++;
+ sample++;
+ out_sample++;
+ }
+ if (sample==samples_per_column) { //finished a column
+ //get root-mean
+ //why does valgrind complain here about uninitialised vars
+ double mean=pow(accum/samples,0.5f);
+ audiodata.push_back(mean);
+ sample=0;
+ samples=0;
+ accum=0.0;
+ }
+ }
+ return out_sample;
+ }
+ void Audio_thumbnailer::print_vector(xmlIO XML){
+ string vdata;
+ int i=0;
+ for (auto sample: audiodata){
+ if (i>0) vdata+=",";
+ vdata+=toString(sample);
+ i++;
+ }
+ XML.addValue("data",vdata);
+ }
+ bool Vamp_node::init(int _channels,int _bits,int _samples, int _rate) {
+ //need these to make sense of data
+ channels=_channels;
+ bits=_bits;
+ samples=_samples;
+
+ features.clear();
+ return analyser.init(soname,id,_channels,_bits,_samples,_rate,outputNo,params);
+
+
+ //attempt to load vamp plugin and prepare to receive frames of data
+ //should the audio analysis contain a vamphost or should it inherit?
+ //maybe neater to contain it in terms of headers etc
+
+ }
+ int Vamp_node::process_frame(uint8_t *data,int samples_in_frame) {
+ analyser.process_frame(data,samples_in_frame);
+ return 1;
+ }
+ void Vamp_node::cleanup() {
+ analyser.cleanup();
+ features=analyser.features;
+ }
+ string Vamp_node::get_features(){
+ string data;
+ for (auto i: features) {
+ data=data+" ["+toString(i.second.number)+":"+toString(i.first);
+ if (i.second.values.size()) {
+ data+=" (";
+ bool first=true;
+ for (auto j: i.second.values) {
+ if (first){
+ first=false;
+ }
+ else data+=",";
+ data=data+toString(j);
+ }
+ data+=") ";
+ }
+ data+="]";
+ }
+ return data;
+ }
+ bool sortsegments(std::pair<int,double> i,std::pair<int,double> j){
+ return (i.second<j.second);
+ }
+
+ bool sortseggrps(std::pair<double,vector<pair<double,int> > > i,std::pair<double,vector<pair<double,int> > > j){
+ return (i.first<j.first);
+ }
+ bool sortgroupmembers(pair<double,int> i,pair<double,int> j){
+ return (i.first<j.first);
+ }
+ void Intensity_segmenter::cleanup(){
+ //algorithm idea:
+ //get average tempo and intensity for each segment and store them
+ //scale by the range to get a value from 0.0 to 1.0
+ //add tempo and intensity according to a weighting
+ //score the results (ie 1st place, 2nd place) to end up with a set of integer numbers
+
+ //how to group with similarity?
+ //segments come with similarity groups
+ // 1 - are the wanted git checksegments less than discovered?
+ // N - do nothing
+ // 2 - get intensity and tempo averages
+ // 2 - count the groups
+ // 3 - are the groups less than the discovered segments?
+ // N - group by intensity as normal
+ // 4 - are the groups less than the wanted levels?
+
+
+ //for (auto a:analysers) a.second.cleanup(); //WHY NOT WORK - its as if the call is const
+ analysers["segmenter"].cleanup();
+ analysers["tempo"].cleanup();
+ analysers["intensity"].cleanup();
+
+ //combine with similarity numbers
+ // 1. count similarity numbers
+
+ map<int,vector<int> > similarities;
+
+ //what do we want to know about these similarities?
+ // how many are there? map.size()
+ // how many members are in each one? map[item].size()
+ // which members are they? auto m: map[item]
+
+ uint32_t i=0;
+ for (auto f:analysers["segmenter"].features) {
+ if (f.second.values.size()) {
+ int group=f.second.values[0];
+ if (similarities.find(group)==similarities.end()){
+ similarities[group]={};
+ }
+ similarities[group].push_back(i);
+ }
+ i++;
+ }
+ /*
+ for (auto s:similarities) {
+ string list="";
+ for (int j=0;j<s.second.size();j++){
+ if (j>0) list+=",";
+ list +=toString(s.second[j]);
+ }
+ cerr<<"group "<<s.first<<" ["<<list<<"]"<<endl;
+ }
+ */
+
+ cerr<<analysers["segmenter"].features.size()<<" segments"<<endl;
+ cerr<<analysers["tempo"].features.size()<<" tempo features"<<endl;
+ cerr<<analysers["intensity"].features.size()<<" intensity features"<<endl;
+ i=0;
+ double min_tempo=9999999.0;
+ double min_intensity=9999999.0;
+ double max_tempo=0.0;
+ double max_intensity=0.0;
+ vector<double> tempos;
+ vector<double> intensities;
+ vector<double> times;
+ auto g=++analysers["segmenter"].features.begin();
+ for (auto f=analysers["segmenter"].features.begin();g!=analysers["segmenter"].features.end();f++,g++,i++){
+ times.push_back(f->first);
+ //integrate tempo and intensity algorithmically
+ double tempo=0;
+ if (analysers["tempo"].features.size()) {
+ double pt=f->first;
+ double pv=analysers["tempo"].get_value(f->first);
+ for (auto u=analysers["tempo"].features.upper_bound(f->first);u!=analysers["tempo"].features.upper_bound(g->first);u++){
+ tempo +=(u->first-pt)*(u->second.values[0]+pv)*0.5f; //area of the slice
+ pt=u->first;
+ pv=u->second.values[0];
+ }
+ tempo +=(g->first-pt)*(analysers["tempo"].get_value(g->first)+pv)*0.5f; //area of the last slice
+ tempo /=g->first-f->first; //average value;
+ }
+ if (tempo>max_tempo) max_tempo=tempo;
+ if (tempo<min_tempo) min_tempo=tempo;
+ tempos.push_back(tempo);
+
+ double intensity=0;
+ if (analysers["intensity"].features.size()) {
+ double pt=f->first;
+ double pv=analysers["intensity"].get_value(f->first);
+ for (auto u=analysers["intensity"].features.upper_bound(f->first);u!=analysers["intensity"].features.upper_bound(g->first);u++){
+ intensity +=(u->first-pt)*(u->second.values[0]+pv)*0.5f; //area of the slice
+ pt=u->first;
+ pv=u->second.values[0];
+ }
+ intensity +=(g->first-pt)*(analysers["intensity"].get_value(g->first)+pv)*0.5f; //area of the last slice
+ intensity /=g->first-f->first; //average value;
+ }
+ if (intensity>max_intensity) max_intensity=intensity;
+ if (intensity<min_intensity) min_intensity=intensity;
+ intensities.push_back(intensity);
+
+ //cerr<<"segment "<<i<<": "<<f->first<<" to "<<g->first<<" average tempo: "<<tempo<<" ,intensity: "<<intensity<<" ,weighted: "<<(tempo*parameters["tempo_weight"]->value)+(intensity*parameters["intensity_weight"]->value)<<endl;
+ }
+ //
+ //
+ //need to calculate the last segment
+ //
+ //
+ //either by adding a bit of code here or by adding a dummy feature at the track duration, previously
+ //
+ //
+ //
+ //
+
+
+ //make relative scale 0.0-1.0 and save weighted totals
+ vector< pair<int,double>> totals;
+ vector<double> totalsmap;
+ for (i=0;i<tempos.size();i++){
+ tempos[i]=(tempos[i]-min_tempo)/(max_tempo-min_tempo);
+ intensities[i]=(intensities[i]-min_intensity)/(max_intensity-min_intensity);
+ totals.push_back(make_pair(i,(tempos[i]*parameters["tempo_weight"]->value)+(intensities[i]*parameters["intensity_weight"]->value)));
+ totalsmap.push_back((tempos[i]*parameters["tempo_weight"]->value)+(intensities[i]*parameters["intensity_weight"]->value));
+ }
+
+ /*
+ //sort and convert to features
+ std::sort(totals.begin(),totals.end(),sortsegments);
+ for (i=0;i<totals.size();i++) {
+ cerr<<"segment "<<totals[i].first<<" average intensity: "<<totals[i].second<<endl;
+ }
+ vector<double> bucketoffsets;
+ for (auto t:totals) bucketoffsets.push_back(0.0);
+ if (parameters["levels"]->value>0.0&&parameters["levels"]->value<totals.size()){
+ //use bucketoffsets to redistribute into smaller number of buckets
+ int numbertoredistribute=totals.size()-((int)parameters["levels"]->value);
+ double numberperbin=((double)numbertoredistribute/totals.size());
+ double toadd=0.5f;
+ int added=0;
+ for (int j=0;j<totals.size();j++){
+ int numbertoadd=min(numbertoredistribute-added,(int)toadd);
+ toadd=(toadd+numberperbin)-numbertoadd;
+ added+=numbertoadd;
+ bucketoffsets[j]=added;
+ }
+ if (numbertoredistribute>0) {
+ cerr<<"reducing number of levels by "<<numbertoredistribute<<", offsets:"<<endl;
+ for (auto o:bucketoffsets) {
+ cerr<<o<<":";
+ }
+ cerr<<endl;
+ }
+ }
+
+ for (i=0;i<totals.size();i++){
+ vampHost::feature f;
+ f.values.push_back((double)i-bucketoffsets[i]);
+ features[times[totals[i].first]]=f;
+ }
+/*
+ /*
+sort intensity totals
+find out how many segments will share levels apart from similarity levels
+start with a structure:
+map<inputnum,vector<pair<tempo,inputnum>>
+start grouping by similarity
+if there are more similarity groups than wantedgroups, start by grouping similarities
+otherwise take biggest similarity groups and split them by intensity
+if there are still too many groups, merge closest smallest groups
+finally sort by intensity to map output
+
+nned to retrieve total intensity by segment
+ */
+ // segment group_intensity seg_intense segment
+ vector<pair<double,vector<pair<double,int> > > > seggrps;
+ vector<pair<double,vector<pair<double,int> > > > oldgrps;
+ for (i=0;i<totalsmap.size();i++){
+ vector<pair<double,int> > data;
+ data.push_back(make_pair(totalsmap[i],i));
+ oldgrps.push_back(make_pair(totalsmap[i],data));
+ }
+
+ for (auto s:similarities){
+ //similarities is a collection of similarity groups in no particular order, referring to segment nos
+ //at this point seggrps is in segment order
+ if (s.second.size()>1){
+ for (int j=s.second.size()-1;j>0;j--){
+ oldgrps[s.second[0]].second.push_back(make_pair(totalsmap[s.second[j]],s.second[j]));
+ //keep running average// should be by area?
+ //seggrps[s.second[0]].first+=(totalsmap[s.second[j]]*(1.0/max(1,(int)seggrps[s.second[0]].second.size()-1)));
+ //double div=seggrps[s.second[0]].second.size()==1?1.0:((double)seggrps[s.second[0]].second.size()-1/(double)seggrps[s.second[0]].second.size());
+ //neat! this gives 1,1/2,2/3,3/4..
+ //seggrps[s.second[0]].first*=div;
+
+ //seggrps.erase(seggrps.begin()+s.second[j]);
+ //after this has happened, seggrpgs indexing can be invalid
+
+ }
+
+
+ //easier is to
+ double avg=0.0f;
+ for (auto p:oldgrps[s.second[0]].second) avg+=p.first;
+ avg/=oldgrps[s.second[0]].second.size();
+ oldgrps[s.second[0]].first=avg;
+
+ }
+ seggrps.push_back(oldgrps[s.second[0]]);
+ }
+
+ //cerr<<"similarities assigned, "<<(totalsmap.size()-seggrps.size())<<" segments merged"<<endl;
+
+ /*
+ i=0;
+ for (auto s:seggrps) {
+ string list="";
+ for (int j=0;j<s.second.size();j++){
+ if (j>0) list+=",";
+ list +=toString(s.second[j].second);
+ }
+ cerr<<"segment "<<i<<" ["<<list<<"]"<<endl;
+ i++;
+ }
+ */
+ //sort the contents by intensity
+ std::sort(seggrps.begin(),seggrps.end(),sortseggrps);
+ //cerr<<"groups sorted by intensity:"<<endl;
+ //possible mergers will be with groups with adjacent intensity
+ i=0;
+ /*
+ for (auto s:seggrps) {
+ string list="";
+ for (int j=0;j<s.second.size();j++){
+ if (j>0) list+=",";
+ list +=toString(s.second[j].second);
+ }
+ cerr<<"segment "<<i<<" ["<<list<<"]"<<endl;
+ i++;
+ }
+ */
+
+ if (((int)parameters["levels"]->value)>0) {
+ if (seggrps.size()>(int)parameters["levels"]->value){
+ while (seggrps.size()>(int)parameters["levels"]->value){
+ //reduce similarity groups
+ //decide the best 2 to merge
+ vector<double> diffs;
+ for (int j=0;j<seggrps.size()-1;j++) diffs.push_back(seggrps[j+1].first-seggrps[j].first);
+ int smallest=0;
+ for (int j=1;j<diffs.size();j++) if (diffs[j]<diffs[smallest]) smallest=j;
+ for (int j=0;j<seggrps[smallest].second.size();j++) {
+ seggrps[smallest+1].second.push_back(seggrps[smallest].second[j]);
+ //cerr<<"copied segment "<<(seggrps[smallest].second[j].second)<<" from group "<<smallest<<" to group "<<(smallest+1)<<endl;
+ }
+ //recalculate intensity average
+ double avg=0.0f;
+ for (auto p:seggrps[smallest+1].second) avg+=p.first;
+ avg/=seggrps[smallest+1].second.size();
+ seggrps[smallest+1].first=avg;
+
+ seggrps.erase(seggrps.begin()+smallest);
+ //cerr<<"removed group "<<smallest<<endl;
+ }
+ cerr<<"intensities merged, "<<seggrps.size()<<" levels remain"<<endl;
+ }
+ //cerr<<seggrps.size()<<" groups, "<<(int)parameters["levels"]->value<<" levels requested, "<<(int)totalsmap.size()<<" original segments"<<endl;
+ if (seggrps.size()<min((int)parameters["levels"]->value,(int)totalsmap.size())){
+ while (seggrps.size()<min((int)parameters["levels"]->value,(int)totalsmap.size())) {
+ //split groups
+ //calculate standard deviation of intensity variation
+ vector<double> devs;
+ for (int j=0;j<seggrps.size()-1;j++) {
+ double avg=0.0;
+ double dev=0.0;
+ for (int k=0;k<seggrps[j].second.size();k++){
+ avg+=seggrps[j].second[k].first;
+ }
+ avg/=seggrps[j].second.size();
+ for (int k=0;k<seggrps[j].second.size();k++){
+ dev+=pow(avg-k<seggrps[j].second[k].first,2.0);
+ }
+ dev/=seggrps[j].second.size();
+ devs.push_back(pow(dev,0.5));
+ }
+ //find group with largest standard deviation
+ int largest=0;
+ for (int j=1;j<devs.size();j++) if (devs[j]>devs[largest]) largest=j;
+ //sanity check: if there are any groups that can be split they will have larger SD than singleton groups
+ //sort members of the group
+ std::sort(seggrps[largest].second.begin(),seggrps[largest].second.end(),sortgroupmembers);
+ //create a new group
+ std::pair<double,vector<pair<double,int> > > newgroup;
+ //cerr<<"splitting group "<<largest<<" with "<<seggrps[largest].second.size()<<" segments: new group will have "<<seggrps[largest].second.size()-(seggrps[largest].second.size()/2)<<" segments"<<endl;
+ for (int j=seggrps[largest].second.size()-1;j>(seggrps[largest].second.size()/2)-1;j--) {
+ newgroup.second.push_back(seggrps[largest].second[j]);
+ seggrps[largest].second.erase(seggrps[largest].second.begin()+j);
+ }
+
+ //refresh averages for the 2 groups
+ double avg=0.0f;
+ for (auto p:seggrps[largest].second) avg+=p.first;
+ avg/=seggrps[largest].second.size();
+ seggrps[largest].first=avg;
+
+ avg=0.0f;
+ for (auto p:newgroup.second) avg+=p.first;
+ avg/=newgroup.second.size();
+ newgroup.first=avg;
+
+ //add the new group
+ seggrps.push_back(newgroup);
+ //cerr<<" added new group with "<<newgroup.second.size()<<" segments"<<endl;
+ }
+ cerr<<"similaritity groups split, "<<seggrps.size()<<" levels total"<<endl;
+ //seggrps are now out of order
+ std::sort(seggrps.begin(),seggrps.end(),sortseggrps);
+
+ }
+
+ }
+
+ map<int,int> outputvalues;
+ for (int j=0;j<seggrps.size();j++){
+ string list="";
+ for (int k=0;k<seggrps[j].second.size();k++){
+ outputvalues[seggrps[j].second[k].second]=j;
+ if (k>0) list+=",";
+ list +=toString(seggrps[j].second[k].second);
+ }
+ //cerr<<"output value: "<<j<<" assigned to ["<<list<<"]"<<endl;
+ }
+
+
+ for (i=0;i<totals.size();i++){
+ vampHost::feature f;
+ f.values.push_back(outputvalues[i]);
+ features[times[totals[i].first]]=f;
+ }
+
+ }
+}
+
+/*
+A data structure to represent segments and their mapping to output levels
+how do we merge the intensity groups with the similarities?
+
+we create a list
+
+
+
+
+... we iterate through the list of segments and place the right output number
+
+
+*/
generated by cgit v1.2.3 (git 2.39.1) at 2025-12-06 10:13:23 +0000