From 89961817f408e921de2c1be6197e2b1ee0f5df98 Mon Sep 17 00:00:00 2001
From: Comment <tim@gray.(none)>
Date: Wed, 29 Jan 2014 23:58:55 +0000
Subject: NT audio framework

---
 NT/src/nodes_audio_analysis.cpp | 475 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 475 insertions(+)
 create mode 100644 NT/src/nodes_audio_analysis.cpp

(limited to 'NT/src/nodes_audio_analysis.cpp')

diff --git a/NT/src/nodes_audio_analysis.cpp b/NT/src/nodes_audio_analysis.cpp
new file mode 100644
index 0000000..1890b8c
--- /dev/null
+++ b/NT/src/nodes_audio_analysis.cpp
@@ -0,0 +1,475 @@
+#include "nodes_audio_analysis.h"
+
+namespace Rotor{
+	bool Audio_thumbnailer::init(int _channels,int _bits,int _samples,int _rate) {
+		//base_audio_processor::init(_channels,_bits,_samples);
+		channels=_channels;
+		bits=_bits;
+		samples=_samples;
+		samples_per_column=samples/width;
+		offset=0x1<<(bits-1); //signed audio
+		scale=1.0/offset;
+
+		out_sample=0; //sample in whole track
+		sample=0;
+		samples=0;
+		accum=0.0;
+		return true;
+	}
+	int Audio_thumbnailer::process_frame(uint8_t *_data,int samples_in_frame){
+		//begin by processing remaining samples
+		//samples per column could be larger than a frame! (probably is)
+		//but all we are doing is averaging
+		int bytes=(bits>>3);
+		int stride=channels*bytes;
+		int in_sample=0;
+		while (in_sample<samples_in_frame) {
+			//continue the column
+			while (sample<samples_per_column&&in_sample<samples_in_frame) {
+				//accumulate samples for this column until we run out of samples
+				for (int i=0;i<channels;i++) {
+					unsigned int this_val=0;
+					for (int j=0;j<bytes;j++) {
+						this_val+=_data[(in_sample*stride)+(i*bytes)+j]<<(j*8);
+					}
+					//convert from integer data format - i.e s16p - to audio signal in -1..1 range
+					//presume 16 bits for now...
+					double val=((double)((int16_t)this_val))*scale;
+					accum+=val*val;
+					samples++;
+				}
+				in_sample++;
+				sample++;
+				out_sample++;
+			}
+			if (sample==samples_per_column) { //finished a column
+				//get root-mean
+				//why does valgrind complain here about uninitialised vars
+				double mean=pow(accum/samples,0.5f);
+				audiodata.push_back(mean);
+				sample=0;
+				samples=0;
+				accum=0.0;
+			}
+		}
+		return out_sample;
+	}
+	void Audio_thumbnailer::print_vector(xmlIO XML){
+		string vdata;
+		int i=0;
+		for (auto sample: audiodata){
+			if (i>0) vdata+=",";
+			vdata+=toString(sample);
+			i++;
+		}
+		XML.addValue("data",vdata);
+	}
+	bool Vamp_node::init(int _channels,int _bits,int _samples, int _rate) {
+		//need these to make sense of data
+		channels=_channels;
+		bits=_bits;
+		samples=_samples;
+
+		features.clear();
+		return analyser.init(soname,id,_channels,_bits,_samples,_rate,outputNo,params);
+
+
+		//attempt to load vamp plugin and prepare to receive frames of data
+		//should the audio analysis contain a vamphost or should it inherit?
+		//maybe neater to contain it in terms of headers etc
+
+	}
+	int Vamp_node::process_frame(uint8_t *data,int samples_in_frame) {
+		analyser.process_frame(data,samples_in_frame);
+		return 1;
+	}
+	void Vamp_node::cleanup() {
+		analyser.cleanup();
+		features=analyser.features;
+	}
+	string Vamp_node::get_features(){
+		string data;
+		for (auto i: features) {
+			data=data+" ["+toString(i.second.number)+":"+toString(i.first);
+			if (i.second.values.size()) {
+				data+=" (";
+				bool first=true;
+				for (auto j: i.second.values) {
+					if (first){
+						first=false;
+					}
+					else data+=",";
+					data=data+toString(j);
+				}
+				data+=") ";
+			}
+			data+="]";
+		}
+		return data;
+	}
+	bool sortsegments(std::pair<int,double> i,std::pair<int,double> j){
+		return (i.second<j.second);
+	}
+
+	bool sortseggrps(std::pair<double,vector<pair<double,int> > > i,std::pair<double,vector<pair<double,int> > > j){
+		return (i.first<j.first);
+	}
+	bool sortgroupmembers(pair<double,int> i,pair<double,int> j){
+		return (i.first<j.first);
+	}
+	void Intensity_segmenter::cleanup(){
+		//algorithm idea:
+		//get average tempo and intensity for each segment and store them
+		//scale by the range to get a value from 0.0 to 1.0
+		//add tempo and intensity according to a weighting
+		//score the results (ie 1st place, 2nd place) to end up with a set of integer numbers
+
+		//how to group with similarity?
+		//segments come with similarity groups
+		// 1 - are the wanted git checksegments less than discovered?
+		// N - do nothing
+		// 2 - get intensity and tempo averages
+		// 2 - count the groups
+		// 3 - are the groups less than the discovered segments?
+		// N - group by intensity as normal
+		// 4 - are the groups less than the wanted levels?
+
+
+		//for (auto a:analysers) a.second.cleanup(); //WHY NOT WORK - its as if the call is const
+		analysers["segmenter"].cleanup();
+		analysers["tempo"].cleanup();
+		analysers["intensity"].cleanup();
+
+		//combine with similarity numbers
+		// 1. count similarity numbers
+
+		map<int,vector<int> > similarities;
+
+		//what do we want to know about these similarities?
+		// how many are there? map.size()
+		// how many members are in each one? map[item].size()
+		// which members are they? auto m: map[item]
+
+		uint32_t i=0;
+		for (auto f:analysers["segmenter"].features) {
+			if (f.second.values.size()) {
+				int group=f.second.values[0];
+				if (similarities.find(group)==similarities.end()){
+					similarities[group]={};
+				}
+				similarities[group].push_back(i);
+			}
+			i++;
+		}
+		/*
+		for (auto s:similarities) {
+			string list="";
+			for (int j=0;j<s.second.size();j++){
+				if (j>0) list+=",";
+				list +=toString(s.second[j]);
+			}
+			cerr<<"group "<<s.first<<" ["<<list<<"]"<<endl;
+		}
+		*/
+
+		cerr<<analysers["segmenter"].features.size()<<" segments"<<endl;
+		cerr<<analysers["tempo"].features.size()<<" tempo features"<<endl;
+		cerr<<analysers["intensity"].features.size()<<" intensity features"<<endl;
+		i=0;
+		double min_tempo=9999999.0;
+		double min_intensity=9999999.0;
+		double max_tempo=0.0;
+		double max_intensity=0.0;
+		vector<double> tempos;
+		vector<double> intensities;
+		vector<double> times;
+		auto g=++analysers["segmenter"].features.begin();
+		for (auto f=analysers["segmenter"].features.begin();g!=analysers["segmenter"].features.end();f++,g++,i++){
+			times.push_back(f->first);
+			//integrate tempo and intensity algorithmically
+			double tempo=0;
+			if (analysers["tempo"].features.size()) {
+				double pt=f->first;
+				double pv=analysers["tempo"].get_value(f->first);
+				for (auto u=analysers["tempo"].features.upper_bound(f->first);u!=analysers["tempo"].features.upper_bound(g->first);u++){
+					tempo +=(u->first-pt)*(u->second.values[0]+pv)*0.5f; //area of the slice
+					pt=u->first;
+					pv=u->second.values[0];
+				}
+				tempo +=(g->first-pt)*(analysers["tempo"].get_value(g->first)+pv)*0.5f; //area of the last slice
+				tempo /=g->first-f->first; //average value;
+			}
+			if (tempo>max_tempo) max_tempo=tempo;
+			if (tempo<min_tempo) min_tempo=tempo;
+			tempos.push_back(tempo);
+			
+			double intensity=0;
+			if (analysers["intensity"].features.size()) {
+				double pt=f->first;
+				double pv=analysers["intensity"].get_value(f->first);
+				for (auto u=analysers["intensity"].features.upper_bound(f->first);u!=analysers["intensity"].features.upper_bound(g->first);u++){
+					intensity +=(u->first-pt)*(u->second.values[0]+pv)*0.5f; //area of the slice
+					pt=u->first;
+					pv=u->second.values[0];
+				}
+				intensity +=(g->first-pt)*(analysers["intensity"].get_value(g->first)+pv)*0.5f; //area of the last slice
+				intensity /=g->first-f->first; //average value;
+			}
+			if (intensity>max_intensity) max_intensity=intensity;
+			if (intensity<min_intensity) min_intensity=intensity;
+			intensities.push_back(intensity);
+
+			//cerr<<"segment "<<i<<": "<<f->first<<" to "<<g->first<<" average tempo: "<<tempo<<" ,intensity: "<<intensity<<" ,weighted: "<<(tempo*parameters["tempo_weight"]->value)+(intensity*parameters["intensity_weight"]->value)<<endl;
+		}
+		//
+		//
+		//need to calculate the last segment
+		//
+		//
+		//either by adding a bit of code here or by adding a dummy feature at the track duration, previously
+		//
+		//
+		//
+		//
+
+
+		//make relative scale 0.0-1.0 and save weighted totals
+		vector< pair<int,double>> totals;
+		vector<double> totalsmap;
+		for (i=0;i<tempos.size();i++){
+			tempos[i]=(tempos[i]-min_tempo)/(max_tempo-min_tempo);
+			intensities[i]=(intensities[i]-min_intensity)/(max_intensity-min_intensity);
+			totals.push_back(make_pair(i,(tempos[i]*parameters["tempo_weight"]->value)+(intensities[i]*parameters["intensity_weight"]->value)));
+			totalsmap.push_back((tempos[i]*parameters["tempo_weight"]->value)+(intensities[i]*parameters["intensity_weight"]->value));
+		}
+
+		/*
+		//sort and convert to features
+		std::sort(totals.begin(),totals.end(),sortsegments);
+		for (i=0;i<totals.size();i++) {
+			cerr<<"segment "<<totals[i].first<<" average intensity: "<<totals[i].second<<endl;
+		}
+		vector<double> bucketoffsets;
+		for (auto t:totals) bucketoffsets.push_back(0.0);
+		if (parameters["levels"]->value>0.0&&parameters["levels"]->value<totals.size()){
+			//use bucketoffsets to redistribute into smaller number of buckets
+			int numbertoredistribute=totals.size()-((int)parameters["levels"]->value);
+			double numberperbin=((double)numbertoredistribute/totals.size());
+			double toadd=0.5f;
+			int added=0;
+			for (int j=0;j<totals.size();j++){
+				int numbertoadd=min(numbertoredistribute-added,(int)toadd);
+				toadd=(toadd+numberperbin)-numbertoadd;
+				added+=numbertoadd;
+				bucketoffsets[j]=added;
+			}
+			if (numbertoredistribute>0) {
+				cerr<<"reducing number of levels by "<<numbertoredistribute<<", offsets:"<<endl;
+				for (auto o:bucketoffsets) {
+					cerr<<o<<":";
+				}
+				cerr<<endl;
+			}
+		}
+
+		for (i=0;i<totals.size();i++){
+			vampHost::feature f;
+			f.values.push_back((double)i-bucketoffsets[i]);
+			features[times[totals[i].first]]=f;
+		}
+/*
+		/*
+sort intensity totals
+find out how many segments will share levels apart from similarity levels
+start with a structure:
+map<inputnum,vector<pair<tempo,inputnum>>
+start grouping by similarity
+if there are more similarity groups than wantedgroups, start by grouping similarities
+otherwise take biggest similarity groups and split them by intensity
+if there are still too many groups, merge closest smallest groups
+finally sort by intensity to map output
+
+nned to retrieve total intensity by segment
+		*/
+		//  segment  group_intensity   seg_intense segment
+		vector<pair<double,vector<pair<double,int> > > > seggrps;
+		vector<pair<double,vector<pair<double,int> > > > oldgrps;
+		for (i=0;i<totalsmap.size();i++){
+		    vector<pair<double,int> > data;
+		    data.push_back(make_pair(totalsmap[i],i));
+			oldgrps.push_back(make_pair(totalsmap[i],data));
+		}
+
+		for (auto s:similarities){
+			//similarities is a collection of similarity groups in no particular order, referring to segment nos
+			//at this point seggrps is in segment order
+			if (s.second.size()>1){
+				for (int j=s.second.size()-1;j>0;j--){
+					oldgrps[s.second[0]].second.push_back(make_pair(totalsmap[s.second[j]],s.second[j]));
+					//keep running average// should be by area?
+					//seggrps[s.second[0]].first+=(totalsmap[s.second[j]]*(1.0/max(1,(int)seggrps[s.second[0]].second.size()-1)));
+					//double div=seggrps[s.second[0]].second.size()==1?1.0:((double)seggrps[s.second[0]].second.size()-1/(double)seggrps[s.second[0]].second.size());
+					//neat! this gives 1,1/2,2/3,3/4..
+					//seggrps[s.second[0]].first*=div;
+
+					//seggrps.erase(seggrps.begin()+s.second[j]);
+					//after this has happened, seggrpgs indexing can be invalid
+					
+				}
+
+
+				//easier is to
+				double avg=0.0f;
+				for (auto p:oldgrps[s.second[0]].second) avg+=p.first;
+				avg/=oldgrps[s.second[0]].second.size();
+				oldgrps[s.second[0]].first=avg;
+
+			}
+			seggrps.push_back(oldgrps[s.second[0]]);
+		}
+
+		//cerr<<"similarities assigned, "<<(totalsmap.size()-seggrps.size())<<" segments merged"<<endl;
+		
+		/*
+		i=0;
+		for (auto s:seggrps) {
+			string list="";
+			for (int j=0;j<s.second.size();j++){
+				if (j>0) list+=",";
+				list +=toString(s.second[j].second);
+			}
+			cerr<<"segment "<<i<<" ["<<list<<"]"<<endl;
+			i++;
+		}
+		*/
+		//sort the contents by intensity
+		std::sort(seggrps.begin(),seggrps.end(),sortseggrps);
+		//cerr<<"groups sorted by intensity:"<<endl;
+		//possible mergers will be with groups with adjacent intensity
+		i=0;
+		/*
+		for (auto s:seggrps) {
+			string list="";
+			for (int j=0;j<s.second.size();j++){
+				if (j>0) list+=",";
+				list +=toString(s.second[j].second);
+			}
+			cerr<<"segment "<<i<<" ["<<list<<"]"<<endl;
+			i++;
+		}
+		*/
+
+		if (((int)parameters["levels"]->value)>0) {
+			if (seggrps.size()>(int)parameters["levels"]->value){
+				while (seggrps.size()>(int)parameters["levels"]->value){
+					//reduce similarity groups
+					//decide the best 2 to merge
+					vector<double> diffs;
+					for (int j=0;j<seggrps.size()-1;j++) diffs.push_back(seggrps[j+1].first-seggrps[j].first);
+					int smallest=0;
+					for (int j=1;j<diffs.size();j++) if (diffs[j]<diffs[smallest]) smallest=j;
+					for (int j=0;j<seggrps[smallest].second.size();j++) {
+						seggrps[smallest+1].second.push_back(seggrps[smallest].second[j]);
+						//cerr<<"copied segment "<<(seggrps[smallest].second[j].second)<<" from group "<<smallest<<" to group "<<(smallest+1)<<endl;
+					}
+					//recalculate intensity average
+					double avg=0.0f;
+					for (auto p:seggrps[smallest+1].second) avg+=p.first;
+					avg/=seggrps[smallest+1].second.size();
+					seggrps[smallest+1].first=avg;
+
+					seggrps.erase(seggrps.begin()+smallest);
+					//cerr<<"removed group "<<smallest<<endl;
+				}
+				cerr<<"intensities merged, "<<seggrps.size()<<" levels remain"<<endl;
+			}
+			//cerr<<seggrps.size()<<" groups, "<<(int)parameters["levels"]->value<<" levels requested, "<<(int)totalsmap.size()<<" original segments"<<endl;
+			if  (seggrps.size()<min((int)parameters["levels"]->value,(int)totalsmap.size())){
+				while (seggrps.size()<min((int)parameters["levels"]->value,(int)totalsmap.size())) {
+					//split groups
+					//calculate standard deviation of intensity variation
+					vector<double> devs;
+					for (int j=0;j<seggrps.size()-1;j++) {
+						double avg=0.0;
+						double dev=0.0;
+						for (int k=0;k<seggrps[j].second.size();k++){
+							avg+=seggrps[j].second[k].first;
+						}
+						avg/=seggrps[j].second.size();
+						for (int k=0;k<seggrps[j].second.size();k++){
+							dev+=pow(avg-k<seggrps[j].second[k].first,2.0);
+						}
+						dev/=seggrps[j].second.size();
+						devs.push_back(pow(dev,0.5));
+					}
+					//find group with largest standard deviation
+					int largest=0;
+					for (int j=1;j<devs.size();j++) if (devs[j]>devs[largest]) largest=j;
+					//sanity check: if there are any groups that can be split they will have larger SD than singleton groups
+					//sort members of the group
+					std::sort(seggrps[largest].second.begin(),seggrps[largest].second.end(),sortgroupmembers);
+					//create a new group
+					std::pair<double,vector<pair<double,int> > > newgroup;
+					//cerr<<"splitting group "<<largest<<" with "<<seggrps[largest].second.size()<<" segments: new group will have "<<seggrps[largest].second.size()-(seggrps[largest].second.size()/2)<<" segments"<<endl;
+					for (int j=seggrps[largest].second.size()-1;j>(seggrps[largest].second.size()/2)-1;j--) {
+						newgroup.second.push_back(seggrps[largest].second[j]);
+						seggrps[largest].second.erase(seggrps[largest].second.begin()+j);
+					}
+
+					//refresh averages for the 2 groups
+					double avg=0.0f;
+					for (auto p:seggrps[largest].second) avg+=p.first;
+					avg/=seggrps[largest].second.size();
+					seggrps[largest].first=avg;
+
+					avg=0.0f;
+					for (auto p:newgroup.second) avg+=p.first;
+					avg/=newgroup.second.size();
+					newgroup.first=avg;
+
+					//add the new group
+					seggrps.push_back(newgroup);
+					//cerr<<" added new group with "<<newgroup.second.size()<<" segments"<<endl;
+				}
+				cerr<<"similaritity groups split, "<<seggrps.size()<<" levels total"<<endl;
+				//seggrps are now out of order
+				std::sort(seggrps.begin(),seggrps.end(),sortseggrps);
+
+			}
+			
+		}
+
+		map<int,int> outputvalues;
+		for (int j=0;j<seggrps.size();j++){
+			string list="";
+			for (int k=0;k<seggrps[j].second.size();k++){
+				outputvalues[seggrps[j].second[k].second]=j;
+				if (k>0) list+=",";
+				list +=toString(seggrps[j].second[k].second);
+			}
+			//cerr<<"output value: "<<j<<" assigned to ["<<list<<"]"<<endl;
+		}
+
+
+		for (i=0;i<totals.size();i++){
+			vampHost::feature f;
+			f.values.push_back(outputvalues[i]);
+			features[times[totals[i].first]]=f;
+		}
+
+	}
+}
+
+/*
+A data structure to represent segments and their mapping to output levels
+how do we merge the intensity groups with the similarities?
+
+we create a list
+
+
+
+
+... we iterate through the list of segments and place the right output number
+
+
+*/
-- 
cgit v1.2.3