path: root/rotord/rotor.cpp

#include "rotor.h"


//float equality
bool fequal(const float u,const float v){
	if (abs(u-v)<.001) return true;
	else return false;
};


using namespace Rotor;

void Render_context::runTask() {
	while (!isCancelled()) {
		int cmd=0;
		mutex.lock();
		if (work_queue.size()){
			cmd=work_queue[0];
			work_queue.pop_front();
		}
		mutex.unlock();
		if (cmd==ANALYSE_AUDIO) {
			state=ANALYSING_AUDIO;
			vector<Base_audio_processor*> processors;
			processors.push_back(audio_thumb);
			vector<Node*> analysers=graph.find_nodes("audio_analysis");
			for (auto a: analysers) {
				processors.push_back(a);
			}
			if (load_audio(audio_filename,processors)) {
				state=AUDIO_READY;
			}
			else {
				//an error occurred: TODO have to clean up allocated data. autoptr?
				state=IDLE;
			}
		}
		sleep(100);
	}
	printf("Rotor: stopping thread\n");
}
void Render_context::add_queue(int item) {
	mutex.lock();
	work_queue.push_back(item);
	mutex.unlock();
}

bool Signal_input::connect(Signal_node* source) {
	if (source->output_type=="signal") {
		connection=(Node*)source;
		return true;
	}
	else return false;
};

bool Signal_output::render(const float duration, const float framerate,string &xml_out){
	cerr << "Rotor: Signal_output rendering " << duration << " seconds at " << framerate << " frames per second" << endl;
	float step=1.0f/framerate;
	float v=0.0f;
	for (float f=0.0f;f<duration;f+=step) {
		float u=get_output(f);
		if (!fequal(u,v)) {
			xml_out+=("<signal time='"+ofToString(f)+"'>"+ofToString(u)+"</signal>\n");
			v=u;
		}
	}
	return true;
}

Command_response Render_context::session_command(const std::vector<std::string>& command){
											//method,id,command1,{command2,}{body}
											//here we allow the controlling server to communicate with running tasks
	Command_response response;
	response.status=HTTPResponse::HTTP_BAD_REQUEST;
	if (command[2]=="audio") {
	    if (command[0]=="PUT") {                                    	//get audio file location and initiate analysis
		if (command.size()>2) {
		    if (state==IDLE) {
												//check file exists
			Poco::File f=Poco::File(command[3]);
			if (f.exists()) {
												//pass to worker thread ??if engine is ready?? ??what if engine has finished but results aren't read??
			    audio_filename=command[3]; 			//for now, store session variables in memory
			    add_queue(ANALYSE_AUDIO);
			    response.description="<status context='"+command[1]+"'>Starting audio analysis: "+command[3]+"</status>\n";
			}
			else {
			    response.status=HTTPResponse::HTTP_NOT_FOUND;
			    response.description="<status context='"+command[1]+"'>File "+command[3]+" not found</status>\n";
			}

		    }
		    else {
			response.status=HTTPResponse::HTTP_BAD_REQUEST;
			response.description="<status context='"+command[1]+"'>Rotor: session busy</status>\n";
		    }
		}
	    }
	     if (command[0]=="GET") {
		     if (state==ANALYSING_AUDIO) {
			     response.status=HTTPResponse::HTTP_OK;
			     response.description="<status context='"+command[1]+"'>Rotor: analysing audio</status>\n";
			     char c[20];
			     sprintf(c,"%02f",progress);
			     response.description+="<progress>"+string(c)+"</progress>\n";
		     }
			if (state==AUDIO_READY) {
				//not sure about this-- should this state be retained?
				//can the data  only be read once?
				//for now
				response.status=HTTPResponse::HTTP_OK;
				response.description="<status context='"+command[1]+"'>Rotor: audio ready</status>\n";
				response.description+="<audio>\n";
				response.description+=audio_thumb->print();
				response.description+="</audio>";
				state=IDLE;
			}
	     }
	     if (command[0]=="DELETE") {
		     //for now
			audio_filename="";
			response.description="<status>1</status>\n";
			response.status=HTTPResponse::HTTP_OK;
	     }
	}
	if (command[2]=="graph") {
		if (command[0]=="GET") {
			if (graph.loaded) {
				response.status=HTTPResponse::HTTP_OK;
				response.description=graph.toString();
			}
			else {
				response.description="<status>Rotor: graph not loaded</status>\n";
			}
		}
		if (command[0]=="PUT") {                                    	//get new graph from file
			if (command.size()>2) {
												//should interrupt whatever is happening?
												//before begining to load from xml
				if (state==IDLE) {					//eventually not like this
					Poco::File f=Poco::File(command[3]);
					if (f.exists()) {
						string graph_filename=command[3];
						if (graph.load(graph_filename)) {
							response.status=HTTPResponse::HTTP_OK;
							//response.description="<status context='"+command[1]+"'>Rotor: loaded graph "+command[3]+"</status>\n";
							response.description=graph.toString();
							//the graph could actually contain an xml object and we could just print it here?
							//or could our nodes even be subclassed from xml nodes?
							//the graph or the audio could load first- have to analyse the audio with vamp after the graph is loaded
							//for now the graph must load 1st
						}
						else {
							response.status=HTTPResponse::HTTP_INTERNAL_SERVER_ERROR; //~/sources/poco-1.4.6-all/Net/include/Poco/Net/HTTPResponse.h
							response.description="<status context='"+command[1]+"'>Rotor: could not load graph "+command[3]+"</status>\n";
						}
					}
					else {
					    response.status=HTTPResponse::HTTP_NOT_FOUND;
					    response.description="<status context='"+command[1]+"'>File "+command[3]+" not found</status>\n";
					}
				}
			}
		}
		if (command[0]=="DELETE") {
			 //for now
			graph=Graph();
			response.description="<status>1</status>\n";
			response.status=HTTPResponse::HTTP_OK;
		}
	}
	if (command[2]=="signal") {
		if (command[0]=="GET") {                                    	//generate xml from 1st signal output
			if (state==IDLE) {
				//direct call for testing
					float framerate=0.0f;
					if (command.size()>2) {
						framerate=ofToFloat(command[3]);
					}
					string signal_xml;
					if (graph.signal_render(framerate,signal_xml)){
						response.status=HTTPResponse::HTTP_OK;
						response.description=signal_xml;
					}
					else {
						response.status=HTTPResponse::HTTP_INTERNAL_SERVER_ERROR;
						response.description="<status context='"+command[1]+"'>Rotor: could not render output signal</status>\n";
					}
				}
				else {
					response.status=HTTPResponse::HTTP_NOT_FOUND;
					response.description="<status context='"+command[1]+"'>Signal output not found</status>\n";
				}
			}
			else {
				response.status=HTTPResponse::HTTP_SERVICE_UNAVAILABLE;
				response.description="<status context='"+command[1]+"'>Rotor: context busy</status>\n";
			}
		}
	if (command[2]=="video") {
		if (command[0]=="GET") {
													//DUMMY RESPONSE
			response.status=HTTPResponse::HTTP_OK;
			response.description="<status context='"+command[1]+"'>DUMMY RESPONSE Rotor: analysing video</status>\n";
			response.description+="<progress>45.2</progress>\n";
		}
		if (command[0]=="PUT") {                                    		//get vide file location and initiate analysis
			if (command.size()>2) {
			    if (state==IDLE) {
													//check file exists
				Poco::File f=Poco::File(command[3]);
				if (f.exists()) {
													//pass to worker thread ??if engine is ready?? ??what if engine has finished but results aren't read??
													//DUMMY RESPONSE
					response.description="<status context='"+command[1]+"'>DUMMY RESPONSE Starting video analysis: "+command[3]+"</status>\n";
				}
				else {
				    response.status=HTTPResponse::HTTP_NOT_FOUND;
				    response.description="<status context='"+command[1]+"'>File "+command[3]+" not found</status>\n";
				}
			    }
			    else {
				response.status=HTTPResponse::HTTP_BAD_REQUEST;
				response.description="<status context='"+command[1]+"'>Rotor: session busy</status>\n";
			    }
			}
		}
		if (command[0]=="DELETE") {
		     //DUMMY RESPONSE
			response.description="<status>DUMMY RESPONSE 1</status>\n";
			response.status=HTTPResponse::HTTP_OK;
	     }

	}
	if (command[2]=="render") {
		if (command[0]=="GET") {
													//DUMMY RESPONSE
			response.status=HTTPResponse::HTTP_OK;
			response.description="<status context='"+command[1]+"'>DUMMY RESPONSE Rotor: rendering video</status>\n";
			response.description+="<progress>25.2</progress>\n";
		}
		if (command[0]=="PUT") {
													//DUMMY RESPONSE
													//SHOULD CHECK REQUIREMENTS
			response.status=HTTPResponse::HTTP_OK;
			response.description="<status context='"+command[1]+"'>DUMMY RESPONSE Rotor: starting render</status>\n";
		}
		if (command[0]=="DELETE") {
													//DUMMY RESPONSE
													//SHOULD CHECK REQUIREMENTS
			response.status=HTTPResponse::HTTP_OK;
			response.description="<status context='"+command[1]+"'>DUMMY RESPONSE Rotor: cancelling render</status>\n";
		}
	}
	return response;
}

//http://blog.tomaka17.com/2012/03/libavcodeclibavformat-tutorial/
//great to use c++11 features

bool Render_context::load_audio(const string &filename,vector<Base_audio_processor*> processors){

	av_register_all();

	AVFrame* frame = avcodec_alloc_frame();
	if (!frame)
	{
		std::cout << "Error allocating the frame" << std::endl;
		return false;
	}

	AVFormatContext* formatContext = NULL;
	if (avformat_open_input(&formatContext, filename.c_str(), NULL, NULL) != 0)
	{
		av_free(frame);
		std::cout << "Error opening the file" << std::endl;
		return false;
	}


	if (avformat_find_stream_info(formatContext, NULL) < 0)
	{
		av_free(frame);
		av_close_input_file(formatContext);
		std::cout << "Error finding the stream info" << std::endl;
		return false;
	}

	AVStream* audioStream = NULL;
	for (unsigned int i = 0; i < formatContext->nb_streams; ++i)
	{
		if (formatContext->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
		{
			audioStream = formatContext->streams[i];
			break;
		}
	}

	if (audioStream == NULL)
	{
		av_free(frame);
		av_close_input_file(formatContext);
		std::cout << "Could not find any audio stream in the file" << std::endl;
		return false;
	}

	AVCodecContext* codecContext = audioStream->codec;

	codecContext->codec = avcodec_find_decoder(codecContext->codec_id);
	if (codecContext->codec == NULL)
	{
		av_free(frame);
		av_close_input_file(formatContext);
		std::cout << "Couldn't find a proper decoder" << std::endl;
		return false;
	}
	else if (avcodec_open2(codecContext, codecContext->codec, NULL) != 0)
	{
		av_free(frame);
		av_close_input_file(formatContext);
		std::cout << "Couldn't open the context with the decoder" << std::endl;
		return false;
	}

	av_dump_format(formatContext, 0, 0, false); //avformat.h line 1256
	int samples = ((formatContext->duration + 5000)*codecContext->sample_rate)/AV_TIME_BASE;

	std::cout << "This stream has " << codecContext->channels << " channels, a sample rate of " << codecContext->sample_rate << "Hz and "<<samples <<" samples" << std::endl;
	std::cout << "The data is in format " <<codecContext->sample_fmt<< " (aka "<< av_get_sample_fmt_name(codecContext->sample_fmt) << ") "<<std::endl;

	for (auto p: processors) {
		if(!p->init(codecContext->channels,16,samples,codecContext->sample_rate) ){
			cerr<<"Plugin failed to initialse"<<endl;
			return false;
		}
	}

	AVPacket packet;
	av_init_packet(&packet);
	int sample_processed=0;

	while (true)
	{
		int ret=av_read_frame(formatContext, &packet);
		if (ret<0) {
			cerr << "finished with code "<<ret <<(ret==AVERROR_EOF?" ,EOF":"")<<endl;
			break;
		}
		if (packet.stream_index == audioStream->index)
		{
			// Try to decode the packet into a frame
			int frameFinished = 0;
			int bytes = avcodec_decode_audio4(codecContext, frame, &frameFinished, &packet);

			// Some frames rely on multiple packets, so we have to make sure the frame is finished before
			// we can use it
			if (frameFinished)
			{
				// frame now has usable audio data in it. How it's stored in the frame depends on the format of
				// the audio. If it's packed audio, all the data will be in frame->data[0]. If it's in planar format,
				// the data will be in frame->data and possibly frame->extended_data. Look at frame->data, frame->nb_samples,
				// frame->linesize, and other related fields on the FFmpeg docs. I don't know how you're actually using
				// the audio data, so I won't add any junk here that might confuse you. Typically, if I want to find
				// documentation on an FFmpeg structure or function, I just type "<name> doxygen" into google (like
				// "AVFrame doxygen" for AVFrame's docs)

				//av_get_channel_layout_string (char *buf, int buf_size, int nb_channels, uint64_t channel_layout)


				//now we can pass the data to the processor(s)
				for (auto p: processors) {
					sample_processed=p->process_frame(frame->data[0],frame->nb_samples);
				}

				mutex.lock();
				progress=((double)sample_processed)/samples;
				mutex.unlock();
			}
		}
		// You *must* call av_free_packet() after each call to av_read_frame() or else you'll leak memory
		av_free_packet(&packet);
	}

	// Some codecs will cause frames to be buffered up in the decoding process. If the CODEC_CAP_DELAY flag
	// is set, there can be buffered up frames that need to be flushed, so we'll do that
	if (codecContext->codec->capabilities & CODEC_CAP_DELAY)
	{
		av_init_packet(&packet);
		// Decode all the remaining frames in the buffer, until the end is reached
		int frameFinished = 0;
		int bytes = avcodec_decode_audio4(codecContext, frame, &frameFinished, &packet);
		while (bytes >= 0 && frameFinished)
		{
			for (auto p: processors) {
				p->process_frame(frame->data[0],frame->nb_samples);
			}
			mutex.lock();
			progress=((double)sample_processed)/samples;
			mutex.unlock();
		}
	}

	cerr << "finished processing: "<<sample_processed << " samples of  "<<samples<<", "<<((double)sample_processed*100)/samples<<"%"<<  std::endl;

	// Clean up!
	for (auto p: processors) {
		p->cleanup();
	}

	av_free(frame);
	avcodec_close(codecContext);
	av_close_input_file(formatContext);

	return true;
}
const string Graph::toString(){
	string xmlgraph;
	if (loaded) {
		xml.copyXmlToString(xmlgraph);
		return xmlgraph;
	}
	else return "";
}
bool Graph::load(string &filename){
	loaded=false;
	printf("loading graph: %s\n",filename.c_str());
	if(xml.loadFile(filename) ){
		init(xml.getAttribute("patchbay","ID","",0),xml.getValue("patchbay","",0));
		if(xml.pushTag("patchbay")) {
			int n1=xml.getNumTags("node");
			for (int i1=0;i1<n1;i1++){
				map<string,string> settings;
				vector<string> attrs;
				xml.getAttributeNames("node",attrs,i1);
				for (auto& attr: attrs) {
					settings[attr]=xml.getAttribute("node",attr,"",i1);
					//cerr << "Got attribute: " << attr << ":" << xml.getAttribute("node",attr,"",i1) << endl;
				}
				settings["description"]=xml.getValue("node","",i1);
				Node* node=factory.create(settings);
				if (node) {
					cerr << "Rotor: created '" << xml.getAttribute("node","type","",i1) << "'" << endl;
					string nodeID=xml.getAttribute("node","ID","",i1);
					nodes[nodeID]=node;
					if(xml.pushTag("node",i1)) {
						int n2=xml.getNumTags("signal_input");
						for (int i2=0;i2<n2;i2++){
							nodes[nodeID]->create_signal_input(xml.getValue("signal_input","",i2));
							string fromID=xml.getAttribute("signal_input","from","",i2);
							if (nodes.find(fromID)!=nodes.end()) {
								if (!nodes[nodeID]->inputs[i2]->connect((Signal_node*)nodes[fromID])){
									cerr << "Rotor: graph loader cannot connect input " << i2 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl;
									return false;
								}
								else cerr << "Rotor: linked input " << i2 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl;
							}
							else cerr << "Rotor: linking input " << i2 << " of node: '" << nodeID << "', cannot find target '" << fromID << "'" << endl;
						}
						xml.popTag();
					}
				}
				else {
					cerr << "Rotor: graph loader cannot find node '" << xml.getAttribute("node","type","",i1) << "'" << endl;
					return false;
				}
			}
			xml.popTag();
		}
		loaded=true;
		return true;
	}
	else return false;
}
Node_factory::Node_factory(){
	//for now, statically load prototype map in constructor
	add_type("audio_analysis",new Audio_analysis());
	add_type("divide",new Signal_divide());
	add_type("==",new Is_new_integer());
	add_type("signal_output",new Signal_output());
}
bool Audio_thumbnailer::init(int _channels,int _bits,int _samples,int _rate) {
	//base_audio_processor::init(_channels,_bits,_samples);
	channels=_channels;
	bits=_bits;
	samples=_samples;
	samples_per_column=samples/width;
	column=0; //point thumbnail bitmap
	out_sample=0; //sample in whole track
	offset=0x1<<(bits-1); //signed audio
	scale=1.0/offset;
	sample=0;
	samples=0;
	accum=0.0;
	return true;
}
int Audio_thumbnailer::process_frame(uint8_t *_data,int samples_in_frame){
	//begin by processing remaining samples
	//samples per column could be larger than a frame! (probably is)
	//but all we are doing is averaging
	int bytes=(bits>>3);
	int stride=channels*bytes;
	int in_sample=0;
	while (in_sample<samples_in_frame&&column<width) {
		//continue the column
		while (sample<samples_per_column&&in_sample<samples_in_frame) {
			//accumulate samples for this column until we run out of samples
			for (int i=0;i<channels;i++) {
				unsigned int this_val=0;
				for (int j=0;j<bytes;j++) {
					this_val+=_data[(in_sample*stride)+(i*bytes)+j]<<(j*8);
				}
				//convert from integer data format - i.e s16p - to audio signal in -1..1 range
				//presume 16 bits for now...
				double val=((double)((int16_t)this_val))*scale;
				accum+=val*val;
				samples++;
			}
			in_sample++;
			sample++;
			out_sample++;
		}
		if (sample==samples_per_column) { //finished a column
			//get root-mean
			double mean=pow(accum/samples,0.5);
			if (column==0) {
				cerr << "first column total: "<< accum << " in " << samples << " samples, average " << (accum/samples)<<endl;
			}
			int colheight=height*mean*0.5;
			int hh=height>>1;
			for (int i=0;i<height;i++) {
				data[i*width+column]=abs(i-hh)<colheight?0xff:0x00;
			}
			column++;
			sample=0;
			samples=0;
			accum=0.0;
		}
	}
	return out_sample;
}
string Audio_thumbnailer::print(){
	string output;
	for (int j=0;j<height;j++) {
		for (int i=0;i<width;i++) {
			output+=data[j*width+i]<0x7f?"0":"1";
		}
		output +="\n";
	}
	return output;
}
bool Audio_analysis::init(int _channels,int _bits,int _samples, int _rate) {
	//need these to make sense of data
	channels=_channels;
	bits=_bits;
	samples=_samples;

	return analyser.init(soname,id,_channels,_bits,_samples,_rate);

	//attempt to load vamp plugin and prepare to receive frames of data
	//should the audio analysis contain a vamphost or should it inherit?
	//maybe neater to contain it in terms of headers etc

}
int Audio_analysis::process_frame(uint8_t *data,int samples_in_frame) {
	analyser.process_frame(data,samples_in_frame);
	return 1;
}
void Audio_analysis::cleanup() {
	analyser.cleanup();
}