path: root/rotord/rotor.cpp

#include "rotor.h"


//float equality
bool fequal(const float u,const float v){
	if (abs(u-v)<.001) return true;
	else return false;
};


using namespace Rotor;

/*
    string soname="qm-vamp-plugins";
    string id="qm-tempotracker";
	string myname="";
	string output="";
	int outputNo=0;
	*/

void Render_context::runTask() {
	while (!isCancelled()) {
		int cmd=0;
		mutex.lock();
		if (work_queue.size()){
			cmd=work_queue[0];
			work_queue.pop_front();
		}
		mutex.unlock();
		if (cmd==ANALYSE_AUDIO) {
			state=ANALYSING_AUDIO;
			//audio_analyser.process(audio_filename);
			//vampHost::runPlugin("","qm-vamp-plugins","qm-tempotracker", "",0, audio_filename, cerr,true);
			vector<base_audio_processor*> proc;
			proc.push_back(audio_thumb);
			if (load_audio(audio_filename,proc)) {
				state=AUDIO_READY;
				//set the response
			}
			else state=IDLE;
		}
		sleep(100);
	}
	printf("Rotor: stopping thread\n");
}
void Render_context::add_queue(int item) {
	mutex.lock();
	work_queue.push_back(item);
	mutex.unlock();
}

bool Signal_input::connect(Signal_node* source) {
	if (source->output_type=="signal") {
		connection=(Node*)source;
		return true;
	}
	else return false;
};

bool Signal_output::render(const float duration, const float framerate,string &xml_out){
	cerr << "Rotor: Signal_output rendering " << duration << " seconds at " << framerate << " frames per second" << endl;
	float step=1.0f/framerate;
	float v=0.0f;
	for (float f=0.0f;f<duration;f+=step) {
		float u=get_output(f);
		if (!fequal(u,v)) {
			xml_out+=("<signal time='"+ofToString(f)+"'>"+ofToString(u)+"</signal>\n");
			v=u;
		}
	}
	return true;
}

Command_response Render_context::session_command(const std::vector<std::string>& command){
										//method,id,command1,{command2,}{body}
										//here we allow the controlling server to communicate with running tasks
	Command_response response;
	response.status=HTTPResponse::HTTP_BAD_REQUEST;
	if (command[2]=="audio") {
	    if (command[0]=="PUT") {                                    	//get audio file location and initiate analysis
		if (command.size()>2) {
		    if (state==IDLE) {
												//check file exists
			Poco::File f=Poco::File(command[3]);
			//std::auto_ptr<std::istream> pStr(URIStreamOpener::defaultOpener().open(command[3]));


			if (f.exists()) {
												//pass to worker thread ??if engine is ready?? ??what if engine has finished but results aren't read??
			    audio_filename=command[3]; 			//for now, store session variables in memory
			    add_queue(ANALYSE_AUDIO);
			    response.description="<status context='"+command[1]+"'>Starting audio analysis: "+command[3]+"</status>\n";
			}
			else {
			    response.status=HTTPResponse::HTTP_NOT_FOUND;
			    response.description="<status context='"+command[1]+"'>File "+command[3]+" not found</status>\n";
			}

		    }
		    else {
			response.status=HTTPResponse::HTTP_BAD_REQUEST;
			response.description="<status context='"+command[1]+"'>Rotor: session busy</status>\n";
		    }
		}
	    }
	     if (command[0]=="GET") {
		     if (state==ANALYSING_AUDIO) {
			     response.status=HTTPResponse::HTTP_OK;
			     response.description="<status context='"+command[1]+"'>Rotor: analysing audio</status>\n";
			     char c[20];
			     sprintf(c,"%02f",progress);
			     response.description+="<progress>"+string(c)+"</progress>\n";
		     }
			if (state==AUDIO_READY) {
				//not sure about this-- should this state be retained?
				//can the data  only be read once?
				//for now
				response.status=HTTPResponse::HTTP_OK;
				response.description="<status context='"+command[1]+"'>Rotor: audio ready</status>\n";
				response.description+="<audio>\n";
				response.description+=audio_thumb->print();
				response.description+="</audio>";
				state=IDLE;
			}
	     }
	     if (command[0]=="DELETE") {
		     //for now
			audio_filename="";
			response.description="<status>1</status>\n";
			response.status=HTTPResponse::HTTP_OK;
	     }
	}
	if (command[2]=="graph") {
		if (command[0]=="GET") {
			if (xml.bDocLoaded) {
				response.status=HTTPResponse::HTTP_OK;
				xml.copyXmlToString(response.description);
			}
			else {
				response.description="<status>Rotor: graph not loaded</status>\n";
			}
		}
		if (command[0]=="PUT") {                                    	//get new graph from file
			if (command.size()>2) {
												//should interrupt whatever is happening?
												//before begining to load from xml
				if (state==IDLE) {					//eventually not like this
					Poco::File f=Poco::File(command[3]);
					if (f.exists()) {
						string graph_filename=command[3];
						if (load_graph(graph_filename)) {
							response.status=HTTPResponse::HTTP_OK;
							//response.description="<status context='"+command[1]+"'>Rotor: loaded graph "+command[3]+"</status>\n";
							string xmlstring;
							xml.copyXmlToString(xmlstring);
							response.description=xmlstring;
							//the graph could actually contain an xml object and we could just print it here?
							//or could our nodes even be subclassed from xml nodes?
						}
						else {
							response.status=HTTPResponse::HTTP_INTERNAL_SERVER_ERROR; //~/sources/poco-1.4.6-all/Net/include/Poco/Net/HTTPResponse.h
							response.description="<status context='"+command[1]+"'>Rotor: could not load graph "+command[3]+"</status>\n";
						}
					}
					else {
					    response.status=HTTPResponse::HTTP_NOT_FOUND;
					    response.description="<status context='"+command[1]+"'>File "+command[3]+" not found</status>\n";
					}
				}
			}
		}
		if (command[0]=="DELETE") {
			 //for now
			graph=Graph();
			response.description="<status>1</status>\n";
			response.status=HTTPResponse::HTTP_OK;
		}
	}
	if (command[2]=="signal") {
		if (command[0]=="GET") {                                    	//generate xml from 1st signal output
			if (state==IDLE) {
				//direct call for testing
					float framerate=0.0f;
					if (command.size()>2) {
						framerate=ofToFloat(command[3]);
					}
					string signal_xml;
					if (graph.signal_render(framerate,signal_xml)){
						response.status=HTTPResponse::HTTP_OK;
						response.description=signal_xml;
					}
					else {
						response.status=HTTPResponse::HTTP_INTERNAL_SERVER_ERROR;
						response.description="<status context='"+command[1]+"'>Rotor: could not render output signal</status>\n";
					}
				}
				else {
					response.status=HTTPResponse::HTTP_NOT_FOUND;
					response.description="<status context='"+command[1]+"'>Signal output not found</status>\n";
				}
			}
			else {
				response.status=HTTPResponse::HTTP_SERVICE_UNAVAILABLE;
				response.description="<status context='"+command[1]+"'>Rotor: context busy</status>\n";
			}
		}
	if (command[2]=="video") {
		if (command[0]=="GET") {
													//DUMMY RESPONSE
			response.status=HTTPResponse::HTTP_OK;
			response.description="<status context='"+command[1]+"'>DUMMY RESPONSE Rotor: analysing video</status>\n";
			response.description+="<progress>45.2</progress>\n";
		}
		if (command[0]=="PUT") {                                    		//get vide file location and initiate analysis
			if (command.size()>2) {
			    if (state==IDLE) {
													//check file exists
				Poco::File f=Poco::File(command[3]);
				if (f.exists()) {
													//pass to worker thread ??if engine is ready?? ??what if engine has finished but results aren't read??
													//DUMMY RESPONSE
					response.description="<status context='"+command[1]+"'>DUMMY RESPONSE Starting video analysis: "+command[3]+"</status>\n";
				}
				else {
				    response.status=HTTPResponse::HTTP_NOT_FOUND;
				    response.description="<status context='"+command[1]+"'>File "+command[3]+" not found</status>\n";
				}
			    }
			    else {
				response.status=HTTPResponse::HTTP_BAD_REQUEST;
				response.description="<status context='"+command[1]+"'>Rotor: session busy</status>\n";
			    }
			}
		}
		if (command[0]=="DELETE") {
		     //DUMMY RESPONSE
			response.description="<status>DUMMY RESPONSE 1</status>\n";
			response.status=HTTPResponse::HTTP_OK;
	     }

	}
	if (command[2]=="render") {
		if (command[0]=="GET") {
													//DUMMY RESPONSE
			response.status=HTTPResponse::HTTP_OK;
			response.description="<status context='"+command[1]+"'>DUMMY RESPONSE Rotor: rendering video</status>\n";
			response.description+="<progress>25.2</progress>\n";
		}
		if (command[0]=="PUT") {
													//DUMMY RESPONSE
													//SHOULD CHECK REQUIREMENTS
			response.status=HTTPResponse::HTTP_OK;
			response.description="<status context='"+command[1]+"'>DUMMY RESPONSE Rotor: starting render</status>\n";
		}
		if (command[0]=="DELETE") {
													//DUMMY RESPONSE
													//SHOULD CHECK REQUIREMENTS
			response.status=HTTPResponse::HTTP_OK;
			response.description="<status context='"+command[1]+"'>DUMMY RESPONSE Rotor: cancelling render</status>\n";
		}
	}
	return response;
}




//http://blog.tomaka17.com/2012/03/libavcodeclibavformat-tutorial/
//great to use c++11 features

bool Render_context::load_audio(const string &filename,vector<base_audio_processor*> processors){
	//load audio data from file
	//what's the best way to use this? the model is background processing, and we want to update a progress bar
	//could pass a function pointer to call when each chunk of data becomes available?
	//should the data processing be the responsibility of an object or a function?
	//in the case of the audio thumbnail, there will be just one place where its kept
	//in the case of audio analysis, the daemon will pass each audio analysis object each chunk of data as it gets it
	//there could even be an array of audio analysis functions to perform simultaneously?
	//how about a vector of  objects that subclass the base audio processor class?

	//1st get parameters and initialise the processors
	//then begin data loop locking progress variable after each frame

	//
	//
	//the example in ffmpeg works, but it isn't one that identifies a codec- it is hard coded to look for a codec for AV_CODEC_ID_MP2
	//it also doesn't load through libavformat - which opens containers- it just loads a naked .mp2 stream
	//
	//

	/*
	av_register_all();

	std::shared_ptr<AVFormatContext> avFormat(avformat_alloc_context(), &avformat_free_context);

	auto avFormatPtr = avFormat.get();
	if (avformat_open_input(&avFormatPtr,filename.c_str(),nullptr, nullptr) != 0) {
		cerr <<"Rotor: Error while calling avformat_open_input (probably invalid file format)" << endl;
		return false;
	}

	if (avformat_find_stream_info(avFormat.get(), nullptr) < 0) {
		cerr << "Rotor: Error while calling avformat_find_stream_info" << endl;
		return false;
	}

	av_dump_format(avFormat.get(), 0, 0, false); //avformat.h line 1256



	AVStream* stream = nullptr;
	for (unsigned int i = 0; i < avFormat->nb_streams; ++i) {
		if (avFormat->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO) {
			// we've found a stream!
			stream = avFormat->streams[i];
			break;
		}
	}
	if (!stream) {
		cerr <<"Rotor: Didn't find any audio stream in the file"<< endl;
		return false;
	}

	// getting the required codec structure
	const auto codec = avcodec_find_decoder(stream->codec->codec_id); //returns AVCodec*
	if (codec == nullptr) {
		cerr <<"Rotor: Audio codec not available"<< endl;
		return false;
	}

	//AVCodecContext??  avFormat->streams[i]->codec

	// allocating a structure
	std::shared_ptr<AVCodecContext> audioCodec(avcodec_alloc_context3(codec), [](AVCodecContext* c) { avcodec_close(c); av_free(c); });

	// extradata???
	// we need to make a copy of videoStream->codec->extradata and give it to the context
	// make sure that this vector exists as long as the avVideoCodec exists
	std::vector<uint8_t> codecContextExtraData;
	codecContextExtraData.assign(stream->codec->extradata, stream->codec->extradata + stream->codec->extradata_size);

	audioCodec->extradata = reinterpret_cast<uint8_t*>(codecContextExtraData.data());
	audioCodec->extradata_size = codecContextExtraData.size();

	// initializing the structure by opening the codec
	if (avcodec_open2(audioCodec.get(), codec, nullptr) < 0) {
		cerr <<"Rotor: Could not open codec"<< endl;
		return false;
	}


	//avcodec.h line 1026


	Packet packet;

// decoding code here
	//cerr << "audio codec context -  sample rate: "<< audioCodec->sample_rate <<", channels: "<<audioCodec->channels<<", sample format: "<<audioCodec->sample_fmt<<endl;

	// suspect why was this in twice 210313
	//do {
	//	if (packet.packet.stream_index != stream->index)
	///		continue;	// the packet is not about the stream we want, let's jump again the start of the loop
	//} while(0);
	//

	// allocating an AVFrame
	std::shared_ptr<AVFrame> avFrame(avcodec_alloc_frame(), &av_free);

	// the current packet of data
	//Packet packet;
	// data in the packet of data already processed
	size_t offsetInData = 0;


	bool foundPacket=false;

	// the decoding loop, running until EOF
	while (true) {
		// reading a packet using libavformat
		if (offsetInData >= packet.packet.size) {
			do {
				packet.reset(avFormat.get());
				if (packet.packet.stream_index != stream->index)
					continue;
			} while(0);
			if (!foundPacket){
				cerr << "audio codec context -  sample rate: "<< audioCodec->sample_rate <<", channels: "<<audioCodec->channels<<", sample format: "<<audioCodec->sample_fmt<<endl;
				foundPacket=true;
			}

		}



		// preparing the packet that we will send to libavcodec
		AVPacket packetToSend;
		packetToSend.data = packet.packet.data + offsetInData;
		packetToSend.size = packet.packet.size - offsetInData;

		// sending data to libavcodec
		int isFrameAvailable = 0;
		//const auto processedLength = avcodec_decode_video2(avVideoCodec.get(), avFrame.get(), &isFrameAvailable, &packetToSend);

		const auto processedLength = avcodec_decode_audio4(audioCodec.get(), avFrame.get(), &isFrameAvailable, &packetToSend);
		//utils.c line 2018

		//unsigned int bitsPerSample = av_get_bytes_per_sample(stream.codecContext->sample_fmt) * CHAR_BIT;    // 16 or 32
		//unsigned int numberOfChannels = stream.codecContext->channels;     // 1 for mono, 2 for stereo, or more
		//unsigned int numberOfSamplesContainedInTheFrame = stream.frame->nb_samples * stream.codecContext->channels;
		//unsigned int numberOfSamplesToPlayPerSecond = stream.codecContext->sample_rate;   // usually 44100 or 48000
		//const void* data = stream.frame->data[0];


		if (processedLength < 0) {
			//av_free_packet(&packet); shouldn't have to because of type safe wrapper
			cerr <<"Error "<<processedLength<< " while processing the data"<< endl;
			return false;
		}
		offsetInData += processedLength;


		// processing the image if available
		if (isFrameAvailable) {
			// display image on the screen

			// sleeping until next frame
			const auto msToWait = avVideoContext->ticks_per_frame * 1000 * avVideoContext->time_base.num / avVideoContext->time_base.den;
			std::this_thread::sleep(std::chrono::milliseconds(msToWait));
		}
		*/


		//http://www.gamedev.net/topic/624876-how-to-read-an-audio-file-with-ffmpeg-in-c/
		// Initialize FFmpeg
		
		//this seems to only read a certain percentage of the sound file? - different for wav and mp3
		//peculiar, because the data comes in as frames and we just read them until all are done
		//is it possible that thhe problem lies with the drawing process only?
		//wav sees 6/10 waves and mp3 sees 3.5/10
		//could this be an aliasing effect i.e. from reading the sample format wrong?
		//I think probably not?
		
		//every format combination seems to miss a different number of samples
		//I don't understand how different formats would make the samples seem to truncate?
		//unless its an aliasing type thing?
		
		//wave.mp3 reads 16.409% of 160913 
		//wave.wav reads 29.706% of 882044
		
		
		
	av_register_all();
	
	//??
	//avcodec_init();
	avcodec_register_all();
	//??

	AVFrame* frame = avcodec_alloc_frame();
	if (!frame)
	{
		std::cout << "Error allocating the frame" << std::endl;
		return false;
	}

	// you can change the file name "01 Push Me to the Floor.wav" to whatever the file is you're reading, like "myFile.ogg" or
	// "someFile.webm" and this should still work
	AVFormatContext* formatContext = NULL;
	if (avformat_open_input(&formatContext, filename.c_str(), NULL, NULL) != 0)
	{
		av_free(frame);
		std::cout << "Error opening the file" << std::endl;
		return false;
	}


	if (avformat_find_stream_info(formatContext, NULL) < 0)
	{
		av_free(frame);
		av_close_input_file(formatContext);
		std::cout << "Error finding the stream info" << std::endl;
		return false;
	}

	AVStream* audioStream = NULL;
	// Find the audio stream (some container files can have multiple streams in them)
	for (unsigned int i = 0; i < formatContext->nb_streams; ++i)
	{
		if (formatContext->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
		{
			audioStream = formatContext->streams[i];
			break;
		}
	}

	if (audioStream == NULL)
	{
		av_free(frame);
		av_close_input_file(formatContext);
		std::cout << "Could not find any audio stream in the file" << std::endl;
		return false;
	}

	AVCodecContext* codecContext = audioStream->codec;

	codecContext->codec = avcodec_find_decoder(codecContext->codec_id);
	if (codecContext->codec == NULL)
	{
		av_free(frame);
		av_close_input_file(formatContext);
		std::cout << "Couldn't find a proper decoder" << std::endl;
		return false;
	}
	else if (avcodec_open2(codecContext, codecContext->codec, NULL) != 0)
	{
		av_free(frame);
		av_close_input_file(formatContext);
		std::cout << "Couldn't open the context with the decoder" << std::endl;
		return false;
	}
	
	//
	// why is the file truncated???
	//if(codecContext->codec->capabilities & CODEC_CAP_TRUNCATED) codecContext->codec->capabilities|=CODEC_FLAG_TRUNCATED;
	//
	//

	av_dump_format(formatContext, 0, 0, false); //avformat.h line 1256
	int samples = ((formatContext->duration + 5000)*codecContext->sample_rate)/AV_TIME_BASE;

	std::cout << "This stream has " << codecContext->channels << " channels, a sample rate of " << codecContext->sample_rate << "Hz and "<<samples <<" samples" << std::endl;
	std::cout << "The data is in format " <<codecContext->sample_fmt<< " (aka "<< av_get_sample_fmt_name(codecContext->sample_fmt) << ") "<<std::endl;

	//we can now tell the processors the format
	//we can work out the number of samples at this point

	for (auto p: processors) {
		p->init(codecContext->channels,16,samples);
	}

	AVPacket packet;
	av_init_packet(&packet);
	int sample_processed=0;
	
	bool diag=true;

	// Read the packets in a loop
	while (true)
	//while(sample_processed<samples)
	{
		int ret=av_read_frame(formatContext, &packet);
		if (ret<0) {
			cerr << "finished with code "<<ret <<(ret==AVERROR_EOF?" ,EOF":"")<<endl;
			break;
		}
		//av_read_frame(formatContext, &packet); //hangs once the packets have been read
		if (packet.stream_index == audioStream->index)
		{
			// Try to decode the packet into a frame
			int frameFinished = 0;
			int bytes = avcodec_decode_audio4(codecContext, frame, &frameFinished, &packet);

			// Some frames rely on multiple packets, so we have to make sure the frame is finished before
			// we can use it
			if (frameFinished)
			{
				// frame now has usable audio data in it. How it's stored in the frame depends on the format of
				// the audio. If it's packed audio, all the data will be in frame->data[0]. If it's in planar format,
				// the data will be in frame->data and possibly frame->extended_data. Look at frame->data, frame->nb_samples,
				// frame->linesize, and other related fields on the FFmpeg docs. I don't know how you're actually using
				// the audio data, so I won't add any junk here that might confuse you. Typically, if I want to find
				// documentation on an FFmpeg structure or function, I just type "<name> doxygen" into google (like
				// "AVFrame doxygen" for AVFrame's docs)
				
				//av_get_channel_layout_string (char *buf, int buf_size, int nb_channels, uint64_t channel_layout)
				
				if (diag) {
					cerr << "first frame: "<<bytes << ", "<<frame->nb_samples<<" samples in "<<av_get_sample_fmt_name(frame->format)<<" format with channel layout "<<frame->channel_layout<<  std::endl;
					diag=false;
				}

				//std::cout << "Got a frame: bytes " << bytes << ", "<<frame->nb_samples<<" samples"<<std::endl;
				//now we can pass the data to the processor(s)
				for (auto p: processors) {
					sample_processed=p->process_frame(frame->data[0],frame->nb_samples);
				}
				
				mutex.lock();
				progress=((double)sample_processed)/samples;
				mutex.unlock();
			}
		}
		// You *must* call av_free_packet() after each call to av_read_frame() or else you'll leak memory
		av_free_packet(&packet);  //crashes here!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! SIGSEV In _int_free (av=0xb4600010, p=0xb46025c8, have_lock=0) at malloc.c:4085 ()
	}

	// Some codecs will cause frames to be buffered up in the decoding process. If the CODEC_CAP_DELAY flag
	// is set, there can be buffered up frames that need to be flushed, so we'll do that
	if (codecContext->codec->capabilities & CODEC_CAP_DELAY)
	{
		av_init_packet(&packet);
		// Decode all the remaining frames in the buffer, until the end is reached
		int frameFinished = 0;
		int bytes = avcodec_decode_audio4(codecContext, frame, &frameFinished, &packet);
		while (bytes >= 0 && frameFinished)
		{
			for (auto p: processors) {
				p->process_frame(frame->data[0],frame->nb_samples);
			}
			mutex.lock();
			progress=((double)sample_processed)/samples;
			mutex.unlock();
		}
	}
	
	cerr << "finished processed: "<<sample_processed << " samples of  "<<samples<<" , "<<((double)sample_processed*100)/samples<<"%"<<  std::endl;
					

	// Clean up!
	av_free(frame);
	avcodec_close(codecContext);
	av_close_input_file(formatContext);

	return true;
}

bool Render_context::load_graph(string &filename){
	printf("loading %s\n",filename.c_str());
	if(xml.loadFile(filename) ){
		graph=Graph(xml.getAttribute("patchbay","ID","",0),xml.getValue("patchbay","",0));
		if(xml.pushTag("patchbay")) {
			int n1=xml.getNumTags("node");
			for (int i1=0;i1<n1;i1++){
				map<string,string> settings;
				vector<string> attrs;
				xml.getAttributeNames("node",attrs,i1);
				for (auto& attr: attrs) {
					settings[attr]=xml.getAttribute("node",attr,"",i1);
					//cerr << "Got attribute: " << attr << ":" << xml.getAttribute("node",attr,"",i1) << endl;
				}
				settings["description"]=xml.getValue("node","",i1);
				Node* node=factory.create(settings);
				if (node) {
					cerr << "Rotor: created '" << xml.getAttribute("node","type","",i1) << "'" << endl;
					string nodeID=xml.getAttribute("node","ID","",i1);
					graph.nodes[nodeID]=node;
					if(xml.pushTag("node",i1)) {
						int n2=xml.getNumTags("signal_input");
						for (int i2=0;i2<n2;i2++){
							graph.nodes[nodeID]->create_signal_input(xml.getValue("signal_input","",i2));
							string fromID=xml.getAttribute("signal_input","from","",i2);
							if (graph.nodes.find(fromID)!=graph.nodes.end()) {
								if (!graph.nodes[nodeID]->inputs[i2]->connect((Signal_node*)graph.nodes[fromID])){
									cerr << "Rotor: graph loader cannot connect input " << i2 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl;
									return false;
								}
								else cerr << "Rotor: linked input " << i2 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl;
							}
							else cerr << "Rotor: linking input " << i2 << " of node: '" << nodeID << "', cannot find target '" << fromID << "'" << endl;
						}
						xml.popTag();
					}
				}
				else {
					cerr << "Rotor: graph loader cannot find node '" << xml.getAttribute("node","type","",i1) << "'" << endl;
					return false;
				}
			}
			xml.popTag();
		}
		return true;
	}
	else return false;
}

Node_factory::Node_factory(){
	//for now, statically load prototype map in constructor
	add_type("audio_analysis",new Audio_analysis());
	add_type("divide",new Signal_divide());
	add_type("==",new Is_new_integer());
	add_type("signal_output",new Signal_output());
}
void audio_thumbnailer::init(int _channels,int _bits,int _samples) {
	//base_audio_processor::init(_channels,_bits,_samples);
	channels=_channels;
	bits=_bits;
	samples=_samples;
	
	cerr << "init audio thumbnailer with "<<channels<<" channels, "<<samples<<" samples of "<< bits<<" bits."<<endl;


	samples_per_column=samples/width;
	column=0; //point thumbnail bitmap
	out_sample=0; //sample in whole track
	offset=0x1<<(bits-1); //signed audio
	scale=1.0/offset;
}
int audio_thumbnailer::process_frame(uint8_t *_data,int samples_in_frame){
	//begin by processing remaining samples
	//samples per column could be larger than a frame! (probably is)
	//but all we are doing is averaging
	int bytes=(bits>>3);
	int stride=channels*bytes;
	int in_sample=0;
	while (in_sample<samples_in_frame&&column<width) {
		int sample=0;
		int samples=0;
		double accum=0;
		while (sample<samples_per_column&&in_sample<samples_in_frame) {
			//accumulate squares for this column
			for (int i=0;i<channels;i++) {
				int this_val=0;
				for (int j=0;j<bytes;j++) {
					this_val+=_data[(sample*stride)+(i*bytes)+j]<<(j*8);
				}
				//convert from integer data format - i.e s16p - to audio signal in -1..1 range
				//don't know how many bytes we are dealing with necessarily?
				//double val=((double)(this_val-offset))*scale;
				if (this_val>offset) this_val=-(this_val-(offset*2));
				double val=((double)this_val)*scale;
				accum+=abs(val); //(val*val);
				samples++;
			}
			in_sample++;
			sample++;
			out_sample++;
		}
		//get root-mean
		double mean=accum/samples; //pow(accum/samples,0.5);
		int colheight=height*mean*0.5;
		int hh=height>>1;
		for (int i=0;i<height;i++) {
			data[i*width+column]=abs(i-hh)<colheight?0xff:0x00;
		}
		column++;
	}
	return out_sample;
}
string audio_thumbnailer::print(){
	string output;
	for (int j=0;j<height;j++) {
		for (int i=0;i<width;i++) {
			output+=data[j*width+i]<0x7f?"0":"1";
		}
		output +="\n";
	}
	return output;
}