path: root/rotord/src/rotor.cpp

#include "rotor.h"
#include "nodes_audio_analysis.h"
#include "nodes_maths.h"
#include "nodes_drawing.h"
#include "nodes_filters.h"

using namespace Rotor;
using Poco::Logger;
Node_factory::Node_factory(){
	//for now, statically load prototype map in constructor
	add_type("time",new Time());
	add_type("track_time",new Track_time());
	add_type("signal_output",new Signal_output());
	add_type("testcard",new Testcard());
	add_type("invert",new Invert());
	add_type("video_cycler",new Video_cycler());
	add_type("signal_colour",new Signal_colour());
	add_type("signal_greyscale",new Signal_greyscale());
	add_type("image_arithmetic",new Image_arithmetic());
	add_type("blend",new Blend());
	add_type("mirror",new Mirror());
	add_type("monochrome",new Monochrome());
	add_type("transform",new Transform());
	add_type("alpha_merge",new Alpha_merge());
	add_type("difference_matte",new Difference_matte());
	//nodes_audio_analysis.h
	add_type("audio_analysis",new Audio_analysis());
	//nodes_maths.h
	add_type("comparison",new Comparison()); //TODO: alias to symbols
	add_type("arithmetic",new Arithmetic()); //TODO: alias to symbols
	add_type("bang",new Is_new_integer());
	add_type("on_off",new On_off());
	add_type("random",new Random());
	//nodes_drawing.h
	add_type("shape",new Shape());
	add_type("hello",new Hello_draw());
	add_type("waves",new Waves());
	//nodes_filters.h
	add_type("blur",new Blur());
	add_type("luma_levels",new Luma_levels());
	add_type("echo_trails",new Echo_trails());
	add_type("rgb_levels",new RGB_levels());
	//video nodes
	add_type("video_loader",new Video_loader());
	add_type("video_output",new Video_output());
	add_type("video_feedback",new Video_feedback());
	add_type("still_image",new Still_image());
}
bool Signal_input::connect(Node* source) {
	connection=dynamic_cast<Signal_node*>(source);
	if (connection)	return true;
	else return false;
}
float Signal_input::get(const Time_spec& time){	//gets input and updates variable
	if (connection){
		return (((Signal_node*)connection)->get_output(time));
	}
	else return 0.0f;
}
bool Image_input::connect(Node* source) {
	connection=dynamic_cast<Image_node*>(source);
	if (connection)	return true;
	else return false;
}
Image* Image_input::get(const Frame_spec& time){	//gets input and updates variable
	if (connection){
		return (((Image_node*)connection)->get_output(time));
	}
	else return nullptr;
}
float Parameter::get(const Time_spec& time){	//gets input and updates variable
	if (connection){
		value = ((Signal_node*)connection)->get_output(time);
	}
	return value;
}
bool Audio_thumbnailer::init(int _channels,int _bits,int _samples,int _rate) {
	//base_audio_processor::init(_channels,_bits,_samples);
	channels=_channels;
	bits=_bits;
	samples=_samples;
	samples_per_column=samples/width;
	column=0; //point thumbnail bitmap
	out_sample=0; //sample in whole track
	offset=0x1<<(bits-1); //signed audio
	scale=1.0f/offset;
	sample=0;
	samples=0;
	accum=0.0;
	return true;
}
int Audio_thumbnailer::process_frame(uint8_t *_data,int samples_in_frame){
	//begin by processing remaining samples
	//samples per column could be larger than a frame! (probably is)
	//but all we are doing is averaging
	int bytes=(bits>>3);
	int stride=channels*bytes;
	int in_sample=0;
	while (in_sample<samples_in_frame&&column<width) {
		//continue the column
		while (sample<samples_per_column&&in_sample<samples_in_frame) {
			//accumulate samples for this column until we run out of samples
			for (int i=0;i<channels;i++) {
				unsigned int this_val=0;
				for (int j=0;j<bytes;j++) {
					this_val+=_data[(in_sample*stride)+(i*bytes)+j]<<(j*8);
				}
				//convert from integer data format - i.e s16p - to audio signal in -1..1 range
				//presume 16 bits for now...
				double val=((double)((int16_t)this_val))*scale;
				accum+=val*val;
				samples++;
			}
			in_sample++;
			sample++;
			out_sample++;
		}
		if (sample==samples_per_column) { //finished a column
			//get root-mean
			double mean=pow(accum/samples,0.5);
			//if (column==0) {
			//	cerr << "first column total: "<< accum << " in " << samples << " samples, average " << (accum/samples)<<endl;
			//}
			int colheight=height*mean*0.5;
			int hh=height>>1;
			for (int i=0;i<height;i++) {
				data[i*width+column]=abs(i-hh)<colheight?0xff:0x00;
			}
			vectordata[column]=mean;
			column++;
			sample=0;
			samples=0;
			accum=0.0;
		}
	}
	return out_sample;
}
string Audio_thumbnailer::print(){
	//base64 encode the image data output it

	stringstream output;
	Poco::Base64Encoder *enc=new Poco::Base64Encoder(output);

	enc->write((char*)data,width*height);
	//tring output;
	/*
	for (int j=0;j<height;j++) {
		for (int i=0;i<width;i++) {
			output+=data[j*width+i]<0x7f?"0":"1";
		}
		output +="\n";
	}
	*/
	enc->close();
	delete enc;
	return output.str();
}
void Audio_thumbnailer::print_vector(xmlIO XML){
	string vdata;
	for (int i=0;i<width;i++){
		if (i>0) vdata+=",";
		vdata+=ofToString(vectordata[i]);
	}
	XML.addValue("data",vdata);
}
bool Audio_analysis::init(int _channels,int _bits,int _samples, int _rate) {
	//need these to make sense of data
	channels=_channels;
	bits=_bits;
	samples=_samples;

	return analyser.init(soname,id,_channels,_bits,_samples,_rate,outputNo,params);


	//attempt to load vamp plugin and prepare to receive frames of data
	//should the audio analysis contain a vamphost or should it inherit?
	//maybe neater to contain it in terms of headers etc

}
int Audio_analysis::process_frame(uint8_t *data,int samples_in_frame) {
	analyser.process_frame(data,samples_in_frame);
	return 1;
}
void Audio_analysis::cleanup() {
	analyser.cleanup();
	//print_features();
}
void Audio_analysis::print_features(){
	for (auto i: analyser.features) {
		cerr<<" ["<<i.second<<":"<<i.first<<"]";
	}
	cerr<<endl;
}
bool Video_output::render(const float duration, const float framerate,const string &output_filename,const string &audio_filename,float& progress,int outW,int outH){

	//
	//setup defaults
	int bitRate=5000000;
	AVCodecID codecId=AV_CODEC_ID_H264; //MPEG4;
	std::string container ="mp4";


    //at the moment it crashes if you render before audio is loaded and also on 2nd render
    libav::exporter exporter;

    float spct=100.0f/duration;

	if (exporter.setup(outW,outH,bitRate,framerate,container)) { //codecId,
		if (exporter.record(output_filename)) {

			libav::audioloader audioloader;

			bool usingaudio=audioloader.setup(audio_filename);
			float *avframe=nullptr;


			Logger& logger = Logger::get("Rotor");
			logger.information("Video_output rendering "+output_filename+": "+ofToString(duration)+" seconds at "+ofToString(framerate)+" fps, audio frame size: "+ofToString(exporter.get_audio_framesize()));
			//25fps video and 43.06640625fps audio? hmm
			//how to get the timecodes correct for the interleaved files

			struct timeval start, end;

		    gettimeofday(&start, NULL);

		    uint16_t *audioframe;
		    int samples_in_frame;

		    if (usingaudio){
			    //does audioloader output interleaved samples?
			    samples_in_frame=(audioloader.codecContext->sample_rate)/framerate;
			    string whether=usingaudio?"Loading":"Cannot load";
			    logger.information(whether+" audio file: "+audio_filename+", each frame contains "+ofToString(samples_in_frame)+" samples at "+ofToString(audioloader.codecContext->sample_rate)+" hz");
			    audioframe=new uint16_t[(samples_in_frame+exporter.get_audio_framesize())*audioloader.codecContext->channels];
			}

			float vstep=1.0f/framerate;
			float v=0.0f;
			float vf=0.0f;
			float af=0.0f;
			int aoffs=0;
			int audioend=0;
			Audio_frame *a;
			while (vf<duration){ //-vstep) {
				uint16_t *audio=nullptr;
				if (usingaudio) {
					uint16_t *audio=audioloader.get_samples(samples_in_frame);
					if (aoffs>0){
						//shift down samples
						int s=0;
						while ((s+aoffs)<audioend) {
							for (int j=0;j<audioloader.codecContext->channels;j++){
								audioframe[s*audioloader.codecContext->channels+j]=audioframe[(s+aoffs)*audioloader.codecContext->channels+j];
							}
							s++;
						}
						aoffs=s;
					}
					for (int i=0;i<samples_in_frame;i++){
						for (int j=0;j<audioloader.codecContext->channels;j++){
							audioframe[(aoffs+i)*audioloader.codecContext->channels+j]=audio[i*audioloader.codecContext->channels+j];
						}
					}
					audioend=aoffs+samples_in_frame;
					aoffs=0;
				    //while (fless(vf+vstep,af+exporter.get_audio_step())) {
				    while (aoffs+exporter.get_audio_framesize()<audioend) {
	                    //insert audio frames until we are only 1 audio frame behind the next video frame
	                    //send audio_framesize() of them through until buffer is used
	                    //pass full buffer within frame_spec for av nodes
	                    exporter.encodeFrame(audioframe+(aoffs*audioloader.codecContext->channels));
	                    af+=exporter.get_audio_step();
	                    aoffs+=exporter.get_audio_framesize();
	                }
	                a=new Audio_frame(audio,audioloader.codecContext->channels,samples_in_frame);
	            }


                //[mp3 @ 0x7fffe40330e0] max_analyze_duration 5000000 reached at 5015510 microseconds
                //[mp3 @ 0x7fffe4033ec0] Insufficient thread locking around avcodec_open/close()
                //[mp3 @ 0x7fffe40330e0] Estimating duration from bitrate, this may be inaccurate
                //[libx264 @ 0x7fffe8003940] using cpu capabilities: MMX2 SSE2Fast SSSE3 FastShuffle SSE4.2
                //[libx264 @ 0x7fffe8003940] profile High, level 3.0
                //[libx264 @ 0x7fffe8003940] 264 - core 123 r2189 35cf912 - H.264/MPEG-4 AVC codec - Copyleft 2003-2012 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=12 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=10 keyint_min=1 scenecut=40 intra_refresh=0 rc_lookahead=10 rc=abr mbtree=1 bitrate=400 ratetol=1.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00
                //Assertion ff_avcodec_locked failed at libavcodec/utils.c:2967

	            //cerr<<"videoloader: "<<vf<<" seconds, vstep "<<vstep<<" ,asking for frame "<<((int)((vf*framerate)+0.5))<<endl

	            Image* i;
	            if (usingaudio) {
			    	i=get_output(Frame_spec(vf,framerate,duration,outW,outH,a));
			    }
			    else i=get_output(Frame_spec(vf,framerate,duration,outW,outH));
			    if (i) {
			    	exporter.encodeFrame(i->RGBdata);
			    }
			    vf+=vstep;
			    progress=vf/duration;
			    if (usingaudio) {delete a;};
			}

			exporter.finishRecord();

			gettimeofday(&end, NULL);

		    float mtime = ((end.tv_sec-start.tv_sec) + (end.tv_usec-start.tv_usec)/1000000.0) + 0.5;

			logger.information("Video_output: rendered "+output_filename+": in "+ofToString(mtime)+" seconds");

			if (usingaudio) {
				audioloader.close();
				delete[] audioframe;
			}



			return true;
		}
	}

	return false;
}

bool Video_loader::load(const string &_filename){
	Logger& logger = Logger::get("Rotor");
    if (isLoaded) {
    	player.cleanup(); ///should be in decoder class?
    	isLoaded=false;
    }
    Poco::Path path;
	string uri="file://"+_filename;
    isLoaded=player.open(uri);
	if (isLoaded){
		logger.information("Video_loader loaded "+_filename+": "\
			+ofToString(player.getNumberOfFrames())+" frames, "\
			+ofToString(player.getFrameRate())+" fps, "\
			+ofToString(player.getWidth())+"x"+ofToString(player.getHeight())\
			+", channels:"+ofToString(player.getNumberOfChannels()));
		return true;
   	}

	logger.error("Video_loader failed to load "+_filename);

    return false;
}
Image* Video_loader::output(const Frame_spec &frame){

	if (isLoaded){
		//this approach is running into the inability to seek when requesting playback speed > 1.
		//need to cache frames so as to avoid asking for a frame other than the next one.
		//need an algorithm to find the previous keyframe and seek forward

		float clipframerate=(parameters["framerate"]->value==0.0f?player.getFrameRate():parameters["framerate"]->value);

		float clipspeed=(clipframerate/frame.framerate)*parameters["speed"]->value;

		int wanted;
		if (attributes["mode"]->intVal==VIDEOFRAMES_frame) {
			wanted=(((int) ((frame.time*frame.framerate)+0.5))%max(1,player.getNumberOfFrames()-1))+1;  //+1 is necessary because 1st frame in a video is number 1?
		}
		if (attributes["mode"]->intVal==VIDEOFRAMES_blend) {
			wanted=(((int) ((frame.time*frame.framerate*clipspeed)+0.5))%max(1,player.getNumberOfFrames()-1))+1;  //+1 is necessary because 1st frame in a video is number 1?
		}

		if (wanted!=lastframe){
			if (!player.fetchFrame(frame.w,frame.h,wanted)) { //seek fail
				Logger& logger = Logger::get("Rotor");
				logger.error("Video_loader failed to seek frame "+ofToString(wanted)+" of "+attributes["filename"]->value);

				if (image.w>0) return &image; //just return the previous frame if possible
				else return nullptr;
			}
			image.setup_fromRGB(frame.w,frame.h,player.pFrameRGB->data[0],player.pFrameRGB->linesize[0]-(frame.w*3));
			lastframe=wanted;
		}
		return &image;
	}
    return nullptr;
};