path: root/rotord/src/rotor.h

#ifndef ROTOR_H
#define ROTOR_H

/*------------------------

Definitions of base classes and types for rotor rendering graph

-------------------------*/

/*------------------------

Aims

-realtime, what does this entail?

	patchbay must be working

	incremental graph editing - examine new graph and make alterations

	window manager

	network stream? rtp?

	realtime architecture - maybe a tiny amount of buffering

	framerate limiting

-undefined number of message types - a mssage is a pointer to a struct

-documentation embedded in nodes

-------------------------*/

/*------------------------

Requirements

-stretch a video to fit a segment

	either use a signal as a playhead {seconds|stretched

	or deliver segment information with a signal


-thoughts on styles

	we can define the properties of a successful style

	1. it has to work with a selection of music in 1 or more genres
	2. it has to work as an edited video
	3, it must not 'look like the same video' with different inputs

	the styles we are working on so far are probably trying to be too original.

	we are in the business of selling commercial art. its about giving people what they know
	they like, not about expanding their consciousnesses.

	can we make a decent techno style that creates something that young/ aspiring techno musicians will relate to?

	can we make a style themed around a fashion i.e goth/emo

	can we make a style about a place or a thing with an image i.e americana, poolside

	this is why I think Sasha's video is the most succesful attempt so far- it takes a
	recognisable style that people understand and deconstructs it.





	http://www.ogre3d.org/forums/viewtopic.php?f=16&t=52936
	chaiscript binding to cairo


HOW TO RE-ARCHITECT TO SUPPORT ARBITRARY TYPES OF NODES???

node types and inputs are templated

Node<Image>

typedef Node<Image> Image_node;

Input<Image>

rather than seperate input types we have

map<string,Input*> inputs

what are the processes needed?

rather than

if (image_inputs[0].connection)

we have

if (inputs["image"].connection)

input=(Image*)inputs["image"].get_output()

???how to define in the graph?

<input type="image" from="blah"/>

inputs: [
	 {
	 	type:"image",
	 	from:"blah"
	 },
	 {
	 	type:"colour",
	 	from:"blah"
	 }
	]


-------------------------*/


#include <unordered_map>
#include <deque>
#include <math.h>
#include <memory>
#include <sys/time.h>
#include <iostream>

#include <json/json.h>
#include <rsvg.h>

#include "Poco/Net/HTTPResponse.h"
#include "Poco/Logger.h"
#include "Poco/File.h"
#include "Poco/Path.h"
#include "Poco/Base64Encoder.h"
#include "Poco/FileStream.h"
#include "Poco/CountingStream.h"
#include "Poco/StreamCopier.h"

#include "xmlIO.h"
#include "utils.h"
#include "cvimage.h"
#include "libavwrapper.h"

//using namespace cv;
namespace Rotor {
	//forward declarations
	class Node;
	class Signal_node;
	class Image_node;
	class Parameter;

	class Audio_frame{
		public:
			Audio_frame(uint16_t *_samples,int _channels,int _numsamples){
				samples=_samples;
				channels=_channels;
				numsamples=_numsamples;
			}
			uint16_t *samples;
			int channels,numsamples;
	};
	class Time_spec{
		public:
			Time_spec(){};
			Time_spec(float _time,float _framerate,float _duration,Audio_frame *_audio=nullptr){ time=_time; framerate=_framerate; duration=_duration; audio=_audio;};
			float time; //num/denom ?
			float framerate;
			float duration;
			Audio_frame *audio;
			Time_spec lastframe() const{
				return Time_spec(time-(1.0f/framerate),framerate,duration);
			}
			Time_spec nextframe() const{
				return Time_spec(time+(1.0f/framerate),framerate,duration);
			}
			int frame(){
				return (int)((time*framerate)+0.5); //rounded to the nearest frame
			}
	};
	class Frame_spec: public Time_spec{
		public:
			Frame_spec(float _time,float _framerate,float _duration,int _w,int _h,Audio_frame *_audio=nullptr)
			{ time=_time; framerate=_framerate; duration=_duration; w=_w; h=_h;audio=_audio;};
			Frame_spec(int _frame,float _framerate,float _duration,int _w,int _h,Audio_frame *_audio=nullptr)
			{ time=((float)_frame)/_framerate; framerate=_framerate; duration=_duration; w=_w; h=_h;audio=_audio;};
			int h,w;
			Frame_spec lastframe() const{
				return Frame_spec(time-(1.0f/framerate),framerate,duration,w,h);
			}
			Frame_spec nextframe() const{
				return Frame_spec(time+(1.0f/framerate),framerate,duration,w,h);
			}
	};
	class Colour{
		public:
			Colour(){
				r=g=b=0;
			}
			Colour(int c){
				r=c&0xFF;
				g=(c&0xFF00)>>8;
				b=(c&0xFF0000)>>16;
			}
			Colour(std::string s){
				r=(uint8_t)hexToChar(s.substr(0,2));
				g=(uint8_t)hexToChar(s.substr(2,2));
				b=(uint8_t)hexToChar(s.substr(4,2));
			}
			float Rfloat(){
				return ((float)r)/255.0f;
			}
			float Gfloat(){
				return ((float)g)/255.0f;
			}
			float Bfloat(){
				return ((float)b)/255.0f;
			}
			uint8_t r,g,b;
	};
	class Command_response{
		public:
		    Command_response() { status=Poco::Net::HTTPResponse::HTTP_OK; }
		    std::string description;
		    Poco::Net::HTTPResponse::HTTPStatus status;
	};
	class Input{
		public:
			Input(const string &_desc,const string &_title): connection(nullptr),description(_desc),title(_title){};
			Node* connection;
			string description;
			string title;
			virtual float get_time_used()=0;
			virtual float get_time_taken()=0;
	};
	class Image_input: public Input{
		public:
			virtual ~Image_input(){};
			bool connect(Node *source);
			Image_input(const string &_desc,const string &_title,Node* _connect): Input(_desc,_title){
				connect(_connect);
			};
			Image* get(const Frame_spec& time);
			float get_time_used();
			float get_time_taken();
	};
	class Signal_input: public Input{
		public:
			virtual ~Signal_input(){};
			bool connect(Node *source);
			Signal_input(const string &_desc,const string &_title,Node* _connect): Input(_desc,_title){
				connect(_connect);
			};
			float get(const Time_spec& time);
			float get_time_used();
			float get_time_taken();
	};
	class Parameter: public Signal_input{
		public:
			virtual ~Parameter(){};
			void init(const float &_val){
				value=_val;
			}
			Parameter(const string &_type,const string &_desc,const string &_title,float _value,float _min,float _max,float _step,Node* _connect): Signal_input(_desc,_title,_connect),value(_value),min(_min),max(_max),step(_step),type(_type){};
			float value,min,max,step;
			float get(const Time_spec& time);
			string type;
	};
	/*
	tributes want to be extended beyond the original vision of a string that can be interpreted in different ways
	could be a template?
	typedef attribute<int32_t> intattribute;
	could be a subclass ie
	attributes=vector<attribute*>
	class lyrics_attribute : public attribut
	if(<dynamic_cast>(lyrics_attribute)attributes["lyrics"]) {
		thislyric=(lyrics_attribute)attributes["lyrics"].findkey(1);
	}

	1) generic way to define and use attributes of different types

	*/
	class Attribute{ //description of a static attribute which can be an enumerated string array
		public:
			virtual ~Attribute(){};
			Attribute(){};
			Attribute(const string &_desc,const string &_title,const string &_value,std::vector<std::string> _vals={},std::string _type="string"): description(_desc),title(_title),value(_value),intVal(0),type(_type){
				vals=_vals;
				init(_value);
			};
			void init(const string &_key){ //inits int value from set::string vals index
				value=_key;
				std::vector<std::string>::iterator it= find(vals.begin(),vals.end(),value);
				if (it!=vals.end()){
					intVal = std::distance(vals.begin(),it)+1; //using 1-index for enums
				}
				else intVal=0;
			}
			void init(const std::vector<std::string> &_vals){
				vals=_vals;
				string s;
				for (auto v:vals) s=s+v+" ";
				cerr<<"array attribute "<<title<<": "<<s<<endl;
			}
			string description,title,value;
			int intVal;
			string type;
			std::vector<std::string> vals;

	};
	class Lyrics_attribute: public Attribute{ //description of a static attribute which can be an enumerated string array
		public:
			virtual ~Lyrics_attribute(){};
			Lyrics_attribute(const string &_desc,const string &_title){
				description=_desc;
				title=_title;
				type="lyrics";
				blank_response="";
			};
			void init(const std::map<float,std::pair<string,float> > _lyrics){
				lyrics=_lyrics;
				for (auto l:lyrics){
					cerr<<l.first<<":"<<l.second.first<<" ("<<l.second.second<<")"<<endl;
				}
				lyrics[-1.0f]=make_pair("",0.0f); //start with an empty entry
			}
			const string &get_lyric(const Time_spec &time){
				if (lyrics.size()) {
					auto l=lyrics.upper_bound(time.time);
					l--;
					//cerr<<(time.time)<<" "<<l->second.first<<","<<(l->first)<<" ("<<(l->second.second)<<")"<<endl;
					if ((time.time>l->first)&&((time.time-l->first) < l->second.second)) return l->second.first;
				}
				return blank_response;
			}
		private:
			std::map<float,std::pair<string,float> > lyrics; //lyrics[startime]=pair<lyric,endtime>
			std::string blank_response;
	};
	class Node{
		public:
			Node(){duplicate_inputs=false;UItype="";};
			virtual Node* clone(map<string,string> &_settings)=0; //pure virtual
			virtual ~Node(){
				for (auto a: attributes) delete a.second;
				for (auto p: parameters) delete p.second;
				for (auto s: inputs) delete s;
			};
			vector<Signal_input*> inputs;		//simple node can have signal inputs, output depends on node type
			unordered_map<string,Parameter*> parameters;		//linked parameters can convert from settings to inputs
			unordered_map<string,Attribute*> attributes;
			void create_signal_input(const string &_desc,const string &_title,Node* _connect=nullptr ) {
				inputs.push_back(new Signal_input(_desc,_title,_connect));
			};
			void create_parameter(const string &_name,const string &_type,const string &_desc,const string &_title,float _value=1.0f,float _min=0.0f,float _max=0.0f,float _step=0.0f,Node* _connect=nullptr) {
				parameters[_name]=new Parameter(_type,_desc,_title,_value,_min,_max,_step,_connect);
			};
			void create_attribute(const string &_attr,const string &_desc,const string &_title,const string &_value,std::vector<std::string> _vals={},std::string _type="string") {
				if (_vals.size()&&_type!="array") _type="enum"; //hack for incomplete attribute types
				attributes[_attr]=new Attribute(_desc,_title,_value,_vals,_type);
			};
			void create_empty_attribute(const string &_attr,const string &_desc,const string &_title,const string &_type="string") {
				//initialiser without initialisation
				if (_type=="lyrics") attributes[_attr]=new Lyrics_attribute(_desc,_title);
				else attributes[_attr]=new Attribute(_desc,_title,"",{},_type);
			};
			void create_attribute(string *alias,const string &_attr,const string &_desc,const string &_title,const string &_value,std::vector<std::string> _vals={},std::string _type="string") {
				create_attribute(_attr,_desc,_title,_value,_vals,_type);
				alias=&(attributes[_attr]->value);
			};
			void create_attribute(int *alias,const string &_attr,const string &_desc,const string &_title,const string &_value,std::vector<std::string> _vals={},std::string _type="string") {
				create_attribute(_attr,_desc,_title,_value,_vals,_type);
				alias=&(attributes[_attr]->intVal);
			};
			virtual void init_attribute(const string &attr){
			};
			virtual void init(){}; //initialise any state variables that expect a linear render
			string description;
			string type;
			string ID;
			string NODEID;
			string title;
			string UItype;
			bool duplicate_inputs;
			string find_setting(map<string,string> &settings,string key,string def=""){ if (settings.find(key)!=settings.end()) return settings[key]; else return def;};
			float find_setting(map<string,string> &settings,string key,float def){ if (settings.find(key)!=settings.end()) return toFloat(settings[key]); else return def;};
			int find_setting(map<string,string> &settings,string key,int def){ if (settings.find(key)!=settings.end()) return toInt(settings[key]); else return def;};
			void base_settings(map<string,string> &settings) {
				description=find_setting(settings,"description");
				type=find_setting(settings,"type");
				ID=find_setting(settings,"ID");
				title=find_setting(settings,"title");
				for (auto a: attributes){
					if (find_setting(settings,a.first,"")!="") {
						attributes[a.first]->init(find_setting(settings,a.first,""));
						cerr<<"setting attribute '"<<a.first<<"' to "<<find_setting(settings,a.first,"")<<" (index: "<<attributes[a.first]->intVal<<")"<<endl;
					}
				}
				for (auto p: parameters){
					if (find_setting(settings,p.first,"")!="") {
						parameters[p.first]->init(find_setting(settings,p.first,0.0f));
						cerr<<"setting parameter '"<<p.first<<"' to "<<find_setting(settings,p.first,0.0f)<<endl;
					}
				}
			}
			void update(const Time_spec &time){
				gettimeofday(&frame_time, NULL);
				for (auto p: parameters){
					p.second->get(time);
				}
			}
			void set_parameter(const std::string &key,const std::string &value){
				if (parameters.find(key)!=parameters.end())	parameters[key]->value=toFloat(value);
			};
			void reset(){
				time_taken=0.0f;
				int();
			}
			void time_frame(){
				struct timeval end_time;
				gettimeofday(&end_time, NULL);
				time_taken+=((end_time.tv_sec-frame_time.tv_sec) + (end_time.tv_usec-frame_time.tv_usec)/1000000.0);
			}
			virtual float get_time_used()=0;
			float time_taken;
		protected:
			struct timeval frame_time;
		};
	class Signal_node: public Node{
	public:
		virtual ~Signal_node(){};
		const float get_output(const Time_spec &time) {
			update(time);
			float o=output(time);
			time_frame();
			return o;
		};
		const float get_time_for_value(const float &value) {
			return 0.0f;
		};
		virtual const float output(const Time_spec &time) { return 0.0f; };
		float get_time_used(){
			float t=time_taken;
			for (auto i:inputs) t-=i->get_time_taken();
			for (auto p:parameters) t-=p.second->get_time_taken();
			return t;
		}
	};
	class Image_node: public Node{
		public:
			virtual ~Image_node(){
				for (auto i: image_inputs) delete i;
			};
			vector<Image_input*> image_inputs;			//image node also has image inputs and outputs
			void create_image_input(const string &_title,const string &_desc,Node* _connect=nullptr) {
				image_inputs.push_back(new Image_input(_desc,_title,_connect));
			};
			Image *get_image_output(const Frame_spec &frame) {
				update((Time_spec)frame);
				image.setup(frame.w,frame.h);
				Image *i=output(frame);
				time_frame();
				return i;
			}
			virtual Image *output(const Frame_spec &frame)=0;
			void clear_output(int w,int h){
				image.setup(w,h);
				image.clear();
			}
			Image image;
			float get_time_used(){
				float t=time_taken;
				for (auto i:inputs) t-=i->get_time_taken();
				for (auto p:parameters) t-=p.second->get_time_taken();
				for (auto i:image_inputs) t-=i->get_time_taken();
				return t;
			}

			private:
				float image_time; //? could be used to detect image reuse?

	};
	class LUT {
			LUT(){
				lut=nullptr;
			};
			~LUT(){if (lut) { delete[] lut;} };
			void generate(float black_in,float white_in,float black_out,float white_out,float gamma){
				//can check here if anything has changed
				if (lut) delete[] lut;
				lut=new unsigned char[256];
				float fltmax=(255.0f/256.0f);
				for (int i=0;i<256;i++){
					lut[i]=(unsigned char)(((pow(min(fltmax,max(0.0f,(((((float)i)/256.0f)-black_in)/(white_in-black_in)))),(1.0/gamma))*(white_out-black_out))+black_out)*255.0f);
				}
			}
			void apply(const cv::Mat& in,cv::Mat &out){ //facility to apply to other images for inherited classes
				out.create(in.rows,in.cols,in.type());
				for (int i=0;i<in.rows*in.cols*in.channels();i++){
					out.data[i]=lut[in.data[i]];
				}
			}
		protected:
			unsigned char *lut;
	};
	//actual nodes-------------------------------------------------
	#define CYCLER_cut 1
	#define CYCLER_mix 2
	#define CYCLER_seconds 1
	#define CYCLER_fraction 2
	#define CYCLER_abs 1
	#define CYCLER_rel 2
	#define CYCLER_stretch 3

	#define CYCLER_screen 3
	#define CYCLER_multiply 4
	#define CYCLER_alpha 5
	#define CYCLER_wrap 6
	#define CYCLER_xor 7
	#define CYCLER_overlay 8
	#define CYCLER_min 9
	#define CYCLER_max 10
	//new mode that remaps sequence length to segments
	//combined with a video mode that maps video length to duration
	//time remapper object
	//takes a signal and turns it into a new timeline
	//
	//to make a movie stretch to a segment..
	//modulus a segmenter into the position controller of a video player
	//
	//how else?
	//q: how to create a relative timeline from a segment?
	//a: you need the begiining and end points to calculate the duration
	//
	//q: where do we have this info/ where do we get it?
	//a: only in the node that generated it?
	//
	//q: what would it take to get this info from place to place
	//a: would need to send featuresets down the line with

	//want to make a limted length transition - i.e. the 1st 2 seconds
	//this creates a similar problem, how do we know the duration of the segment and where we are in it
	//could there be fast way to interrogate a signal in reverse, i.e lookup the time for which a signal reaches a certain value?
	//from the POV of a feature set, yes, we could
	//from the POV of lots of other node types, no it would be hard
	//support an error code? return 0.0 if the node doesn't support reverse query
	 class Video_cycler: public Image_node {
		public:
			Video_cycler(){
				create_image_input("Image input","Image input");
				create_signal_input("Selector","Selector input");
				create_attribute("mode","Cycling mode","Mode","cut",{"cut","mix","screen","multiply","alpha","wrap","xor","overlay","min","max"});
				create_attribute("length_mode","Transition length mode","Length mode","fraction",{"seconds","fraction"});
				create_attribute("time_mode","Time mode","time mode","absolute",{"absolute","relative"});
				create_parameter("transition_length","number","transition length","Transition length",1.0f,0.0f,0.0f);
				title="Video cycler";
				description="Cycles through video inputs according to selector signal";
				duplicate_inputs=true;
				NODEID="93dd9d76-2d09-11e3-9589-5bbbeea1b304";
			}
			Video_cycler(map<string,string> &settings):Video_cycler() {
				base_settings(settings);
			};
			~Video_cycler(){};
			bool load(const string &filename);
			void init(){
				segment=0;
				segment_start=0.0f;
				prevseg_start=0.0f;
				lastframe=0;
			}
			Image *output(const Frame_spec &frame){

				//work out timing
				//can this be a general purpose function of node
				float ph=inputs[0]->get((Time_spec)frame);
				int seg=((int)ph);
				int thisframe=((Time_spec)frame).frame();
				if(thisframe==lastframe||thisframe==lastframe+1){
					if (segment!=seg){
						//start of new segment
						prevseg=segment;
						prevseg_start=segment_start;
						segment=seg;
						segment_start=((Time_spec)frame).time;
						//find segment end
						Time_spec testframe=(Time_spec)frame.nextframe();
						while ((int)inputs[0]->get(testframe)==seg&&testframe.time<frame.duration){
							testframe=testframe.nextframe();
						}
						//nextseg=(int)inputs[0]->get(testframe);
						segment_end=((Time_spec)testframe).time;
						//cerr<<"Video_cycler: segment "<<seg<<" preroll: "<<prevseg_start<<" start: "<<segment_start<<" end: "<<segment_end<<endl;
					}
				}
				else {
					if (thisframe==0){
						segment_start=0.0f;
						segment=seg;
					}
					else {
						//find segment start
						Time_spec testframe=(Time_spec)frame.lastframe();
						while ((int)inputs[0]->get(testframe)==seg&&testframe.frame()>1){
							testframe=testframe.lastframe();
						}
						segment_start=testframe.time;
						segment=seg;
						if ((int)inputs[0]->get(testframe)!=seg){
							prevseg=(int)inputs[0]->get(testframe);
							//handle previous segment
							while ((int)inputs[0]->get(testframe)==prevseg&&testframe.frame()>1){
								testframe=testframe.lastframe();
							}
							prevseg_start=testframe.time;
						}

						testframe=(Time_spec)frame.nextframe();
						while ((int)inputs[0]->get(testframe)==seg&&testframe.time<frame.duration){
							testframe=testframe.nextframe();
						}
						segment_end=((Time_spec)testframe).time;

						//cerr<<"Video_cycler: segment "<<seg<<" start: "<<segment_start<<" end: "<<segment_end<<endl;

					}
				}
				lastframe=thisframe;
				//float start_time=(((Time_spec)frame).time-segment_start);
				//float end_time=(((Time_spec)frame).time-segment_end);

				float in_time=(((Time_spec)frame).time-segment_start);
				//time in seconds for the incoming sequence: starts at 0
				float out_time=(((Time_spec)frame).time-prevseg_start);
				 //time in seconds for the outgoing sequence:
								//starts at how many seconds the outgoing has got to
								//so we need the previous segmente duration


				Frame_spec inframe=frame;
				Frame_spec outframe=frame;
				switch (attributes["time_mode"]->intVal){
					case CYCLER_abs:
						inframe=frame;
						break;
					case CYCLER_rel:
						inframe=Frame_spec(in_time,frame.framerate,frame.duration,frame.w,frame.h,frame.audio);
						outframe=Frame_spec(out_time,frame.framerate,frame.duration,frame.w,frame.h,frame.audio);
						break;
				}

				//to x fade between segments in relative mode, need to know the end of the segment as well as the beginning?
				//or keep an index of segment times?

				//we don't want to mix from this input to the next
				//we want to mix out the previous segment whose start time was the previous segment (or the end one)
				//awkward, its getting close to the point where we just need access to all of the segments data everywhere?
				//we have 2 times here

				if (attributes["mode"]->intVal>=CYCLER_mix&&image_inputs.size()>1){
					int im1=seg%image_inputs.size();
					int im2=prevseg%image_inputs.size();

					float f;

					switch (attributes["length_mode"]->intVal){
						case CYCLER_seconds:
							f=min(1.0f,in_time/parameters["transition_length"]->value);
							break;
						case CYCLER_fraction:
							f=min(1.0f,(in_time/(segment_end-segment_start))/parameters["transition_length"]->value);
							break;
					}

					//cerr<<f<<" of input "<<(seg%image_inputs.size())<<" & "<<(1.0f-f)<<" of input "<<(prevseg%image_inputs.size())<<endl;


					Image *in1=image_inputs[im1]->get(attributes["time_mode"]->intVal==CYCLER_abs?frame:inframe);
					if (in1){
						if (f<1.0f) {
							Image *in2=image_inputs[im2]->get(attributes["time_mode"]->intVal==CYCLER_abs?frame:outframe);
							if (in2){
								image=(*in1);
								image*=f;
								Image i2=(*in2);
								i2*=(1.0f-f);
								switch(attributes["mode"]->intVal){
									case CYCLER_screen:
										image+=i2;
										break;
									case CYCLER_multiply:
										image*=i2;
										break;
									case CYCLER_xor:
										image^=i2;
										break;
									case CYCLER_alpha:
										image=image.alpha_blend(i2);
										break;
									case CYCLER_wrap:
										image=image.add_wrap(i2);
										break;
									case CYCLER_overlay:
										image=image.overlay(i2);
										break;
									case CYCLER_min:
										image=image.min(i2);
										break;
									case CYCLER_max:
										image=image.max(i2);
										break;
									case CYCLER_mix: //has to be last because of initialser of *in? go figure

										image+=i2;
										break;
								}
								
								return &image;
							}
						}
						return in1;
					}
					return nullptr;
				}
				//cut mode
				//for (uint32_t i=0;i<image_inputs.size();i++){ //this skipped a beat for some reason
				int whichinput=(((int)inputs[0]->get((Time_spec)frame)))%image_inputs.size(); //+i
				Image *in=image_inputs[whichinput]->get(inframe);
				if (in) return in;
				//}
				return nullptr;
			}
			Video_cycler* clone(map<string,string> &_settings) { return new Video_cycler(_settings);};
		private:
			float segment_start,segment_end;
			float prevseg_start;
			int segment,prevseg;
			int lastframe;
	};
#define VIDEOFRAMES_frame 1
#define VIDEOFRAMES_blend 2
#define VIDEOTIME_play 1
#define VIDEOTIME_stretch 2
	class Base_video: public Image_node {
		public:
			Base_video(){
				create_signal_input("playhead","Playhead");
				//floating point control of playback time
				//if signal is connected it overrides normal playback
				//time_mode dictates whether control is seconds, or duration
				create_parameter("speed","number","video playback speed","Speed",1.0f,0.0f,0.0f);
				create_parameter("framerate","number","framerate override","Frame rate",0.0f,0.0f,0.0f);
				create_attribute("frame_mode","frame mode","Frame mode","frame",{"frame","blend"});
				create_attribute("time_mode","time mode","Time mode","play",{"play","stretch"});
				create_attribute("media_id","media_id","media_id","media_id"); //for rotorW
				NODEID="e92255a0-447a-11e3-b0ce-3fc7ff4bdac9";
				isLoaded=false;
			};
			bool load(const string &filename){
				Poco::Logger& logger = Poco::Logger::get("Rotor");
			    isLoaded=player.open(filename);
				if (isLoaded){
					logger.information("libav::decoder loaded "+filename+": "\
						+toString(player.get_number_frames())+" frames, "\
						+toString(player.get_framerate())+" fps, "\
						+toString(player.get_width())+"x"+toString(player.get_height())\
						+", channels:"+toString(player.get_number_channels()));
					lastframe=-2;
					return true;
			   	}
				logger.error("libav::decoder failed to load "+filename);
			    return false;
			}
			bool get_frame(float wanted,const Frame_spec &frame){
				if (attributes["frame_mode"]->intVal==VIDEOFRAMES_blend){
					if (((int)wanted)!=Base_video::lastframe){
						//get a new pair of frames possibly by switching the next one
						//darn peculiar, as if copying wasn't actually copying
						if ((Base_video::lastframe==(((int)wanted)-1))&&(in2.w>0)) {
							in1=in2;
						}
						else {
							player.fetch_frame(frame.w,frame.h,(int)wanted);
							//use a temp image because setup_fromRGB just copies pointer
							temp.setup_fromRGB(frame.w,frame.h,player.frame->Data[0],player.frame->Linesize[0]-(frame.w*3));
							in1=temp;
						}
						player.fetch_frame(frame.w,frame.h,((int)wanted+1)%max(1,player.get_number_frames()));
						temp.setup_fromRGB(frame.w,frame.h,player.frame->Data[0],player.frame->Linesize[0]-(frame.w*3));
						in2=temp;
						lastframe=wanted;
					}
					float amt=1.0f-(wanted-((int)wanted));
					//cout<<"video loader time:"<<frame.time<<" frames "<<((int)wanted)<<" (x"<<amt<<"),"<<(((int)wanted+1)%max(1,player.get_number_frames()))<<endl;
					image=in1;
					image*=amt;
					//Image in2t=in2; //DOES NOT WORK, copies pointer by assignation
					in2t=in2;
					in2t*=(1.0f-amt);
					image+=in2t;
				}
				else {
					if (((int)wanted)!=Base_video::lastframe){
						if (!player.fetch_frame(frame.w,frame.h,((int)wanted))) { //seek fail
							Poco::Logger& logger = Poco::Logger::get("Rotor");
							logger.error("Video_loader failed to seek frame "+toString(wanted)+" of "+attributes["filename"]->value);

							if (image.w>0) return &image; //just return the previous frame if possible
							else return nullptr;
						}
						image.setup_fromRGB(frame.w,frame.h,player.frame->Data[0],player.frame->Linesize[0]-(frame.w*3));
					}
				}
				return true;
			}
			Image *output(const Frame_spec &frame){
				if (isLoaded){
					float clipframerate=(parameters["framerate"]->value==0.0f?player.get_framerate():parameters["framerate"]->value);
					float clipspeed=(clipframerate/frame.framerate)*parameters["speed"]->value;
					float wanted=0.0f;
					if (inputs[0]->connection) {
					//using playhead
					//should speed affect it?
					//if you want absolute control then you just want absolute control?
						switch (attributes["frame_mode"]->intVal){
							case VIDEOTIME_play:
								wanted=fmod(inputs[0]->get((Time_spec)frame)*frame.framerate*clipspeed,(float)player.get_number_frames());
								break;
							case VIDEOTIME_stretch:
								wanted=fmod(fmod(inputs[0]->get((Time_spec)frame),1.0f)*((float)player.get_number_frames())*clipspeed,(float)player.get_number_frames());
								break;
						}
					}
					else {
						switch (attributes["frame_mode"]->intVal){
							case VIDEOTIME_play:
								wanted=fmod(frame.time*frame.framerate*clipspeed,(float)player.get_number_frames());
								break;
							case VIDEOTIME_stretch:
								wanted=fmod((frame.time/frame.duration)*((float)player.get_number_frames())*clipspeed,(float)player.get_number_frames());
								break;
						}
					}
					if (get_frame(wanted,frame)) return &image;
				}
			    return nullptr;
			};
		protected:
			Image in1,in2,in2t,temp; //for blend frames;
			libav::video_decoder player;
			int lastframe;
			bool isLoaded;
			string media_path;
	};
	//relative timelines used to stretch video
	//1. make a video position input for video node - seconds and stretch modes
	//2. video mode that maps to duration- timeline remapping from cycler and others
	class Video_loader: public Base_video {
		public:
			Video_loader(){
				create_attribute("filename","name of video file to load","File name","");
				create_attribute("media_id","media_id","media_id","media_id"); //for rotorW
				//create_attribute("media_path","media_path","media_path","media_path"); //for rotorW? NO
				title="Video loader";
				description="Loads a video file";
				NODEID="5b64b8ca-2d0a-11e3-92ed-4b7420b40040";
			};
			Video_loader(map<string,string> &settings): Video_loader() {
				base_settings(settings);
				media_path=find_setting(settings,"media_path","");
				//for (auto a:attributes) cerr<<"Video loader: "<<a.first<<" set to "<<a.second->value<<endl;
				if (attributes["filename"]->value!="") {
					isLoaded=load(media_path+attributes["filename"]->value);
				}

			};
			~Video_loader(){};
			//void init_attribute(const string &attr){
			//	if (attr=="filename") {
			//		isLoaded=load(media_path+attributes[attr]->value);
			//	}
			//};
			Video_loader* clone(map<string,string> &_settings) { return new Video_loader(_settings);};
			bool isLoaded;
		private:
			string *filename;
	};
	class Video_bank: public Base_video {
		public:
			//manage a bank of video inclusing transitions
			//blend mode etc
			//there may be a total of 4 frames required
			//and 2 videos open
			//is there any way to re-use the parts which prepare frames
			Video_bank(){
				create_attribute("filenames","names of video files to load","File names","",{},"array");
				NODEID="73616e66-4306-11e3-981e-74d02b29f6a6";
				title="Video bank";
				description="Loads a banks of video files";
				clip_loaded=-1;
				isLoaded=false;
				segment=0;
				segment_start=0.0f;
				lastframe=0;
			};
			Video_bank(map<string,string> &settings): Video_bank() {
				base_settings(settings);
				media_path=find_setting(settings,"media_path","");
			};
			~Video_bank(){};
			bool load(int v){
				if (players[v].loaded) return true;
				Poco::Logger& logger = Poco::Logger::get("Rotor");
				players[v]=libav::video_decoder();
				string filename=media_path+attributes["filenames"]->vals[v];
			    isLoaded=players[v].open(filename);
				if (isLoaded){
					logger.information("libav::decoder loaded "+filename+": "\
						+toString(players[v].get_number_frames())+" frames, "\
						+toString(players[v].get_framerate())+" fps, "\
						+toString(players[v].get_width())+"x"+toString(players[v].get_height())\
						+", channels:"+toString(players[v].get_number_channels()));
					lastframe=-2;
					return true;
			   	}
				logger.error("libav::decoder failed to load "+filename);
			    return false;
			}
			bool get_frame(float wanted,const Frame_spec &frame){
				if (attributes["frame_mode"]->intVal==VIDEOFRAMES_blend){
					if (((int)wanted)!=Base_video::lastframe){
						//get a new pair of frames possibly by switching the next one
						//darn peculiar, as if copying wasn't actually copying
						if ((Base_video::lastframe==(((int)wanted)-1))&&(in2.w>0)) {
							in1=in2;
						}
						else {
							players[clip_loaded].fetch_frame(frame.w,frame.h,(int)wanted);
							//use a temp image because setup_fromRGB just copies pointer
							temp.setup_fromRGB(frame.w,frame.h,players[clip_loaded].frame->Data[0],players[clip_loaded].frame->Linesize[0]-(frame.w*3));
							in1=temp;
						}
						players[clip_loaded].fetch_frame(frame.w,frame.h,((int)wanted+1)%max(1,players[clip_loaded].get_number_frames()));
						temp.setup_fromRGB(frame.w,frame.h,players[clip_loaded].frame->Data[0],players[clip_loaded].frame->Linesize[0]-(frame.w*3));
						in2=temp;
						lastframe=wanted;
					}
					float amt=1.0f-(wanted-((int)wanted));
					//cout<<"video loader time:"<<frame.time<<" frames "<<((int)wanted)<<" (x"<<amt<<"),"<<(((int)wanted+1)%max(1,player.get_number_frames()))<<endl;
					image=in1;
					image*=amt;
					//Image in2t=in2; //DOES NOT WORK, copies pointer by assignation
					in2t=in2;
					in2t*=(1.0f-amt);
					image+=in2t;
				}
				else {
					if (((int)wanted)!=Base_video::lastframe){
						if (!players[clip_loaded].fetch_frame(frame.w,frame.h,((int)wanted))) { //seek fail
							Poco::Logger& logger = Poco::Logger::get("Rotor");
							logger.error("Video_loader failed to seek frame "+toString(wanted)+" of "+attributes["filename"]->value);

							if (image.w>0) return &image; //just return the previous frame if possible
							else return nullptr;
						}
						image.setup_fromRGB(frame.w,frame.h,players[clip_loaded].frame->Data[0],players[clip_loaded].frame->Linesize[0]-(frame.w*3));
					}
				}
				return true;
			}
			Image *output(const Frame_spec &frame){
				if (!inputs[0]->connection){ //default to single loader
					if (!isLoaded){
						if (Base_video::load(media_path+attributes["filenames"]->vals[0] )) {
							isLoaded=true;
						}
						else {
							isLoaded=false;
						}
					}
					if (isLoaded){
						return Base_video::output(frame);
					}
					return nullptr;
				}
				if (attributes["filenames"]->vals.size()){
					float ph=inputs[0]->get((Time_spec)frame);
					int seg=((int)ph);
					int wv=seg%attributes["filenames"]->vals.size();
					players.resize(attributes["filenames"]->vals.size());
					ph=seg==0?ph:fmod(ph,seg);
					if (clip_loaded!=wv){
						if (load(wv)) {
							clip_loaded=wv;
							isLoaded=true;
						}
						//else {
							//cerr<<"Video bank could not load "<<(media_path+attributes["filenames"]->vals[wv])<<endl;
						//	clip_loaded=-1;
						//}
					}
					if (isLoaded){
						int wanted=0.0f;
						int thisframe=((Time_spec)frame).frame();
						float clipframerate=(parameters["framerate"]->value==0.0f?players[clip_loaded].get_framerate():parameters["framerate"]->value);
						float clipspeed=(clipframerate/frame.framerate)*parameters["speed"]->value;
						switch (attributes["time_mode"]->intVal){
							case VIDEOTIME_play:
								if(thisframe==lastframe||thisframe==lastframe+1){
									if (segment!=seg){
										//start of new segment
										//cerr<<"Video_bank: segment "<<seg<<" started: "<<((Time_spec)frame).time<<endl;
										segment=seg;
										segment_start=((Time_spec)frame).time;
									}
								}
								else {
									if (thisframe==0){
										segment_start=0.0f;
										segment=seg;
									}
									else {
										//find segment start
										Time_spec testframe=(Time_spec)frame.lastframe();
										while ((int)inputs[0]->get(testframe)==seg&&testframe.frame()>1){
											testframe=testframe.lastframe();

										}
										segment_start=testframe.time;
										segment=seg;
									}
								}
								wanted=(((Time_spec)frame).time-segment_start)*clipspeed*frame.framerate;
								lastframe=thisframe;
								break;
							case VIDEOTIME_stretch:
								wanted=ph*players[clip_loaded].get_number_frames();
								break;
						}
						if (get_frame(wanted,frame)) return &image;
					}

				}
				return nullptr;
			}
			Video_bank* clone(map<string,string> &_settings) { return new Video_bank(_settings);};
		private:
			vector<libav::video_decoder> players;
			int clip_loaded;
			float segment_start;
			int segment;
			int lastframe;
	};
	class Video_output: public Image_node {
	//Video_output 'presents' the output movie. Aspect ratio, bars, fadein/fadeout would happen here
		public:
			Video_output(){
				create_image_input("image to output","Image input");
				create_attribute("begin_mode","mode to begin movie","Begin mode","cut",{"cut","blank silence","fade peak"});
				create_attribute("end_mode","mode to end movie","End mode","cut",{"cut","blank silence","fade peak"});
				title="Video output";
				description="Output to video";
				start_silence=start_peak=end_silence=end_peak=-1.0f;
				silence_threshold=0.01f;
				NODEID="693d2220-2d0a-11e3-9312-232908c3cc33";
			};
			Video_output(map<string,string> &settings):Video_output() {
				base_settings(settings);
			};
			~Video_output(){ };
			void create_envelope(const vector<float> &audio){
				if (audio.size()){
					uint32_t i=1;
					while (i<audio.size()&&audio[i]<silence_threshold) i++;
					start_silence=((float)i)/audio.size();
					while (i<audio.size()&&audio[i]>audio[i-1]) i++;
					start_peak=((float)i-1)/audio.size();
					i=audio.size();
					while (i>0&&audio[i]<silence_threshold) i--;
					end_silence=((float)i)/audio.size();
					while (i<audio.size()&&audio[i]>audio[i+1]) i--;
					end_peak=((float)i+1)/audio.size();
					cerr<<"Video_output sound envelope: silence - "<<start_silence<<" : peak "<<start_peak<<" : peak "<<end_peak<<" silence - "<<end_silence<<endl;
				}
				else cerr<<"Video_output sound envelope: no data"<<endl;
			}
			Image *output(const Frame_spec &frame){
				Image *in=image_inputs[0]->get(frame);
				if (in){
					//make copy of the image, for feedback
					//optimise?
					float amount=1.0f;
					float track_time=frame.time/frame.duration;
					if (attributes["begin_mode"]->value=="fade peak"||attributes["begin_mode"]->value=="blank silence"){
						if (track_time<start_silence){
							amount=0.0f;
						}
						else if (track_time<start_peak&&attributes["begin_mode"]->value=="fade peak"&&start_peak>start_silence){
							amount = (track_time-start_silence)/(start_peak-start_silence);
						}
					}
					if (attributes["end_mode"]->value=="fade peak"||attributes["end_mode"]->value=="blank silence"){
						if (track_time>end_silence){
							amount=0.0f;
						}
						else if (track_time>end_peak&&attributes["end_mode"]->value=="fade peak"&&end_silence>end_peak){
							amount = 1.0f-((track_time-end_peak)/(end_silence-end_peak));
						}
					}
					if (amount<(1.0f/256.0f)){
						image.clear();
					}
					image=(*in);
					if (amount<(255.0f/256.0f)){
						image*=amount;
					}
					//seems to be outputting correctly but not saving frames
					return &image;
				}
				return nullptr;
			};
			Video_output* clone(map<string,string> &_settings) { return new Video_output(_settings);};

		private:
			float silence_threshold;
			float start_silence;
			float start_peak;
			float end_silence;
			float end_peak;
	};
	class Video_feedback: public Image_node {
		public:
			Video_feedback(){
				title="Video feedback";
				description="Repeats output of the last frame";
				feedback=nullptr;
				NODEID="78edfa28-2d0a-11e3-86c7-9f266fabb10c";
			};
			Video_feedback(map<string,string> &settings):Video_feedback() {
				base_settings(settings);
			};
			~Video_feedback(){ };
			void set_feedback(Image *iptr){
				feedback=iptr;
			}
			Image *output(const Frame_spec &frame){
				if (feedback->RGBdata){
					return feedback;
				}
				image.setup(frame.w,frame.h);
				image.clear();
				return &image;
			};
			Video_feedback* clone(map<string,string> &_settings) { return new Video_feedback(_settings);};
		private:
			Image *feedback;
	};
		//-------------------------------------------------------------------
	class Node_factory{
		public:
			Node_factory();
			~Node_factory(){
				for (auto t:type_map) delete t.second;
			}
			void add_type(string type,Node* proto){
				type_map[type]=proto;
				type_map[type]->type=type;
			};
			void add_type(string type,Node* proto,vector<Rotor::Node*> &category){
				add_type(type,proto);
				category.push_back(proto);
			};
			Node *create(map<string,string> &settings){
				if (settings.find("type")!=settings.end()) {
					if (type_map.find(settings["type"])!=type_map.end()) {
						return type_map[settings["type"]]->clone(settings);
					}
				}
				return NULL;
			};
			bool list_node(const string &t,xmlIO XML);
			bool list_node(const string &t,Json::Value &JSON);
			void list_node(Rotor::Node* type,xmlIO XML,int i=0);
			Json::Value list_node(Rotor::Node* type);
			void list_nodes(xmlIO XML);
			void list_nodes(Json::Value &JSON);
			void list_categories(xmlIO XML);
			void list_categories(Json::Value &JSON);
		private:
			map<string,Node*> type_map;
			map<string,vector<Rotor::Node*> > category;
	};
}

#endif