path: root/rotord/src/rotor.h

#ifndef ROTOR_H
#define ROTOR_H

/*------------------------

Definitions of base classes and types for rotor rendering graph

-------------------------*/

/*------------------------

Aims

-realtime, what does this entail?

	patchbay must be working

	incremental graph editing - examine new graph and make alterations

	window manager

	network stream? rtp?

	realtime architecture - maybe a tiny amount of buffering

	framerate limiting

-undefined number of message types - a mssage is a pointer to a struct

-documentation embedded in nodes

-------------------------*/

/*------------------------

Requirements

-stretch a video to fit a segment

	either use a signal as a playhead {seconds|stretched

	or deliver segment information with a signal


-thoughts on styles

	we can define the properties of a successful style

	1. it has to work with a selection of music in 1 or more genres
	2. it has to work as an edited video
	3, it must not 'look like the same video' with different inputs

	the styles we are working on so far are probably trying to be too original.

	we are in the business of selling commercial art. its about giving people what they know
	they like, not about expanding their consciousnesses.

	can we make a decent techno style that creates something that young/ aspiring techno musicians will relate to?

	can we make a style themed around a fashion i.e goth/emo

	can we make a style about a place or a thing with an image i.e americana, poolside

	this is why I think Sasha's video is the most succesful attempt so far- it takes a
	recognisable style that people understand and deconstructs it.

-------------------------*/


#include <unordered_map>
#include <deque>
#include <math.h>
#include <memory>
#include <sys/time.h>
#include <iostream>

#include <json/json.h>
#include <rsvg.h>

#include "Poco/Net/HTTPResponse.h"
#include "Poco/Logger.h"
#include "Poco/File.h"
#include "Poco/Path.h"
#include "Poco/Base64Encoder.h"
#include "Poco/FileStream.h"
#include "Poco/CountingStream.h"
#include "Poco/StreamCopier.h"

#include "xmlIO.h"
#include "utils.h"
#include "cvimage.h"
#include "libavwrapper.h"

//using namespace cv;
namespace Rotor {
	//forward declarations
	class Node;
	class Signal_node;
	class Image_node;
	class Parameter;

	class Audio_frame{
		public:
			Audio_frame(uint16_t *_samples,int _channels,int _numsamples){
				samples=_samples;
				channels=_channels;
				numsamples=_numsamples;
			}
			uint16_t *samples;
			int channels,numsamples;
	};
	class Time_spec{
		public:
			Time_spec(){};
			Time_spec(float _time,float _framerate,float _duration,Audio_frame *_audio=nullptr){ time=_time; framerate=_framerate; duration=_duration; audio=_audio;};
			float time; //num/denom ?
			float framerate;
			float duration;
			Audio_frame *audio;
			Time_spec lastframe() const{
				return Time_spec(time-(1.0f/framerate),framerate,duration);
			}
			int frame(){
				return (int)((time*framerate)+0.5); //rounded to the nearest frame
			}
	};
	class Frame_spec: public Time_spec{
		public:
			Frame_spec(float _time,float _framerate,float _duration,int _w,int _h,Audio_frame *_audio=nullptr)
			{ time=_time; framerate=_framerate; duration=_duration; w=_w; h=_h;audio=_audio;};
			Frame_spec(int _frame,float _framerate,float _duration,int _w,int _h,Audio_frame *_audio=nullptr)
			{ time=((float)_frame)/_framerate; framerate=_framerate; duration=_duration; w=_w; h=_h;audio=_audio;};
			int h,w;
			Frame_spec lastframe() const{
				return Frame_spec(time-(1.0f/framerate),framerate,duration,w,h);
			}
	};
	class Colour{
		public:
			Colour(){
				r=g=b=0;
			}
			Colour(int c){
				r=c&0xFF;
				g=(c&0xFF00)>>8;
				b=(c&0xFF0000)>>16;
			}
			Colour(std::string s){
				r=(uint8_t)hexToChar(s.substr(0,2));
				g=(uint8_t)hexToChar(s.substr(2,2));
				b=(uint8_t)hexToChar(s.substr(4,2));
			}
			float Rfloat(){
				return ((float)r)/255.0f;
			}
			float Gfloat(){
				return ((float)g)/255.0f;
			}
			float Bfloat(){
				return ((float)b)/255.0f;
			}
			uint8_t r,g,b;
	};
	class Command_response{
		public:
		    Command_response() { status=Poco::Net::HTTPResponse::HTTP_OK; }
		    std::string description;
		    Poco::Net::HTTPResponse::HTTPStatus status;
	};
	class Input{
		public:
			Input(const string &_desc,const string &_title): connection(nullptr),description(_desc),title(_title){};
			Node* connection;
			string description;
			string title;
	};
	class Image_input: public Input{
		public:
			virtual ~Image_input(){};
			bool connect(Node *source);
			Image_input(const string &_desc,const string &_title,Node* _connect): Input(_desc,_title){
				connect(_connect);
			};
			Image* get(const Frame_spec& time);
	};
	class Signal_input: public Input{
		public:
			virtual ~Signal_input(){};
			bool connect(Node *source);
			Signal_input(const string &_desc,const string &_title,Node* _connect): Input(_desc,_title){
				connect(_connect);
			};
			float get(const Time_spec& time);
	};
	class Parameter: public Signal_input{
		public:
			virtual ~Parameter(){};
			void init(const float &_val){
				value=_val;
			}
			Parameter(const string &_type,const string &_desc,const string &_title,float _value,float _min,float _max,float _step,Node* _connect): Signal_input(_desc,_title,_connect),value(_value),min(_min),max(_max),step(_step),type(_type){};
			float value,min,max,step;
			float get(const Time_spec& time);
			string type;
	};
	class Attribute{ //description of a static attribute which can be an enumerated string array
		public:
			virtual ~Attribute(){};
			Attribute(const string &_desc,const string &_title,const string &_value,std::vector<std::string> _vals={},std::string _type="string"): description(_desc),title(_title),value(_value),intVal(0),type(_type){
				vals=_vals;
				init(_value);
			};
			void init(const string &_key){ //inits int value from set::string vals index
				value=_key;
				std::vector<std::string>::iterator it= find(vals.begin(),vals.end(),value);
				if (it!=vals.end()){
					intVal = std::distance(vals.begin(),it)+1; //using 1-index for enums
				}
				else intVal=0;
			}
			void init(const std::vector<std::string> &_vals){
				vals=_vals;
				string s;
				for (auto v:vals) s=s+v+" ";
				cerr<<"array attribute "<<title<<": "<<s<<endl;
			}
			string description,title,value;
			int intVal;
			string type;
			std::vector<std::string> vals;

	};
	class Node{
		public:
			Node(){duplicate_inputs=false;};
			virtual Node* clone(map<string,string> &_settings)=0; //pure virtual
			virtual ~Node(){
				for (auto a: attributes) delete a.second;
				for (auto p: parameters) delete p.second;
				for (auto s: inputs) delete s;
			};
			vector<Signal_input*> inputs;		//simple node can have signal inputs, output depends on node type
			unordered_map<string,Parameter*> parameters;		//linked parameters can convert from settings to inputs
			unordered_map<string,Attribute*> attributes;
			void create_signal_input(const string &_desc,const string &_title,Node* _connect=nullptr ) {
				inputs.push_back(new Signal_input(_desc,_title,_connect));
			};
			void create_parameter(const string &_name,const string &_type,const string &_desc,const string &_title,float _value=1.0f,float _min=0.0f,float _max=0.0f,float _step=0.0f,Node* _connect=nullptr) {
				parameters[_name]=new Parameter(_type,_desc,_title,_value,_min,_max,_step,_connect);
			};
			void create_attribute(const string &_attr,const string &_desc,const string &_title,const string &_value,std::vector<std::string> _vals={},std::string _type="string") {
				if (_vals.size()&&_type!="array") _type="enum"; //hack for incomplete attribute types
				attributes[_attr]=new Attribute(_desc,_title,_value,_vals,_type);
			};
			void create_attribute(string *alias,const string &_attr,const string &_desc,const string &_title,const string &_value,std::vector<std::string> _vals={},std::string _type="string") {
				create_attribute(_attr,_desc,_title,_value,_vals,_type);
				alias=&(attributes[_attr]->value);
			};
			void create_attribute(int *alias,const string &_attr,const string &_desc,const string &_title,const string &_value,std::vector<std::string> _vals={},std::string _type="string") {
				create_attribute(_attr,_desc,_title,_value,_vals,_type);
				alias=&(attributes[_attr]->intVal);
			};
			virtual void init_attribute(const string &attr){
			};
			string description;
			string type;
			string ID;
			string UID;
			string title;
			bool duplicate_inputs;
			string find_setting(map<string,string> &settings,string key,string def=""){ if (settings.find(key)!=settings.end()) return settings[key]; else return def;};
			float find_setting(map<string,string> &settings,string key,float def){ if (settings.find(key)!=settings.end()) return toFloat(settings[key]); else return def;};
			int find_setting(map<string,string> &settings,string key,int def){ if (settings.find(key)!=settings.end()) return toInt(settings[key]); else return def;};
			void base_settings(map<string,string> &settings) {
				description=find_setting(settings,"description");
				type=find_setting(settings,"type");
				ID=find_setting(settings,"ID");
				title=find_setting(settings,"title");
				for (auto a: attributes){
					if (find_setting(settings,a.first,"")!="") {
						attributes[a.first]->init(find_setting(settings,a.first,""));
						cerr<<"setting attribute '"<<a.first<<"' to "<<find_setting(settings,a.first,"")<<" (index: "<<attributes[a.first]->intVal<<")"<<endl;
					}
				}
				for (auto p: parameters){
					if (find_setting(settings,p.first,"")!="") {
						parameters[p.first]->init(find_setting(settings,p.first,0.0f));
						cerr<<"setting parameter '"<<p.first<<"' to "<<find_setting(settings,p.first,0.0f)<<endl;
					}
				}
			}
			void update(const Time_spec &time){
				for (auto p: parameters){
					p.second->get(time);
				}
			}
			void set_parameter(const std::string &key,const std::string &value){
				if (parameters.find(key)!=parameters.end())	parameters[key]->value=toFloat(value);
			};
		};
	class Signal_node: public Node{
	public:
		virtual ~Signal_node(){};
		const float get_output(const Time_spec &time) {
			update(time);
			return output(time);
		};
		const float get_time_for_value(const float &value) {
			return 0.0f;
		};
		virtual const float output(const Time_spec &time) { return 0.0f; };
	};
	class Image_node: public Node{
		public:
			virtual ~Image_node(){
				for (auto i: image_inputs) delete i;
			};
			vector<Image_input*> image_inputs;			//image node also has image inputs and outputs
			void create_image_input(const string &_title,const string &_desc,Node* _connect=nullptr) {
				image_inputs.push_back(new Image_input(_desc,_title,_connect));
			};
			Image *get_image_output(const Frame_spec &frame) {
				image.setup(frame.w,frame.h);
				update((Time_spec)frame);
				return output(frame);
			}
			virtual Image *output(const Frame_spec &frame)=0;
			Image image;
			private:
				float image_time; //? could be used to detect image reuse?

	};
	class LUT {
			LUT(){
				lut=nullptr;
			};
			~LUT(){if (lut) { delete[] lut;} };
			void generate(float black_in,float white_in,float black_out,float white_out,float gamma){
				//can check here if anything has changed
				if (lut) delete[] lut;
				lut=new unsigned char[256];
				float fltmax=(255.0f/256.0f);
				for (int i=0;i<256;i++){
					lut[i]=(unsigned char)(((pow(min(fltmax,max(0.0f,(((((float)i)/256.0f)-black_in)/(white_in-black_in)))),(1.0/gamma))*(white_out-black_out))+black_out)*255.0f);
				}
			}
			void apply(const cv::Mat& in,cv::Mat &out){ //facility to apply to other images for inherited classes
				out.create(in.rows,in.cols,in.type());
				for (int i=0;i<in.rows*in.cols*in.channels();i++){
					out.data[i]=lut[in.data[i]];
				}
			}
		protected:
			unsigned char *lut;
	};
	//actual nodes-------------------------------------------------
	#define CYCLER_cut 1
	#define CYCLER_mix 2
	#define CYCLER_abs 1
	#define CYCLER_rel 2
	#define CYCLER_stretch 3
	//new mode that remaps sequence length to segments
	//combined with a video mode that maps video length to duration
	//time remapper object
	//takes a signal and turns it into a new timeline
	//
	//to make a movie stretch to a segment..
	//modulus a segmenter into the position controller of a video player
	//
	//how else?
	//q: how to create a relative timeline from a segment?
	//a: you need the begiining and end points to calculate the duration
	//
	//q: where do we have this info/ where do we get it?
	//a: only in the node that generated it?
	//
	//q: what would it take to get this info from place to place
	//a: would need to send featuresets down the line with

	//want to make a limted length transition - i.e. the 1st 2 seconds
	//this creates a similar problem, how do we know the duration of the segment and where we are in it
	//could there be fast way to interrogate a signal in reverse, i.e lookup the time for which a signal reaches a certain value?
	//from the POV of a feature set, yes, we could
	//from the POV of lots of other node types, no it would be hard
	//support an error code? return 0.0 if the node doesn't support reverse query
	 class Video_cycler: public Image_node {
		public:
			Video_cycler(){
				create_image_input("Image input","Image input");
				create_signal_input("Selector","Selector input");
				create_attribute("mode","Cycling mode {cut|mix}","Mode","cut",{"cut","mix"});
				create_attribute("length_mode","Transition length mode","Length mode","seconds",{"seconds","fraction"});
				//create_attribute("time_mode","Time mode {abs|rel}","time mode","abs",{"abs","rel","stretch"});
				create_parameter("transition_length","number","transition length","Transition length",-1.0f,0.0f,0.0f);
				title="Video cycler";
				description="Cycles through video inputs according to selector signal";
				duplicate_inputs=true;
				UID="93dd9d76-2d09-11e3-9589-5bbbeea1b304";
				segment=0;
				segment_start=0.0f;
				lastframe=0;
			}
			Video_cycler(map<string,string> &settings):Video_cycler() {
				base_settings(settings);
			};
			~Video_cycler(){};
			bool load(const string &filename);
			Image *output(const Frame_spec &frame){
				if (attributes["mode"]->intVal==CYCLER_mix&&image_inputs.size()>1){
					int im1=((int)inputs[0]->get((Time_spec)frame))%image_inputs.size();
					int im2=(im1+1)%image_inputs.size();

					//work out timing
					float ph=inputs[0]->get((Time_spec)frame);
					int seg=((int)ph);
					int thisframe=((Time_spec)frame).frame();
					if(thisframe==lastframe||thisframe==lastframe+1){
						if (segment!=seg){
							//start of new segment
							cerr<<"Video_cycler: segment "<<seg<<" started: "<<((Time_spec)frame).time<<endl;
							segment=seg;
							segment_start=((Time_spec)frame).time;
						}
					}
					else {
						if (thisframe==0){
							segment_start=0.0f;
							segment=seg;
						}
						else {
							//find segment start
							Time_spec testframe=(Time_spec)frame.lastframe();
							while ((int)inputs[0]->get(testframe)==seg&&testframe.frame()>0){
								testframe=testframe.lastframe();

							}
							segment_start=testframe.time;
							segment=seg;
						}
					}
					float wanted=(((Time_spec)frame).time-segment_start);

					float f=fmod(inputs[0]->get((Time_spec)frame),1.0f);
					Image *in1=image_inputs[im1]->get(frame);
					if (in1){
						Image *in2=image_inputs[im2]->get(frame);
						if (in2){
							image=(*in1);
							image*=(1.0f-f);
							Image i2=(*in2);
							i2*=f;
							image+=i2;
							return &image;
						}
						return in1;
					}
					return nullptr;
				}
				//cut mode
				for (uint32_t i=0;i<image_inputs.size();i++){
					int whichinput=((int)inputs[0]->get((Time_spec)frame)+i)%image_inputs.size();
					Image *in=image_inputs[whichinput]->get(frame);
					if (in) return in;
				}
				return nullptr;
			}
			Video_cycler* clone(map<string,string> &_settings) { return new Video_cycler(_settings);};
		private:
			float segment_start;
			int segment;
			int lastframe;
	};
#define VIDEOFRAMES_frame 1
#define VIDEOFRAMES_blend 2
#define VIDEOTIME_play 1
#define VIDEOTIME_stretch 2
	class Base_video: public Image_node {
		public:
			Base_video(){
				create_signal_input("playhead","Playhead");
				//floating point control of playback time
				//if signal is connected it overrides normal playback
				//time_mode dictates whether control is seconds, or duration
				create_parameter("speed","number","video playback speed","Speed",1.0f,0.0f,0.0f);
				create_parameter("framerate","number","framerate override","Frame rate",0.0f,0.0f,0.0f);
				create_attribute("frame_mode","frame mode","Frame mode","frame",{"frame","blend"});
				create_attribute("time_mode","time mode","Time mode","play",{"play","stretch"});
				create_attribute("media_id","media_id","media_id","media_id"); //for rotorW
				UID="e92255a0-447a-11e3-b0ce-3fc7ff4bdac9";
				isLoaded=false;
			};
			bool load(const string &filename){
				Poco::Logger& logger = Poco::Logger::get("Rotor");
			    isLoaded=player.open(filename);
				if (isLoaded){
					logger.information("libav::decoder loaded "+filename+": "\
						+toString(player.get_number_frames())+" frames, "\
						+toString(player.get_framerate())+" fps, "\
						+toString(player.get_width())+"x"+toString(player.get_height())\
						+", channels:"+toString(player.get_number_channels()));
					lastframe=-2;
					return true;
			   	}
				logger.error("libav::decoder failed to load "+filename);
			    return false;
			}
			bool get_frame(float wanted,const Frame_spec &frame){
				if (attributes["frame_mode"]->intVal==VIDEOFRAMES_blend){
					if (((int)wanted)!=Base_video::lastframe){
						//get a new pair of frames possibly by switching the next one
						//darn peculiar, as if copying wasn't actually copying
						if ((Base_video::lastframe==(((int)wanted)-1))&&(in2.w>0)) {
							in1=in2;
						}
						else {
							player.fetch_frame(frame.w,frame.h,(int)wanted);
							//use a temp image because setup_fromRGB just copies pointer
							temp.setup_fromRGB(frame.w,frame.h,player.frame->Data[0],player.frame->Linesize[0]-(frame.w*3));
							in1=temp;
						}
						player.fetch_frame(frame.w,frame.h,((int)wanted+1)%max(1,player.get_number_frames()));
						temp.setup_fromRGB(frame.w,frame.h,player.frame->Data[0],player.frame->Linesize[0]-(frame.w*3));
						in2=temp;
						lastframe=wanted;
					}
					float amt=1.0f-(wanted-((int)wanted));
					//cout<<"video loader time:"<<frame.time<<" frames "<<((int)wanted)<<" (x"<<amt<<"),"<<(((int)wanted+1)%max(1,player.get_number_frames()))<<endl;
					image=in1;
					image*=amt;
					//Image in2t=in2; //DOES NOT WORK, copies pointer by assignation
					in2t=in2;
					in2t*=(1.0f-amt);
					image+=in2t;
				}
				else {
					if (((int)wanted)!=Base_video::lastframe){
						if (!player.fetch_frame(frame.w,frame.h,((int)wanted))) { //seek fail
							Poco::Logger& logger = Poco::Logger::get("Rotor");
							logger.error("Video_loader failed to seek frame "+toString(wanted)+" of "+attributes["filename"]->value);

							if (image.w>0) return &image; //just return the previous frame if possible
							else return nullptr;
						}
						image.setup_fromRGB(frame.w,frame.h,player.frame->Data[0],player.frame->Linesize[0]-(frame.w*3));
					}
				}
				return true;
			}
			Image *output(const Frame_spec &frame){
				if (isLoaded){
					float clipframerate=(parameters["framerate"]->value==0.0f?player.get_framerate():parameters["framerate"]->value);
					float clipspeed=(clipframerate/frame.framerate)*parameters["speed"]->value;
					float wanted=0.0f;
					if (inputs[0]->connection) {
					//using playhead
					//should speed affect it?
					//if you want absolute control then you just want absolute control?
						switch (attributes["frame_mode"]->intVal){
							case VIDEOTIME_play:
								wanted=fmod(inputs[0]->get((Time_spec)frame)*frame.framerate*clipspeed,(float)player.get_number_frames());
								break;
							case VIDEOTIME_stretch:
								wanted=fmod(fmod(inputs[0]->get((Time_spec)frame),1.0f)*((float)player.get_number_frames())*clipspeed,(float)player.get_number_frames());
								break;
						}
					}
					else {
						switch (attributes["frame_mode"]->intVal){
							case VIDEOTIME_play:
								wanted=fmod(frame.time*frame.framerate*clipspeed,(float)player.get_number_frames());
								break;
							case VIDEOTIME_stretch:
								wanted=fmod((frame.time/frame.duration)*((float)player.get_number_frames())*clipspeed,(float)player.get_number_frames());
								break;
						}
					}
					if (get_frame(wanted,frame)) return &image;
				}
			    return nullptr;
			};
		protected:
			Image in1,in2,in2t,temp; //for blend frames;
			libav::video_decoder player;
			int lastframe;
			bool isLoaded;
	};
	//relative timelines used to stretch video
	//1. make a video position input for video node - seconds and stretch modes
	//2. video mode that maps to duration- timeline remapping from cycler and others
	class Video_loader: public Base_video {
		public:
			Video_loader(){
				create_attribute("filename","name of video file to load","File name","");
				create_attribute("media_id","media_id","media_id","media_id"); //for rotorW
				create_attribute("media_path","media_path","media_path","media_path"); //for rotorW
				title="Video loader";
				description="Loads a video file";
				UID="5b64b8ca-2d0a-11e3-92ed-4b7420b40040";
			};
			Video_loader(map<string,string> &settings): Video_loader() {
				base_settings(settings);
				for (auto a:attributes) cerr<<"Video loader: "<<a.first<<" set to "<<a.second->value<<endl;
				if (attributes["filename"]->value!="") {
					isLoaded=load(find_setting(settings,"media_path","")+attributes["filename"]->value);
				}

			};
			~Video_loader(){};
			void init_attribute(const string &attr){
				if (attr=="filename") {
					isLoaded=load(attributes["media_path"]->value+attributes[attr]->value);
				}
			};
			Video_loader* clone(map<string,string> &_settings) { return new Video_loader(_settings);};
			bool isLoaded;
		private:
			string *filename;
	};
	class Video_bank: public Base_video {
		public:
			//manage a bank of video inclusing transitions
			//blend mode etc
			//there may be a total of 4 frames required
			//and 2 videos open
			//is there any way to re-use the parts which prepare frames
			Video_bank(){
				create_attribute("filenames","names of video files to load","File names","",{},"array");
				UID="73616e66-4306-11e3-981e-74d02b29f6a6";
				title="Video bank";
				description="Loads a banks of video files";
				clip_loaded=-1;
				isLoaded=false;
				segment=0;
				segment_start=0.0f;
				lastframe=0;
			};
			Video_bank(map<string,string> &settings): Video_bank() {
				base_settings(settings);
				media_path=find_setting(settings,"media_path","");
			};
			Image *output(const Frame_spec &frame){
				if (!inputs[0]->connection){ //default to single loader
					if (!isLoaded){
						if (load(media_path+attributes["filenames"]->vals[0] )) {
							isLoaded=true;
						}
						else {
							isLoaded=false;
						}
					}
					if (isLoaded){
						return Base_video::output(frame);
					}
					return nullptr;
				}
				if (attributes["filenames"]->vals.size()){
					float ph=inputs[0]->get((Time_spec)frame);
					int seg=((int)ph);
					int wv=seg%attributes["filenames"]->vals.size();
					ph=seg==0?ph:fmod(ph,seg);
					if (clip_loaded!=wv){
						if (load(media_path+attributes["filenames"]->vals[wv] )) {
							clip_loaded=wv;
							isLoaded=true;
						}
						else {
							//cerr<<"Video bank could not load "<<(media_path+attributes["filenames"]->vals[wv])<<endl;
							clip_loaded=-1;
						}
					}
					if (isLoaded){
						int tests=0;
						int wanted=0.0f;
						int thisframe=((Time_spec)frame).frame();
						float clipframerate=(parameters["framerate"]->value==0.0f?player.get_framerate():parameters["framerate"]->value);
						float clipspeed=(clipframerate/frame.framerate)*parameters["speed"]->value;
						switch (attributes["time_mode"]->intVal){
							case VIDEOTIME_play:
								if(thisframe==lastframe||thisframe==lastframe+1){
									if (segment!=seg){
										//start of new segment
										cerr<<"Video_bank: segment "<<seg<<" started: "<<((Time_spec)frame).time<<endl;
										segment=seg;
										segment_start=((Time_spec)frame).time;
									}
								}
								else {
									if (thisframe==0){
										segment_start=0.0f;
										segment=seg;
									}
									else {
										//find segment start
										Time_spec testframe=(Time_spec)frame.lastframe();
										while ((int)inputs[0]->get(testframe)==seg&&testframe.frame()>0){
											testframe=testframe.lastframe();

										}
										segment_start=testframe.time;
										segment=seg;
									}
								}
								wanted=(((Time_spec)frame).time-segment_start)*clipspeed*frame.framerate;
								lastframe=thisframe;
								break;
							case VIDEOTIME_stretch:
								wanted=ph*player.get_number_frames();
								break;
						}
						if (Base_video::get_frame(wanted,frame)) return &image;
					}

				}
				return nullptr;
			}
			Video_bank* clone(map<string,string> &_settings) { return new Video_bank(_settings);};
			~Video_bank(){

			};
		private:
			int clip_loaded;
			string media_path;
			float segment_start;
			int segment;
			int lastframe;
	};
	class Video_output: public Image_node {
	//Video_output 'presents' the output movie. Aspect ratio, bars, fadein/fadeout would happen here
		public:
			Video_output(){
				create_image_input("image to output","Image input");
				create_attribute("begin_mode","mode to begin movie","Begin mode","cut",{"cut","blank silence","fade peak"});
				create_attribute("end_mode","mode to end movie","End mode","cut",{"cut","blank silence","fade peak"});
				title="Video output";
				description="Output to video";
				start_silence=start_peak=end_silence=end_peak=-1.0f;
				silence_threshold=0.01f;
				UID="693d2220-2d0a-11e3-9312-232908c3cc33";
			};
			Video_output(map<string,string> &settings):Video_output() {
				base_settings(settings);
			};
			~Video_output(){ };
			void create_envelope(const vector<float> &audio){
				if (audio.size()){
					uint32_t i=1;
					while (i<audio.size()&&audio[i]<silence_threshold) i++;
					start_silence=((float)i)/audio.size();
					while (i<audio.size()&&audio[i]>audio[i-1]) i++;
					start_peak=((float)i-1)/audio.size();
					i=audio.size();
					while (i>0&&audio[i]<silence_threshold) i--;
					end_silence=((float)i)/audio.size();
					while (i<audio.size()&&audio[i]>audio[i+1]) i--;
					end_peak=((float)i+1)/audio.size();
					cerr<<"Video_output sound envelope: silence - "<<start_silence<<" : peak "<<start_peak<<" : peak "<<end_peak<<" silence - "<<end_silence<<endl;
				}
				else cerr<<"Video_output sound envelope: no data"<<endl;
			}
			Image *output(const Frame_spec &frame){
				Image *in=image_inputs[0]->get(frame);
				if (in){
					//make copy of the image, for feedback
					//optimise?
					float amount=1.0f;
					float track_time=frame.time/frame.duration;
					if (attributes["begin_mode"]->value=="fade peak"||attributes["begin_mode"]->value=="blank silence"){
						if (track_time<start_silence){
							amount=0.0f;
						}
						else if (track_time<start_peak&&attributes["begin_mode"]->value=="fade peak"&&start_peak>start_silence){
							amount = (track_time-start_silence)/(start_peak-start_silence);
						}
					}
					if (attributes["end_mode"]->value=="fade peak"||attributes["end_mode"]->value=="blank silence"){
						if (track_time>end_silence){
							amount=0.0f;
						}
						else if (track_time>end_peak&&attributes["end_mode"]->value=="fade peak"&&end_silence>end_peak){
							amount = 1.0f-((track_time-end_peak)/(end_silence-end_peak));
						}
					}
					if (amount<(1.0f/254.0f)){
						image.clear();
					}
					image=(*in);
					if (amount<1.0f){
						image*=amount;
					}
					//seems to be outputting correctly but not saving frames
					return &image;
				}
				return nullptr;
			};
			Video_output* clone(map<string,string> &_settings) { return new Video_output(_settings);};

		private:
			float silence_threshold;
			float start_silence;
			float start_peak;
			float end_silence;
			float end_peak;
	};
	class Video_feedback: public Image_node {
		public:
			Video_feedback(){
				title="Video feedback";
				description="Repeats output of the last frame";
				feedback=nullptr;
				UID="78edfa28-2d0a-11e3-86c7-9f266fabb10c";
			};
			Video_feedback(map<string,string> &settings):Video_feedback() {
				base_settings(settings);
			};
			~Video_feedback(){ };
			void set_feedback(Image *iptr){
				feedback=iptr;
			}
			Image *output(const Frame_spec &frame){
				if (feedback->RGBdata){
					return feedback;
				}
				image.setup(frame.w,frame.h);
				image.clear();
				return &image;
			};
			Video_feedback* clone(map<string,string> &_settings) { return new Video_feedback(_settings);};
		private:
			Image *feedback;
	};
		//-------------------------------------------------------------------
	class Node_factory{
		public:
			Node_factory();
			~Node_factory(){
				for (auto t:type_map) delete t.second;
			}
			void add_type(string type,Node* proto){
				type_map[type]=proto;
				type_map[type]->type=type;
			};
			void add_type(string type,Node* proto,vector<Rotor::Node*> &category){
				add_type(type,proto);
				category.push_back(proto);
			};
			Node *create(map<string,string> &settings){
				if (settings.find("type")!=settings.end()) {
					if (type_map.find(settings["type"])!=type_map.end()) {
						return type_map[settings["type"]]->clone(settings);
					}
				}
				return NULL;
			};
			bool list_node(const string &t,xmlIO XML);
			bool list_node(const string &t,Json::Value &JSON);
			void list_node(Rotor::Node* type,xmlIO XML,int i=0);
			Json::Value list_node(Rotor::Node* type);
			void list_nodes(xmlIO XML);
			void list_nodes(Json::Value &JSON);
			void list_categories(xmlIO XML);
			void list_categories(Json::Value &JSON);
		private:
			map<string,Node*> type_map;
			map<string,vector<Rotor::Node*> > category;
	};
}

#endif