path: root/rotord/src/rotor.h

#ifndef ROTOR_H
#define ROTOR_H

/*------------------------

Definitions of base classes and types for rotor rendering graph

-------------------------*/

#include <unordered_map>
#include <deque>
#include <math.h>
#include <memory>
#include <sys/time.h>
#include <iostream>

#include <json/json.h>

#include "Poco/Net/HTTPResponse.h"
#include "Poco/Logger.h"
#include "Poco/File.h"
#include "Poco/Path.h"
#include "Poco/Base64Encoder.h"
#include "Poco/FileStream.h"
#include "Poco/CountingStream.h"
#include "Poco/StreamCopier.h"

#include "xmlIO.h"
#include "utils.h"
#include "cvimage.h"
#include "libavwrapper.h"

//using namespace cv;
namespace Rotor {
	//forward declarations
	class Node;
	class Signal_node;
	class Image_node;
	class Parameter;

	class Audio_frame{
		public:
			Audio_frame(uint16_t *_samples,int _channels,int _numsamples){
				samples=_samples;
				channels=_channels;
				numsamples=_numsamples;
			}
			uint16_t *samples;
			int channels,numsamples;
	};
	class Time_spec{
		public:
			Time_spec(){};
			Time_spec(float _time,float _framerate,float _duration,Audio_frame *_audio=nullptr){ time=_time; framerate=_framerate; duration=_duration; audio=_audio;};
			float time; //num/denom ?
			float framerate;
			float duration;
			Audio_frame *audio;
			Time_spec lastframe() const{
				return Time_spec(time-(1.0f/framerate),framerate,duration);
			}
			int frame(){
				return (int)((time*framerate)+0.5); //rounded to the nearest frame
			}
	};
	class Frame_spec: public Time_spec{
		public:
			Frame_spec(float _time,float _framerate,float _duration,int _w,int _h,Audio_frame *_audio=nullptr)
			{ time=_time; framerate=_framerate; duration=_duration; w=_w; h=_h;audio=_audio;};
			Frame_spec(int _frame,float _framerate,float _duration,int _w,int _h,Audio_frame *_audio=nullptr)
			{ time=((float)_frame)/_framerate; framerate=_framerate; duration=_duration; w=_w; h=_h;audio=_audio;};
			int h,w;
			Frame_spec lastframe(){
				return Frame_spec(time-(1.0f/framerate),framerate,duration,w,h);
			}
	};
	class Colour{
		public:
			Colour(){
				r=g=b=0;
			}
			Colour(int c){
				r=c&0xFF;
				g=(c&0xFF00)>>8;
				b=(c&0xFF0000)>>16;
			}
			Colour(std::string s){
				r=(uint8_t)hexToChar(s.substr(0,2));
				g=(uint8_t)hexToChar(s.substr(2,2));
				b=(uint8_t)hexToChar(s.substr(4,2));
			}
			float Rfloat(){
				return ((float)r)/255.0f;
			}
			float Gfloat(){
				return ((float)g)/255.0f;
			}
			float Bfloat(){
				return ((float)b)/255.0f;
			}
			uint8_t r,g,b;
	};
	class Command_response{
		public:
		    Command_response() { status=Poco::Net::HTTPResponse::HTTP_OK; }
		    std::string description;
		    Poco::Net::HTTPResponse::HTTPStatus status;
	};
	class Input{
		public:
			Input(const string &_desc,const string &_title): connection(nullptr),description(_desc),title(_title){};
			Node* connection;
			string description;
			string title;
	};
	class Image_input: public Input{
		public:
			virtual ~Image_input(){};
			bool connect(Node *source);
			Image_input(const string &_desc,const string &_title,Node* _connect): Input(_desc,_title){
				connect(_connect);
			};
			Image* get(const Frame_spec& time);
	};
	class Signal_input: public Input{
		public:
			virtual ~Signal_input(){};
			bool connect(Node *source);
			Signal_input(const string &_desc,const string &_title,Node* _connect): Input(_desc,_title){
				connect(_connect);
			};
			float get(const Time_spec& time);
	};
	class Parameter: public Signal_input{
		public:
			virtual ~Parameter(){};
			void init(const float &_val){
				value=_val;
			}
			Parameter(const string &_type,const string &_desc,const string &_title,float _value,float _min,float _max,Node* _connect): Signal_input(_desc,_title,_connect),value(_value),min(_min),max(_max),type(_type){};
			float value,min,max;
			float get(const Time_spec& time);
			string type;
	};
	class Attribute{ //description of a static attribute which can be an enumerated string array
		public:
			virtual ~Attribute(){};
			Attribute(const string &_desc,const string &_title,const string &_value,std::vector<std::string> _vals={}): description(_desc),title(_title),value(_value),intVal(0){
				vals=_vals;
				init(_value);
			};
			void init(const string &_key){ //inits int value from set::string vals index
				value=_key;
				std::vector<std::string>::iterator it=it = find(vals.begin(),vals.end(),value);
				if (it!=vals.end()){
					intVal = std::distance(vals.begin(),it)+1; //using 1-index for enums
				}
				else intVal=0;
			}
			string value,description,title;
			std::vector<std::string> vals;
			int intVal;
	};
	class Node{
		public:
			Node(){duplicate_inputs=false;};
			virtual Node* clone(map<string,string> &_settings)=0; //pure virtual
			virtual ~Node(){
				for (auto a: attributes) delete a.second;
				for (auto p: parameters) delete p.second;
				for (auto s: inputs) delete s;
			};
			vector<Signal_input*> inputs;		//simple node can have signal inputs, output depends on node type
			unordered_map<string,Parameter*> parameters;		//linked parameters can convert from settings to inputs
			unordered_map<string,Attribute*> attributes;
			void create_signal_input(const string &_desc,const string &_title,Node* _connect=nullptr ) {
				inputs.push_back(new Signal_input(_desc,_title,_connect));
			};
			void create_parameter(const string &_name,const string &_type,const string &_desc,const string &_title,float _value=1.0f,float _min=0.0f,float _max=0.0f,Node* _connect=nullptr) {
				parameters[_name]=new Parameter(_type,_desc,_title,_value,_min,_max,_connect);
			};
			void create_attribute(const string &_attr,const string &_desc,const string &_title,const string &_value,std::vector<std::string> _vals={}) {
				attributes[_attr]=new Attribute(_desc,_title,_value,_vals);
			};
			void create_attribute(string *alias,const string &_attr,const string &_desc,const string &_title,const string &_value,std::vector<std::string> _vals={}) {
				attributes[_attr]=new Attribute(_desc,_title,_value,_vals);
				alias=&(attributes[_attr]->value);
			};
			void create_attribute(int *alias,const string &_attr,const string &_desc,const string &_title,const string &_value,std::vector<std::string> _vals={}) {
				attributes[_attr]=new Attribute(_desc,_title,_value,_vals);
				alias=&(attributes[_attr]->intVal);
			};
			string description;
			string type;
			string ID;
			string title;
			bool duplicate_inputs;
			string find_setting(map<string,string> &settings,string key,string def=""){ if (settings.find(key)!=settings.end()) return settings[key]; else return def;};
			float find_setting(map<string,string> &settings,string key,float def){ if (settings.find(key)!=settings.end()) return toFloat(settings[key]); else return def;};
			int find_setting(map<string,string> &settings,string key,int def){ if (settings.find(key)!=settings.end()) return toInt(settings[key]); else return def;};
			void base_settings(map<string,string> &settings) {
				description=find_setting(settings,"description");
				type=find_setting(settings,"type");
				ID=find_setting(settings,"ID");
				title=find_setting(settings,"title");
				for (auto a: attributes){
					if (find_setting(settings,a.first,"")!="") {
						attributes[a.first]->init(find_setting(settings,a.first,""));
						cerr<<"setting attribute '"<<a.first<<"' to "<<find_setting(settings,a.first,"")<<" (index: "<<attributes[a.first]->intVal<<")"<<endl;
					}
				}
				for (auto p: parameters){
					if (find_setting(settings,p.first,"")!="") {
						parameters[p.first]->init(find_setting(settings,p.first,0.0f));
						cerr<<"setting parameter '"<<p.first<<"' to "<<find_setting(settings,p.first,0.0f)<<endl;
					}
				}
			}
			void update(const Time_spec &time){
				for (auto p: parameters){
					p.second->get(time);
				}
			}
			void set_parameter(const std::string &key,const std::string &value){
				if (parameters.find(key)!=parameters.end())	parameters[key]->value=toFloat(value);
			};
		};
	class Signal_node: public Node{
	public:
		virtual ~Signal_node(){};
		const float get_output(const Time_spec &time) {
			update(time);
			return output(time);
		};
		virtual const float output(const Time_spec &time) { return 0.0f; };
	};
	class Image_node: public Node{
		public:
			virtual ~Image_node(){
				for (auto i: image_inputs) delete i;
			};
			vector<Image_input*> image_inputs;			//image node also has image inputs and outputs
			void create_image_input(const string &_title,const string &_desc,Node* _connect=nullptr) {
				image_inputs.push_back(new Image_input(_desc,_title,_connect));
			};
			Image *get_image_output(const Frame_spec &frame) {
				image.setup(frame.w,frame.h);
				update((Time_spec)frame);
				return output(frame);
			}
			virtual const Image *output(const Frame_spec &frame)=0;
			Image image;
			private:
				float image_time; //? could be used to detect image reuse?

	};
	class LUT {
			LUT(){
				lut=nullptr;
			};
			~LUT(){if (lut) { delete[] lut;} };
			void generate(float black_in,float white_in,float black_out,float white_out,float gamma){
				//can check here if anything has changed
				if (lut) delete[] lut;
				lut=new unsigned char[256];
				float fltmax=(255.0f/256.0f);
				for (int i=0;i<256;i++){
					lut[i]=(unsigned char)(((pow(min(fltmax,max(0.0f,(((((float)i)/256.0f)-black_in)/(white_in-black_in)))),(1.0/gamma))*(white_out-black_out))+black_out)*255.0f);
				}
			}
			void apply(const cv::Mat& in,cv::Mat &out){ //facility to apply to other images for inherited classes
				out.create(in.rows,in.cols,in.type());
				for (int i=0;i<in.rows*in.cols*in.channels();i++){
					out.data[i]=lut[in.data[i]];
				}
			}
		protected:
			unsigned char *lut;
	};
	class Audio_processor: public Signal_node {
		public:
			virtual Audio_processor(){};
			virtual ~Audio_processor(){};
			virtual int process_frame(uint8_t *data,int samples)=0;
			virtual bool init(int _channels,int _bits,int _samples,int _rate)=0;
			virtual void cleanup()=0;
			virtual void print_summary(){};
			virtual string get_features(){};
			int channels,bits,samples,rate;
	};
	//actual nodes-------------------------------------------------
	class Time: public Signal_node {
		public:
			Time(){
				title="Time";
				description="Outputs the time in seconds as a signal";
			};
			Time(map<string,string> &settings): Time() {
				base_settings(settings);
			};
			Time* clone(map<string,string> &_settings) { return new Time(_settings);};
			const float output(const Time_spec &time) {
				return time.time;
			}
	};
	class Track_time: public Signal_node {
		public:
			Track_time(){
				title="Track time";
				description="Outputs the fraction of the track as a signal";
			};
			Track_time(map<string,string> &settings): Track_time() {
				base_settings(settings);
			};
			Track_time* clone(map<string,string> &_settings) { return new Track_time(_settings);};
			const float output(const Time_spec &time) {
				return time.time/time.duration;
			}
	};
	class At_track_time: public Signal_node {
		public:
			At_track_time(){
				create_signal_input("signal","Signal Input");
				create_parameter("time","number","Track time to evaluate","Time",0.0f);
				title="@Track time";
				description="Gets input from a different point in the track";
			};
			At_track_time(map<string,string> &settings): At_track_time() {
				base_settings(settings);
			};
			At_track_time* clone(map<string,string> &_settings) { return new At_track_time(_settings);};
			const float output(const Time_spec &time) {
				Time_spec t=Time_spec(parameters["time"]->value*time.duration,time.framerate,time.duration);
				return inputs[0]->get(t);
			}
	};
	class Signal_output: public Signal_node {
		public:
			Signal_output(){
				create_signal_input("signal","Signal Input");
				title="Signal output";
				description="Outputs a signal to xml for testing";
			};
			Signal_output(map<string,string> &settings): Signal_output() {
				base_settings(settings);
			};
			Signal_output* clone(map<string,string> &_settings) { return new Signal_output(_settings);};
			bool render(const float duration, const float framerate,string &xml_out);
			const float output(const Time_spec &time) {
					return inputs[0]->get(time);
			}
	};
	class Testcard: public Image_node {
		public:
			Testcard(){
				//internal testing node only
			};
			Testcard(map<string,string> &settings): Testcard() {
				base_settings(settings);
			};
			~Testcard(){};
			Testcard* clone(map<string,string> &_settings) { return new Testcard(_settings);};
			Image *output(const Frame_spec &frame){
				float hs=(255.0f/frame.h);
				for (int i=0;i<frame.h;i++){
					for (int j=0;j<frame.w;j++){
						image.RGBdata[(i*frame.w+j)*3]=(uint8_t)((int)((i+(frame.time*25.0f)*hs))%255);
						image.RGBdata[((i*frame.w+j)*3)+1]=(uint8_t)((int)((j+(frame.time*100.0f)*hs))%255);
						image.RGBdata[((i*frame.w+j)*3)+2]=(uint8_t)(0);
						//image->Adata[i*frame.w+j]=(uint8_t)255;
						//image->Zdata[i*frame.w+j]=(uint16_t)512; //1.0 in fixed point 8.8 bits
					}
				}
				return &image;
			}
		private:

	};
	class Invert: public Image_node {
		public:
			Invert(){
				create_image_input("Image to invert","Image input");
				create_parameter("invert","number","Invert when greater than 0.0","Negative",1.0f,0.0f,1.0f);
				title="Negative";
				description="Inverts the input picture";
			};
			Invert(map<string,string> &settings) :Invert() {
				base_settings(settings);
			};
			~Invert(){};
			Invert* clone(map<string,string> &_settings) { return new Invert(_settings);};
			Image *output(const Frame_spec &frame){
				Image *in=image_inputs[0]->get(frame);
				if (in) {
					if (parameters["invert"]->value>0.0f){
						for (int i=0;i<in->w*in->h*3;i++) {
                            image.RGBdata[i]=255-in->RGBdata[i];
                        }
                        return &image;
					}
					return in;
				}
				return nullptr;
			}
		private:
	};
	#define CYCLER_cut 1
	#define CYCLER_mix 2
	//new mode that remaps sequence length to segments
	//combined with a video mode that maps video length to duration
	//or
	 class Video_cycler: public Image_node {
		public:
			Video_cycler(){
				create_image_input("Image input","Image input");
				create_signal_input("Selector","Selector input");
				create_attribute("mode","Cycling mode {cut|mix}","Mode","cut",{"cut","mix"});
				create_attribute("time_mode","Time mode {abs|rel}","time mode","abs",{"abs","rel"});
				title="Video cycler";
				description="Cycles through video inputs according to selector signal";
				duplicate_inputs=true;
			}
			Video_cycler(map<string,string> &settings):Video_cycler() {
				base_settings(settings);
			};
			~Video_cycler(){};
			bool load(const string &filename);
			Image *output(const Frame_spec &frame){
				if (attributes["mode"]->intVal==CYCLER_mix&&image_inputs.size()>1){
					int im1=((int)inputs[0]->get((Time_spec)frame))%image_inputs.size();
					int im2=(im1+1)%image_inputs.size();
					float f=fmod(inputs[0]->get((Time_spec)frame),1.0f);
					Image *in1=image_inputs[im1]->get(frame);
					if (in1){
						Image *in2=image_inputs[im2]->get(frame);
						if (in2){
							image=(*in1);
							image*=(1.0f-f);
							Image i2=(*in2);
							i2*=f;
							image+=i2;
							return &image;
						}
						return in1;
					}
					return nullptr;
				}
				//cut mode
				for (int i=0;i<image_inputs.size();i++){
					int whichinput=((int)inputs[0]->get((Time_spec)frame)+i)%image_inputs.size();
					Image *in=image_inputs[whichinput]->get(frame);
					if (in) return in;
				}
				return nullptr;
			}
			Video_cycler* clone(map<string,string> &_settings) { return new Video_cycler(_settings);};
	};
	class Signal_colour: public Image_node {
		public:
			Signal_colour(){
				create_signal_input("Selector","Selector input");
				create_attribute("palette","palette list of web colours","Colour palette","000000");
				title="Signal colour";
				description="Cycles through a palette of background colours according to selector signal";
			};
			Signal_colour(map<string,string> &settings):Signal_colour() {
				base_settings(settings);
				for (int i=0;i<attributes["palette"]->value.size()/6;i++){
					palette.push_back(Colour(attributes["palette"]->value.substr(i*6,6)));
				}
				prevcol=-1;
			};
			~Signal_colour(){};
			Image *output(const Frame_spec &frame){
				if (palette.size()) {
					int col=((int)inputs[0]->get((Time_spec)frame))%palette.size();
					//if (col!=prevcol){ //how about when starting a new render?
						for (int i=0;i<image.w*image.h;i++){
							image.RGBdata[i*3]=palette[col].r;
							image.RGBdata[i*3+1]=palette[col].g;
							image.RGBdata[i*3+2]=palette[col].b;
						}
						prevcol=col;
					//}
					return &image;
				}
				return nullptr;
			}
			Signal_colour* clone(map<string,string> &_settings) { return new Signal_colour(_settings);};
		private:
			vector<Rotor::Colour> palette;
			int prevcol;
	};
	class Signal_greyscale: public Image_node {
		//Draws signal bars in greyscale
		public:
			Signal_greyscale(){
				create_signal_input("Signal","Signal input");
				title="Signal greyscale";
				description="Renders signal level as greyscale background";
			};
			Signal_greyscale(map<string,string> &settings):Signal_greyscale() {
				base_settings(settings);
				prevcol=-1;
			};
			~Signal_greyscale(){};
			Image *output(const Frame_spec &frame){
				uint8_t col=((uint8_t)(inputs[0]->get((Time_spec)frame)*255.0f));
				if (col!=prevcol){ //how about when starting a new render?
					for (int i=0;i<image.w*image.h*3;i++){
						image.RGBdata[i]=col;
					}
					prevcol=col;
				}
				return &image;

			}
			Signal_greyscale* clone(map<string,string> &_settings) { return new Signal_greyscale(_settings);};
		private:
			uint8_t prevcol;
	};
	#define ARITHMETIC_plus 1
	#define ARITHMETIC_minus 2
	#define ARITHMETIC_multiply 3
	#define ARITHMETIC_divide 4
	#define ARITHMETIC_modulo 5
	class Image_arithmetic: public Image_node {
		public:
			Image_arithmetic(){
				create_image_input("image input","Image input");
				create_parameter("value","number","Value or signal for operation","Value",1.0f);
				create_attribute("operator","operator for image","Operator","+",{"+","-","*","/"});
				title="Image arithmetic";
				description="Performs arithmetic on an image with a signal or value";
			};
			Image_arithmetic(map<string,string> &settings):Image_arithmetic() {
				base_settings(settings);
			}
			~Image_arithmetic(){};
			Image *output(const Frame_spec &frame){
				Image *in=image_inputs[0]->get(frame);
				if (in){
                    switch (attributes["operator"]->intVal) {
                    	case ARITHMETIC_plus:
                    		image=(*in); //could be poss without copy?
                    		image+=parameters["value"]->value;
                    		break;
						case ARITHMETIC_minus:
							image=(*in);
							image-=parameters["value"]->value;
                    		break;
						case ARITHMETIC_multiply:
							image=(*in);
							image*=parameters["value"]->value;
                    		break;
						case ARITHMETIC_divide:
							image=(*in);
							image/=parameters["value"]->value;
                    		break;
                    }
				}
				return &image;
			}
			Image_arithmetic* clone(map<string,string> &_settings) { return new Image_arithmetic(_settings);};
		private:
	};
	#define BLEND_blend 1
	#define BLEND_screen 2
	#define BLEND_multiply 3
	#define BLEND_alpha 4
	#define BLEND_wrap 5
	#define BLEND_xor 6
	class Blend: public Image_node {
		public:
			Blend(){
				create_image_input("image input 1","Image input 1");
				create_image_input("image input 2","Image input 2");
				create_parameter("amount","number","amount to blend input 2","Blend amount",0.5f,0.0f,1.0f);
				create_attribute("mode","Blend mode","Blend mode","blend",{"blend","screen","multiply","alpha","wrap","xor"});
				title ="Blend";
				description="Blend images in various modes";
			};
			Blend(map<string,string> &settings):Blend() {
				base_settings(settings);
			};
			~Blend(){};
			Blend* clone(map<string,string> &_settings) { return new Blend(_settings);};
			Image *output(const Frame_spec &frame){
				Image *in1=image_inputs[0]->get(frame);
				if (in1){
					Image *in2=image_inputs[1]->get(frame);
                	if (in2) {
						image=*(in1);
						switch(attributes["mode"]->intVal){
							case BLEND_screen:
								image+=(*in2);
								break;
							case BLEND_multiply:
								image*=(*in2);
								break;
							case BLEND_xor:
								image^=(*in2);
								break;
							case BLEND_alpha:
								image=image.alpha_blend(*in2);
								break;
							case BLEND_wrap:
								image=image.add_wrap(*in2);
								break;
							case BLEND_blend: //has to be last because of initialser of *in? go figure

								image*=(1.0f-parameters["amount"]->value);
								/* //problem here with leak
								//opencv handles not being released
								Image *in=(*in2)*parameters["amount"]->value;
								image+=(*in);
								delete in;
								*/
								in=(*in2); //removed allocator
								in*=parameters["amount"]->value;
								image+=in;
								break;
						}
						return &image;
                	}
                    //if there aren't 2 image inputs connected just return the first
                    return in1;
                }
				return nullptr;
			}
		private:
			Image in;
	};
	#define MIRROR_horiz 1
	#define MIRROR_vert 2
	#define MIRROR_horizR 3
	#define MIRROR_vertR 4
	class Mirror: public Image_node {
		public:
			Mirror(){
				create_image_input("image input","Image input");
				create_attribute("mode","Mirror mode","Mirror mode","horiz",{"horiz","vert","horizR","vertR"});
				title="Mirror";
				description="Mirror video across a central axis";
			};
			Mirror(map<string,string> &settings):Mirror() {
				base_settings(settings);
			};
			~Mirror(){ };
			Mirror* clone(map<string,string> &_settings) { return new Mirror(_settings);};
			Image *output(const Frame_spec &frame){
				Image *in=image_inputs[0]->get(frame);
				if (in){
					//copy incoming image **writable
					image=(*in);
					//could be more efficient here by only copying once
					switch (attributes["mode"]->intVal) {
						case MIRROR_horiz:
							for (int i=0;i<image.w/2;i++){
								for (int j=0;j<image.h;j++){
									for (int k=0;k<3;k++){
										image.RGBdata[(((j*image.w)+((image.w/2)+i))*3)+k]=image.RGBdata[(((j*image.w)+((image.w/2)-i))*3)+k];
									}
								}
							}
							break;
						case MIRROR_vert:
							for (int i=0;i<image.w;i++){
								for (int j=0;j<image.h/2;j++){
									for (int k=0;k<3;k++){
										image.RGBdata[((((image.h/2+j)*image.w)+i)*3)+k]=image.RGBdata[((((image.h/2-j)*image.w)+i)*3)+k];
									}
								}
							}
							break;
						case MIRROR_horizR:
							for (int i=0;i<image.w/2;i++){
								for (int j=0;j<image.h;j++){
									for (int k=0;k<3;k++){
										image.RGBdata[(((j*image.w)+((image.w/2)-i))*3)+k]=image.RGBdata[(((j*image.w)+((image.w/2)+i))*3)+k];
									}
								}
							}
							break;
						case MIRROR_vertR:
							for (int i=0;i<image.w;i++){
								for (int j=0;j<image.h/2;j++){
									for (int k=0;k<3;k++){
										image.RGBdata[((((image.h/2-j)*image.w)+i)*3)+k]=image.RGBdata[((((image.h/2+j)*image.w)+i)*3)+k];
									}
								}
							}
							break;
					}
                    return &image;
			    }
				return nullptr;
			}
		private:
	};
	class Monochrome: public Image_node {
		public:
			Monochrome(){
				create_image_input("image input","Image input");
				title="Monochrome";
				description="Render video greyscale";
			};
			Monochrome(map<string,string> &settings):Monochrome() {
				base_settings(settings);
			};
			~Monochrome(){
			};
			Monochrome* clone(map<string,string> &_settings) { return new Monochrome(_settings);};
			Image *output(const Frame_spec &frame){
				Image *in=image_inputs[0]->get(frame);
				if (in){
					for (int i=0;i<image.w;i++){
						for (int j=0;j<image.h;j++){
							uint8_t luma=0;
							for (int l=0;l<3;l++) luma+=pixels.mono_weights[l][in->RGBdata[(((j*image.w)+i)*3)+l]];
							for (int k=0;k<3;k++) image.RGBdata[(((j*image.w)+i)*3)+k]=luma;
						}
					}
                    return &image;
			    }
				return nullptr;
			}
		private:
	};
	class Alpha_merge: public Image_node {
		public:
			Alpha_merge(){
				create_image_input("image input","Image input");
				create_image_input("alpha input","Alpha input");
				title="Alpha merge";
				description="Alpha merge two images";
			};
			Alpha_merge(map<string,string> &settings):Alpha_merge() {
				base_settings(settings);
			};
			~Alpha_merge(){};
			Alpha_merge* clone(map<string,string> &_settings) { return new Alpha_merge(_settings);};
			Image *output(const Frame_spec &frame){
				Image *in1=image_inputs[0]->get(frame);
				if (in1){
					//copy incoming image **writable
					Image *in2=image_inputs[1]->get(frame);
					if (in2) {
						image=(*in1);
                    	image.alpha_merge(*in2);
                    	return &image;
                    }
                    //if there aren't 2 image inputs connected just return the first
                    return in1;
                }
				return nullptr;
			}
		private:
	};
	class Difference_matte: public Image_node {
		public:
			Difference_matte(){
				create_image_input("image input","Image input");
				create_image_input("background input","Background input");
				create_parameter("threshold","number","Difference threshold","Threshold",0.2f,0.0f,1.0f);
				create_parameter("feather","number","Feather width","Feather",0.1f,0.0f,1.0f);
				create_parameter("weight_h","number","H component weight","Weight H",0.5f,0.0f,1.0f);
				create_parameter("weight_s","number","S component weight","Weight S",0.5f,0.0f,1.0f);
				create_parameter("weight_v","number","V component weight","Weight V",0.5f,0.0f,1.0f);
				create_parameter("blursize","number","Blur size","Blur size",2.0f,0.0f,10.0f);
				create_attribute("mode","Output {image|alpha}","output mode","alpha",{"image","alpha"});
				title="Difference matte";
				description="Create an alpha channel using a background reference picture";
				LUT=nullptr;
			};
			Difference_matte(map<string,string> &settings):Difference_matte() {
				base_settings(settings);
			};
			~Difference_matte(){if (LUT) delete[] LUT;};
			Difference_matte* clone(map<string,string> &_settings) { return new Difference_matte(_settings);};
			Image *output(const Frame_spec &frame){
				Image *in1=image_inputs[0]->get(frame);
				if (in1){
					Image *in2=image_inputs[1]->get(frame);
					if (in2) {
						generate_LUT();

						/*
						cv::cvtColor(in1->rgb,greyfg,CV_RGB2GRAY);
						cv::cvtColor(in2->rgb,greybg,CV_RGB2GRAY);
						cv::absdiff(greyfg,greybg,greyDiff);

						//parameters["threshold"]->value
						cv::threshold(greyDiff,mask,parameters["threshold"]->value,255,CV_THRESH_BINARY); //int block_size=3, double param1=5); //int blockSize, int offset=0,bool invert=false, bool gauss=false);

						//cv::adaptiveThreshold(greyDiff,mask,255,CV_ADAPTIVE_THRESH_GAUSSIAN_C,CV_THRESH_BINARY, 3,5); //int block_size=3, double param1=5); //int blockSize, int offset=0,bool invert=false, bool gauss=false);
						*/

						cv::cvtColor(in1->rgb, hsv1, CV_RGB2HSV);
                    	cv::cvtColor(in2->rgb, hsv2, CV_RGB2HSV);

						mask.create(frame.h,frame.w,CV_8UC1);
						lutmask.create(frame.h,frame.w,CV_8UC1);

						//get euclidean distance in HSV space
						int dist,d;
						uint8_t m;

						float weights[3] = {parameters["weight_h"]->value,parameters["weight_s"]->value,parameters["weight_v"]->value};
						float weight_total=255.0f/pow(pow(weights[0]*255,2)+pow(weights[1]*255,2)+pow(weights[2]*255,2),0.5);

						for (int i=0;i<frame.w*frame.h;i++){
						    dist=0;
							for (int j=0;j<3;j++){
								d=((int)hsv1.data[i*3+j])-((int)hsv2.data[i*3+j]);
								dist+=(d*d)*weights[j];
							}
							uint8_t id=(uint8_t)(sqrt((float)dist)*weight_total);
							mask.data[i]=id;
						}

					/*

						for (int i=0;i<frame.w*frame.h;i++){
						    dist=0;
							for (int j=0;j<3;j++){
								d=((int)hsv1.data[i*3+j])-((int)hsv2.data[i*3+j]);
								dist+=(abs(d))*weights[j];
							}
							uint8_t id=(uint8_t)(((float)dist)/weight_total);
							m=LUT[id];
							mask.data[i]=m;
						}
						*/

						//cv::bilateralFilter(mask,filtmask, 4,8,2 );
						//cv::GaussianBlur(mask,filtmask,cv::Size( 4, 4 ), 2, 2);

						int ksize=max((ceil(parameters["blursize"]->value/2.0)*2)+1,1.0);
						//nb this doesn't do the intended: create 'continuously variable' blur
						cv::GaussianBlur(mask,filtmask,cvSize(ksize,ksize),parameters["blursize"]->value);


						for (int i=0;i<frame.w*frame.h;i++){
							lutmask.data[i]=LUT[filtmask.data[i]];
						}


						image=(*in1);
						if (attributes["mode"]->value=="image"){
							cv::cvtColor(lutmask, image.rgb, CV_GRAY2RGB);
						}
						else image.alpha_from_cv(lutmask);
                    	return &image;



                    }
                    //if there aren't 2 image inputs connected just return the first
                    return in1;
                }
				return nullptr;
			}
			void generate_LUT(){
				//can check here if anything has changed
				//cerr<<"generating LUT: threshold "<<parameters["threshold"]->value<<", feather "<<parameters["feather"]->value<<endl;
				if (LUT) delete[] LUT;
				LUT=new uint8_t[256];
				float fltmax=(255.0f/256.0f);
				float minf=max(0.0f,parameters["threshold"]->value-(parameters["feather"]->value*0.5f));
				float maxf=min(1.0f,parameters["threshold"]->value+(parameters["feather"]->value*0.5f));
				for (int i=0;i<256;i++){
					LUT[i]=(uint8_t)(min(1.0f,max(0.0f,((((float)i)/255.0f)-minf)/(maxf-minf)))*255.0f);
				//	cerr<<((int)LUT[i])<<" ";
				}
				//cerr<<endl;
			}
		private:
			cv::Mat greyfg,greybg,greyDiff,mask,filtmask,lutmask;
			cv::Mat hsv1,hsv2;
			uint8_t *LUT;
	};
#define VIDEOFRAMES_frame 1
#define VIDEOFRAMES_blend 2
#define VIDEOTIME_play 1
#define VIDEOTIME_stretch 2
	//relative timelines used to stretch video
	//1. make a video position input for video node - seconds and stretch modes
	//2. video mode that maps to duration- timeline remapping from cycler and others
	class Video_loader: public Image_node {
		public:
			Video_loader(){
				create_parameter("speed","number","video playback speed","Speed",1.0f,0.0f,0.0f);
				create_parameter("framerate","number","framerate override","Frame rate",0.0f,0.0f,0.0f);
				create_attribute("filename","name of video file to load","File name","");
				create_attribute("frame_mode","frame mode","Frame mode","frame",{"frame","blend"});
				create_attribute("time_mode","time mode","Time mode","play",{"play","stretch"});
				title="Video loader";
				description="Loads a video file";
			};
			Video_loader(map<string,string> &settings): Video_loader() {
				base_settings(settings);
				isLoaded=false;
				if (attributes["filename"]->value!="") {
					load(find_setting(settings,"media_path","")+attributes["filename"]->value);
				}

			};
			~Video_loader(){};
			bool load(const string &filename){
				Poco::Logger& logger = Poco::Logger::get("Rotor");
			    if (isLoaded) {
			    	player.cleanup(); ///should be in decoder class?
			    	isLoaded=false;
			    }
			    isLoaded=player.open(filename);
				if (isLoaded){
					logger.information("Video_loader loaded "+filename+": "\
						+toString(player.get_number_frames())+" frames, "\
						+toString(player.get_framerate())+" fps, "\
						+toString(player.get_width())+"x"+toString(player.get_height())\
						+", channels:"+toString(player.get_number_channels()));
					lastframe=-2;
					return true;
			   	}
				logger.error("Video_loader failed to load "+filename);
			    return false;
			}
			Image *output(const Frame_spec &frame){
				if (isLoaded){
					float clipframerate=(parameters["framerate"]->value==0.0f?player.get_framerate():parameters["framerate"]->value);
					float clipspeed=(clipframerate/frame.framerate)*parameters["speed"]->value;
					float wanted;
					switch (attributes["frame_mode"]->intVal){
						case VIDEOTIME_play:
							wanted=fmod(frame.time*frame.framerate*clipspeed,(float)player.get_number_frames());
							break;
						case VIDEOTIME_stretch:
							wanted=fmod(frame.time*frame.framerate*clipspeed,(float)player.get_number_frames());
							break;
					}
					if (attributes["frame_mode"]->intVal==VIDEOFRAMES_blend){
						if (((int)wanted)!=lastframe){
							//get a new pair of frames possibly by switching the next one
							//darn peculiar, as if copying wasn't actually copying
							if ((lastframe==(((int)wanted)-1))&&(in2.w>0)) {
								in1=in2;
							}
							else {
								player.fetch_frame(frame.w,frame.h,(int)wanted);
								//use a temp image because setup_fromRGB just copies pointer
								temp.setup_fromRGB(frame.w,frame.h,player.frame->Data[0],player.frame->Linesize[0]-(frame.w*3));
								in1=temp;
							}
							player.fetch_frame(frame.w,frame.h,((int)wanted+1)%max(1,player.get_number_frames()));
							temp.setup_fromRGB(frame.w,frame.h,player.frame->Data[0],player.frame->Linesize[0]-(frame.w*3));
							in2=temp;
							lastframe=wanted;
						}
						float amt=1.0f-(wanted-((int)wanted));
						//cout<<"video loader time:"<<frame.time<<" frames "<<((int)wanted)<<" (x"<<amt<<"),"<<(((int)wanted+1)%max(1,player.get_number_frames()))<<endl;
						image=in1;
						image*=amt;
						//Image in2t=in2; //DOES NOT WORK, copies pointer by assignation
						in2t=in2;
						in2t*=(1.0f-amt);
						image+=in2t;
					}
					else {
						if (((int)wanted)!=lastframe){
							if (!player.fetch_frame(frame.w,frame.h,((int)wanted))) { //seek fail
								Poco::Logger& logger = Poco::Logger::get("Rotor");
								logger.error("Video_loader failed to seek frame "+toString(wanted)+" of "+attributes["filename"]->value);

								if (image.w>0) return &image; //just return the previous frame if possible
								else return nullptr;
							}
							image.setup_fromRGB(frame.w,frame.h,player.frame->Data[0],player.frame->Linesize[0]-(frame.w*3));
						}
					}
					return &image;
				}
			    return nullptr;
			};
			Video_loader* clone(map<string,string> &_settings) { return new Video_loader(_settings);};
			bool isLoaded;
		private:
			//ffmpegsource::decoder player;
			libav::video_decoder player;
			int lastframe;
			Image in1,in2,in2t,temp; //for blend frames;
	};
	class Video_output: public Image_node {
	//Video_output 'presents' the output movie. Aspect ratio, bars, fadein/fadeout would happen here
		public:
			Video_output(){
				create_image_input("image to output","Image input");
				create_attribute("begin_mode","mode to begin movie","Begin mode","cut",{"cut","blank silence","fade peak"});
				create_attribute("end_mode","mode to end movie","End mode","cut",{"cut","blank silence","fade peak"});
				title="Video output";
				description="Output to video";
				start_silence=start_peak=end_silence=end_peak=-1.0f;
				silence_threshold=0.01f;
			};
			Video_output(map<string,string> &settings):Video_output() {
				base_settings(settings);
			};
			~Video_output(){ };
			void create_envelope(const vector<float> &audio){
				if (audio.size()){
					int i=0;
					while (i<audio.size()&&audio[i]<silence_threshold) i++;
					start_silence=((float)i)/audio.size();
					while (i<audio.size()&&audio[i]>audio[i-1]) i++;
					start_peak=((float)i-1)/audio.size();
					i=audio.size();
					while (i>-1&&audio[i]<silence_threshold) i--;
					end_silence=((float)i)/audio.size();
					while (i<audio.size()&&audio[i]>audio[i+1]) i--;
					end_peak=((float)i+1)/audio.size();
					cerr<<"Video_output sound envelope: silence - "<<start_silence<<" : peak "<<start_peak<<" : peak "<<end_peak<<" silence - "<<end_silence<<endl;
				}
				else cerr<<"Video_output sound envelope: no data"<<endl;
			}
			Image *output(const Frame_spec &frame){
				Image *in=image_inputs[0]->get(frame);
				if (in){
					//make copy of the image, for feedback
					//optimise?
					float amount=1.0f;
					float track_time=frame.time/frame.duration;
					if (attributes["begin_mode"]->value=="fade peak"||attributes["begin_mode"]->value=="blank silence"){
						if (track_time<start_silence){
							amount=0.0f;
						}
						else if (track_time<start_peak&&attributes["begin_mode"]->value=="fade peak"&&start_peak>start_silence){
							amount = (track_time-start_silence)/(start_peak-start_silence);
						}
					}
					if (attributes["end_mode"]->value=="fade peak"||attributes["end_mode"]->value=="blank silence"){
						if (track_time>end_silence){
							amount=0.0f;
						}
						else if (track_time>end_peak&&attributes["end_mode"]->value=="fade peak"&&end_silence>end_peak){
							amount = 1.0f-((track_time-end_peak)/(end_silence-end_peak));
						}
					}
					if (amount<(1.0f/254.0f)){
						image.clear();
					}
					image=(*in);
					if (amount<1.0f){
						image*=amount;
					}
					//seems to be outputting correctly but not saving frames
					return &image;
				}
				return nullptr;
			};
			Video_output* clone(map<string,string> &_settings) { return new Video_output(_settings);};

		private:
			float silence_threshold;
			float start_silence;
			float start_peak;
			float end_silence;
			float end_peak;
	};
	class Video_feedback: public Image_node {
		public:
			Video_feedback(){
				title="Video feedback";
				description="Repeats output of the last frame";
				feedback=nullptr;
			};
			Video_feedback(map<string,string> &settings):Video_feedback() {
				base_settings(settings);
			};
			~Video_feedback(){ };
			void set_feedback(Image *iptr){
				feedback=iptr;
			}
			Image *output(const Frame_spec &frame){
				if (feedback->RGBdata){
					return feedback;
				}
				image.setup(frame.w,frame.h);
				image.clear();
				return &image;
			};
			Video_feedback* clone(map<string,string> &_settings) { return new Video_feedback(_settings);};
		private:
			Image *feedback;
	};
		//-------------------------------------------------------------------
	class Node_factory{
		public:
			Node_factory();
			~Node_factory(){
				for (auto t:type_map) delete t.second;
			}
			void add_type(string type,Node* proto){
				type_map[type]=proto;
				type_map[type]->type=type;
			};
			void add_type(string type,Node* proto,vector<Rotor::Node*> &category){
				add_type(type,proto);
				category.push_back(proto);
			};
			Node *create(map<string,string> &settings){
				if (settings.find("type")!=settings.end()) {
					if (type_map.find(settings["type"])!=type_map.end()) {
						return type_map[settings["type"]]->clone(settings);
					}
				}
				return NULL;
			};
			bool list_node(const string &t,xmlIO XML){
				for (auto& type: type_map) {
					if (type.first==t) {
						list_node(type.second,XML);
						return true;
					}
				}
				XML.addValue("error","Node /"+t+"/ not found");
			};
			void list_node(Rotor::Node* type,xmlIO XML,int i=0){
					XML.addTag("node");
					XML.addAttribute("node","type",type->type,i);
					XML.addAttribute("node","inputs",type->duplicate_inputs?"expandable":"fixed",i);
					XML.addAttribute("node","title",type->title,i);
					XML.addAttribute("node","description",type->description,i);
					if (dynamic_cast<Signal_node*> (type)!=nullptr) XML.addAttribute("node","output","signal",i);
					if (dynamic_cast<Image_node*> (type)!=nullptr) XML.addAttribute("node","output","image",i);
					XML.pushTag("node",i);
					//if (type->description!="") {
					//	XML.addTag("description");
					//	XML.setValue("description",type->description,0);
					//}
					int j=0;
					for (auto& input: type->inputs) {
						XML.addTag("signal_input");
						XML.addAttribute("signal_input","title",input->title,j);
						XML.addAttribute("signal_input","description",input->description,j);
						j++;
					}
					j=0;
					if (dynamic_cast<Image_node*> (type)!=nullptr) {
						for (auto& input: (dynamic_cast<Image_node*>(type))->image_inputs) {
							XML.addTag("image_input");
							XML.addAttribute("image_input","title",input->title,j);
							XML.addAttribute("image_input","description",input->description,j);
							j++;
						}
					}
					j=0;
					for (auto& parameter: type->parameters) {
						XML.addTag("parameter");
						XML.addAttribute("parameter","name",parameter.first,j);
						XML.addAttribute("parameter","type",parameter.second->type,j);
						XML.addAttribute("parameter","title",parameter.second->title,j);
						XML.addAttribute("parameter","description",parameter.second->description,j);
						XML.addAttribute("parameter","value",parameter.second->value,j);
						XML.addAttribute("parameter","min",parameter.second->min,j);
						XML.addAttribute("parameter","max",parameter.second->max,j);
						j++;
					}
					j=0;
					for (auto& attribute: type->attributes) {
						XML.addTag("attribute");
						XML.addAttribute("attribute","name",attribute.first,j);
						XML.addAttribute("attribute","title",attribute.second->title,j);
						XML.addAttribute("attribute","description",attribute.second->description,j);
						XML.addAttribute("attribute","value",attribute.second->value,j);
						if (attribute.second->vals.size()){ //document attribute enumeration
							XML.addAttribute("attribute","type","enum",j);
							XML.pushTag("attribute",j);
							int k=0;
							for (auto val: attribute.second->vals){
								XML.addTag("option");
								XML.addAttribute("option","value",val,k);
								k++;
							}
							XML.popTag();
						}
						else XML.addAttribute("attribute","type","string",j);
						j++;
					}
					XML.popTag();
			}
			void list_nodes(xmlIO XML){
				int i=0;
				for (auto& type: type_map) {
					if (type.second->description!="") { //blank description = internal/ testing node
						list_node(type.second,XML,i);
						i++;
					}
				}
			}
			void list_categories(xmlIO XML){
				int i=0;
				for (auto& category: categories) {
					XML.addTag("category");
					XML.addAttribute("category","name",category.first,i);
					XML.pushTag("category",i);
					int j=0;
					for (auto& node: category.second){
						list_node(node,XML,j);
						j++;
					}
					XML.popTag();
					i++;
				}
			}
			void list_categories(Json::Value &JSON){
				JSON["categories"]=Json::arrayValue;
				for (auto& _category: categories) {
					Json::Value category;
					category["name"]=_category.first;
					category["nodes"]=Json::arrayValue;
					for (auto& _node: _category.second){
						Json::Value node;
						node["type"]=_node->type;
						node["title"]=_node->title;
						node["inputs"]=_node->duplicate_inputs?"expandable":"fixed";
						if (dynamic_cast<Signal_node*> (_node)!=nullptr) node["output"]="signal";
						if (dynamic_cast<Image_node*> (_node)!=nullptr) node["output"]="image";
						node["description"]=_node->description;
						if (_node->inputs.size()){
							node["signal_inputs"]=Json::arrayValue;
							for (auto& input: _node->inputs) {
								Json::Value signal_input;
								signal_input["title"]=input->title;
								signal_input["description"]=input->description;
								node["signal_inputs"].append(signal_input);
							}
						}
						if (dynamic_cast<Image_node*> (_node)!=nullptr) {
							if ((dynamic_cast<Image_node*>(_node))->image_inputs.size()){
								node["image_inputs"]=Json::arrayValue;
								for (auto& input: (dynamic_cast<Image_node*>(_node))->image_inputs) {
									Json::Value image_input;
									image_input["title"]=input->title;
									image_input["description"]=input->description;
									node["image_inputs"].append(image_input);
								}
							}
						}
						if (_node->parameters.size()){
							node["parameters"]=Json::arrayValue;
							for (auto& param: _node->parameters) {
								Json::Value parameter;
								parameter["name"]=param.first;
								parameter["type"]=param.second->type;
								parameter["title"]=param.second->title;
								parameter["description"]=param.second->description;
								parameter["value"]=param.second->value;
								parameter["min"]=param.second->min;
								parameter["max"]=param.second->max;
								node["parameters"].append(parameter);
							}
						}
						if (_node->attributes.size()){
							node["attributes"]=Json::arrayValue;
							for (auto& attr: _node->attributes) {
								Json::Value attribute;
								attribute["name"]=attr.first;
								attribute["title"]=attr.second->title;
								attribute["description"]=attr.second->description;
								attribute["value"]=attr.second->value;
								if (attr.second->vals.size()){ //document attribute enumeration
									attribute["type"]="enum";
									attribute["options"]=Json::arrayValue;
									for (auto val: attr.second->vals){
										attribute["options"].append(val);
									}
								}
								else attribute["type"]="string";
								node["attributes"].append(attribute);
							}
						}
						category["nodes"].append(node);
					}
					JSON["categories"].append(category);
				}
			}
			void list_nodes(Json::Value &JSON){
				JSON["nodeslist"]=Json::arrayValue;
				for (auto& type: type_map) {
					if (type.second->description!="") { //blank description = internal/ testing node
						Json::Value node;
						node["type"]=type.first;
						node["title"]=type.second->title;
						node["inputs"]=type.second->duplicate_inputs?"expandable":"fixed";
						if (dynamic_cast<Signal_node*> (type.second)!=nullptr) node["output"]="signal";
						if (dynamic_cast<Image_node*> (type.second)!=nullptr) node["output"]="image";
						node["description"]=type.second->description;
						if (type.second->inputs.size()){
							node["signal_inputs"]=Json::arrayValue;
							for (auto& input: type.second->inputs) {
								Json::Value signal_input;
								signal_input["title"]=input->title;
								signal_input["description"]=input->description;
								node["signal_inputs"].append(signal_input);
							}
						}
						if (dynamic_cast<Image_node*> (type.second)!=nullptr) {
							if ((dynamic_cast<Image_node*>(type.second))->image_inputs.size()){
								node["image_inputs"]=Json::arrayValue;
								for (auto& input: (dynamic_cast<Image_node*>(type.second))->image_inputs) {
									Json::Value image_input;
									image_input["title"]=input->title;
									image_input["description"]=input->description;
									node["image_inputs"].append(image_input);
								}
							}
						}
						if (type.second->parameters.size()){
							node["parameters"]=Json::arrayValue;
							for (auto& param: type.second->parameters) {
								Json::Value parameter;
								parameter["name"]=param.first;
								parameter["type"]=param.second->type;
								parameter["title"]=param.second->title;
								parameter["description"]=param.second->description;
								parameter["value"]=param.second->value;
								parameter["min"]=param.second->min;
								parameter["max"]=param.second->max;
								node["parameters"].append(parameter);
							}
						}
						if (type.second->attributes.size()){
							node["attributes"]=Json::arrayValue;
							for (auto& attr: type.second->attributes) {
								Json::Value attribute;
								attribute["name"]=attr.first;
								attribute["title"]=attr.second->title;
								attribute["description"]=attr.second->description;
								attribute["value"]=attr.second->value;
								if (attr.second->vals.size()){ //document attribute enumeration
									attribute["type"]="enum";
									attribute["options"]=Json::arrayValue;
									for (auto val: attr.second->vals){
										attribute["options"].append(val);
									}
								}
								else attribute["type"]="string";
								node["attributes"].append(attribute);
							}
						}
						JSON["nodeslist"].append(node);
					}
				}
			}
		private:
			unordered_map<string,Node*> type_map;
			unordered_map<string,vector<Rotor::Node*> > categories;
	};
}

#endif