/* nodes can have many inputs but only 1 output image nodes that use an image as input can pass on the incoming image only if its unchanged. */ #include #include #include #include #include "Poco/Net/HTTPServer.h" #include "Poco/Net/HTTPResponse.h" #include "Poco/UUID.h" #include "Poco/UUIDGenerator.h" #include "Poco/Notification.h" #include "Poco/NotificationCenter.h" #include "Poco/Observer.h" #include "Poco/ThreadPool.h" #include "Poco/Thread.h" #include "Poco/Task.h" #include "Poco/Runnable.h" #include "Poco/Mutex.h" #include "Poco/Random.h" #include "Poco/AutoPtr.h" #include "Poco/File.h" #include "Poco/Base64Encoder.h" #include "Poco/Path.h" #include using Poco::UUID; using Poco::UUIDGenerator; using Poco::Net::HTTPResponse; /* extern "C" { #include #include #include #include #include #include #include #include #include //#include stops the compiler error but causes a linker error. does libavcodec need to be statically linked? #include #include //#include } */ #include "ofxMovieExporter.h" #define AUDIO_INBUF_SIZE 20480 #define AUDIO_REFILL_THRESH 4096 #include "vampHost.h" #include "xmlIO.h" #include "libavaudioloader.h" #include "libavexporter.h" #include "gstvideoloader.h" namespace Rotor { #define IDLE 0 #define ANALYSING_AUDIO 1 #define AUDIO_READY 2 #define CREATING_PREVIEW 3 #define PREVIEW_READY 4 #define RENDERING 5 #define RENDER_READY 6 #define ANALYSE_AUDIO 1 #define PREVIEW 2 #define RENDER 3 #ifndef FEQUAL //float equality bool fequal(const float u,const float v){ if (abs(u-v)<.001) return true; else return false; }; bool flessorequal(const float u,const float v){ //v is less or equal to u if (u-v>-.001) return true; else return false; }; #define FEQUAL #endif //forward declaration class Node; class Signal_node; class Image_node; //http://blog.tomaka17.com/2012/03/libavcodeclibavformat-tutorial/ struct Packet { explicit Packet(AVFormatContext* ctxt = nullptr) { av_init_packet(&packet); packet.data = nullptr; packet.size=0; if (ctxt) reset(ctxt); } Packet(Packet&& other) : packet(std::move(other.packet)) { other.packet.data = nullptr; } ~Packet() { if (packet.data) av_free_packet(&packet); } void reset(AVFormatContext* ctxt) { if (packet.data) av_free_packet(&packet); if (av_read_frame(ctxt, &packet) < 0) packet.data = nullptr; } AVPacket packet; }; class Image{ public: Image(){ zero(); }; Image(int _w,int _h){ zero(); setup(_w,_h); }; ~Image() { free(); }; void free(){ if (RGBdata&&ownsRGBdata) delete[] RGBdata; if (Adata&&ownsAdata) delete[] Adata; if (Zdata&&ownsZdata) delete[] Zdata; zero(); } void zero(){ RGBdata=nullptr; Adata=nullptr; Zdata=nullptr; w=0; h=0; ownsRGBdata=ownsAdata=ownsZdata=false; } bool setup(int _w,int _h){ //set up with internal data if (w!=_w||h!=_h||!ownsRGBdata||!ownsAdata||!ownsZdata){ free(); w=_w; h=_h; RGBdata=new uint8_t[w*h*3]; Adata=new uint8_t[w*h]; Zdata=new uint16_t[w*h]; ownsRGBdata=ownsAdata=ownsZdata=true; return true; } else return false; } bool setup_fromRGB(int _w,int _h,uint8_t *pRGBdata){ //possibility of just resetting pointer? if (w!=_w||h!=_h||ownsRGBdata||!ownsAdata||!ownsZdata){ free(); w=_w; h=_h; RGBdata=pRGBdata; Adata=new uint8_t[w*h]; Zdata=new uint16_t[w*h]; ownsRGBdata=false; ownsAdata=ownsZdata=true; return true; } return false; } uint8_t *RGBdata; uint8_t *Adata; uint16_t *Zdata; int h,w; bool ownsRGBdata,ownsAdata,ownsZdata; //better done through auto_ptr? }; class Time_spec{ public: Time_spec(float _time,float _framerate){ time=_time; framerate=_framerate; }; float time; float framerate; Time_spec lastframe() const{ return Time_spec(time-(1.0f/framerate),framerate); } }; class Frame_spec{ public: Frame_spec(float _time,float _framerate,int _w,int _h){ time=_time; framerate=_framerate; w=_w; h=_h;}; float time; //this hould probably be implemented with a num/denom scheme eventually for accuracy float framerate; int h,w; Frame_spec lastframe(){ return Frame_spec(time-(1.0f/framerate),framerate,w,h); } }; class Render_status{ public: int id; float progress; }; class Render_requirements{ public: int num_performances; int num_clips; }; class Command_response{ public: Command_response() { status=Poco::Net::HTTPResponse::HTTP_OK; } std::string description; Poco::Net::HTTPResponse::HTTPStatus status; }; class Input{ public: Input(const string &_desc): connection(nullptr),description(_desc){}; Node* connection; string description; }; class Image_input: public Input{ public: bool connect(Image_node *source); Image_input(const string &_desc): Input(_desc){}; }; class Signal_input: public Input{ public: bool connect(Signal_node *source); Signal_input(const string &_desc): Input(_desc){}; }; class Node{ public: virtual Node* clone(map &_settings)=0; UUID uid; //every usable node has a UUID int id; vector inputs; //simple node can have signal inputs, output depends on node type void create_signal_input(const string &description) {inputs.push_back(new Signal_input(description));}; string description; string type; string output_type; string ID; string check(map &settings,string key,string def=""){ if (settings.find(key)!=settings.end()) return settings[key]; else return def;}; void base_settings(map &settings) { description=check(settings,"description"); type=check(settings,"type"); output_type=check(settings,"output"); ID=check(settings,"ID"); } }; class Signal_node: public Node{ public: virtual const float get_output(const Time_spec &time) { return 0.0f; }; }; class Image_node: public Node{ public: vector image_inputs; //image node also has image inputs and outputs void create_image_input(const string &description) {image_inputs.push_back(new Image_input(description));}; virtual Image *get_output(const Frame_spec &frame)=0; Image *get_preview(const Frame_spec &frame); Image *image; //this can be privately allocated or just passed on as the node see fit private: float image_time; }; class Base_audio_processor: public Signal_node { public: virtual int process_frame(uint8_t *data,int samples)=0; virtual bool init(int _channels,int _bits,int _samples,int _rate)=0; virtual void cleanup()=0; int channels,bits,samples,rate; }; //actual nodes------------------------------------------------- class Audio_analysis: public Base_audio_processor { public: Audio_analysis(){}; Audio_analysis(map &settings) { base_settings(settings); soname=check(settings,"soname"); id=check(settings,"id"); outputNo=ofToInt(check(settings,"output","0")); }; Audio_analysis* clone(map &_settings) { return new Audio_analysis(_settings);}; bool init(int _channels,int _bits,int _samples,int _rate); void cleanup(); int process_frame(uint8_t *data,int samples_in_frame); const float get_output(const Time_spec &time) { if (analyser.features.size()) { auto i=analyser.features.lower_bound(time.time); if (i!=analyser.features.end()){ float lk=i->first; int ln=i->second; if (i++!=analyser.features.end()){ float uk=i->first; return (((time.time-lk)/(uk-lk))+ln); } else return (float)ln; } } return 0.0f; } void print_features(); private: string soname,id; int outputNo; vampHost::Analyser analyser; }; class Signal_divide: public Signal_node { public: Signal_divide(){}; Signal_divide(map &settings) { base_settings(settings); divide_amount=ofToFloat(check(settings,"amount")); }; Signal_divide* clone(map &_settings) { return new Signal_divide(_settings);}; const float get_output(const Time_spec &time) { if (inputs.size()) { //there should there be a way to specify number of inputs in the code rather than in xml if (inputs[0]->connection) { return (((Signal_node*)inputs[0]->connection)->get_output(time))/divide_amount; } } return 0.0f; } float divide_amount; }; class Is_new_integer: public Signal_node { public: Is_new_integer(){}; Is_new_integer(map &settings) { base_settings(settings); }; Is_new_integer* clone(map &_settings) { return new Is_new_integer(_settings);}; const float get_output(const Time_spec &time) { if (inputs[0]->connection) { float s1=(((Signal_node*)(inputs[0]->connection))->get_output(time)); float s2=(((Signal_node*)(inputs[0]->connection))->get_output(time.lastframe())); if (((int)s1)>((int)s2)) { return 1.0f; } } return 0.0f; } }; class Signal_output: public Signal_node { public: Signal_output(){}; Signal_output(map &settings) { base_settings(settings); }; Signal_output* clone(map &_settings) { return new Signal_output(_settings);}; bool render(const float duration, const float framerate,string &xml_out); const float get_output(const Time_spec &time) { if (inputs[0]->connection) { return ((Signal_node*)(inputs[0]->connection))->get_output(time); } else return 0.0f; } }; class Testcard: public Image_node { public: Testcard(){}; Testcard(map &settings) { base_settings(settings); image=new Image(); }; ~Testcard(){ delete image;}; Testcard* clone(map &_settings) { return new Testcard(_settings);}; Image *get_output(const Frame_spec &frame){ if (image->setup(frame.w,frame.h)) { } //always create testcard //float ws=(255.0f/frame.w); float hs=(255.0f/frame.h); for (int i=0;iRGBdata[(i*frame.w+j)*3]=(uint8_t)((int)((i+(frame.time*25.0f)*hs))%255); image->RGBdata[((i*frame.w+j)*3)+1]=(uint8_t)((int)((j+(frame.time*100.0f)*hs))%255); image->RGBdata[((i*frame.w+j)*3)+2]=(uint8_t)(0); image->Adata[i*frame.w+j]=(uint8_t)255; image->Zdata[i*frame.w+j]=(uint16_t)512; //1.0 in fixed point 8.8 bits } } return image; } private: Image *image; //is an image generator }; class Invert: public Image_node { public: Invert(){}; Invert(map &settings) { base_settings(settings); image=new Image(); }; ~Invert(){ delete image;}; Invert* clone(map &_settings) { return new Invert(_settings);}; Image *get_output(const Frame_spec &frame){ if (inputs.size()) { if (inputs[0]->connection) { Time_spec requested=Time_spec(frame.time,frame.framerate); if (fequal((((Signal_node*)inputs[0]->connection)->get_output(requested)),1.0f)) { //invert=!invert; } } } if (image_inputs[0]->connection) { } return image; } private: Image *image; //is an image generator //bool invert; }; class Video_output: public Image_node { public: Video_output(){}; Video_output(map &settings) { base_settings(settings); exporter=new libav::Exporter(); }; ~Video_output(){ delete exporter; }; Image *get_output(const Frame_spec &frame){ if (image_inputs[0]->connection) { return ((Image_node*)(image_inputs[0]->connection))->get_output(frame); } else return nullptr; }; Video_output* clone(map &_settings) { return new Video_output(_settings);}; bool render(const float duration, const float framerate,const string &output_filename,const string &audio_filename); private: //ofxMovieExporter *exporter; libav::Exporter *exporter; libav::Audioloader audioloader; }; class Video_input: public Image_node { public: Video_input(){}; Video_input(map &settings) { base_settings(settings); player=new ofGstVideoPlayer(); image=new Image(); }; ~Video_input(){ delete player; delete image;}; bool load(const string &filename); Image *get_output(const Frame_spec &frame); Video_input* clone(map &_settings) { return new Video_input(_settings);}; private: ofGstVideoPlayer *player; Image *image; }; //------------------------------------------------------------------- class Node_factory{ public: Node_factory(); void add_type(string type,Node* proto){ type_map[type]=proto; }; Node *create(map &settings){ if (settings.find("type")!=settings.end()) { if (type_map.find(settings["type"])!=type_map.end()) { return type_map[settings["type"]]->clone(settings); } } return NULL; }; private: unordered_map type_map; }; class Graph{ public: Graph(){duration=10.0f;loaded = false;}; Graph(const string& _uid,const string& _desc){init(_uid,_desc);}; void init(const string& _uid,const string& _desc){ uid=_uid;description=_desc;duration=10.0f;}; string uid; //every version of a graph has a UUID, no particular need to actually read its data(?) //?? is it faster than using strings?? string description; std::unordered_map nodes; vector find_nodes(const string &type); Node* find_node(const string &type); bool signal_render(string &signal_xml,const float framerate); bool video_render(const string &output_filename,const string &audio_filename,const float framerate); int load(Poco::UUID uid); bool load(string &graph_filename); UUID save(); //save to DB, returns UUID of saved graph bool loaded; float duration; const string toString(); private: Node_factory factory; xmlIO xml; }; class Audio_thumbnailer: public Base_audio_processor { public: Audio_thumbnailer(){ height=128; width=512; //fit data=new uint8_t[height*width]; memset(data,0,height*width); }; ~Audio_thumbnailer(){ delete[] data; }; Audio_thumbnailer* clone(map &_settings) { return new Audio_thumbnailer();}; bool init(int _channels,int _bits,int _samples,int _rate); void cleanup(){}; int process_frame(uint8_t *data,int samples_in_frame); string print(); uint8_t *data; int height,width,samples_per_column; int column,out_sample,sample,samples; int offset; double scale,accum; }; class Render_context: public Poco::Task { //Poco task object //manages a 'patchbay' //high level interfaces for the wizard //and low level interface onto the graph public: Render_context(const std::string& name): Task(name) { audio_thumb=new Audio_thumbnailer(); state=IDLE; output_framerate=25.0f; }; void runTask(); void add_queue(int item); Command_response session_command(const std::vector& command); Render_status get_status(); void cancel(); //interrupt locking process int make_preview(int nodeID, float time); //starts a frame preview - returns status code - how to retrieve? bool load_audio(const string &filename,vector processors); Render_requirements get_requirements(); bool load_video(const string &nodeID,const string &filename);//can be performance or clip private: int state; double progress; //for a locking process: audio analysis or rendering //thread only does one thing at once std::deque work_queue; Poco::Mutex mutex; //lock for access from parent thread std::string audio_filename; std::string output_filename; Audio_thumbnailer *audio_thumb; vampHost::QMAnalyser audio_analyser; Graph graph; Node_factory factory; float output_framerate; }; } /* coding style Types begin with capitals 'New_type' variables/ instances use lower case with underscore as a seperator */