#include "graph.h" using namespace Rotor; using Poco::Logger; const string Graph::graphToString(){ string xmlgraph; if (loaded) { xml.copyXmlToString(xmlgraph); return xmlgraph; } else return ""; } vector Graph::find_nodes(const string &type){ vector found; for (std::unordered_map::iterator it=nodes.begin();it!=nodes.end();++it) { if (it->second->type==type) found.push_back(it->second); } return found; }; Node* Graph::find_node(const string &type){ for (std::unordered_map::iterator it=nodes.begin();it!=nodes.end();++it) { if (it->second->type==type) return it->second; } return nullptr; //can be tested against }; bool Graph::signal_render(xmlIO &XML,const string &node,const float framerate) { if (nodes.find(node)!=nodes.end()){ Signal_node *signal_output=dynamic_cast(nodes[node]); if (signal_output) { //return signal_output->render(duration,framerate,signal_xml); XML.addValue("signal_duration",duration); XML.addValue("signal_framerate",framerate); float sig=0.0f; string val=""; for (float i=0;iget_output(Time_spec(i,framerate,duration))+1.0f)/10.0f; if (!fequal(sig,s)){ val+=toString(i)+","+toString(s)+" "; sig=s; } } XML.addValue("signal",val); return true; } cerr<<"Error: /"<(nodes[node])){ XML.addValue("features",dynamic_cast(nodes[node])->get_features()); return true; } XML.addValue("ERROR","node /"+node+"/ is not an Audio processor"); return false; } XML.addValue("ERROR","could not find node /"+node+"/"); return false; } bool Graph::preview(xmlIO &XML,string &node,string &_format,int frame,int w,int h){ if (nodes.find(node)!=nodes.end()){ float t=frame/framerate; XML.addTag("preview"); XML.addAttribute("preview","frame",toString(frame),0); XML.addAttribute("preview","nodeID",node,0); XML.pushTag("preview"); if (dynamic_cast(nodes[node])){ Time_spec ts=Time_spec(t,framerate,0.0f); XML.addValue("signal",dynamic_cast(nodes[node])->get_output(ts)); } if (dynamic_cast(nodes[node])){ Frame_spec fs=Frame_spec(t,framerate,0.0f,w,h); Image *img=dynamic_cast(nodes[node])->get_image_output(fs); vector buf; string format=(_format==""?".png":_format); if (cv::imencode(format,img->rgb,buf)){ //, const vector& params=vector()) stringstream output; Poco::Base64Encoder *enc=new Poco::Base64Encoder(output); enc->write((char*)buf.data(),buf.size()); enc->close(); delete enc; XML.addValue("image",output.str()); XML.addAttribute("image","format",format,0); } } XML.popTag(); return true; } return false; } bool Graph::video_render(const string &output_filename,const float framerate,int start, int stop) { if (output_filename.size()==0) return false; //https://www.adobe.com/devnet/video/articles/mp4_movie_atom.html //https://www.google.ie/search?q=ffmbc&aq=f&oq=ffmbc&aqs=chrome.0.57j0l2j60j0j60.4360j0&sourceid=chrome&ie=UTF-8#q=ffmbc+git //vector loaders=find_nodes("video_loader"); //for (auto i:loaders){ // if (!dynamic_cast(i)->isLoaded) { // cerr<<"Rotor: all loaders must be populated before rendering"<(find_node("video_output")); if (audio_filename!=""){ //BETTER WAY TO KNOW IF WE ARE USING AUDIO? video_output->create_envelope(audio_thumb->audiodata); } for (auto f: find_nodes("video_feedback")){ video_output->clear_output(outW,outH); (dynamic_cast(f))->set_feedback(&(video_output->image)); } // //setup defaults std::string container; Poco::StringTokenizer t(output_filename,"."); if (t.count()>1){ container="."+t[t.count()-1]; } else container=".mp4"; libav::exporter exporter; Image* i; if (exporter.setup(outW,outH,bitRate,framerate,container,use_fragmentation)) { //codecId, if (exporter.record(output_filename)) { libav::audio_decoder audioloader; bool usingaudio=audioloader.open(audio_filename); Logger& logger = Logger::get("Rotor"); logger.information("Video_output rendering "+output_filename+": "+toString(duration)+" seconds at "+toString(framerate)+" fps, audio frame size: "+toString(exporter.get_audio_framesize())); //25fps video and 43.06640625fps audio? hmm //how to get the timecodes correct for the interleaved files struct timeval _start, _end; gettimeofday(&_start, NULL); uint16_t *audioframe=nullptr; uint16_t *audio=nullptr; int samples_in_frame; for (auto n:nodes) n.second->reset(); if (usingaudio){ samples_in_frame=(audioloader.get_sample_rate())/framerate; string whether=usingaudio?"Loading":"Cannot load"; logger.information(whether+" audio file: "+audio_filename+", each frame contains "+toString(samples_in_frame)+" samples at "+toString(audioloader.get_sample_rate())+" hz"); audioframe=new uint16_t[(samples_in_frame+exporter.get_audio_framesize())*audioloader.get_number_channels()]; audio=new uint16_t[samples_in_frame*audioloader.get_number_channels()]; } float vstep=1.0f/framerate; float vf=start*vstep; float af=start*vstep; int aoffs=0; int audioend=0; Audio_frame *a; int64_t sample_start=(start*audioloader.get_sample_rate())/framerate; while (vf0){ //shift down samples int s=0; while ((s+aoffs)get_image_output(Frame_spec(vf,framerate,duration,outW,outH,a)); } else i=video_output->get_image_output(Frame_spec(vf,framerate,duration,outW,outH)); if (i) { exporter.encodeFrame(i->RGBdata); } vf+=vstep; //mutex.lock(); progress=vf/duration; //mutex.unlock(); if (usingaudio) {delete a;}; } //exporter.encodeFrame(i->RGBdata,true); //final keyframe; exporter.finishRecord(); gettimeofday(&_end, NULL); float mtime = ((_end.tv_sec-_start.tv_sec) + (_end.tv_usec-_start.tv_usec)/1000000.0); logger.information("Video_output: rendered "+output_filename+": in "+toString(mtime)+" seconds"); logger.information("compression codec took "+toString(mtime-video_output->time_taken)+" seconds"); for (auto n:nodes) { logger.information(n.second->type+" node '"+n.first+"' took "+toString(n.second->get_time_used())+" seconds"); } if (usingaudio) { audioloader.cleanup(); delete[] audioframe; delete[] audio; } return true; } } return false; } cerr<<"Rotor: video output node not found"<64&&h>48){ outW=w; outH=h; return true; } else return false; } bool Graph::load(string data,string media_path){ if (xml.loadFromBuffer(data)){ return parseXml(media_path); } return parseJson(data,media_path); return false; } bool Graph::loadFile(string &filename,string media_path){ //if (loaded) printf("loading graph: %s\n",(filename).c_str()); if (xml.loadFile(filename)){ return parseXml(media_path); } Poco::FileInputStream fis(filename); Poco::CountingInputStream countingIstr(fis); std::string str; Poco::StreamCopier::copyToString(countingIstr, str); return parseJson(str,media_path); } bool Graph::check_audio(string audio,string path){ if (audio!="") { Poco::File f=Poco::File(path+audio); if (f.exists()) { audio_filename=path+audio; audio_loaded=true; cerr<<"Rotor: loading "< settings; vector attrs; settings["type"]=jnodes[i]["type"].asString(); //attributes settings["media_path"]=media_path; for (uint32_t m=0;mattributes.find(attribute)!=node->attributes.end()){ Attribute *attr=node->attributes.find(attribute)->second; if (attr->type=="enum"){ val=jnodes[i]["attributes"][m]["value"].asString(); attr->init(val); } if (attr->type=="string") { val=jnodes[i]["attributes"][m]["value"].asString(); attr->init(val); } if (attr->type=="array"){ std::vector vals; for (uint32_t i5 = 0; i5 < jnodes[i]["attributes"][m]["value"].size(); i5++ ) { vals.push_back(jnodes[i]["attributes"][m]["value"][i5].asString()); } attr->init(vals); } if (attr->type=="lyrics"){ std::map > lyrics; for (auto k:jnodes[i]["attributes"][m]["value"]){ if (k.size()>2&&k[0].isString()&&k[1].isNumeric()&&k[2].isNumeric()) { lyrics[k[1].asFloat()]=std::make_pair(k[0].asString(),k[2].asFloat()); } } ((Lyrics_attribute*)attr)->init(lyrics); } node->init_attribute(attribute); //cerr << "Rotor: setting attribute '"<type<<" to "<type<<" to "<inputs.size()>j) { string fromID=jnodes[i]["signal_inputs"][j]["from"].asString(); if (fromID!=""){ if (nodes.find(fromID)!=nodes.end()) { if (!nodes[nodeID]->inputs[j]->connect((Signal_node*)nodes[fromID])){ cerr << "ERROR: graph loader cannot connect input " << j << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; return false; } else cerr << "Rotor: linked input " << j << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; } else cerr << "ERROR: linking input " << j << " of node: '" << nodeID << "', cannot find target '" << fromID << "'" << endl; } } else cerr << "ERROR: input " << j << " of node: '" << nodeID << "' does not exist" << endl; } //image inputs if (dynamic_cast(nodes[nodeID])!=nullptr) { //handle expandable inputs if ((((Image_node*)nodes[nodeID])->image_inputs.size()duplicate_inputs){ string desc=((Image_node*)nodes[nodeID])->image_inputs[0]->description; string title=((Image_node*)nodes[nodeID])->image_inputs[0]->title; while(((Image_node*)nodes[nodeID])->image_inputs.size()create_image_input(desc,title); cerr<<"creating an image input"<image_inputs.size()<=k) { if (nodes[nodeID]->duplicate_inputs) { while(((Image_node*)nodes[nodeID])->image_inputs.size()<=k){ ((Image_node*)nodes[nodeID])->create_image_input(settings["description"],settings["title"]); } } } if (((Image_node*)nodes[nodeID])->image_inputs.size()>k) { string fromID=jnodes[i]["image_inputs"][k]["from"].asString(); if (fromID!=""){ if (nodes.find(fromID)!=nodes.end()) { if (dynamic_cast(nodes[fromID])!=nullptr) { if (!dynamic_cast(nodes[nodeID])->image_inputs[k]->connect((Image_node*)nodes[fromID])){ cerr << "ERROR: graph loader cannot connect image input " << k << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; return false; } else cerr << "Rotor: linked image input " << k << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; } else cerr << "ERROR: cannot link image input "<< k << " of node '" << nodeID << "' to node '" << fromID << "' : not an image node" << endl; } else cerr << "ERROR: linking image input " << k << " of node: '" << nodeID << "', cannot find target '" << fromID << "'" << endl; } } else cerr << "ERROR: image input number " << k << " of node: '" << nodeID << "' does not exist" << endl; } } //parameters for (uint32_t l=0;lparameters.find(parameter)!=nodes[nodeID]->parameters.end()) { float val=jnodes[i]["parameters"][l]["value"].asFloat(); if (val!=nodes[nodeID]->parameters.find(parameter)->second->value){ nodes[nodeID]->parameters.find(parameter)->second->value=val; cerr << "Rotor: set parameter '"<parameters[parameter]->connect(nodes[fromID])){ cerr << "ERROR: graph loader cannot connect parameter " << parameter << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; return false; } else cerr << "Rotor: linked parameter " << parameter << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; } else if (fromID!="") cerr << "Rotor: linking parameter " << parameter << " of node: '" << nodeID << "', cannot find target '" << fromID << "'" << endl; } else cerr << "ERROR: cannot find parameter '" << parameter << "' of "< settings; vector attrs; xml.getAttributeNames("node",attrs,i1); for (auto& attr: attrs) { settings[attr]=xml.getAttribute("node",attr,"",i1); //cerr << "Got attribute: " << attr << ":" << xml.getAttribute("node",attr,"",i1) << endl; } settings["media_path"]=media_path; Node* node=factory.create(settings); if (node) { string nodeID=xml.getAttribute("node","ID","",i1); if (nodes.find(nodeID)==nodes.end()){ string nodetype=xml.getAttribute("node","type","",i1); cerr << "Rotor: creating node '"<create_signal_input(xml.getAttribute("signal_input","description","",i2),xml.getAttribute("signal_input","title","",i2)); if ((nodes[nodeID])->inputs.size()>i2) { string fromID=xml.getAttribute("signal_input","from","",i2); if (nodes.find(fromID)!=nodes.end()) { if (!nodes[nodeID]->inputs[i2]->connect((Signal_node*)nodes[fromID])){ cerr << "Rotor: graph loader cannot connect input " << i2 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; return false; } else cerr << "Rotor: linked input " << i2 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; } else cerr << "Rotor: linking input " << i2 << " of node: '" << nodeID << "', cannot find target '" << fromID << "'" << endl; } else cerr << "Rotor: input " << i2 << " of node: '" << nodeID << "' does not exist" << endl; } uint32_t n3=xml.getNumTags("image_input"); for (uint32_t i3=0;i3image_inputs.size()duplicate_inputs){ string desc=((Image_node*)nodes[nodeID])->image_inputs[0]->description; string title=((Image_node*)nodes[nodeID])->image_inputs[0]->title; while(((Image_node*)nodes[nodeID])->image_inputs.size()create_image_input(desc,title); } } //((Image_node*)nodes[nodeID])->create_image_input(xml.getValue("image_input","",i3)); if (((Image_node*)nodes[nodeID])->image_inputs.size()>i3) { string fromID=xml.getAttribute("image_input","from","",i3); if (nodes.find(fromID)!=nodes.end()) { if (!(((Image_node*)nodes[nodeID])->image_inputs[i3]->connect((Image_node*)nodes[fromID]))){ cerr << "Rotor: graph loader cannot connect image input " << i3 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; return false; } else cerr << "Rotor: linked image input " << i3 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; } else cerr << "Rotor: linking image input " << i3 << " of node: '" << nodeID << "', cannot find target '" << fromID << "'" << endl; } else cerr << "Rotor: image input number " << i3 << " of node: '" << nodeID << "' does not exist" << endl; } int n4=xml.getNumTags("parameter"); for (int i4=0;i4parameters.find(parameter)!=nodes[nodeID]->parameters.end()) { string val=xml.getAttribute("parameter","value","",i4); if (val!="") nodes[nodeID]->parameters.find(parameter)->second->value=toFloat(val); string fromID=xml.getAttribute("parameter","from","",i4); if (nodes.find(fromID)!=nodes.end()) { if (!nodes[nodeID]->parameters[parameter]->connect(nodes[fromID])){ cerr << "Rotor: graph loader cannot connect parameter input " << i4 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; return false; } else cerr << "Rotor: linked parameter input " << i4 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; } else if (fromID!="") cerr << "Rotor: linking parameter input " << i4 << " of node: '" << nodeID << "', cannot find target '" << fromID << "'" << endl; } else cerr << "Rotor: cannot find parameter input '" << parameter << "' of "<parameters.find(parameter)!=nodes[nodeID]->parameters.end()) { string val=xml.getAttribute("parameter_input","value","",i4); if (val!="") nodes[nodeID]->parameters.find(parameter)->second->value=toFloat(val); string fromID=xml.getAttribute("parameter_input","from","",i4); if (nodes.find(fromID)!=nodes.end()) { if (!nodes[nodeID]->parameters[parameter]->connect(nodes[fromID])){ cerr << "Rotor: graph loader cannot connect parameter input " << i4 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; return false; } else cerr << "Rotor: linked parameter input " << i4 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; } else cerr << "Rotor: linking parameter input " << i4 << " of node: '" << nodeID << "', cannot find target '" << fromID << "'" << endl; } else cerr << "Rotor: cannot find parameter input '" << parameter << "' of "<set_parameter(xml.getAttribute("parameter","name","",i5),xml.getAttribute("parameter","value","",i5)); //} //if (n5>0) cerr << "Rotor: found " << n5 << " extra parameters for node '" << nodeID << "'" << endl; //support attributes in tags n4=xml.getNumTags("attribute"); for (int i4=0;i4attributes.find(attribute)!=nodes[nodeID]->attributes.end()) { string val=xml.getAttribute("attribute","value","",i4); if (val!="") nodes[nodeID]->attributes.find(attribute)->second->value=val; string type=xml.getAttribute("attribute","type","",i4); if (nodes[nodeID]->attributes.find(attribute)->second->type=="array"){ if(xml.pushTag("attribute",i4)) { int n5=xml.getNumTags("value"); std::vector vals; for (int i5=0;i5attributes.find(attribute)->second->init(vals); xml.popTag(); } } } else cerr << "Rotor: cannot find attribute '" << attribute << "' of "< processors){ if (filename.size()==0) return false; Logger& logger = Logger::get("Rotor"); logger.information("Analysing "+filename+" seed:"+toString(analysis_seed)); //audio_loaded=false; libav::audio_decoder loader; if (!loader.open(filename)) { logger.error("ERROR: Could not open audio: "+filename); return false; } duration=loader.get_duration(); int rate = loader.get_sample_rate(); int samples = loader.get_number_samples(); int channels= loader.get_number_channels(); int bits = loader.get_bit_depth(); for (auto p: processors) { if(!p->init(channels,bits,samples,rate) ){ logger.error("ERROR: Audio plugin failed to initialse"); return false; } } bool finished=false; uint16_t *audio=new uint16_t[1024*loader.get_number_channels()]; uint64_t sample=0; srand(analysis_seed); while (!finished&&!cancelled) { if (loader.get_samples(audio,sample,1024)) { //now we can pass the data to the processor(s) for (auto p: processors) { p->process_frame((uint8_t*)audio,1024); } sample+=1024; //mutex.lock(); progress=((float)sample)/samples; //atomic on 64 bit? //mutex.unlock(); } else finished=true; } loader.cleanup(); for (auto p: processors) { p->cleanup(); p->print_summary(); } logger.information("Finished audio analysis"); //audio_loaded=true; return true; } bool Graph::load_video(const string &nodeID,const string &filename){ //this is a good standard example of how to find //a node of a specific type by ID and do something if (nodes.find(nodeID)!=nodes.end()){ if (nodes[nodeID]->type=="video_loader") { if (((Video_loader*)nodes[nodeID])->load(filename)) { return true; } } } return false; }