#include "rotor.h" //float equality bool fequal(const float u,const float v){ if (abs(u-v)<.001) return true; else return false; }; using namespace Rotor; void Render_context::runTask() { while (!isCancelled()) { int cmd=0; mutex.lock(); if (work_queue.size()){ cmd=work_queue[0]; work_queue.pop_front(); } mutex.unlock(); if(cmd==ANALYSE_AUDIO) { state=ANALYSING_AUDIO; vector processors; processors.push_back(audio_thumb); vector analysers=graph.find_nodes("audio_analysis"); for (auto a: analysers) { processors.push_back(dynamic_cast(a)); } if (load_audio(audio_filename,processors)) { state=AUDIO_READY; } else { //an error occurred: TODO have to clean up allocated data. autoptr? state=IDLE; } } if(cmd==RENDER) { state=RENDERING; if(graph.video_render(output_filename,audio_filename,output_framerate)){ state=RENDER_READY; } else { //an error occurred: TODO have to clean up allocated data. autoptr? state=IDLE; } } sleep(100); } printf("Rotor: stopping thread\n"); } void Render_context::add_queue(int item) { mutex.lock(); work_queue.push_back(item); mutex.unlock(); } bool Signal_input::connect(Signal_node* source) { if (source->output_type=="signal") { connection=(Node*)source; return true; } else return false; } bool Image_input::connect(Image_node* source) { if (source->output_type=="image") { connection=(Node*)source; return true; } else return false; } bool Signal_output::render(const float duration, const float framerate,string &xml_out){ //testing signal routes cerr << "Rotor: Signal_output rendering " << duration << " seconds at " << framerate << " frames per second" << endl; float step=1.0f/framerate; float v=0.0f; for (float f=0.0f;f"+ofToString(u)+"\n"); v=u; } } return true; } Command_response Render_context::session_command(const std::vector& command){ //method,id,command1,{command2,}{body} //here we allow the controlling server to communicate with running tasks Command_response response; response.status=HTTPResponse::HTTP_BAD_REQUEST; if (command[2]=="audio") { if (command[0]=="PUT") { //get audio file location and initiate analysis if (command.size()>2) { if (state==IDLE) { //check file exists Poco::File f=Poco::File(command[3]); if (f.exists()) { //pass to worker thread ??if engine is ready?? ??what if engine has finished but results aren't read?? audio_filename=command[3]; //for now, store session variables in memory add_queue(ANALYSE_AUDIO); response.status=HTTPResponse::HTTP_OK; response.description="Starting audio analysis: "+command[3]+"\n"; } else { response.status=HTTPResponse::HTTP_NOT_FOUND; response.description="File "+command[3]+" not found\n"; } } else { response.status=HTTPResponse::HTTP_BAD_REQUEST; response.description="Rotor: session busy\n"; } } } if (command[0]=="GET") { if (state==ANALYSING_AUDIO) { response.status=HTTPResponse::HTTP_OK; response.description="Rotor: analysing audio\n"; char c[20]; sprintf(c,"%02f",progress); response.description+=""+string(c)+"\n"; } if (state==AUDIO_READY) { //not sure about this-- should this state be retained? //can the data only be read once? //for now response.status=HTTPResponse::HTTP_OK; response.description="Rotor: audio ready\n"; response.description+=""; state=IDLE; } } if (command[0]=="DELETE") { //for now audio_filename=""; response.description="1\n"; response.status=HTTPResponse::HTTP_OK; } } if (command[2]=="graph") { if (command[0]=="GET") { if (graph.loaded) { response.status=HTTPResponse::HTTP_OK; response.description=graph.toString(); } else { response.description="Rotor: graph not loaded\n"; } } if (command[0]=="PUT") { //get new graph from file if (command.size()>2) { //should interrupt whatever is happening? //before begining to load from xml if (state==IDLE) { //eventually not like this Poco::File f=Poco::File(command[3]); if (f.exists()) { string graph_filename=command[3]; if (graph.load(graph_filename)) { response.status=HTTPResponse::HTTP_OK; //response.description="Rotor: loaded graph "+command[3]+"\n"; response.description=graph.toString(); //the graph could actually contain an xml object and we could just print it here? //or could our nodes even be subclassed from xml nodes? //the graph or the audio could load first- have to analyse the audio with vamp after the graph is loaded //for now the graph must load 1st } else { response.status=HTTPResponse::HTTP_INTERNAL_SERVER_ERROR; //~/sources/poco-1.4.6-all/Net/include/Poco/Net/HTTPResponse.h response.description="Rotor: could not load graph "+command[3]+"\n"; } } else { response.status=HTTPResponse::HTTP_NOT_FOUND; response.description="File "+command[3]+" not found\n"; } } } } if (command[0]=="DELETE") { //for now graph=Graph(); response.description="1\n"; response.status=HTTPResponse::HTTP_OK; } } if (command[2]=="signal") { if (command[0]=="GET") { //generate xml from 1st signal output if (state==IDLE) { //direct call for testing float framerate=25.0f; if (command.size()>2) { framerate=ofToFloat(command[3]); } string signal_xml; if (graph.signal_render(signal_xml,framerate)){ response.status=HTTPResponse::HTTP_OK; response.description=signal_xml; } else { response.status=HTTPResponse::HTTP_INTERNAL_SERVER_ERROR; response.description="Rotor: could not render output signal\n"; } } else { response.status=HTTPResponse::HTTP_NOT_FOUND; response.description="Signal output not found\n"; } } else { response.status=HTTPResponse::HTTP_SERVICE_UNAVAILABLE; response.description="Rotor: context busy\n"; } } if (command[2]=="video") { if (command[0]=="GET") { //DUMMY RESPONSE response.status=HTTPResponse::HTTP_OK; response.description="DUMMY RESPONSE Rotor: analysing video\n"; response.description+="45.2\n"; } if (command[0]=="PUT") { //get vide file location and initiate analysis if (command.size()>2) { if (state==IDLE) { //check file exists Poco::File f=Poco::File(command[3]); if (f.exists()) { //pass to worker thread ??if engine is ready?? ??what if engine has finished but results aren't read?? //DUMMY RESPONSE response.description="DUMMY RESPONSE Starting video analysis: "+command[3]+"\n"; } else { response.status=HTTPResponse::HTTP_NOT_FOUND; response.description="File "+command[3]+" not found\n"; } } else { response.status=HTTPResponse::HTTP_BAD_REQUEST; response.description="Rotor: session busy\n"; } } } if (command[0]=="DELETE") { //DUMMY RESPONSE response.description="DUMMY RESPONSE 1\n"; response.status=HTTPResponse::HTTP_OK; } } if (command[2]=="render") { if (command[0]=="GET") { //DUMMY RESPONSE response.status=HTTPResponse::HTTP_OK; response.description="DUMMY RESPONSE Rotor: rendering video\n"; response.description+="25.2\n"; } if (command[0]=="PUT") { if (command.size()>2) { if (state==IDLE) { output_filename=command[3]; if (command.size()>3) { // output_framerate=ofToFloat(command[4]); } add_queue(RENDER); response.status=HTTPResponse::HTTP_OK; response.description="Starting render: "+command[3]+"\n"; } else { response.status=HTTPResponse::HTTP_BAD_REQUEST; response.description="Rotor: session busy\n"; } } else { response.status=HTTPResponse::HTTP_BAD_REQUEST; response.description="Rotor: no output file specified\n"; } } if (command[0]=="DELETE") { //DUMMY RESPONSE //SHOULD CHECK REQUIREMENTS response.status=HTTPResponse::HTTP_OK; response.description="DUMMY RESPONSE Rotor: cancelling render\n"; } } return response; } //http://blog.tomaka17.com/2012/03/libavcodeclibavformat-tutorial/ //great to use c++11 features bool Render_context::load_audio(const string &filename,vector processors){ av_register_all(); AVFrame* frame = avcodec_alloc_frame(); if (!frame) { std::cout << "Error allocating the frame" << std::endl; return false; } AVFormatContext* formatContext = NULL; if (avformat_open_input(&formatContext, filename.c_str(), NULL, NULL) != 0) { av_free(frame); std::cout << "Error opening the file" << std::endl; return false; } if (avformat_find_stream_info(formatContext, NULL) < 0) { av_free(frame); avformat_close_input(&formatContext); std::cout << "Error finding the stream info" << std::endl; return false; } AVStream* audioStream = NULL; for (unsigned int i = 0; i < formatContext->nb_streams; ++i) { if (formatContext->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO) { audioStream = formatContext->streams[i]; break; } } if (audioStream == NULL) { av_free(frame); avformat_close_input(&formatContext); std::cout << "Could not find any audio stream in the file" << std::endl; return false; } AVCodecContext* codecContext = audioStream->codec; codecContext->codec = avcodec_find_decoder(codecContext->codec_id); if (codecContext->codec == NULL) { av_free(frame); avformat_close_input(&formatContext); std::cout << "Couldn't find a proper decoder" << std::endl; return false; } else if (avcodec_open2(codecContext, codecContext->codec, NULL) != 0) { av_free(frame); avformat_close_input(&formatContext); std::cout << "Couldn't open the context with the decoder" << std::endl; return false; } av_dump_format(formatContext, 0, 0, false); //avformat.h line 1256 int samples = ((formatContext->duration + 5000)*codecContext->sample_rate)/AV_TIME_BASE; graph.duration=((float)formatContext->duration)/AV_TIME_BASE; std::cout << "This stream has " << codecContext->channels << " channels, a sample rate of " << codecContext->sample_rate << "Hz and "<sample_fmt<< " (aka "<< av_get_sample_fmt_name(codecContext->sample_fmt) << ") "<init(codecContext->channels,16,samples,codecContext->sample_rate) ){ cerr<<"Plugin failed to initialse"<index) { // Try to decode the packet into a frame int frameFinished = 0; //int bytes = avcodec_decode_audio4(codecContext, frame, &frameFinished, &packet); // Some frames rely on multiple packets, so we have to make sure the frame is finished before // we can use it if (frameFinished) { // frame now has usable audio data in it. How it's stored in the frame depends on the format of // the audio. If it's packed audio, all the data will be in frame->data[0]. If it's in planar format, // the data will be in frame->data and possibly frame->extended_data. Look at frame->data, frame->nb_samples, // frame->linesize, and other related fields on the FFmpeg docs. I don't know how you're actually using // the audio data, so I won't add any junk here that might confuse you. Typically, if I want to find // documentation on an FFmpeg structure or function, I just type " doxygen" into google (like // "AVFrame doxygen" for AVFrame's docs) //av_get_channel_layout_string (char *buf, int buf_size, int nb_channels, uint64_t channel_layout) //now we can pass the data to the processor(s) for (auto p: processors) { p->process_frame(frame->data[0],frame->nb_samples); } sample_processed+=frame->nb_samples; mutex.lock(); progress=((double)sample_processed)/samples; mutex.unlock(); } } // You *must* call av_free_packet() after each call to av_read_frame() or else you'll leak memory av_free_packet(&packet); } // Some codecs will cause frames to be buffered up in the decoding process. If the CODEC_CAP_DELAY flag // is set, there can be buffered up frames that need to be flushed, so we'll do that if (codecContext->codec->capabilities & CODEC_CAP_DELAY) { av_init_packet(&packet); // Decode all the remaining frames in the buffer, until the end is reached int frameFinished = 0; int bytes = avcodec_decode_audio4(codecContext, frame, &frameFinished, &packet); while (bytes >= 0 && frameFinished) { for (auto p: processors) { p->process_frame(frame->data[0],frame->nb_samples); } mutex.lock(); progress=((double)sample_processed)/samples; mutex.unlock(); } } cerr << "finished processing: "<cleanup(); } av_free(frame); avcodec_close(codecContext); avformat_close_input(&formatContext); return true; } const string Graph::toString(){ string xmlgraph; if (loaded) { xml.copyXmlToString(xmlgraph); return xmlgraph; } else return ""; } bool Graph::load(string &filename){ loaded=false; printf("loading graph: %s\n",filename.c_str()); if(xml.loadFile(filename) ){ init(xml.getAttribute("patchbay","ID","",0),xml.getValue("patchbay","",0)); if(xml.pushTag("patchbay")) { int n1=xml.getNumTags("node"); for (int i1=0;i1 settings; vector attrs; xml.getAttributeNames("node",attrs,i1); for (auto& attr: attrs) { settings[attr]=xml.getAttribute("node",attr,"",i1); //cerr << "Got attribute: " << attr << ":" << xml.getAttribute("node",attr,"",i1) << endl; } settings["description"]=xml.getValue("node","",i1); Node* node=factory.create(settings); if (node) { cerr << "Rotor: created '" << xml.getAttribute("node","type","",i1) << "'" << endl; string nodeID=xml.getAttribute("node","ID","",i1); nodes[nodeID]=node; if(xml.pushTag("node",i1)) { int n2=xml.getNumTags("signal_input"); for (int i2=0;i2create_signal_input(xml.getValue("signal_input","",i2)); string fromID=xml.getAttribute("signal_input","from","",i2); if (nodes.find(fromID)!=nodes.end()) { if (!nodes[nodeID]->inputs[i2]->connect((Signal_node*)nodes[fromID])){ cerr << "Rotor: graph loader cannot connect input " << i2 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; return false; } else cerr << "Rotor: linked input " << i2 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; } else cerr << "Rotor: linking input " << i2 << " of node: '" << nodeID << "', cannot find target '" << fromID << "'" << endl; } int n3=xml.getNumTags("image_input"); for (int i3=0;i3create_image_input(xml.getValue("image_input","",i3)); string fromID=xml.getAttribute("image_input","from","",i3); if (nodes.find(fromID)!=nodes.end()) { if (!(((Image_node*)nodes[nodeID])->image_inputs[i3]->connect((Image_node*)nodes[fromID]))){ cerr << "Rotor: graph loader cannot connect image input " << i3 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; return false; } else cerr << "Rotor: linked input " << i3 << " of node '" << nodeID << "' to node '" << fromID << "'" << endl; } else cerr << "Rotor: linking input " << i3 << " of node: '" << nodeID << "', cannot find target '" << fromID << "'" << endl; } xml.popTag(); } } else { cerr << "Rotor: graph loader cannot find node '" << xml.getAttribute("node","type","",i1) << "'" << endl; return false; } } xml.popTag(); } loaded=true; return true; } else return false; } Node_factory::Node_factory(){ //for now, statically load prototype map in constructor add_type("audio_analysis",new Audio_analysis()); add_type("divide",new Signal_divide()); add_type("bang",new Is_new_integer()); add_type("signal_output",new Signal_output()); add_type("testcard",new Testcard()); add_type("video_output",new Video_output()); } bool Audio_thumbnailer::init(int _channels,int _bits,int _samples,int _rate) { //base_audio_processor::init(_channels,_bits,_samples); channels=_channels; bits=_bits; samples=_samples; samples_per_column=samples/width; column=0; //point thumbnail bitmap out_sample=0; //sample in whole track offset=0x1<<(bits-1); //signed audio scale=1.0/offset; sample=0; samples=0; accum=0.0; return true; } int Audio_thumbnailer::process_frame(uint8_t *_data,int samples_in_frame){ //begin by processing remaining samples //samples per column could be larger than a frame! (probably is) //but all we are doing is averaging int bytes=(bits>>3); int stride=channels*bytes; int in_sample=0; while (in_sample>1; for (int i=0;i"+ofToString(u)+"\n"); v=u; } } return true; */ /* bool Video_output::render(const float duration, const float framerate,const string &output_filename,const string &audio_filename){ //render out the network //set up output context //then iterate through frames //querying graph at each frame av_register_all(); AVCodec *codec; AVCodecContext *c= NULL; int i, out_size, size, x, y, outbuf_size; FILE *f; AVFrame *picture; uint8_t *outbuf, *picture_buf; cerr << "Rotor: rendering " << output_filename << " , " << duration << " seconds at " << framerate << " frames per second" << endl; codec = avcodec_find_encoder(AV_CODEC_ID_H264); if (!codec) { cerr<< "codec not found" << endl; return false; } c= avcodec_alloc_context3(codec); picture= avcodec_alloc_frame(); // put sample parameters / c->bit_rate = 400000; // resolution must be a multiple of two / c->width = 640; c->height = 480; // frames per second / c->time_base= (AVRational){1,25}; c->gop_size = 10; // emit one intra frame every ten frames / c->max_b_frames=1; c->pix_fmt = PIX_FMT_YUV420P; //AV_PIX_FMT_RGB24 AVDictionary *options; //= NULL; causes a forward declaration error!? options=NULL; // open it / if (avcodec_open2(c, codec, &options) < 0) { cerr << "could not open codec" << endl; return false; } f = fopen(output_filename.c_str(), "wb"); if (!f) { cerr << "could not open "<< output_filename<width * c->height; picture_buf = malloc((size * 3) / 2); // size for YUV 420 / picture->data[0] = picture_buf; picture->data[1] = picture->data[0] + size; picture->data[2] = picture->data[1] + size / 4; picture->linesize[0] = c->width; picture->linesize[1] = c->width / 2; picture->linesize[2] = c->width / 2; // encode 1 second of video / for(i=0;i<250;i++) { fflush(stdout); // prepare a dummy image / // Y / for(y=0;yheight;y++) { for(x=0;xwidth;x++) { picture->data[0][y * picture->linesize[0] + x] = x + y + i * 3; } } // Cb and Cr / for(y=0;yheight/2;y++) { for(x=0;xwidth/2;x++) { picture->data[1][y * picture->linesize[1] + x] = 128 + y + i * 2; picture->data[2][y * picture->linesize[2] + x] = 64 + x + i * 5; } } // encode the image / out_size = avcodec_encode_video(c, outbuf, outbuf_size, picture); printf("encoding frame %3d (size=%5d)\n", i, out_size); fwrite(outbuf, 1, out_size, f); } // get the delayed frames / for(; out_size; i++) { fflush(stdout); out_size = avcodec_encode_video(c, outbuf, outbuf_size, NULL); printf("write frame %3d (size=%5d)\n", i, out_size); fwrite(outbuf, 1, out_size, f); } // add sequence end code to have a real mpeg file / outbuf[0] = 0x00; outbuf[1] = 0x00; outbuf[2] = 0x01; outbuf[3] = 0xb7; fwrite(outbuf, 1, 4, f); fclose(f); free(picture_buf); free(outbuf); avcodec_close(c); av_free(c); av_free(picture); printf("\n"); return true; } */ bool Video_output::render(const float duration, const float framerate,const string &output_filename,const string &audio_filename){ // //setup defaults int outW=640; int outH=480; int bitRate=4000000; int frameRate=25; AVCodecID codecId=AV_CODEC_ID_MPEG4; std::string container ="mov"; if (exporter->setup(outW,outH,bitRate,frameRate,codecId,container)) { if (exporter->record(output_filename)) { cerr << "Rotor: Video_output rendering " << duration << " seconds at " << framerate << " fps" << endl; float step=1.0f/framerate; float v=0.0f; for (float f=0.0f;fencodeFrame(get_output(Frame_spec(f,framerate,outW,outH))->RGBdata,outW,outH); } exporter->finishRecord(); return true; } } return false; } //new version from libav examples /* AVOutputFormat *fmt; AVFormatContext *oc; AVStream *audio_st, *video_st; double audio_pts, video_pts; int i; //Initialize libavcodec, and register all codecs and formats. // av_register_all(); //think about this: when to register and unregister? //Autodetect the output format from the name. default is MPEG. // fmt = av_guess_format(NULL, output_filename.c_str(), NULL); if (!fmt) { printf("Could not deduce output format from file extension: using MPEG.\n"); fmt = av_guess_format("mpeg", NULL, NULL); } if (!fmt) { cerr << "Rotor: could not find suitable output format" << endl; return false; } //Allocate the output media context. // oc = avformat_alloc_context(); if (!oc) { cerr <<"Rotor: memory error"<< endl; return false; } oc->oformat = fmt; snprintf(oc->filename, sizeof(oc->filename), "%s", filename); //Add the audio and video streams using the default format codecs * and initialize the codecs. // video_st = NULL; audio_st = NULL; if (fmt->video_codec != AV_CODEC_ID_NONE) { video_st = add_video_stream(oc, fmt->video_codec); } if (fmt->audio_codec != AV_CODEC_ID_NONE) { audio_st = add_audio_stream(oc, fmt->audio_codec); } //Now that all the parameters are set, we can open the audio and * video codecs and allocate the necessary encode buffers. // if (video_st) open_video(oc, video_st); if (audio_st) open_audio(oc, audio_st); av_dump_format(oc, 0, filename, 1); //open the output file, if needed // if (!(fmt->flags & AVFMT_NOFILE)) { if (avio_open(&oc->pb, filename, AVIO_FLAG_WRITE) < 0) { cerr <<"Could not open "<pts.val * audio_st->time_base.num / audio_st->time_base.den; else audio_pts = 0.0; if (video_st) video_pts = (double)video_st->pts.val * video_st->time_base.num / video_st->time_base.den; else video_pts = 0.0; if ((!audio_st || audio_pts >= STREAM_DURATION) && (!video_st || video_pts >= STREAM_DURATION)) break; //write interleaved audio and video frames // if (!video_st || (video_st && audio_st && audio_pts < video_pts)) { write_audio_frame(oc, audio_st); } else { write_video_frame(oc, video_st); } } //Write the trailer, if any. The trailer must be written before you // close the CodecContexts open when you wrote the header; otherwise // av_write_trailer() may try to use memory that was freed on // av_codec_close(). // //av_write_trailer(oc); //Close each codec. // if (video_st) close_video(oc, video_st); if (audio_st) close_audio(oc, audio_st); //Free the streams. // for (i = 0; i < oc->nb_streams; i++) { av_freep(&oc->streams[i]->codec); av_freep(&oc->streams[i]); } if (!(fmt->flags & AVFMT_NOFILE)) //Close the output file. // avio_close(oc->pb); //free the stream // av_free(oc); return true; */