From 89961817f408e921de2c1be6197e2b1ee0f5df98 Mon Sep 17 00:00:00 2001 From: Comment Date: Wed, 29 Jan 2014 23:58:55 +0000 Subject: NT audio framework --- NT/src/graph.cpp | 65 ++++ NT/src/graph.h | 10 +- NT/src/libavwrapper.cpp | 724 ++++++++++++++++++++++++++++++++++++++++ NT/src/libavwrapper.h | 239 +++++++++++++ NT/src/nodes.h | 1 + NT/src/nodes_audio_analysis.cpp | 475 ++++++++++++++++++++++++++ NT/src/nodes_audio_analysis.h | 329 ++++++++++++++++++ NT/src/rendercontext.cpp | 171 +++++----- NT/src/rendercontext.h | 33 +- NT/src/rotor.h | 1 + NT/src/rotord.o | Bin 136232 -> 0 bytes NT/src/tinyxml.o | Bin 499688 -> 0 bytes NT/src/tinyxmlerror.o | Bin 213368 -> 0 bytes NT/src/tinyxmlparser.o | Bin 355616 -> 0 bytes NT/src/vampHost.cpp | 416 +++++++++++++++++++++++ NT/src/vampHost.h | 95 ++++++ 16 files changed, 2456 insertions(+), 103 deletions(-) create mode 100644 NT/src/libavwrapper.cpp create mode 100644 NT/src/libavwrapper.h create mode 100644 NT/src/nodes_audio_analysis.cpp create mode 100644 NT/src/nodes_audio_analysis.h delete mode 100644 NT/src/rotord.o delete mode 100644 NT/src/tinyxml.o delete mode 100644 NT/src/tinyxmlerror.o delete mode 100644 NT/src/tinyxmlparser.o create mode 100644 NT/src/vampHost.cpp create mode 100644 NT/src/vampHost.h (limited to 'NT/src') diff --git a/NT/src/graph.cpp b/NT/src/graph.cpp index 50db271..ad89061 100644 --- a/NT/src/graph.cpp +++ b/NT/src/graph.cpp @@ -18,6 +18,71 @@ bool Graph::check_audio(string audio,string path){ } return false; } +bool Graph::load_audio(const std::string &filename) { + Logger& logger = Logger::get(id); + if (filename.size()==0) { + logger.information("ERROR: empty filename for audio analysis"); + return false; + } + libav::audio_decoder loader; + if (!loader.open(filename)) { + logger.error("ERROR: Could not open audio: "+filename); + return false; + } + logger.information("Analysing "+filename); + + duration=loader.get_duration(); + + int rate = loader.get_sample_rate(); + int samples = loader.get_number_samples(); + int channels= loader.get_number_channels(); + int bits = loader.get_bit_depth(); + + vector processors; + //processors.push_back(audio_thumb); + for (auto a: nodes) { + if (dynamic_cast(a.second)){ + processors.push_back(dynamic_cast(a.second)); + } + } + for (auto p: processors) { + if(!p->init(channels,bits,samples,rate) ){ + logger.error("ERROR: Audio plugin failed to initialse"); + return false; + } + } + + bool finished=false; + uint16_t *audio=new uint16_t[1024*loader.get_number_channels()]; + uint64_t sample=0; + + while (!finished&&!cancelled) + { + if (loader.get_samples(audio,sample,1024)) { + //now we can pass the data to the processor(s) + for (auto p: processors) { + p->process_frame((uint8_t*)audio,1024); + } + sample+=1024; + //mutex.lock(); + progress=((double)sample)/samples; //atomic on 64 bit? + //mutex.unlock(); + + } + else finished=true; + } + + loader.cleanup(); + + for (auto p: processors) { + p->cleanup(); + p->print_summary(); + } + + logger.information("Finished audio analysis"); + //audio_loaded=true; + return true; +} bool Graph::load_file(std::string filename,std::string media_path){ Logger& logger = Logger::get(id); Poco::File f=Poco::File(filename); diff --git a/NT/src/graph.h b/NT/src/graph.h index cef2fb8..00aee48 100644 --- a/NT/src/graph.h +++ b/NT/src/graph.h @@ -56,7 +56,7 @@ namespace Rotor { //--context// Json::Value preview(std::string &node ,std::string &format,int frame,int w,int h); bool check_audio (std::string audio ,std::string path); //--context// Json::Value print_features (std::string &node); - //bool load_audio(const std::string &filename, std::vector processors); + bool load_audio(const std::string &filename); //bool load_video(const std::string &node_id,const std::string &filename); //load audio and video should be methods of the nodes themselves? bool loaded; @@ -69,11 +69,17 @@ namespace Rotor { //Log_name=_Log_name; //} - //Audio_thumbnailer *audio_thumb; + //Audio_thumbnail *audio_thumb; + //how does audio thumbnail fit in with the new plan? + //a graph has A audio thumbnail to represent THE audio track? + //is this important any more? I don't think Josep uses it? + + //also need to review workflow with audio private: std::string id; std::unordered_map nodes; + double duration; }; } diff --git a/NT/src/libavwrapper.cpp b/NT/src/libavwrapper.cpp new file mode 100644 index 0000000..4a8315f --- /dev/null +++ b/NT/src/libavwrapper.cpp @@ -0,0 +1,724 @@ +#include "libavwrapper.h" + +Poco::Mutex mutex; + +#include +#include +#include + +using namespace std; +using namespace Poco; + +static bool b_is_one_time_inited=false; +static int sws_flags = SWS_BICUBIC; + +void libav::maybeInitFFMpegLib() +{ + if (b_is_one_time_inited) + return; + FFMS_Init(0, 0); //should do for all + //av_register_all(); + //avcodec_register_all(); + avformat_network_init(); + b_is_one_time_inited = true; +} + + void libav::video_decoder::cleanup(){ + if (loaded) { + Mutex::ScopedLock lock(mutex); + FFMS_DestroyVideoSource(source); + loaded=false; + } +} + +bool libav::video_decoder::open(const std::string& filename){ + cleanup(); + Mutex::ScopedLock lock(mutex); + Poco::File f=Poco::File(filename); + if (!f.exists()) { + cerr<<"ERROR: "<1){ + for (uint32_t i=0;iEncodedWidth; + h=propframe->EncodedHeight; + //propframe->EncodedPixelFormat; + + if (FFMS_SetOutputFormatV2(source, pixfmts, propframe->EncodedWidth, propframe->EncodedHeight, FFMS_RESIZER_BICUBIC, &err)) { + std::cerr<<"ffmpegsource: "<w=w; + this->h=h; + this->bitRate=bitRate; + this->frameRate=frameRate; + this->container=container; + + if (NULL != sws_ctx) { + sws_freeContext(sws_ctx); + sws_ctx = NULL; + } + + sws_ctx = sws_getContext(w, h, AV_PIX_FMT_RGB24, + w, h, AV_PIX_FMT_YUV420P, + sws_flags, NULL, NULL, NULL); + + return true; +} + +bool libav::exporter::record(std::string filename){ + + + // allocate the output media context // + avformat_alloc_output_context2(&oc, NULL, NULL, filename.c_str()); + + + if (!oc) { + printf("Could not deduce output format from file extension: using MPEG.\n"); + avformat_alloc_output_context2(&oc, NULL, "mpeg", filename.c_str()); + } + if (!oc) { + return false; + } + fmt = oc->oformat; + + av_opt_set(oc->priv_data, "profile", "baseline", AV_OPT_SEARCH_CHILDREN); + + + // Add the audio and video streams using the default format codecs + // * and initialize the codecs. // + video_st = NULL; + audio_st = NULL; + + fmt->video_codec=AV_CODEC_ID_H264; //AV_CODEC_ID_MPEG4 + fmt->audio_codec=AV_CODEC_ID_AAC; //guessing, 011013 + + + if (fmt->video_codec != AV_CODEC_ID_NONE) { + video_st = add_stream(oc, &video_codec, fmt->video_codec); + } + if (fmt->audio_codec != AV_CODEC_ID_NONE) { + //audio_st = add_stream(oc, &audio_codec, fmt->audio_codec); + audio_st = add_stream(oc, &audio_codec, fmt->audio_codec); + } + + + + //set initial video params + video_st->codec->width=w; + video_st->codec->height=h; + video_st->codec->time_base.num = 1;//codecCtx->ticks_per_frame; + video_st->codec->time_base.den = frameRate; + video_st->time_base = video_st->codec->time_base; + //audioStream->time_base = codecCtx->time_base; //???has the capability of crashing + + video_st->codec->gop_size = 75; /* emit one intra frame every 75 frames */ + video_st->codec->pix_fmt = PIX_FMT_YUV420P; + + if (bitRate){ + cerr<<"Libav::exporter setting bitrate to "<codec->bit_rate = bitRate; //need to deal with resolution etc + video_st->codec->rc_max_rate = bitRate; + video_st->codec->rc_min_rate = 0; + + //added 131113 + video_st->codec->rc_buffer_size = bitRate * 2; + } + + //av_dict_set(&options, "profile", "baseline", 0); + //video_st->codec->flags2 = CODEC_FLAG2_FASTPSKIP; + video_st->codec->profile = FF_PROFILE_H264_BASELINE; + video_st->codec->level = 13; + + + //how to set profile????? 190913 + //this causes a crash + //av_opt_set(video_st->codec->priv_data,"profile",FF_PROFILE_H264_CONSTRAINED_BASELINE, 0); + // + // http://libav-users.943685.n4.nabble.com/Libav-user-encoding-bitrate-wrong-td4433740.html + // http://libav-users.943685.n4.nabble.com/Libav-user-H264-encoding-set-target-average-bitrate-td3912529.html + // http://trac.ffmpeg.org/wiki/x264EncodingGuide + + + + // Now that all the parameters are set, we can open the audio and + // * video codecs and allocate the necessary encode buffers. // + + if (video_st) { + if (!open_video(oc, video_codec, video_st)){ + return false; + } + } + if (audio_st) { + audioframesize=open_audio(oc, audio_codec, audio_st); + audiostep=((double)audioframesize)/(audio_st->codec->sample_rate); + std::cerr << "opened audio codec with "<flags & AVFMT_NOFILE)) { + Mutex::ScopedLock lock(mutex); + int ret = avio_open(&oc->pb, filename.c_str(), AVIO_FLAG_WRITE); + if (ret < 0) { + std::cerr <<"Could not open " << filename.c_str() << std::endl; + return false; + } + } + + AVDictionary *opts = NULL; // "create" an empty dictionary + + // https://libav.org/avprobe.html#MOV_002fMP4_002fISMV + + //THIS DOES SEEM TO SET CONTAINER OPTS= AS MOOV_SIZE MAKES THE MOVIE BANJAXED + //av_dict_set(&opts, "moov_size", "20000", 0); + if (fragmentation) { + av_dict_set(&opts, "movflags","frag_keyframe", 0); //makes a video watchable in chrome, quicktime & mplayer + //av_dict_set(&opts, "movflags","empty_moov",1); + } + + + // Write the stream header, if any. // + int ret = avformat_write_header(oc, &opts); + if (ret < 0) { + std::cerr <<"Error occurred when opening output file:" < + //mov_write_moov_tag(AVIOContext *pb, MOVMuxContext *mov, AVFormatContext *s) + /*ret = mov_write_moov_tag(oc, NULL,NULL); + if (ret < 0) { + std::cerr <<"Error occurred when writing moov atom " <pts = 0; + + outputframe=0; + + return true; +} +bool libav::exporter::encodeFrame(unsigned char *pixels,uint16_t *samples){ + // Compute current audio and video time. // + if (audio_st) + audio_pts = (double)audio_st->pts.val * audio_st->time_base.num / audio_st->time_base.den; + else + audio_pts = 0.0; + + if (video_st) + video_pts = (double)video_st->pts.val * video_st->time_base.num / + video_st->time_base.den; + else + video_pts = 0.0; + + // write interleaved audio and video frames // + if (!video_st || (video_st && audio_st && audio_pts < video_pts)) { + write_audio_frame(oc, audio_st, samples); + } else { + write_video_frame(oc, video_st, pixels); + + frame->pts += av_rescale_q(1, video_st->codec->time_base, video_st->time_base); + } + + //std::cerr << "encoded frame " << outputframe << std::endl; + outputframe++; + + return true; +} +bool libav::exporter::encodeFrame(unsigned char *pixels,AVPacket *audio){ + // Compute current audio and video time. // + if (audio_st) + audio_pts = (double)audio_st->pts.val * audio_st->time_base.num / audio_st->time_base.den; + else + audio_pts = 0.0; + + if (video_st) + video_pts = (double)video_st->pts.val * video_st->time_base.num / + video_st->time_base.den; + else + video_pts = 0.0; + + // write interleaved audio and video frames // + if (!video_st || (video_st && audio_st && audio_pts < video_pts)) { + write_audio_frame(oc, audio_st, audio); + } else { + write_video_frame(oc, video_st, pixels); + + frame->pts += av_rescale_q(1, video_st->codec->time_base, video_st->time_base); + } + + //std::cerr << "encoded frame " << outputframe << std::endl; + outputframe++; + + return true; +} +bool libav::exporter::encodeFrame(unsigned char *pixels,bool keyframe){ + video_pts = (double)video_st->pts.val * video_st->time_base.num / video_st->time_base.den; + write_video_frame(oc, video_st, pixels,keyframe); + frame->pts += av_rescale_q(1, video_st->codec->time_base, video_st->time_base); + outputframe++; + return true; +} +bool libav::exporter::encodeFrame(uint16_t *samples){ + audio_pts = (double)audio_st->pts.val * audio_st->time_base.num / audio_st->time_base.den; + write_audio_frame(oc, audio_st, samples); + return true; +} +void libav::exporter::finishRecord(){ + + av_write_trailer(oc); + // Close each codec. // + if (video_st) + close_video(oc, video_st); + if (audio_st) + close_audio(oc, audio_st); + + if (!(fmt->flags & AVFMT_NOFILE)) { + // Close the output file. // + Mutex::ScopedLock lock(mutex); + avio_close(oc->pb); + } + + // free the stream // + avformat_free_context(oc); +} + +AVStream* libav::exporter::add_stream(AVFormatContext *oc, AVCodec **codec,enum AVCodecID codec_id) + { + //removed some of the repeats from record() 141113 + AVCodecContext *c; + AVStream *st; + + // find the encoder // + *codec = avcodec_find_encoder(codec_id); + if (!(*codec)) { + //fprintf(stderr, "Could not find encoder for '%s'\n", + // avcodec_get_name(codec_id)); + exit(1); + } + + st = avformat_new_stream(oc, *codec); + if (!st) { + //fprintf(stderr, "Could not allocate stream\n"); + exit(1); + } + st->id = oc->nb_streams-1; + c = st->codec; + + switch ((*codec)->type) { + case AVMEDIA_TYPE_AUDIO: + st->id = 1; + c->sample_fmt = AV_SAMPLE_FMT_S16; + c->bit_rate = 160000; + c->sample_rate = 44100; + c->channels = 2; + c->channel_layout=AV_CH_LAYOUT_STEREO; + break; + + case AVMEDIA_TYPE_VIDEO: + c->codec_id = codec_id; + + //c->bit_rate = bitRate; //need to deal with resolution etc + + // c->rc_max_rate = bitRate; + //c->rc_min_rate = 0; + //c->rc_buffer_size = Profile()->m_videoMaxVopSize; //?? + + /* + all taken out 091113 to try to stop making incompatible movies + */ + + + // Resolution must be a multiple of two. // + //c->width = 352; + //c->height = 288; + + + // timebase: This is the fundamental unit of time (in seconds) in terms + // * of which frame timestamps are represented. For fixed-fps content, + // * timebase should be 1/framerate and timestamp increments should be + // * identical to 1. // + c->time_base.den = frameRate; + c->time_base.num = 1; + // c->gop_size = 12; // emit one intra frame every twelve frames at most // + c->pix_fmt = AV_PIX_FMT_YUV420P; //ADDED HARDCODED TJR 280513 + if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) { + // just for testing, we also add B frames // + c->max_b_frames = 2; + } + if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO) { + // Needed to avoid using macroblocks in which some coeffs overflow. + // * This does not happen with normal video, it just happens here as + // * the motion of the chroma plane does not match the luma plane. // + c->mb_decision = 2; + } + + break; + + default: + break; + } + + // Some formats want stream headers to be separate. // + if (oc->oformat->flags & AVFMT_GLOBALHEADER) + c->flags |= CODEC_FLAG_GLOBAL_HEADER; + + return st; + } + +bool libav::exporter::open_video(AVFormatContext *oc, AVCodec *codec, AVStream *st) + { + + int ret; + AVCodecContext *c = st->codec; + + + // open the codec // + { + Mutex::ScopedLock lock(mutex); + ret = avcodec_open2(c, codec, NULL); //OPTIONS CAN GO HERE + + } + if (ret < 0) { + //string e=av_err2str(ret); + cerr<<"Could not open video codec, avcodec_open2 returned "<pix_fmt, c->width, c->height); + if (ret < 0) { + //av_err2str(ret)<< + cerr<<"Could not allocate picture"<pix_fmt != AV_PIX_FMT_YUV420P) { + ret = avpicture_alloc(&src_picture, AV_PIX_FMT_RGB24, c->width, c->height); + if (ret < 0) { + //<codec->width,st->codec->height)); + return true; + } + + int libav::exporter::open_audio(AVFormatContext *oc, AVCodec *codec, AVStream *st) + { + AVCodecContext *c; + int ret; + + c = st->codec; + + // open it // + mutex.lock(); + ret = avcodec_open2(c, codec, NULL); + mutex.unlock(); + if (ret < 0) { + //fprintf(stderr, "Could not open audio codec: %s\n", av_err2str(ret)); + exit(1); + } + + // init signal generator // + t = 0; + tincr = 2 * M_PI * 110.0 / c->sample_rate; + // increment frequency by 110 Hz per second // + tincr2 = 2 * M_PI * 110.0 / c->sample_rate / c->sample_rate; + + if (c->codec->capabilities & CODEC_CAP_VARIABLE_FRAME_SIZE) + audio_input_frame_size = 10000; + else + audio_input_frame_size = c->frame_size; + + /* + samples = av_malloc(audio_input_frame_size * + av_get_bytes_per_sample(c->sample_fmt) * + c->channels); + if (!samples) { + //fprintf(stderr, "Could not allocate audio samples buffer\n"); + exit(1); + } + */ + return audio_input_frame_size; + } + + void libav::exporter::write_audio_frame(AVFormatContext *oc, AVStream *st,uint16_t *samples) + { + AVCodecContext *c; + AVPacket pkt = { 0 }; // data and size must be 0; + AVFrame *frame = avcodec_alloc_frame(); + int got_packet, ret; + + //av_init_packet(&pkt); 111013 NOT NECESSARY + c = st->codec; + + //get_audio_frame(samples, audio_input_frame_size, c->channels); + frame->nb_samples = audio_input_frame_size; + uint8_t *sampleptr; + int bufsize=audio_input_frame_size * av_get_bytes_per_sample(c->sample_fmt) *c->channels; + if (samples) { + sampleptr=(uint8_t*)samples; + } + else { + sampleptr=new uint8_t[bufsize]; + memset(sampleptr,0,bufsize); + } + + avcodec_fill_audio_frame(frame, c->channels, c->sample_fmt, + sampleptr, + audio_input_frame_size * + av_get_bytes_per_sample(c->sample_fmt) * + c->channels, 0); //; + //frame->sample_rate=44100; //hard coded input rate- nope, this doesn't help + //frame->format=AV_SAMPLE_FMT_S16P; + //?? why is ffmpeg reporting fltp as the sample format??? doesn't seem to have an effect to change this though + ret = avcodec_encode_audio2(c, &pkt, frame, &got_packet); + if (!samples) { + delete[] sampleptr; + } + if (ret < 0) { + //fprintf(stderr, "Error encoding audio frame: %s\n", av_err2str(ret)); + exit(1); + } + + if (!got_packet) + return; + + pkt.stream_index = st->index; + + //avcodec_free_frame(&frame); ///removed 091013 + + + // Write the compressed frame to the media file. // + ret = av_interleaved_write_frame(oc, &pkt); + av_free_packet(&pkt); + if (ret != 0) { + //fprintf(stderr, "Error while writing audio frame: %s\n", + // av_err2str(ret)); + exit(1); + } + + } + + void libav::exporter::write_audio_frame(AVFormatContext *oc, AVStream *st,AVPacket *pkt) + { + + pkt->stream_index = st->index; + + // Write the compressed frame to the media file. // + int ret = av_interleaved_write_frame(oc, pkt); + if (ret != 0) { + //fprintf(stderr, "Error while writing audio frame: %s\n", + // av_err2str(ret)); + exit(1); + } + //avcodec_free_frame(&frame); + av_free_packet(pkt); ///added 091013 + } + + void libav::exporter::close_audio(AVFormatContext *oc, AVStream *st) + { + mutex.lock(); + avcodec_close(st->codec); + mutex.unlock(); + + } + + void libav::exporter::write_video_frame(AVFormatContext *oc, AVStream *st, uint8_t *pixels, bool keyframe) + { + int ret; + + AVCodecContext *c = st->codec; + + avpicture_fill(&src_picture, pixels, PIX_FMT_RGB24, c->width,c->height); + //avpicture_fill(&dst_picture, outPixels, PIX_FMT_YUV420P, c->width,c->height); + + sws_scale(sws_ctx, src_picture.data, src_picture.linesize, 0, c->height, dst_picture.data, dst_picture.linesize); + //fill_yuv_image(&dst_picture, frame_count, c->width, c->height); + if (oc->oformat->flags & AVFMT_RAWPICTURE) { + // Raw video case - directly store the picture in the packet // + AVPacket pkt; + //av_init_packet(&pkt); ///removed 101013 NOT NECESSARY + + pkt.flags |= AV_PKT_FLAG_KEY; + pkt.stream_index = st->index; + pkt.data = dst_picture.data[0]; + pkt.size = sizeof(AVPicture); + + ret = av_interleaved_write_frame(oc, &pkt); + av_free_packet(&pkt); ///added 091013 + } else { + AVPacket pkt = { 0 }; + int got_packet; + + if (keyframe) pkt.flags |= AV_PKT_FLAG_KEY; + + ret = avcodec_encode_video2(c, &pkt, frame, &got_packet); + if (ret < 0) { + //fprintf(stderr, "Error encoding video frame: %s\n", av_err2str(ret)); + exit(1); + } + // If size is zero, it means the image was buffered. // + + if (!ret && got_packet && pkt.size) { + pkt.stream_index = st->index; + + // Write the compressed frame to the media file. // + ret = av_interleaved_write_frame(oc, &pkt); + } else { + ret = 0; + } + av_free_packet(&pkt); ///added 091013 + } + + + + + if (ret != 0) { + //fprintf(stderr, "Error while writing video frame: %s\n", av_err2str(ret)); + exit(1); + } + frame_count++; + + //avcodec_free_frame(&frame); + } + + void libav::exporter::close_video(AVFormatContext *oc, AVStream *st) + { + mutex.lock(); + //avcodec_close(st->codec); //change 0706 to trace 2nd render issue + avcodec_close(audio_st->codec); + avcodec_close(video_st->codec); + // + // + + + //av_free(src_picture.data[0]); //removed to explore weird 2nd render crash.. seems to WORK -- seems that the picture data is owned elsewhere + av_free(dst_picture.data[0]); + av_free(frame); //removed to explore crash 2nd time render + //gives *** Error in `./rotord': corrupted double-linked list: 0x00007fd8b005bd60 *** + //where is frame initialised??? + //moved to destructor + + + av_free(outPixels); //SIGSEV here??? + mutex.unlock(); + } \ No newline at end of file diff --git a/NT/src/libavwrapper.h b/NT/src/libavwrapper.h new file mode 100644 index 0000000..162a77e --- /dev/null +++ b/NT/src/libavwrapper.h @@ -0,0 +1,239 @@ +#ifndef libavwrapper_H +#define libavwrapper_H + +#ifndef UINT64_C +#define UINT64_C(c) (c ## ULL) +#endif + +#include "Poco/Mutex.h" +#include "Poco/ScopedLock.h" +#include "Poco/StringTokenizer.h" +#include "Poco/File.h" + +extern Poco::Mutex mutex; //application wide mutex + +extern "C" { +#include +#include +#include +#include +#include +} + +#include +#include +#include +#include +#include +#include + +#include + + +namespace libav { + // Some libavcodec calls are not reentrant + void maybeInitFFMpegLib(); + + class video_decoder + { + public: + video_decoder(){ + maybeInitFFMpegLib(); + pixfmts[0] = FFMS_GetPixFmt("rgb24"); + pixfmts[1] = -1; + h=0; + w=0; + source=NULL; + loaded=false; + err.Buffer = errmsg; + err.BufferSize = sizeof(errmsg); + err.ErrorType = FFMS_ERROR_SUCCESS; + err.SubType = FFMS_ERROR_SUCCESS; + } + ~video_decoder(){ + cleanup(); + } + void cleanup(); + bool open(const std::string& filename); + double get_framerate(){ + if (loaded) return (((double)props->FPSNumerator)/((double)props->FPSDenominator)); + else return -1.0; + } + int get_number_frames(){ + if (loaded) return props->NumFrames; + else return -1; + } + int get_number_channels(){ + return 3; //this is what we convert to + } + int get_width(){ + return w; + } + int get_height(){ + return h; + } + bool fetch_frame(int width,int height,int wanted){ + if (FFMS_SetOutputFormatV2(source, pixfmts, width, height, FFMS_RESIZER_BICUBIC, &err)) { + std::cerr<<"ffmpegsource: "<NumFrames, &err); + if (frame == NULL) { + std::cerr<<"ffmpegsource: "<SampleFormat; + else return 0; + } + int get_sample_rate(){ + if (props) return props->SampleRate; + else return 0; + } + int get_bit_depth(){ + if (props) return props->BitsPerSample; + else return 0; + } + int get_number_channels(){ + if (props) return props->Channels; + else return 0; + } + int get_number_samples(){ + if (props) return props->NumSamples; + else return 0; + } + int64_t get_channel_layout(){ + if (props) return props->ChannelLayout; + else return 0; + } + double get_duration(){ + if (props) return ((double)props->NumSamples)/props->SampleRate; + else return 0; + } + bool get_samples(void *buf,int64_t start, int64_t count){ + if (source) { + if (FFMS_GetAudio(source, buf, start, count, &err)) { + std::cerr<<"ffmpegsource: "<>3); + int stride=channels*bytes; + int in_sample=0; + while (in_sample0) vdata+=","; + vdata+=toString(sample); + i++; + } + XML.addValue("data",vdata); + } + bool Vamp_node::init(int _channels,int _bits,int _samples, int _rate) { + //need these to make sense of data + channels=_channels; + bits=_bits; + samples=_samples; + + features.clear(); + return analyser.init(soname,id,_channels,_bits,_samples,_rate,outputNo,params); + + + //attempt to load vamp plugin and prepare to receive frames of data + //should the audio analysis contain a vamphost or should it inherit? + //maybe neater to contain it in terms of headers etc + + } + int Vamp_node::process_frame(uint8_t *data,int samples_in_frame) { + analyser.process_frame(data,samples_in_frame); + return 1; + } + void Vamp_node::cleanup() { + analyser.cleanup(); + features=analyser.features; + } + string Vamp_node::get_features(){ + string data; + for (auto i: features) { + data=data+" ["+toString(i.second.number)+":"+toString(i.first); + if (i.second.values.size()) { + data+=" ("; + bool first=true; + for (auto j: i.second.values) { + if (first){ + first=false; + } + else data+=","; + data=data+toString(j); + } + data+=") "; + } + data+="]"; + } + return data; + } + bool sortsegments(std::pair i,std::pair j){ + return (i.second > > i,std::pair > > j){ + return (i.first i,pair j){ + return (i.first > similarities; + + //what do we want to know about these similarities? + // how many are there? map.size() + // how many members are in each one? map[item].size() + // which members are they? auto m: map[item] + + uint32_t i=0; + for (auto f:analysers["segmenter"].features) { + if (f.second.values.size()) { + int group=f.second.values[0]; + if (similarities.find(group)==similarities.end()){ + similarities[group]={}; + } + similarities[group].push_back(i); + } + i++; + } + /* + for (auto s:similarities) { + string list=""; + for (int j=0;j0) list+=","; + list +=toString(s.second[j]); + } + cerr<<"group "< tempos; + vector intensities; + vector times; + auto g=++analysers["segmenter"].features.begin(); + for (auto f=analysers["segmenter"].features.begin();g!=analysers["segmenter"].features.end();f++,g++,i++){ + times.push_back(f->first); + //integrate tempo and intensity algorithmically + double tempo=0; + if (analysers["tempo"].features.size()) { + double pt=f->first; + double pv=analysers["tempo"].get_value(f->first); + for (auto u=analysers["tempo"].features.upper_bound(f->first);u!=analysers["tempo"].features.upper_bound(g->first);u++){ + tempo +=(u->first-pt)*(u->second.values[0]+pv)*0.5f; //area of the slice + pt=u->first; + pv=u->second.values[0]; + } + tempo +=(g->first-pt)*(analysers["tempo"].get_value(g->first)+pv)*0.5f; //area of the last slice + tempo /=g->first-f->first; //average value; + } + if (tempo>max_tempo) max_tempo=tempo; + if (tempofirst; + double pv=analysers["intensity"].get_value(f->first); + for (auto u=analysers["intensity"].features.upper_bound(f->first);u!=analysers["intensity"].features.upper_bound(g->first);u++){ + intensity +=(u->first-pt)*(u->second.values[0]+pv)*0.5f; //area of the slice + pt=u->first; + pv=u->second.values[0]; + } + intensity +=(g->first-pt)*(analysers["intensity"].get_value(g->first)+pv)*0.5f; //area of the last slice + intensity /=g->first-f->first; //average value; + } + if (intensity>max_intensity) max_intensity=intensity; + if (intensityfirst<<" to "<first<<" average tempo: "<value)+(intensity*parameters["intensity_weight"]->value)<> totals; + vector totalsmap; + for (i=0;ivalue)+(intensities[i]*parameters["intensity_weight"]->value))); + totalsmap.push_back((tempos[i]*parameters["tempo_weight"]->value)+(intensities[i]*parameters["intensity_weight"]->value)); + } + + /* + //sort and convert to features + std::sort(totals.begin(),totals.end(),sortsegments); + for (i=0;i bucketoffsets; + for (auto t:totals) bucketoffsets.push_back(0.0); + if (parameters["levels"]->value>0.0&¶meters["levels"]->valuevalue); + double numberperbin=((double)numbertoredistribute/totals.size()); + double toadd=0.5f; + int added=0; + for (int j=0;j0) { + cerr<<"reducing number of levels by "<> +start grouping by similarity +if there are more similarity groups than wantedgroups, start by grouping similarities +otherwise take biggest similarity groups and split them by intensity +if there are still too many groups, merge closest smallest groups +finally sort by intensity to map output + +nned to retrieve total intensity by segment + */ + // segment group_intensity seg_intense segment + vector > > > seggrps; + vector > > > oldgrps; + for (i=0;i > data; + data.push_back(make_pair(totalsmap[i],i)); + oldgrps.push_back(make_pair(totalsmap[i],data)); + } + + for (auto s:similarities){ + //similarities is a collection of similarity groups in no particular order, referring to segment nos + //at this point seggrps is in segment order + if (s.second.size()>1){ + for (int j=s.second.size()-1;j>0;j--){ + oldgrps[s.second[0]].second.push_back(make_pair(totalsmap[s.second[j]],s.second[j])); + //keep running average// should be by area? + //seggrps[s.second[0]].first+=(totalsmap[s.second[j]]*(1.0/max(1,(int)seggrps[s.second[0]].second.size()-1))); + //double div=seggrps[s.second[0]].second.size()==1?1.0:((double)seggrps[s.second[0]].second.size()-1/(double)seggrps[s.second[0]].second.size()); + //neat! this gives 1,1/2,2/3,3/4.. + //seggrps[s.second[0]].first*=div; + + //seggrps.erase(seggrps.begin()+s.second[j]); + //after this has happened, seggrpgs indexing can be invalid + + } + + + //easier is to + double avg=0.0f; + for (auto p:oldgrps[s.second[0]].second) avg+=p.first; + avg/=oldgrps[s.second[0]].second.size(); + oldgrps[s.second[0]].first=avg; + + } + seggrps.push_back(oldgrps[s.second[0]]); + } + + //cerr<<"similarities assigned, "<<(totalsmap.size()-seggrps.size())<<" segments merged"<0) list+=","; + list +=toString(s.second[j].second); + } + cerr<<"segment "<0) list+=","; + list +=toString(s.second[j].second); + } + cerr<<"segment "<value)>0) { + if (seggrps.size()>(int)parameters["levels"]->value){ + while (seggrps.size()>(int)parameters["levels"]->value){ + //reduce similarity groups + //decide the best 2 to merge + vector diffs; + for (int j=0;jvalue<<" levels requested, "<<(int)totalsmap.size()<<" original segments"<value,(int)totalsmap.size())){ + while (seggrps.size()value,(int)totalsmap.size())) { + //split groups + //calculate standard deviation of intensity variation + vector devs; + for (int j=0;jdevs[largest]) largest=j; + //sanity check: if there are any groups that can be split they will have larger SD than singleton groups + //sort members of the group + std::sort(seggrps[largest].second.begin(),seggrps[largest].second.end(),sortgroupmembers); + //create a new group + std::pair > > newgroup; + //cerr<<"splitting group "<(seggrps[largest].second.size()/2)-1;j--) { + newgroup.second.push_back(seggrps[largest].second[j]); + seggrps[largest].second.erase(seggrps[largest].second.begin()+j); + } + + //refresh averages for the 2 groups + double avg=0.0f; + for (auto p:seggrps[largest].second) avg+=p.first; + avg/=seggrps[largest].second.size(); + seggrps[largest].first=avg; + + avg=0.0f; + for (auto p:newgroup.second) avg+=p.first; + avg/=newgroup.second.size(); + newgroup.first=avg; + + //add the new group + seggrps.push_back(newgroup); + //cerr<<" added new group with "< outputvalues; + for (int j=0;j0) list+=","; + list +=toString(seggrps[j].second[k].second); + } + //cerr<<"output value: "< { + public: + Audio_processor(){}; + virtual ~Audio_processor(){}; + virtual int process_frame(uint8_t *data,int samples)=0; + virtual bool init(int _channels,int _bits,int _samples,int _rate)=0; + virtual void cleanup()=0; + virtual void print_summary(){}; + virtual string get_features(){return "";}; + int channels,bits,samples,rate; + }; + //actual nodes------------------------------------------------- + class Audio_thumbnailer: public Audio_processor { + public: + Audio_thumbnailer(){ + height=128; + width=512; //fit + + //trying to reduce valgrind errors + out_sample=0; //sample in whole track + sample=0; + samples=0; + accum=0.0; + }; + ~Audio_thumbnailer(){}; + Audio_thumbnailer* clone(Json::Value& _settings) { return new Audio_thumbnailer();}; + bool init(int _channels,int _bits,int _samples,int _rate); + void cleanup(){}; + int process_frame(uint8_t *data,int samples_in_frame); + void print_vector(xmlIO XML); + const double get_output(const Frame_parameters &frame){return 0.0;}; + vector audiodata; + int height,width,samples_per_column; + int out_sample,sample,samples; + int offset; + double scale,accum; + }; + class Vamp_node: public Audio_processor { + //base class for vamp plugin hosts + public: + Vamp_node(){ + create_attribute("mode","Data output mode","Mode","timeline",{"timeline","timesteps","valueline","values"}); + }; + bool init(int _channels,int _bits,int _samples,int _rate); + void cleanup(); + int process_frame(uint8_t *data,int samples_in_frame); + const double output(const Time_spec &time) { + if (features.size()) { + auto i=features.upper_bound(time.time); //the first element in the container whose key is considered to go after k + double uk; + double v1,v2; + v1=v2=0.0; + if (i==features.end()) { + uk=time.duration; + } + else { + uk=i->first; + if (i->second.values.size()) v2=i->second.values[0]; + } + i--; + double lk=i->first; + int ln=i->second.number; + if (i->second.values.size()) v1=i->second.values[0]; + switch (attributes["mode"]->intVal){ + case VAMPHOST_Timeline: + return (((time.time-lk)/(uk-lk))+ln); + case VAMPHOST_Timesteps: + return (double)ln; + case VAMPHOST_Valueline: + return ((time.time-lk)/(uk-lk))+v1; //((((time.time-lk)/(uk-lk))*(v2-v1))+v1); + case VAMPHOST_Values: + return v1; + } + //} + //return (--features.end())->second.values[0]; + } + return 0.0; + } + string get_features(); + void print_summary(){ + cerr<<"vamp plugin "< params; + map features; + private: + vampHost::Analyser analyser; + }; + class Audio_analysis: public Vamp_node { + //vamp node that allows the user to choose a plugin + public: + Audio_analysis(){ + //create_attribute("soname","Plugin library to use","Plugin library","vamp-example-plugins"); + //create_attribute("id","ID of Plugin to use","Plugin ID","percussiononsets"); + //create_attribute("analyser","Analyser Plugin to use","Analyser plugin","barbeattracker",{"barbeattracker","segmenter"}); + create_parameter("outputNo","number","Plugin output to use","Output number",0.0); + //title="Audio analysis"; + //description="Analyse audio and output"; + NODEID="b769f54e-2d0b-11e3-87dd-f73fc7b1c636"; + }; + Audio_analysis(map &settings):Audio_analysis() { + base_settings(settings); + soname=find_setting(settings,"soname"); + id=find_setting(settings,"id"); + outputNo=find_setting(settings,"outputNo",0); + }; + ~Audio_analysis(){}; + Audio_analysis* clone(map &_settings) { return new Audio_analysis(_settings);}; + private: + }; + class Audio_analysis2: public Vamp_node { + //reworked the plugin loader + public: + Audio_analysis2(){ + //create_attribute("soname","Plugin library to use","Plugin library","vamp-example-plugins",{"horiz","vert","horizR","vertR"}); + //create_attribute("id","ID of Plugin to use","Plugin ID","percussiononsets",{"horiz","vert","horizR","vertR"}); + create_attribute("analyser","Analyser Plugin to use","Analyser plugin","barbeattracker",{"adaptivespectrum","barbeattracker","chromagram","dwt","mfcc","onsetdetector","segmenter","similarity","tempotracker","transcription"}); + create_parameter("outputNo","number","Plugin output to use","Output number",0.0); + title="Audio analysis"; + description="Analyse audio and output"; + NODEID="b769f54e-2d0b-11e3-87dd-f73fc7b1c636"; + plugins.push_back(make_pair("qm-adaptivespectrogram","qm-vamp-plugins")); + plugins.push_back(make_pair("qm-barbeattracker","qm-vamp-plugins")); + plugins.push_back(make_pair("qm-chromagram","qm-vamp-plugins")); + plugins.push_back(make_pair("qm-dwt","qm-vamp-plugins")); + plugins.push_back(make_pair("qm-mfcc","qm-vamp-plugins")); + plugins.push_back(make_pair("qm-onsetdetector","qm-vamp-plugins")); + plugins.push_back(make_pair("qm-segmenter","qm-vamp-plugins")); + plugins.push_back(make_pair("qm-similarity","qm-vamp-plugins")); + plugins.push_back(make_pair("qm-tempotracker","qm-vamp-plugins")); + plugins.push_back(make_pair("qm-transcription","qm-vamp-plugins")); + }; + Audio_analysis2(map &settings):Audio_analysis2() { + base_settings(settings); + soname=plugins[attributes["analyser"]->intVal-1].second; + id=plugins[attributes["analyser"]->intVal-1].first; + outputNo=find_setting(settings,"outputNo",0); + }; + ~Audio_analysis2(){}; + void init_attribute(const string &attr){ + if (attr=="analyser") { + soname=plugins[attributes["analyser"]->intVal-1].second; + id=plugins[attributes["analyser"]->intVal-1].first; + } + if (attr=="outputNo") { + outputNo=parameters["outputNo"]->value; + } + }; + Audio_analysis2* clone(map &_settings) { return new Audio_analysis2(_settings);}; + private: + vector< pair> plugins; + }; + class Act_segmenter: public Vamp_node { + //vamp node that applies a ruleset to manage a set of acts via a cycler + public: + Act_segmenter(){ + create_parameter("outputNo","number","Plugin output to use","Output number",0.0); + create_parameter("acts","number","Number of acts defined","Acts",1.0); + title="Act manager"; + description="Applies a ruleset to manage acts based on segments"; + NODEID="c55359a2-2d0b-11e3-8a3d-53fa9c2b8859"; + }; + Act_segmenter(map &settings):Act_segmenter() { + base_settings(settings); + soname="qm-vamp-plugins"; + id="qm-segmenter"; + outputNo=find_setting(settings,"outputNo",0); + }; + ~Act_segmenter(){}; + Act_segmenter* clone(map &_settings) { return new Act_segmenter(_settings);}; + void cleanup(){ + Vamp_node::cleanup(); + map acts; //temporary storage for new set of features + vector act_count; + for (int i=0;i<(int)parameters["acts"]->value;i++) act_count.push_back(0); + + if (features.size()<=(uint32_t)parameters["acts"]->value+1){ + //iteratively split segments and refresh durations + //this could work well on the original data + //pick the longest and split it in two + //refresh durations - this can be a function + //keep going + //finally copy out + while (features.size()<(uint32_t)parameters["acts"]->value+1){ + map durations; + map times; + int i=0; + for (map::iterator f=features.begin();f!=features.end();++f){ + auto g=f; + if (++g!=features.end()){ + durations[i]=g->first-f->first; + times[i]=f->first; + } + i++; + } + double f=0; + int n=-1; + for (auto d: durations){ + if (d.second>f){ + f=d.second; + n=d.first; + } + } + features[times[n]+(durations[n]/2)]=features[times[n]]; + } + int j=0; + for (auto f: features){ + f.second.number=j; + acts[f.first]=f.second; + j++; + } + } + else { //there are extra segments to assign to acts + //assign act numbers to segments + + //work through the segments + //have a list of segments that are to be filled + //start at the ends and work inwards + //have a list of how many times each segment has been used + + //start with a set of the segments numbers + //(the aim: to access the segment numbers by the time that they start) + map segments=features; + segments.erase(--segments.end()); + + //assign the acts from the beginning and end in towards the middle + //when all slots have bveen used it picks the middle item and uses surrrounding items again + //there is a limit of item use based on how near it is to the edge + bool take_start=true; + while (segments.size()){ + int act=-1; + vampHost::feature f; + double t; + if (take_start){ + f=(*segments.begin()).second; + t=(*segments.begin()).first; + segments.erase(segments.begin()); + for (int i=0;i<(int)parameters["acts"]->value-1;i++) { + if (act_count[i]==0||(act_count[i+1]>act_count[i]&&act_count[i]value-1;i>0;i--) { + if (act_count[i]==0||(act_count[i-1]>act_count[i]&&act_count[i]<(int)parameters["acts"]->value-i)) { + act=i; + break; + } + } + + } + if (act==-1){ //all slots are filled equally + act=(int)parameters["acts"]->value/2; + } + act_count[act]++; + f.number=act; + acts[t]=f; + take_start=!take_start; + } + } + //add last item back on + acts[(*--features.end()).first]=features[(*--features.end()).first]; + features=acts; + } + private: + + }; + class Intensity_segmenter: public Vamp_node { + //vamp node that applies a ruleset to manage a set of acts via a cycler + public: + Intensity_segmenter(){ + title="Intensity segmenter"; + description="Combines the output of segmentation, tempo, and intensity analysis plugins"; + NODEID="6ce236b6-4080-11e3-90b7-74d02b29f6a6"; + analysers["segmenter"]=vampHost::Analyser(); + analysers["tempo"]=vampHost::Analyser(); + analysers["intensity"]=vampHost::Analyser(); + create_parameter("intensity_weight","number","intensity weight","Intensity weighting",1.0); + create_parameter("tempo_weight","number","tempo weight","Tempo weighting",1.0); + create_parameter("levels","number","levels","Number of intensity levels",0.0); + }; + Intensity_segmenter(map &settings):Intensity_segmenter() { + base_settings(settings); + }; + ~Intensity_segmenter(){}; + Intensity_segmenter* clone(map &_settings) { return new Intensity_segmenter(_settings);}; + bool init(int _channels,int _bits,int _samples,int _rate) { + features.clear(); + return analysers["segmenter"].init("qm-vamp-plugins","qm-segmenter",_channels,_bits,_samples,_rate,0,params)\ + &&analysers["tempo"].init("qm-vamp-plugins","qm-tempotracker",_channels,_bits,_samples,_rate,2,params)\ + &&analysers["intensity"].init("bbc-vamp-plugins","bbc-intensity",_channels,_bits,_samples,_rate,0,params); + } + int process_frame(uint8_t *data,int samples_in_frame) { + //for (auto a:analysers) a.second.process_frame(data,samples_in_frame); //WHY NOT WORK + analysers["segmenter"].process_frame(data,samples_in_frame); + analysers["tempo"].process_frame(data,samples_in_frame); + analysers["intensity"].process_frame(data,samples_in_frame); + return 1; + } + void cleanup(); //all the fun happens here + void print_summary(){ + cerr<<"Intensity segmenter found "< analysers; + }; +} + + + +#endif diff --git a/NT/src/rendercontext.cpp b/NT/src/rendercontext.cpp index 3309b16..a03723f 100644 --- a/NT/src/rendercontext.cpp +++ b/NT/src/rendercontext.cpp @@ -5,6 +5,64 @@ using namespace std; using Poco::Net::HTTPResponse; using Poco::Logger; +void Render_context::runTask() { + while (!isCancelled()) { + Session_task cmd; + mutex.lock(); + if (work_queue.size()){ + cmd=work_queue[0]; + work_queue.pop_front(); + graph.cancelled=false; + } + mutex.unlock(); + if(cmd.task==ANALYSE_AUDIO) { + state=ANALYSING_AUDIO; + graph.load_audio(cmd.body); + state=IDLE; + } + if(cmd.task==RENDER) { + state=RENDERING; + renders[cmd.uid]=Render_status(RENDERING); + if(graph.video_render(output_filename,output_framerate,start,stop)){ + state=IDLE; + if (graph.cancelled) renders[cmd.uid].status=CANCELLED; + else renders[cmd.uid].status=RENDER_READY; + } + else { + //an error occurred: TODO have to clean up allocated data. autoptr? + cerr<<"Rotor: render failed"<1){ - int w=toInt(t1[0]); - int h=toInt(t1[1]); - if (set_resolution(w,h)){ - logger.information("resolution set to "+t1[0]+"x"+t1[1]); - XML.addValue("status","resolution set to "+t1[0]+"x"+t1[1]); - status=HTTPResponse::HTTP_OK; - } - else { - logger.error("ERROR: invalid resolution request: "+t1[0]+"x"+t1[1]); - XML.addValue("error","invalid resolution request: "+t1[0]+"x"+t1[1]); - } + if (set_profile(command.body)){ + logger.information("profile set to "+command.body); + XML.addValue("status","profile set to "+command.body); + status=HTTPResponse::HTTP_OK; + } + else { + logger.error("ERROR: invalid profile request: "+command.body); + XML.addValue("error","invalid profile request: "+command.body); } } else { @@ -69,65 +122,6 @@ void Render_context::session_command(const Session_command& command,xmlIO& XML,H } } } - if (command.commands[1]=="bitrate") { - if (command.method=="PUT") { - int b=toInt(command.body); - if (graph.set_bitrate(b)){ - logger.information("bitrate set to "+command.body); - XML.addValue("status","bitrate set to "+command.body); - status=HTTPResponse::HTTP_OK; - } - else { - logger.error("ERROR: Could not set bitrate set to "+command.body); - XML.addValue("error","Could not set bitrate set to "+command.body); - } - } - else { - status=HTTPResponse::HTTP_BAD_REQUEST; - logger.error("ERROR: Bad request"); - XML.addValue("error","Bad request"); - } - } - if (command.commands[1]=="fragmentation") { - if (command.method=="PUT") { - bool f=(toInt(command.body)!=0); - if (graph.set_fragmentation(f)){ - string fs=f?"on":"off"; - logger.information("MP4 fragmentation "+fs); - XML.addValue("status","MP4 fragmentation "+fs); - status=HTTPResponse::HTTP_OK; - } - else { - logger.error("ERROR: Could not set MP4 fragmentation"); - XML.addValue("error","Could not set MP4 fragmentation"); - } - } - else { - status=HTTPResponse::HTTP_BAD_REQUEST; - logger.error("ERROR: Bad request"); - XML.addValue("error","Bad request"); - } - } - if (command.commands[1]=="dash") { - if (command.method=="PUT") { - bool f=(toInt(command.body)!=0); - if (graph.set_dash(f)){ - string fs=f?"on":"off"; - logger.information("MPEG-DASH output "+fs); - XML.addValue("status","MPEG-DASH output "+fs); - status=HTTPResponse::HTTP_OK; - } - else { - logger.error("ERROR: Could not set MPEG-DASH output"); - XML.addValue("error","Could not set MPEG-DASH output"); - } - } - else { - status=HTTPResponse::HTTP_BAD_REQUEST; - logger.error("ERROR: Bad request"); - XML.addValue("error","Bad request"); - } - } if (command.commands[1]=="audio") { if (command.method=="PUT") { //get audio file location and initiate analysis if (command.body!="") { @@ -135,7 +129,6 @@ void Render_context::session_command(const Session_command& command,xmlIO& XML,H graph.audio_filename=media_dir+command.body; //for now, store session variables in memory //check file exists Poco::File f=Poco::File(graph.audio_filename); if (f.exists()) { - //pass to worker thread ??if engine is ready?? ??what if engine has finished but results aren't read?? add_queue(Session_task(command.uid,ANALYSE_AUDIO)); status=HTTPResponse::HTTP_OK; logger.information("Audio analysis: "+command.body); @@ -162,6 +155,7 @@ void Render_context::session_command(const Session_command& command,xmlIO& XML,H sprintf(c,"%02f",graph.progress); XML.addValue("progress",string(c)); } + /* else if (graph.audio_loaded) { //not sure about this-- should this state be retained? //can the data only be read once? @@ -175,6 +169,7 @@ void Render_context::session_command(const Session_command& command,xmlIO& XML,H logger.error("ERROR: audio thumbnail requested but no audio loaded"); XML.addValue("error","No audio loaded"); } + */ } if (command.method=="DELETE") { if (state==IDLE) { @@ -216,29 +211,25 @@ void Render_context::session_command(const Session_command& command,xmlIO& XML,H //before begining to load from xml if (state==IDLE) { //eventually not like this if (command.body!="") { - graph_filename=""; - graph_body=""; - if (Poco::File(graph_dir+command.body).exists()) { - graph_filename=graph_dir+command.body; - add_queue(Session_task(command.uid,LOAD_GRAPH)); + string graph_filename=graph_dir+command.body; + string graph_body=""; + if (Poco::File(graph_filename).exists()) {; + add_queue(Session_task(command.uid,LOAD_GRAPH,graph_filename)); status=HTTPResponse::HTTP_OK; logger.information("Loading graph from file: "+command.body); XML.addValue("status","Loading graph from file: "+command.body); //XML.addValue("render_id",command.uid); process ID? } else { - xmlIO xml; bool readable=true; - if (!xml.loadFromBuffer(command.body)){ - Json::Value root; // will contains the root value after parsing. - Json::Reader reader; - if ( !reader.parse( command.body, root ) ) - { - status=HTTPResponse::HTTP_BAD_REQUEST; - logger.error("ERROR: Could not load graph"); - XML.addValue("error","Could not load graph"); - readable=false; - } + Json::Value root; // will contains the root value after parsing. + Json::Reader reader; + if ( !reader.parse( command.body, root ) ) + { + status=HTTPResponse::HTTP_BAD_REQUEST; + logger.error("ERROR: Could not load graph"); + XML.addValue("error","Could not load graph"); + readable=false; } if (readable) { graph_body=command.body; diff --git a/NT/src/rendercontext.h b/NT/src/rendercontext.h index a84cad8..d1acf55 100644 --- a/NT/src/rendercontext.h +++ b/NT/src/rendercontext.h @@ -11,6 +11,7 @@ TJR Jan 2014 -------------------------*/ #include "Poco/Task.h" +#include "Poco/StringTokenizer.h" #include "rotor.h" #include "graph.h" @@ -48,6 +49,12 @@ namespace Rotor { Session_command(){body="";}; string uid,method,id,body; vector commands; + }; + class Session_task { + public: + Session_task(const string &_uid="",int _task=0):uid(_uid),task(_task) {}; + string uid; + int task; }; class Render_status { public: @@ -63,8 +70,9 @@ namespace Rotor { }; class Render_context: public Poco::Task { public: - Render_context(const std::string& _id): Task(_id) { + Render_context(const std::string& _id,const std::string& _media_dir=""): Task(_id) { id=_id; + media_dir=_media_dir; graph.init(id); //set up log AutoPtr splitterChannel(new SplitterChannel()); @@ -75,6 +83,8 @@ namespace Rotor { AutoPtr formatter(new PatternFormatter("%d-%m-%Y %H:%M:%S %s: %t")); AutoPtr formattingChannel(new FormattingChannel(formatter, splitterChannel)); Logger& logger = Logger::create(id, formattingChannel, Message::PRIO_TRACE); + + //load profiles logger.information("started thread"); } ~Render_context(){ @@ -82,14 +92,10 @@ namespace Rotor { cancel(); logger.information("stopped thread"); } - void runTask() { - while (!isCancelled()) { - sleep(100); - } - } - bool set_preset(std::string& _preset){ - if (presets.find(_preset)==presets.end()) return false; - preset=_preset; + void add_queue(Session_task item); + bool set_profile(const std::string& _profile){ + if (profiles.find(_profile)==profiles.end()) return false; + profile=_profile; return true; } void session_command(const Session_command& command,xmlIO& XML,Poco::Net::HTTPResponse::HTTPStatus& status); @@ -105,10 +111,15 @@ namespace Rotor { std::string text_render(std::string node_id=""); Graph graph; private: + int state; std::string id; std::unordered_map renders; - std::unordered_map presets; - std::string preset; + std::unordered_map profiles; + std::string profile; + std::string media_dir; + + std::deque work_queue; + Poco::Mutex mutex; }; }; diff --git a/NT/src/rotor.h b/NT/src/rotor.h index f687694..1ed02dc 100644 --- a/NT/src/rotor.h +++ b/NT/src/rotor.h @@ -23,6 +23,7 @@ What next? #include #include "xmlIO.h" +#include "libavwrapper.h" #include "Poco/Logger.h" #include "Poco/Channel.h" diff --git a/NT/src/rotord.o b/NT/src/rotord.o deleted file mode 100644 index e73e551..0000000 Binary files a/NT/src/rotord.o and /dev/null differ diff --git a/NT/src/tinyxml.o b/NT/src/tinyxml.o deleted file mode 100644 index 811f6ab..0000000 Binary files a/NT/src/tinyxml.o and /dev/null differ diff --git a/NT/src/tinyxmlerror.o b/NT/src/tinyxmlerror.o deleted file mode 100644 index 49bba63..0000000 Binary files a/NT/src/tinyxmlerror.o and /dev/null differ diff --git a/NT/src/tinyxmlparser.o b/NT/src/tinyxmlparser.o deleted file mode 100644 index 1b4101c..0000000 Binary files a/NT/src/tinyxmlparser.o and /dev/null differ diff --git a/NT/src/vampHost.cpp b/NT/src/vampHost.cpp new file mode 100644 index 0000000..a70e795 --- /dev/null +++ b/NT/src/vampHost.cpp @@ -0,0 +1,416 @@ +#include "vampHost.h" + + +void vampHost::printFeatures(int frame, int sr, int output, + Plugin::FeatureSet features, ostream& out, bool useFrames) +{ + if (features[output].size()) { + cout << "." << features[output].size(); + } + for (unsigned int i = 0; i < features[output].size(); ++i) { + + if (useFrames) { + + int displayFrame = frame; + + if (features[output][i].hasTimestamp) { + displayFrame = RealTime::realTime2Frame + (features[output][i].timestamp, sr); + } + + out << displayFrame; + + if (features[output][i].hasDuration) { + displayFrame = RealTime::realTime2Frame + (features[output][i].duration, sr); + out << "," << displayFrame; + } + + out << ":"; + + } else { + + RealTime rt = RealTime::frame2RealTime(frame, sr); + + if (features[output][i].hasTimestamp) { + rt = features[output][i].timestamp; + } + + out << rt.toString(); + + if (features[output][i].hasDuration) { + rt = features[output][i].duration; + out<< "," << rt.toString(); + } + + out << ":"; + } + + for (unsigned int j = 0; j < features[output][i].values.size(); ++j) { + out<< " " << features[output][i].values[j]; + } + out << " " << features[output][i].label; + + out << endl; + } + +} + + + +void vampHost::rotorGetFeatures(int frame, int sr, int output,Plugin::FeatureSet features, vector& out, double& progress) +{ + if (features[output].size()) { + cout << "." << features[output].size(); + } + for (unsigned int i = 0; i < features[output].size(); ++i) { + + + + int displayFrame = frame; + + if (features[output][i].hasTimestamp) { + displayFrame = RealTime::realTime2Frame + (features[output][i].timestamp, sr); + } + + cout << displayFrame; + + + cout << endl; + } + +} + + + +void vampHost::getTimestamps(int output,Plugin::FeatureSet features, vector& out){ + + /* + vamp-simple-host qm-vamp-plugins:qm-tempotracker 01.wav + + 0.046439908: 156.61 bpm + 0.429569160: 156.61 bpm + 0.812698412: 161.50 bpm + 1.184217686: 152.00 bpm + + + vamp-simple-host qm-vamp-plugins:qm-segmenter 01.wav + + 0.000000000: 4 4 + 23.800000000: 6 6 + 44.600000000: 5 5 + 55.000000000: 7 7 + 72.800000000: 1 1 + 90.600000000: 2 2 + 109.200000000: 5 5 + 116.000000000: 3 3 + 143.800000000: 5 5 + 153.400000000: 3 3 + 163.000000000: 8 8 + + seems to be FP seconds then another metric + for now we can just take the first part + + features[output][i].timestamp is of type RealTime: represents time values to nanosecond precision + int sec; + int nsec; + 1 sec = 10^9 nanosec + + actually maybe this would be the way to go for rotor- avoiding rounding errors etc + for now - ideally will get a double representation + + features[output][i].values is a vector of doubles + a description + WE DON'T CARE ABOUT ANYTHING <.01 seconds + + static long realTime2Frame(const RealTime &r, unsigned int sampleRate); + + get a vector of doubles out, using frames, presuming data has a timestamp + + + this is crashing with "Aborted (core dumped)" + if we check for timestamp + + */ + + cout << "." << features[output].size(); + + //if (!features[output][0].hasTimestamp) { + // cerr << output << " channel, getTimestamps: error, featureset doesn't support timestamp" << endl; + //}_ + //else { + for (unsigned int i = 0; i < features[output].size(); ++i) { + out.push_back( ((double)RealTime::realTime2Frame(features[output][i].timestamp, 1000))*.001f); + cout << "feature found.\n"; + } + //} +} + +bool vampHost::Analyser::init(const string &soname,const string &id,const int &_channels,const int &_bits,const int &_samples,const int &_rate,int _outputNo,const map ¶ms){ + + //stuff that only happens once + channels =_channels; + samples=_samples; + rate=_rate; + bits=_bits; + outputNo=_outputNo; + //output=_output; + + //http://www.mega-nerd.com/libsndfile/api.html#note1 + //libsndfile returns -1..1 for fp data + bytes=(bits>>3); + stride=channels*bytes; + scale=(1.0/pow(2.0,bits)); + + features.clear(); //in case of reuse + features[0.0]=feature(); + + loader = PluginLoader::getInstance(); + key = loader->composePluginKey(soname, id); + plugin = loader->loadPlugin(key, _rate, PluginLoader::ADAPT_ALL_SAFE); + if (!plugin) { + cerr << ": ERROR: Failed to load plugin \"" << id + << "\" from library \"" << soname << "\"" << endl; + return false; + } + + cerr << "Running plugin: \"" << plugin->getIdentifier() << "\"... Domain:"; + + if (plugin->getInputDomain() == Plugin::FrequencyDomain) { + cerr << "frequency" << endl; + } + else { + + cerr << "time" << endl; + + } + + blockSize = plugin->getPreferredBlockSize(); + stepSize = plugin->getPreferredStepSize(); + + if (blockSize == 0) { + blockSize = 1024; + } + if (stepSize == 0) { + if (plugin->getInputDomain() == Plugin::FrequencyDomain) { + stepSize = blockSize/2; + } else { + stepSize = blockSize; + } + } + else if (stepSize > blockSize) { + cerr << "WARNING: stepSize " << stepSize << " > blockSize " << blockSize << ", resetting blockSize to "; + if (plugin->getInputDomain() == Plugin::FrequencyDomain) { + blockSize = stepSize * 2; + } else { + blockSize = stepSize; + } + cerr << blockSize << endl; + } + overlapSize = blockSize - stepSize; + currentStep = 0; + finalStepsRemaining = max(1, (blockSize / stepSize) - 1); // at end of file, this many part-silent frames needed after we hit EOF + + plugbuf = new float*[channels]; + for (int c = 0; c < channels; ++c) plugbuf[c] = new float[blockSize + 2]; + + cerr << "Using block size = " << blockSize << ", step size = " + << stepSize << endl; + + // The channel queries here are for informational purposes only -- + // a PluginChannelAdapter is being used automatically behind the + // scenes, and it will take case of any channel mismatch + + int minch = plugin->getMinChannelCount(); + int maxch = plugin->getMaxChannelCount(); + cerr << "Plugin accepts " << minch << " -> " << maxch << " channel(s)" << endl; + cerr << "Sound file has " << channels << " (will mix/augment if necessary)" << endl; + + Plugin::OutputList outputs = plugin->getOutputDescriptors(); + Plugin::OutputDescriptor od; + + RealTime rt; + PluginWrapper *wrapper = 0; + RealTime adjustment = RealTime::zeroTime; + + if (outputs.empty()) { + cerr << "ERROR: Plugin has no outputs!" << endl; + return false; + } + + if (outputNo < 0) { + for (size_t oi = 0; oi < outputs.size(); ++oi) { + if (outputs[oi].identifier == output) { + outputNo = oi; + break; + } + } + if (outputNo < 0) { + cerr << "ERROR: Non-existent output \"" << output << "\" requested" << endl; + return false; + } + } + else { + if (int(outputs.size()) <= outputNo) { + cerr << "ERROR: Output " << outputNo << " requested, but plugin has only " << outputs.size() << " output(s)" << endl; + return false; + } + } + od = outputs[outputNo]; + cerr << "Output number "<setParameter(i.first,i.second); + cerr << "Set plugin parameter: "<initialise(channels, stepSize, blockSize)) { + cerr << "ERROR: Plugin initialise (channels = " << channels + << ", stepSize = " << stepSize << ", blockSize = " + << blockSize << ") failed." << endl; + return false; + } + + cerr << "Vamphost Plugin initialised: (channels = " << channels + << ", stepSize = " << stepSize << ", blockSize = " + << blockSize << ")" << endl; + + wrapper = dynamic_cast(plugin); + if (wrapper) { + // See documentation for + // PluginInputDomainAdapter::getTimestampAdjustment + PluginInputDomainAdapter *ida =wrapper->getWrapper(); + if (ida) adjustment = ida->getTimestampAdjustment(); + } + + //everything is prepared to start consuming data in blocks + + in_block=0; + blocks_processed=0; + currentStep=0; + + featureNo=0; + + return true; +} +void vampHost::Analyser::process_frame(uint8_t *data,int samples_in_frame){ + + int sample=0; + + uint16_t *_data=(uint16_t*)data; + //process the whole frame which may be f>1getIdentifier()<<" processed block "<process(plugbuf, rt); + + double t; + + for (unsigned int i = 0; i < feat[outputNo].size(); ++i) { + feature f; + f.number=featureNo; + f.values=feat[outputNo][i].values; + //fix for plugins that don't set timestamp properly + t=((double)feat[outputNo][i].timestamp.sec)+(((double)feat[outputNo][i].timestamp.nsec)*.000000001); + if (t<.01) t=((rt.sec)+(rt.nsec)*.000000001); + features[t]=f; + featureNo++; + } + + //if (feat[outputNo].size()>0) cerr<<"vamphost: "<0) cerr<getIdentifier()<<" step:"<process(plugbuf, rt); + + for (unsigned int i = 0; i < feat[outputNo].size(); ++i) { + feature f; + f.number=featureNo; + f.values=feat[outputNo][i].values; + features[((double)feat[outputNo][i].timestamp.sec)+(((double)feat[outputNo][i].timestamp.nsec)*.000000001)]=f; + featureNo++; + } + + feat=plugin->getRemainingFeatures(); + + for (unsigned int i = 0; i < feat[outputNo].size(); ++i) { + feature f; + f.number=featureNo; + f.values=feat[outputNo][i].values; + features[((double)feat[outputNo][i].timestamp.sec)+(((double)feat[outputNo][i].timestamp.nsec)*.000000001)]=f; + featureNo++; + } + + //always make a final blank feature at the end + feature f; + f.number=featureNo; + f.values={}; + features[((double)rt.sec)+(((double)rt.nsec)*.000000001f)]=f; + + //cerr<getIdentifier()<<" found "<<(features.size()-1)<<" features"<first; + double v1,v2; + v1=v2=0.0; + if (i->second.values.size()) v2=i->second.values[0]; + i--; + double lk=i->first; + if (i->second.values.size()) v1=i->second.values[0]; + + return ((((time-lk)/(uk-lk))*(v2-v1))+v1); + } + } + return 0.0; +} diff --git a/NT/src/vampHost.h b/NT/src/vampHost.h new file mode 100644 index 0000000..a8b0bbb --- /dev/null +++ b/NT/src/vampHost.h @@ -0,0 +1,95 @@ +#include +#include +#include + +#include "Poco/Mutex.h" + +#include +#include +#include +#include +#include "libavwrapper.h" + +#include +#include + +#include "system.h" + +#include + +/* +line 366: is returnValue the fail/succeed return value? +*/ + +using namespace std; + +using Vamp::Plugin; +using Vamp::PluginHostAdapter; +using Vamp::RealTime; +using Vamp::HostExt::PluginLoader; +using Vamp::HostExt::PluginWrapper; +using Vamp::HostExt::PluginInputDomainAdapter; + +#define HOST_VERSION "1.5" + +namespace vampHost { + struct feature{ + feature():number(0){}; + int number; + vector values; + }; + class Settings{ + public: + Settings(string _so="",string _filter="",string _input="") { + soname=_so; + filtername=_filter; + inputFile=_input; + } + string soname; + string filtername; + string inputFile; + }; + class QMAnalyser{ + public: + int process(const string soundfile); + double get_progress(); + vector beats; + private: + double progress; + Poco::Mutex mutex; //lock for progress data + }; + class Analyser{ + //can load any vamp analysis plugin and present its data with a unified interface + public: + bool init(const string &soname,const string &id,const int &_channels,const int &_bits,const int &_samples,const int &_rate,int outputNo,const map ¶ms); + void process_frame(uint8_t *data,int samples_in_frame); + void cleanup(); + double get_value(const double &time); + //map features; + map features; + //map + //this is the best way to store features: because map allows to search for the key below and above the present time + + private: + PluginLoader *loader; + PluginLoader::PluginKey key; + Plugin *plugin; + RealTime rt; + int channels,bits,samples,rate; + int bytes,stride; + double scale; + int blockSize,stepSize,overlapSize,finalStepsRemaining,currentStep,outputNo; + int in_block,blocks_processed; + string output; + float **plugbuf; + + int featureNo; + + }; + + string getQMBeats(const string soundfile); + void printFeatures(int, int, int, Plugin::FeatureSet, ostream &, bool frames); + void getTimestamps(int output,Plugin::FeatureSet features, vector& out); + void rotorGetFeatures(int frame, int sr, int output,Plugin::FeatureSet features, vector& out, double& progress); + +} -- cgit v1.2.3