#include "ffplayer.h" #include #include #if(QT_VERSION_MAJOR > 5) #include #endif #include #include using namespace std; AVFmt::AVFmt(QByteArray url) { fmt_ctx = avformat_alloc_context(); int res = avformat_open_input(&fmt_ctx, url.constData(), 0, 0); if(res < 0) { err = "Open input fail: "+errStr(res); return; } res = avformat_find_stream_info(fmt_ctx, 0); if(res < 0) { err = "Find stream info fail: "+errStr(res); return; } start = fmt_ctx->start_time; qInfo() << "Fmt Start" << start << "Dur" << fmt_ctx->duration; for(uint ss=0; ssnb_streams; ss++) if(fmt_ctx->streams[ss]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {vi.idx = ss; break;} for(uint ss=0; ssnb_streams; ss++) if(fmt_ctx->streams[ss]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {au.idx = ss; break;} if(vi.idx == -1) qWarning()<<"Didn't find a Video Stream"; else { auto viStream = fmt_ctx->streams[vi.idx]; auto decoder = avcodec_find_decoder(viStream->codecpar->codec_id); if(decoder==0) { vi.idx = -1; qCritical()<<"avcodec_find_decoder fail"; } else { vi.ctx = avcodec_alloc_context3(decoder); avcodec_parameters_to_context(vi.ctx, viStream->codecpar); vi.ctx->thread_count = 0; if((res = avcodec_open2(vi.ctx, decoder, 0)) < 0) { vi.idx = -1; qCritical()<<"avcodec_open2 fail:"<width*4+63)/64*64; dstStride[3] = dstStride[0] * vi.ctx->height; sws_ctx = sws_getContext(vi.ctx->width, vi.ctx->height, vi.ctx->pix_fmt, vi.ctx->width, vi.ctx->height, AV_PIX_FMT_RGB32, SWS_FAST_BILINEAR, 0, 0, 0); if(vi.ctx->color_primaries==AVCOL_PRI_BT2020) { res = sws_setColorspaceDetails(sws_ctx, sws_getCoefficients(SWS_CS_BT2020), 0, sws_getCoefficients(SWS_CS_DEFAULT), 0, -0x2200, 0x1c000, 0x18000); //0x12800, 0x40000 if(res < 0) qInfo()<<"sws_setColorspaceDetails fail:"<time_base.num * 1000000; vi.time_den = viStream->time_base.den; vi.start = viStream->start_time * vi.time_num / vi.time_den; frmDur = viStream->r_frame_rate.den * 1000000 / viStream->r_frame_rate.num; qInfo()<<"Video idx" << vi.idx << "Start" << vi.start << "dur" << viStream->duration * vi.time_num / vi.time_den << "frame dur" << frmDur; if(frmDur <= 0) frmDur = 16666; } } } if(au.idx == -1) qInfo()<<"Couldn't find a Audio Stream"; else { auto auStream = fmt_ctx->streams[au.idx]; #if(QT_VERSION_MAJOR > 5) QAudioDevice audioDevice = QMediaDevices::defaultAudioOutput(); #else QAudioDeviceInfo audioDevice = QAudioDeviceInfo::defaultOutputDevice(); #endif if(audioDevice.isNull()) { au.idx = -1; qInfo()<<"Couldn't find a Audio Device"; } else { AVCodecParameters *codecpar = auStream->codecpar; AVSampleFormat packed_sample_fmt = av_get_packed_sample_fmt((AVSampleFormat)codecpar->format); qInfo()<<"audio codecpar:"; qInfo()<<" sample_rate"<sample_rate; qInfo()<<" frame_size"<frame_size; qInfo()<<" sample_fmt"<format); qInfo()<<" bits per coded/raw sample"<bits_per_coded_sample<bits_per_raw_sample; sample_rate = codecpar->sample_rate; QAudioFormat audioFmt; audioFmt.setSampleRate(sample_rate); audioFmt.setChannelCount(2); #if(QT_VERSION_MAJOR > 5) if(packed_sample_fmt==AV_SAMPLE_FMT_FLT) audioFmt.setSampleFormat(QAudioFormat::Float); else if(packed_sample_fmt==AV_SAMPLE_FMT_S16) audioFmt.setSampleFormat(QAudioFormat::Int16); else if(packed_sample_fmt==AV_SAMPLE_FMT_S32) audioFmt.setSampleFormat(QAudioFormat::Int32); else if(packed_sample_fmt==AV_SAMPLE_FMT_S64) { packed_sample_fmt = AV_SAMPLE_FMT_S32; audioFmt.setSampleFormat(QAudioFormat::Int32); } else if(packed_sample_fmt==AV_SAMPLE_FMT_DBL) { packed_sample_fmt = AV_SAMPLE_FMT_FLT; audioFmt.setSampleFormat(QAudioFormat::Float); } else { packed_sample_fmt = AV_SAMPLE_FMT_S16; audioFmt.setSampleFormat(QAudioFormat::Int16); } #else audioFmt.setCodec("audio/pcm"); audioFmt.setByteOrder(QAudioFormat::LittleEndian); if(packed_sample_fmt==AV_SAMPLE_FMT_FLT) { audioFmt.setSampleType(QAudioFormat::Float); audioFmt.setSampleSize(32); } else if(packed_sample_fmt==AV_SAMPLE_FMT_S16) { audioFmt.setSampleType(QAudioFormat::SignedInt); audioFmt.setSampleSize(16); } else if(packed_sample_fmt==AV_SAMPLE_FMT_S32) { audioFmt.setSampleType(QAudioFormat::SignedInt); audioFmt.setSampleSize(32); } else if(packed_sample_fmt==AV_SAMPLE_FMT_S64) { packed_sample_fmt = AV_SAMPLE_FMT_S32; audioFmt.setSampleType(QAudioFormat::SignedInt); audioFmt.setSampleSize(32); } else if(packed_sample_fmt==AV_SAMPLE_FMT_DBL) { packed_sample_fmt = AV_SAMPLE_FMT_FLT; audioFmt.setSampleType(QAudioFormat::Float); audioFmt.setSampleSize(32); } else { packed_sample_fmt = AV_SAMPLE_FMT_S16; audioFmt.setSampleType(QAudioFormat::SignedInt); audioFmt.setSampleSize(16); } #endif sink = new QAudioSink(audioDevice, audioFmt); sinkWriter = sink->start(); if(sinkWriter==nullptr) { au.idx = -1; qCritical()<<"Couldn't Start Audio Output"<error(); } else { const AVCodec *decoder = avcodec_find_decoder(codecpar->codec_id); if(decoder==nullptr) { au.idx = -1; qWarning()<<"Couldn't found Audio Decoder"; } else { au.ctx = avcodec_alloc_context3(decoder); avcodec_parameters_to_context(au.ctx, codecpar); if(avcodec_open2(au.ctx, decoder, nullptr) < 0) { au.idx = -1; qWarning()<<"Couldn't open aCodecCtx"; } else { #if(QT_VERSION_MAJOR < 6) out_ch_layout.order = AV_CHANNEL_ORDER_NATIVE; out_ch_layout.nb_channels = 2; out_ch_layout.u.mask = AV_CH_LAYOUT_STEREO; #endif swr_alloc_set_opts2(&swr_ctx, &out_ch_layout, packed_sample_fmt, audioFmt.sampleRate(), &au.ctx->ch_layout, au.ctx->sample_fmt, au.ctx->sample_rate, 0, nullptr); if(swr_init(swr_ctx) < 0) { au.idx = -1; qCritical()<<"Couldn't init swrCtx"; } else { pcm_sample_size = audioFmt.bytesPerFrame(); au.time_num = auStream->time_base.num * 1000000; au.time_den = auStream->time_base.den; au.start = auStream->start_time * au.time_num / au.time_den; qInfo()<<"Audio Idx" << au.idx << "Start" << au.start<< "Dur" << auStream->duration * au.time_num / au.time_den; buf_size = sink->bytesFree(); qInfo()<<"Audio buffer size"< -1) au.thd = new thread(&AVFmt::mainAuDecode, this); if(vi.idx > -1) vi.thd = new thread(&AVFmt::mainViDecode, this); } AVFmt::~AVFmt() { if(read_thd!=nullptr) { acts.lockAddNtf(new Act{0, nullptr, 'Q'}); read_thd->join(); delete read_thd; qInfo() << "~ read thread"; } if(au.thd!=nullptr) { au.thd->join(); delete au.thd; qInfo() << "~ au thread"; } if(vi.thd!=nullptr) { vi.thd->join(); delete vi.thd; qInfo() << "~ vi thread"; } av_packet_free(&packet); avformat_close_input(&fmt_ctx); sws_freeContext(sws_ctx); swr_free(&swr_ctx); qInfo() << "~ AVContext"; } void AVFmt::mainRead() { int wait = 0; int res = 0; while(true) { auto act = wait==0 ? acts.lockPoll() : acts.take(wait); while(act!=nullptr) { if(act->act=='S') { if(au.idx!=-1) { unique_lock lock(au.pkts.mtx); au.pkts.delAll(); au.pkts.add(new Packet{0, 0, 'R'}); lock.unlock(); au.pkts.cv.notify_all(); } if(vi.idx!=-1) { unique_lock lock(vi.pkts.mtx); vi.pkts.delAll(); vi.pkts.add(new Packet{0, 0, 'R'}); lock.unlock(); vi.pkts.cv.notify_all(); } av_seek_frame(fmt_ctx, -1, act->arg, act->arg==0 ? AVSEEK_FLAG_FRAME : AVSEEK_FLAG_BACKWARD); } else if(act->act=='Q') { if(au.idx!=-1) { unique_lock lock(au.pkts.mtx); au.pkts.delAll(); au.pkts.add(new Packet{0, 0, 'Q'}); lock.unlock(); au.pkts.cv.notify_all(); } if(vi.idx!=-1) { unique_lock lock(vi.pkts.mtx); vi.pkts.delAll(); vi.pkts.add(new Packet{0, 0, 'Q'}); lock.unlock(); vi.pkts.cv.notify_all(); } delete act; return; } delete act; act = acts.lockPoll(); } if(res < 0) {wait = 500; continue;} if(vi.pkts.size > 120 || au.pkts.size > 200) {wait = 100; continue;} res = av_read_frame(fmt_ctx, packet); if(res < 0) { if(au.idx > -1) au.pkts.lockAddNtf(new Packet{0, 0, 0}); if(vi.idx > -1) vi.pkts.lockAddNtf(new Packet{0, 0, 0}); if(res!=AVERROR_EOF) { qCritical()<<"Read Packet fail:"<stream_index == vi.idx) { vi.pkts.lockAddNtf(new Packet{packet, 0, 0}); packet = av_packet_alloc(); } else if(packet->stream_index == au.idx) { au.pkts.lockAddNtf(new Packet{packet, 0, 0}); packet = av_packet_alloc(); } wait = 0; } } static void imgCleanupHandler(void *info) { delete [] (uchar*)info; } void AVFmt::mainViDecode() { uint8_t *dst[4]{0}; while(true) { if(vi_frms.size >= 7) { unique_lock lock(vi.pkts.mtx); if(vi.pkts.first && vi.pkts.first->act=='Q') return; vi.pkts.cv.wait_for(lock, chrono::milliseconds(20)); continue; } auto pkt = vi.pkts.take(); if(pkt->act=='R') { avcodec_flush_buffers(vi.ctx); vi.deLoop = 0; vi.stop = false; } else if(pkt->act=='Q') { delete pkt; return; } else if(!vi.stop) { int res = avcodec_send_packet(vi.ctx, pkt->pkt); if(res == AVERROR_EOF) goto end; if(res < 0) qCritical()<<"Video send packet fail:"<data, vi.frm->linesize, 0, vi.ctx->height, dst, dstStride); QImage img(dst[0], vi.ctx->width, vi.ctx->height, dstStride[0], QImage::Format_ARGB32, imgCleanupHandler, dst[0]); { lock_guard lock(vi_frms.mtx); if(vi.stop) goto end; if(vi.frm->pkt_duration) viLmt = (vi.frm->best_effort_timestamp + vi.frm->pkt_duration) * vi.time_num / vi.time_den; else viLmt = vi.frm->best_effort_timestamp * vi.time_num / vi.time_den + frmDur; vi_frms.add(new Img{img, vi.frm->best_effort_timestamp * vi.time_num / vi.time_den, viLmt, vi.deLoop, 0}); } } if(pkt->pkt==0) { avcodec_flush_buffers(vi.ctx); vi.deLoop++; } } end: delete pkt; } } void AVFmt::mainAuDecode() { while(true) { if(au_frms.size >= 9) { unique_lock lock(au.pkts.mtx); if(au.pkts.first && au.pkts.first->act=='Q') return; au.pkts.cv.wait_for(lock, chrono::milliseconds(20)); continue; } auto pkt = au.pkts.take(); if(pkt->act=='R') { avcodec_flush_buffers(au.ctx); au.deLoop = 0; au.stop = false; } else if(pkt->act=='Q') { delete pkt; return; } else if(!au.stop) { int res = avcodec_send_packet(au.ctx, pkt->pkt); if(res == AVERROR_EOF) goto end; if(res < 0) qCritical()<<"Audio send packet fail:"<nb_samples<=0) continue; auto pcm = new uint8_t[au.frm->nb_samples * pcm_sample_size]; int samp_cnt = swr_convert(swr_ctx, &pcm, au.frm->nb_samples, (const uint8_t **)&au.frm->data, au.frm->nb_samples); if(samp_cnt<=0) { delete [] pcm; qWarning()<<"swr convert fail:"< lock(au_frms.mtx); if(au.stop) { delete [] pcm; goto end; } au_frms.add(new Pcm{pcm, samp_cnt * pcm_sample_size, au.frm->best_effort_timestamp * au.time_num / au.time_den, (au.frm->best_effort_timestamp + au.frm->pkt_duration) * au.time_num / au.time_den, au.deLoop, 0}); } } if(pkt->pkt==0) { avcodec_flush_buffers(au.ctx); au.deLoop++; } } end: delete pkt; } } void AVFmt::checkAndUpd(FFPlayer *player, qint64 now_epoch) { if(vi.isNull()) { playAudio(au_frms.lockGet(), player, now_epoch); return; } if(au.isNull()) { player->hasImg = false; auto viFrm = vi_frms.lockGet(); if(viFrm==0) return; if(start_epoch==0) { if(vi.start > viFrm->st) { vi.start = viFrm->st; if(start > viFrm->st) start = viFrm->st; } start_epoch = now_epoch - start + 12500; return; } qint64 cur = now_epoch - start_epoch; checkFrm: if(viFrm->loop!=playLoop) { playLoop = viFrm->loop; start_epoch = now_epoch - start + 12500; qInfo()<<"Restart"; return; } if(viFrm->et < cur - 8334) { if(viFrm->next) { qWarning() << "Drop video frame. st-cur" << (viFrm->st - cur)/1000 << "st" << viFrm->st/1000 << "cur" << cur/1000 << "Video Only"; viFrm = vi_frms.lockDelGetNext(); goto checkFrm; } qInfo() << "Video Adjust Start. st-cur" << (viFrm->st - cur)/1000 << "st" << viFrm->st/1000 << "cur" << cur/1000 << "Video Only"; cur = viFrm->st; start_epoch = now_epoch - cur; } if(viFrm->st > cur) return; player->img = viFrm->img; player->hasImg = true; player->viCurTime = viFrm->st; delete vi_frms.lockPoll(); return; } player->hasImg = false; auto auFrm = au_frms.lockGet(); auto viFrm = vi_frms.lockGet(); if(start_epoch==0) { if(viFrm==0) { if(vi.start < start+500000) return; } else if(vi.start > viFrm->st) { vi.start = viFrm->st; if(start > viFrm->st) start = viFrm->st; } if(auFrm==0) { if(au.start < start+500000) return; } else if(au.start > auFrm->st) { au.start = auFrm->st; if(start > auFrm->st) start = auFrm->st; } start_epoch = now_epoch - (start - 12500); playAudio(auFrm, player, now_epoch, vi.pkts.lockGet()==0 ? INT64_MAX : viLmt); return; } if(viFrm==0) { if(vi.pkts.lockGet()==0) playAudio(auFrm, player, now_epoch); return; } if(viFrm->loop!=playLoop) { playAudio(auFrm, player, now_epoch); return; } qint64 cur = now_epoch - start_epoch; if(auFrm && auFrm->st < cur && auFrm->loop==playLoop) { qInfo() << "Audio Adjust Start. st-cur" << (auFrm->st - cur)/1000 << "st" << auFrm->st/1000 << "cur" << cur/1000; cur = auFrm->st; start_epoch = now_epoch - cur; } qint64 curPre = cur - 8334; while(viFrm->et < curPre && viFrm->next && viFrm->next->loop==playLoop) { qWarning() << "Drop video frame. st-cur" << (viFrm->st - cur)/1000 << "st" << viFrm->st/1000 << "cur" << cur/1000; viFrm = vi_frms.lockDelGetNext(); } auto bytesFree = sink->bytesFree(); bool isIdle = bytesFree==buf_size || sink->state()==QAudio::IdleState; if(viFrm->et < curPre) { if(isIdle) { qInfo() << "Video Adjust Start. st-cur" << (viFrm->st - cur)/1000 << "st" << viFrm->st/1000 << "cur" << cur/1000; cur = viFrm->st; start_epoch = now_epoch - cur; } else qInfo() << "Video et < cur. st-cur" << (viFrm->st - cur)/1000 << "st" << viFrm->st/1000 << "cur" << cur/1000; } if(auFrm && bytesFree >= auFrm->size && auFrm->loop==playLoop) whiteAudio(auFrm, player, isIdle, cur, vi.deLoop!=playLoop || vi.pkts.lockGet()==0 ? INT64_MAX : viLmt); if(viFrm->st > cur) return; player->img = viFrm->img; player->hasImg = true; player->viCurTime = viFrm->st; delete vi_frms.lockPoll(); } void AVFmt::playAudio(Pcm *auFrm, FFPlayer *player, qint64 now_epoch, qint64 lmt) { if(auFrm==0) return; auto bytesFree = sink->bytesFree(); if(bytesFree < auFrm->size) return; bool isIdle = bytesFree==buf_size || sink->state()==QAudio::IdleState; qint64 cur = now_epoch - start_epoch; if(start_epoch==0) { if(au.start > auFrm->st) { au.start = auFrm->st; if(start > auFrm->st) start = auFrm->st; } cur = start - 12500; start_epoch = now_epoch - cur; } else if(auFrm->loop!=playLoop) { if(! isIdle) return; playLoop = auFrm->loop; cur = start - 12500; start_epoch = now_epoch - cur; qInfo()<<"Restart"; } if(auFrm->st < cur) { qInfo() << "Audio Adjust Start. st-cur" << (auFrm->st - cur)/1000 << "st" << auFrm->st/1000 << "cur" << cur/1000 << "isIdle" << isIdle; cur = auFrm->st; start_epoch = now_epoch - cur; } whiteAudio(auFrm, player, isIdle, cur, lmt); return; } void AVFmt::whiteAudio(Pcm *auFrm, FFPlayer *player, bool isIdle, qint64 cur, qint64 lmt) { if(auFrm->st >= lmt) return; if(isIdle) { auto delay = auFrm->st - cur; if(delay > 50000) return; else if(delay > 999) { auto size = (delay * sample_rate / 1000000) * pcm_sample_size; char *data = new char[size](); sinkWriter->write(data, size); delete [] data; } } do { sinkWriter->write((char*)auFrm->data, auFrm->size); player->auCurTime = auFrm->st; auFrm = au_frms.lockDelGetNext(); } while(auFrm && auFrm->data && sink->bytesFree() >= auFrm->size && auFrm->st < lmt); } FFPlayer::FFPlayer() { thread = new QThread(); moveToThread(thread); connect(thread, &QThread::finished, thread, &QThread::deleteLater); connect(thread, &QThread::finished, this, &FFPlayer::deleteLater); connect(this, &FFPlayer::emQuit, this, &FFPlayer::onQuit); connect(this, &FFPlayer::emOpen, this, &FFPlayer::onOpen); connect(this, &FFPlayer::emClose, this, &FFPlayer::onClose); connect(this, &FFPlayer::emPlay, this, &FFPlayer::onPlay); connect(this, &FFPlayer::emPause, this, &FFPlayer::onPause); connect(this, &FFPlayer::emStop, this, &FFPlayer::onStop); connect(this, &FFPlayer::emVol, this, &FFPlayer::onVol); thread->start(); } FFPlayer::~FFPlayer() { qInfo()<<"~ FFPlayer"; } void FFPlayer::onQuit() { onClose(); thread->quit(); } void FFPlayer::onOpen(QByteArray url) { onClose(); ctx = new AVFmt(url); if(mVol!=1.0 && ctx->sink) ctx->sink->setVolume(mVol); if(ctx->err.isEmpty()) status = Paused; else { qCritical()<err; emit emError(ctx->err); delete ctx; ctx = 0; } } void FFPlayer::onClose() { status = Closed; if(audioDaemonTimerId!=0) { killTimer(audioDaemonTimerId); audioDaemonTimerId = 0; } if(ctx==0) return; if(ctx->sink) { ctx->sink->stop(); delete ctx->sink; } delete ctx; ctx = 0; emit emUpd(QImage()); } void FFPlayer::onPlay() { if(ctx==0) return; status = Playing; if(ctx->sink) ctx->sink->resume(); startAudioDaemon(); } void FFPlayer::onPause() { if(ctx==0) return; status = Paused; if(ctx->sink) ctx->sink->suspend(); } void FFPlayer::onStop() { if(audioDaemonTimerId!=0) { killTimer(audioDaemonTimerId); audioDaemonTimerId = 0; } if(ctx==0) return; if(ctx->sink) { ctx->sink->stop(); ctx->sinkWriter = ctx->sink->start(); } lock_guard lock(ctx->acts.mtx); ctx->vi.stop = true; ctx->au.stop = true; status = Paused; ctx->start_epoch = 0; ctx->playLoop = 0; ctx->au_frms.lockDelAll(); ctx->vi_frms.lockDelAll(); img = QImage(); ctx->acts.add(new Act{0, 0, 'S'}); ctx->acts.cv.notify_all(); } void FFPlayer::onVol(qreal vol) { mVol = vol; if(ctx && ctx->sink) ctx->sink->setVolume(vol); } void FFPlayer::updFrame() { if(ctx==0) return; if(status!=Playing) { if(img.isNull()) { lock_guard lock(ctx->vi_frms.mtx); viSize = ctx->vi_frms.size; auSize = ctx->au_frms.size; if(ctx->vi_frms.first==0) return; img = ctx->vi_frms.first->img; emit emUpd(img); } return; } auto now_epoch = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); dur = now_epoch - last_epoch; last_epoch = now_epoch; if(dur>100000) return; if(dur<4000) { qWarning()<<"FFPlayer UpdFrame Fail: dur"<checkAndUpd(this, now_epoch); viSize = ctx->vi_frms.size; auSize = ctx->au_frms.size; if(hasImg) emit emUpd(img); startAudioDaemon(); } void FFPlayer::timerEvent(QTimerEvent *event) { if(event->timerId()!=audioDaemonTimerId) return; if(ctx==0 || status!=Playing) { if(audioDaemonTimerId!=0) { killTimer(audioDaemonTimerId); audioDaemonTimerId = 0; } return; } auto now_epoch = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); auto dur = now_epoch - last_epoch; if(dur<=40000 && ! ctx->vi.isNull()) return; ctx->checkAndUpd(this, now_epoch); viSize = ctx->vi_frms.size; auSize = ctx->au_frms.size; if(hasImg) emit emUpd(img); }