#include "ffplayer.h" #include #include #if(QT_VERSION_MAJOR > 5) #include #endif #include #include using namespace std; AVFmt::AVFmt(QByteArray url) { qInfo()<<"Path"<start_time; qInfo() << "Fmt Start" << start << "Dur" << fmt_ctx->duration; for(uint ss=0; ssnb_streams; ss++) if(fmt_ctx->streams[ss]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {vi.idx = ss; break;} for(uint ss=0; ssnb_streams; ss++) if(fmt_ctx->streams[ss]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {au.idx = ss; break;} if(vi.idx == -1) qWarning()<<"Didn't find a Video Stream"; else { auto viStream = fmt_ctx->streams[vi.idx]; const AVCodec *decoder = avcodec_find_decoder(viStream->codecpar->codec_id); if(decoder==nullptr) { vi.idx = -1; qInfo()<<"Couldn't found Video Decoder"; } else { vi.ctx = avcodec_alloc_context3(decoder); avcodec_parameters_to_context(vi.ctx, viStream->codecpar); vi.ctx->thread_count = 0; if(avcodec_open2(vi.ctx, decoder, nullptr) < 0) { vi.idx = -1; qInfo()<<"Couldn't open Video Codec Ctx"; } else { sws_ctx = sws_getContext(vi.ctx->width, vi.ctx->height, vi.ctx->pix_fmt, vi.ctx->width, vi.ctx->height, AV_PIX_FMT_RGB24, SWS_FAST_BILINEAR, nullptr, nullptr, nullptr); qInfo()<width<<"x"<height<framerate.num<<"/"<framerate.den<<"color_primaries"<color_primaries<<"colorspace"<colorspace; if(vi.ctx->color_primaries==AVCOL_PRI_BT2020) { res = sws_setColorspaceDetails(sws_ctx, sws_getCoefficients(SWS_CS_BT2020), 0, sws_getCoefficients(SWS_CS_DEFAULT), 0, -0x2200, 0x1c000, 0x18000); //0x12800, 0x40000 qInfo()<<"sws_setColorspaceDetails"<time_base.num * 1000000; vi.time_den = viStream->time_base.den; vi.start = viStream->start_time * vi.time_num / vi.time_den; qInfo()<<"Video Idx" << vi.idx << "Start" << vi.start << "Dur" << viStream->duration * vi.time_num / vi.time_den; } } } if(au.idx == -1) qInfo()<<"Couldn't find a Audio Stream"; else { auto auStream = fmt_ctx->streams[au.idx]; #if(QT_VERSION_MAJOR > 5) QAudioDevice audioDevice = QMediaDevices::defaultAudioOutput(); #else QAudioDeviceInfo audioDevice = QAudioDeviceInfo::defaultOutputDevice(); #endif if(audioDevice.isNull()) { au.idx = -1; qInfo()<<"Couldn't find a Audio Device"; } else { AVCodecParameters *codecpar = auStream->codecpar; AVSampleFormat packed_sample_fmt = av_get_packed_sample_fmt((AVSampleFormat)codecpar->format); qInfo()<<"audio codecpar:"; qInfo()<<" sample_rate"<sample_rate; qInfo()<<" frame_size"<frame_size; qInfo()<<" sample_fmt"<format); qInfo()<<" bits per coded/raw sample"<bits_per_coded_sample<bits_per_raw_sample; sample_rate = codecpar->sample_rate; QAudioFormat audioFmt; audioFmt.setSampleRate(sample_rate); audioFmt.setChannelCount(2); #if(QT_VERSION_MAJOR > 5) if(packed_sample_fmt==AV_SAMPLE_FMT_FLT) audioFmt.setSampleFormat(QAudioFormat::Float); else if(packed_sample_fmt==AV_SAMPLE_FMT_S16) audioFmt.setSampleFormat(QAudioFormat::Int16); else if(packed_sample_fmt==AV_SAMPLE_FMT_S32) audioFmt.setSampleFormat(QAudioFormat::Int32); else if(packed_sample_fmt==AV_SAMPLE_FMT_S64) { packed_sample_fmt = AV_SAMPLE_FMT_S32; audioFmt.setSampleFormat(QAudioFormat::Int32); } else if(packed_sample_fmt==AV_SAMPLE_FMT_DBL) { packed_sample_fmt = AV_SAMPLE_FMT_FLT; audioFmt.setSampleFormat(QAudioFormat::Float); } else { packed_sample_fmt = AV_SAMPLE_FMT_S16; audioFmt.setSampleFormat(QAudioFormat::Int16); } #else audioFmt.setCodec("audio/pcm"); audioFmt.setByteOrder(QAudioFormat::LittleEndian); if(packed_sample_fmt==AV_SAMPLE_FMT_FLT) { audioFmt.setSampleType(QAudioFormat::Float); audioFmt.setSampleSize(32); } else if(packed_sample_fmt==AV_SAMPLE_FMT_S16) { audioFmt.setSampleType(QAudioFormat::SignedInt); audioFmt.setSampleSize(16); } else if(packed_sample_fmt==AV_SAMPLE_FMT_S32) { audioFmt.setSampleType(QAudioFormat::SignedInt); audioFmt.setSampleSize(32); } else if(packed_sample_fmt==AV_SAMPLE_FMT_S64) { packed_sample_fmt = AV_SAMPLE_FMT_S32; audioFmt.setSampleType(QAudioFormat::SignedInt); audioFmt.setSampleSize(32); } else if(packed_sample_fmt==AV_SAMPLE_FMT_DBL) { packed_sample_fmt = AV_SAMPLE_FMT_FLT; audioFmt.setSampleType(QAudioFormat::Float); audioFmt.setSampleSize(32); } else { packed_sample_fmt = AV_SAMPLE_FMT_S16; audioFmt.setSampleType(QAudioFormat::SignedInt); audioFmt.setSampleSize(16); } #endif sink = new QAudioSink(audioDevice, audioFmt); sinkWriter = sink->start(); if(sinkWriter==nullptr) { au.idx = -1; qCritical()<<"Couldn't Start Audio Output"<error(); } else { const AVCodec *decoder = avcodec_find_decoder(codecpar->codec_id); if(decoder==nullptr) { au.idx = -1; qWarning()<<"Couldn't found Audio Decoder"; } else { au.ctx = avcodec_alloc_context3(decoder); avcodec_parameters_to_context(au.ctx, codecpar); if(avcodec_open2(au.ctx, decoder, nullptr) < 0) { au.idx = -1; qWarning()<<"Couldn't open aCodecCtx"; } else { swr_ctx = swr_alloc_set_opts(nullptr, AV_CH_LAYOUT_STEREO, packed_sample_fmt, audioFmt.sampleRate(), au.ctx->channel_layout, au.ctx->sample_fmt, au.ctx->sample_rate, 0, nullptr); if(swr_init(swr_ctx) < 0) { au.idx = -1; qCritical()<<"Couldn't init swrCtx"; } else { pcm_sample_size = audioFmt.bytesPerFrame(); au.time_num = auStream->time_base.num * 1000000; au.time_den = auStream->time_base.den; au.start = auStream->start_time * au.time_num / au.time_den; qInfo()<<"Audio Idx" << au.idx << "Start" << au.start<< "Dur" << auStream->duration * au.time_num / au.time_den; buf_size = sink->bytesFree(); qInfo()<<"Audio buffer size"< -1) au.thd = new thread(&AVFmt::mainAuDecode, this); if(vi.idx > -1) vi.thd = new thread(&AVFmt::mainViDecode, this); } AVFmt::~AVFmt() { if(read_thd!=nullptr) { acts.lockAddNtf(new Act{0, nullptr, 'Q'}); read_thd->join(); delete read_thd; qInfo() << "~ read thread"; } if(au.thd!=nullptr) { au.thd->join(); delete au.thd; qInfo() << "~ au thread"; } if(vi.thd!=nullptr) { vi.thd->join(); delete vi.thd; qInfo() << "~ vi thread"; } av_packet_free(&packet); avformat_close_input(&fmt_ctx); sws_freeContext(sws_ctx); swr_free(&swr_ctx); qInfo() << "~ AVContext"; } void AVFmt::mainRead() { int wait = 0; int res = 0; while(true) { auto act = wait==0 ? acts.lockPoll() : acts.take(wait); while(act!=nullptr) { if(act->act=='S') { if(au.idx!=-1) { unique_lock lock(au.pkts.mtx); au.pkts.delAll(); au.pkts.add(new Packet{nullptr, nullptr, 'R'}); lock.unlock(); au.pkts.cv.notify_all(); } if(vi.idx!=-1) { unique_lock lock(vi.pkts.mtx); vi.pkts.delAll(); vi.pkts.add(new Packet{nullptr, nullptr, 'R'}); lock.unlock(); vi.pkts.cv.notify_all(); } av_seek_frame(fmt_ctx, -1, act->arg, AVSEEK_FLAG_BACKWARD); } else if(act->act=='Q') { if(au.idx!=-1) { unique_lock lock(au.pkts.mtx); au.pkts.delAll(); au.pkts.add(new Packet{nullptr, nullptr, 'Q'}); lock.unlock(); au.pkts.cv.notify_all(); } if(vi.idx!=-1) { unique_lock lock(vi.pkts.mtx); vi.pkts.delAll(); vi.pkts.add(new Packet{nullptr, nullptr, 'Q'}); lock.unlock(); vi.pkts.cv.notify_all(); } delete act; return; } delete act; act = acts.lockPoll(); } if(res < 0) {wait = 500; continue;} if(vi.pkts.size > 120 || au.pkts.size > 200) {wait = 100; continue;} res = av_read_frame(fmt_ctx, packet); if(res < 0) { if(au.idx > -1) au.pkts.lockAddNtf(new Packet{nullptr, nullptr, 0}); if(vi.idx > -1) vi.pkts.lockAddNtf(new Packet{nullptr, nullptr, 0}); if(res!=AVERROR_EOF) { qCritical()<<"Read Packet fail:"<stream_index == vi.idx) { vi.pkts.lockAddNtf(new Packet{packet, nullptr, 0}); packet = av_packet_alloc(); } else if(packet->stream_index == au.idx) { au.pkts.lockAddNtf(new Packet{packet, nullptr, 0}); packet = av_packet_alloc(); } wait = 0; } } void AVFmt::mainViDecode() { while(true) { if(vi_frms.size >= 9) { unique_lock lock(vi.pkts.mtx); if(vi.pkts.first!=nullptr && vi.pkts.first->act=='Q') return; vi.pkts.cv.wait_for(lock, chrono::milliseconds(20)); continue; } auto pkt = vi.pkts.take(); if(pkt->act=='R') { avcodec_flush_buffers(vi.ctx); vi.stop = false; } else if(pkt->act=='Q') { delete pkt; return; } else if(!vi.stop) { int res = avcodec_send_packet(vi.ctx, pkt->pkt); if(res == AVERROR_EOF) goto end; if(res < 0) qCritical()<<"Video send packet fail:"<width, vi.ctx->height, QImage::Format_RGB888); uint8_t *dst[4]{img.bits()}; int dstStride[4]{(int)img.bytesPerLine()}; sws_scale(sws_ctx, vi.frm->data, vi.frm->linesize, 0, vi.ctx->height, dst, dstStride); { lock_guard lock(vi_frms.mtx); if(vi.stop) goto end; viLmt = (vi.frm->best_effort_timestamp + vi.frm->pkt_duration) * vi.time_num / vi.time_den; vi_frms.add(new Img{img, vi.frm->best_effort_timestamp * vi.time_num / vi.time_den, viLmt, vi.deLoop, nullptr}); } } if(pkt->pkt==nullptr) { avcodec_flush_buffers(vi.ctx); vi.deLoop++; } } end: delete pkt; } } void AVFmt::mainAuDecode() { while(true) { if(au_frms.size >= 9) { unique_lock lock(au.pkts.mtx); if(au.pkts.first!=nullptr && au.pkts.first->act=='Q') return; au.pkts.cv.wait_for(lock, chrono::milliseconds(20)); continue; } auto pkt = au.pkts.take(); if(pkt->act=='R') { avcodec_flush_buffers(au.ctx); au.stop = false; } else if(pkt->act=='Q') { delete pkt; return; } else if(!au.stop) { int res = avcodec_send_packet(au.ctx, pkt->pkt); if(res == AVERROR_EOF) goto end; if(res < 0) qCritical()<<"Audio send packet fail:"<nb_samples<=0) continue; auto pcm = new uint8_t[au.frm->nb_samples * pcm_sample_size]; int samp_cnt = swr_convert(swr_ctx, &pcm, au.frm->nb_samples, (const uint8_t **)&au.frm->data, au.frm->nb_samples); if(samp_cnt<=0) { delete [] pcm; qWarning()<<"swr convert fail:"< lock(au_frms.mtx); if(au.stop) { delete [] pcm; goto end; } au_frms.add(new Pcm{pcm, samp_cnt * pcm_sample_size, au.frm->best_effort_timestamp * au.time_num / au.time_den, (au.frm->best_effort_timestamp + au.frm->pkt_duration) * au.time_num / au.time_den, au.deLoop, nullptr}); } } if(pkt->pkt==nullptr) { avcodec_flush_buffers(au.ctx); au.deLoop++; } } end: delete pkt; } } void AVFmt::checkAndUpd(FFPlayer *player, qint64 now_epoch) { if(vi.isNull()) { playAudio(au_frms.lockGet(), player, now_epoch); return; } if(au.isNull()) { player->hasImg = false; auto viFrm = vi_frms.lockGet(); if(viFrm==nullptr) return; if(start_epoch==0) { if(vi.start > viFrm->st) { vi.start = viFrm->st; if(start > viFrm->st) start = viFrm->st; } start_epoch = now_epoch - start + 12500; return; } qint64 cur = now_epoch - start_epoch; checkFrm: if(viFrm->loop!=curLoop) { curLoop = viFrm->loop; start_epoch = now_epoch - start + 12500; qInfo()<<"Restart"; return; } if(viFrm->et < cur - 8334) { if(viFrm->next!=nullptr) { qWarning() << "Drop video frame. st-cur" << (viFrm->st - cur)/1000 << "st" << viFrm->st/1000 << "cur" << cur/1000 << "Video Only"; viFrm = vi_frms.lockDelGetNext(); goto checkFrm; } qInfo() << "Video Adjust Start. st-cur" << (viFrm->st - cur)/1000 << "st" << viFrm->st/1000 << "cur" << cur/1000 << "Video Only"; cur = viFrm->st; start_epoch = now_epoch - cur; } if(viFrm->st > cur) return; player->img = viFrm->img; player->hasImg = true; player->viCurTime = viFrm->st; delete vi_frms.lockPoll(); return; } player->hasImg = false; auto auFrm = au_frms.lockGet(); auto viFrm = vi_frms.lockGet(); if(start_epoch==0) { if(viFrm==nullptr) { if(vi.start < start+500000) return; } else if(vi.start > viFrm->st) { vi.start = viFrm->st; if(start > viFrm->st) start = viFrm->st; } if(auFrm==nullptr) { if(au.start < start+500000) return; } else if(au.start > auFrm->st) { au.start = auFrm->st; if(start > auFrm->st) start = auFrm->st; } start_epoch = now_epoch - (start - 12500); playAudio(auFrm, player, now_epoch, vi.deLoop!=curLoop || vi.pkts.first==nullptr ? INT64_MAX : viLmt); return; } if(viFrm==nullptr) { if(vi.deLoop!=curLoop || vi.pkts.lockGet()==nullptr) playAudio(auFrm, player, now_epoch); return; } if(viFrm->loop!=curLoop) { playAudio(auFrm, player, now_epoch); return; } qint64 cur = now_epoch - start_epoch; if(auFrm!=nullptr && auFrm->st < cur && auFrm->loop==curLoop) { qInfo() << "Audio Adjust Start. st-cur" << (auFrm->st - cur)/1000 << "st" << auFrm->st/1000 << "cur" << cur/1000; cur = auFrm->st; start_epoch = now_epoch - cur; } qint64 curPre = cur - 8334; while(viFrm->et < curPre && viFrm->next!=nullptr && viFrm->next->loop==curLoop) { qWarning() << "Drop video frame. st-cur" << (viFrm->st - cur)/1000 << "st" << viFrm->st/1000 << "cur" << cur/1000; viFrm = vi_frms.lockDelGetNext(); } auto bytesFree = sink->bytesFree(); bool isIdle = bytesFree==buf_size || sink->state()==QAudio::IdleState; if(viFrm->et < curPre) { if(isIdle) { qInfo() << "Video Adjust Start. st-cur" << (viFrm->st - cur)/1000 << "st" << viFrm->st/1000 << "cur" << cur/1000; cur = viFrm->st; start_epoch = now_epoch - cur; } else qInfo() << "Video et < cur. st-cur" << (viFrm->st - cur)/1000 << "st" << viFrm->st/1000 << "cur" << cur/1000; } if(auFrm!=nullptr && bytesFree >= auFrm->size && auFrm->loop==curLoop) whiteAudio(auFrm, player, isIdle, cur, vi.deLoop!=curLoop || vi.pkts.lockGet()==nullptr ? INT64_MAX : viLmt); if(viFrm->st > cur) return; player->img = viFrm->img; player->hasImg = true; player->viCurTime = viFrm->st; delete vi_frms.lockPoll(); } void AVFmt::playAudio(Pcm *auFrm, FFPlayer *player, qint64 now_epoch, qint64 lmt) { if(auFrm==nullptr) return; auto bytesFree = sink->bytesFree(); if(bytesFree < auFrm->size) return; bool isIdle = bytesFree==buf_size || sink->state()==QAudio::IdleState; qint64 cur = now_epoch - start_epoch; if(start_epoch==0) { if(au.start > auFrm->st) { au.start = auFrm->st; if(start > auFrm->st) start = auFrm->st; } cur = start - 12500; start_epoch = now_epoch - cur; } else if(auFrm->loop!=curLoop) { if(! isIdle) return; curLoop = auFrm->loop; cur = start - 12500; start_epoch = now_epoch - cur; qInfo()<<"Restart"; } if(auFrm->st < cur) { qInfo() << "Audio Adjust Start. st-cur" << (auFrm->st - cur)/1000 << "st" << auFrm->st/1000 << "cur" << cur/1000 << "isIdle" << isIdle; cur = auFrm->st; start_epoch = now_epoch - cur; } whiteAudio(auFrm, player, isIdle, cur, lmt); return; } void AVFmt::whiteAudio(Pcm *auFrm, FFPlayer *player, bool isIdle, qint64 cur, qint64 lmt) { if(auFrm->st >= lmt) return; if(isIdle) { auto delay = auFrm->st - cur; if(delay > 50000) return; else if(delay > 999) { auto size = (delay * sample_rate / 1000000) * pcm_sample_size; char *data = new char[size](); sinkWriter->write(data, size); delete [] data; } } do { sinkWriter->write((char*)auFrm->data, auFrm->size); player->auCurTime = auFrm->st; auFrm = au_frms.lockDelGetNext(); } while(auFrm!=nullptr && auFrm->data!=nullptr && sink->bytesFree() >= auFrm->size && auFrm->st < lmt); } FFPlayer::FFPlayer() { thread = new QThread(); moveToThread(thread); connect(thread, &QThread::finished, thread, &QThread::deleteLater); connect(thread, &QThread::finished, this, &FFPlayer::deleteLater); connect(this, &FFPlayer::emQuit, this, &FFPlayer::onQuit); connect(this, &FFPlayer::emOpen, this, &FFPlayer::onOpen); connect(this, &FFPlayer::emClose, this, &FFPlayer::onClose); connect(this, &FFPlayer::emPlay, this, &FFPlayer::onPlay); connect(this, &FFPlayer::emStop, this, &FFPlayer::onStop); connect(this, &FFPlayer::emVol, this, &FFPlayer::onVol); thread->start(); } FFPlayer::~FFPlayer() { qInfo()<<"~ FFPlayer"; } void FFPlayer::onQuit() { onClose(); thread->quit(); } void FFPlayer::onOpen(QByteArray url) { onClose(); ctx = new AVFmt(url); if(mVol!=1.0 && ctx->sink!=nullptr) ctx->sink->setVolume(mVol); if(ctx->err.isEmpty()) status = Paused; else { qCritical()<err; emit emError(ctx->err); delete ctx; ctx = nullptr; } } void FFPlayer::onClose() { status = Closed; if(audioDaemonTimerId!=0) { killTimer(audioDaemonTimerId); audioDaemonTimerId = 0; } if(ctx==nullptr) return; if(ctx->sink!=nullptr) { ctx->sink->stop(); delete ctx->sink; } delete ctx; ctx = nullptr; emit emUpd(QImage()); } void FFPlayer::onPlay() { if(ctx==nullptr) return; status = Playing; startAudioDaemon(); } void FFPlayer::onStop() { if(audioDaemonTimerId!=0) { killTimer(audioDaemonTimerId); audioDaemonTimerId = 0; } if(ctx==nullptr) return; if(ctx->sink!=nullptr) { ctx->sink->stop(); ctx->sinkWriter = ctx->sink->start(); } lock_guard lock(ctx->acts.mtx); ctx->vi.stop = true; ctx->au.stop = true; status = Paused; ctx->start_epoch = 0; ctx->au_frms.lockDelAll(); ctx->vi_frms.lockDelAll(); img = QImage(); ctx->acts.add(new Act{0, nullptr, 'S'}); ctx->acts.cv.notify_all(); } void FFPlayer::onVol(qreal vol) { mVol = vol; if(ctx!=nullptr && ctx->sink!=nullptr) ctx->sink->setVolume(vol); } void FFPlayer::updFrame() { if(ctx==nullptr) return; if(status!=Playing) { if(img.isNull()) { lock_guard lock(ctx->vi_frms.mtx); viSize = ctx->vi_frms.size; auSize = ctx->au_frms.size; if(ctx->vi_frms.first==nullptr) return; img = ctx->vi_frms.first->img; emit emUpd(img); } return; } auto now_epoch = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); dur = now_epoch - last_epoch; last_epoch = now_epoch; if(dur>100000) return; if(dur<4000) { qWarning()<<"FFPlayer UpdFrame Fail: dur"<checkAndUpd(this, now_epoch); viSize = ctx->vi_frms.size; auSize = ctx->au_frms.size; if(hasImg) emit emUpd(img); startAudioDaemon(); } void FFPlayer::timerEvent(QTimerEvent *event) { if(event->timerId()!=audioDaemonTimerId) return; if(ctx==nullptr || status!=Playing) { if(audioDaemonTimerId!=0) { killTimer(audioDaemonTimerId); audioDaemonTimerId = 0; } return; } auto now_epoch = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); auto dur = now_epoch - last_epoch; if(dur<=40000 && ! ctx->vi.isNull()) return; ctx->checkAndUpd(this, now_epoch); viSize = ctx->vi_frms.size; auSize = ctx->au_frms.size; if(hasImg) emit emUpd(img); }