videoframe.cpp

Go to the documentation of this file.

/*
    Copyright © 2014-2019 by The qTox Project Contributors
 
    This file is part of qTox, a Qt-based graphical interface for Tox.
 
    qTox is libre software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
 
    qTox is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
 
    You should have received a copy of the GNU General Public License
    along with qTox.  If not, see <http://www.gnu.org/licenses/>.
*/
 
#include "videoframe.h"
 
extern "C" {
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
}
 
// Initialize static fields
VideoFrame::AtomicIDType VideoFrame::frameIDs{0};
 
std::unordered_map<VideoFrame::IDType, QMutex> VideoFrame::mutexMap{};
std::unordered_map<VideoFrame::IDType, std::unordered_map<VideoFrame::IDType, std::weak_ptr<VideoFrame>>>
    VideoFrame::refsMap{};
 
QReadWriteLock VideoFrame::refsLock{};
 
VideoFrame::VideoFrame(IDType sourceID, AVFrame* sourceFrame, QRect dimensions, int pixFmt,
                       bool freeSourceFrame)
    : frameID(frameIDs++)
    , sourceID(sourceID)
    , sourceDimensions(dimensions)
    , sourceFrameKey(getFrameKey(dimensions.size(), pixFmt, sourceFrame->linesize[0]))
    , freeSourceFrame(freeSourceFrame)
{
 
    // We override the pixel format in the case a deprecated one is used
    switch (pixFmt) {
    case AV_PIX_FMT_YUVJ420P: {
        sourcePixelFormat = AV_PIX_FMT_YUV420P;
        sourceFrame->color_range = AVCOL_RANGE_MPEG;
        break;
    }
 
    case AV_PIX_FMT_YUVJ411P: {
        sourcePixelFormat = AV_PIX_FMT_YUV411P;
        sourceFrame->color_range = AVCOL_RANGE_MPEG;
        break;
    }
 
    case AV_PIX_FMT_YUVJ422P: {
        sourcePixelFormat = AV_PIX_FMT_YUV422P;
        sourceFrame->color_range = AVCOL_RANGE_MPEG;
        break;
    }
 
    case AV_PIX_FMT_YUVJ444P: {
        sourcePixelFormat = AV_PIX_FMT_YUV444P;
        sourceFrame->color_range = AVCOL_RANGE_MPEG;
        break;
    }
 
    case AV_PIX_FMT_YUVJ440P: {
        sourcePixelFormat = AV_PIX_FMT_YUV440P;
        sourceFrame->color_range = AVCOL_RANGE_MPEG;
        break;
    }
 
    default: {
        sourcePixelFormat = pixFmt;
        sourceFrame->color_range = AVCOL_RANGE_UNSPECIFIED;
    }
    }
 
    frameBuffer[sourceFrameKey] = sourceFrame;
}
 
VideoFrame::VideoFrame(IDType sourceID, AVFrame* sourceFrame, bool freeSourceFrame)
    : VideoFrame(sourceID, sourceFrame, QRect{0, 0, sourceFrame->width, sourceFrame->height},
                 sourceFrame->format, freeSourceFrame)
{
}
 
VideoFrame::~VideoFrame()
{
    // Release frame
    frameLock.lockForWrite();
 
    deleteFrameBuffer();
 
    frameLock.unlock();
 
    // Delete tracked reference
    refsLock.lockForRead();
 
    if (refsMap.count(sourceID) > 0) {
        QMutex& sourceMutex = mutexMap[sourceID];
 
        sourceMutex.lock();
        refsMap[sourceID].erase(frameID);
        sourceMutex.unlock();
    }
 
    refsLock.unlock();
}
 
bool VideoFrame::isValid()
{
    frameLock.lockForRead();
    bool retValue = frameBuffer.size() > 0;
    frameLock.unlock();
 
    return retValue;
}
 
std::shared_ptr<VideoFrame> VideoFrame::trackFrame()
{
    // Add frame to tracked reference list
    refsLock.lockForRead();
 
    if (refsMap.count(sourceID) == 0) {
        // We need to add a new source to our reference map, obtain write lock
        refsLock.unlock();
        refsLock.lockForWrite();
    }
 
    QMutex& sourceMutex = mutexMap[sourceID];
 
    sourceMutex.lock();
 
    std::shared_ptr<VideoFrame> ret{this};
 
    refsMap[sourceID][frameID] = ret;
 
    sourceMutex.unlock();
    refsLock.unlock();
 
    return ret;
}
 
void VideoFrame::untrackFrames(const VideoFrame::IDType& sourceID, bool releaseFrames)
{
    refsLock.lockForWrite();
 
    if (refsMap.count(sourceID) == 0) {
        // No tracking reference exists for source, simply return
        refsLock.unlock();
 
        return;
    }
 
    if (releaseFrames) {
        QMutex& sourceMutex = mutexMap[sourceID];
 
        sourceMutex.lock();
 
        for (auto& frameIterator : refsMap[sourceID]) {
            std::shared_ptr<VideoFrame> frame = frameIterator.second.lock();
 
            if (frame) {
                frame->releaseFrame();
            }
        }
 
        sourceMutex.unlock();
    }
 
    refsMap[sourceID].clear();
 
    mutexMap.erase(sourceID);
    refsMap.erase(sourceID);
 
    refsLock.unlock();
}
 
void VideoFrame::releaseFrame()
{
    frameLock.lockForWrite();
 
    deleteFrameBuffer();
 
    frameLock.unlock();
}
 
const AVFrame* VideoFrame::getAVFrame(QSize frameSize, const int pixelFormat, const bool requireAligned)
{
    if (!frameSize.isValid()) {
        frameSize = sourceDimensions.size();
    }
 
    // Since we are retrieving the AVFrame* directly, we merely need to pass the arguement through
    const std::function<AVFrame*(AVFrame * const)> converter = [](AVFrame* const frame) {
        return frame;
    };
 
    // We need an explicit null pointer holding object to pass to toGenericObject()
    AVFrame* nullPointer = nullptr;
 
    // Returns std::nullptr case of invalid generation
    return toGenericObject(frameSize, pixelFormat, requireAligned, converter, nullPointer);
}
 
QImage VideoFrame::toQImage(QSize frameSize)
{
    if (!frameSize.isValid()) {
        frameSize = sourceDimensions.size();
    }
 
    // Converter function (constructs QImage out of AVFrame*)
    const std::function<QImage(AVFrame * const)> converter = [&](AVFrame* const frame) {
        return QImage{*(frame->data), frameSize.width(), frameSize.height(), *(frame->linesize),
                      QImage::Format_RGB888};
    };
 
    // Returns an empty constructed QImage in case of invalid generation
    return toGenericObject(frameSize, AV_PIX_FMT_RGB24, false, converter, QImage{});
}
 
ToxYUVFrame VideoFrame::toToxYUVFrame(QSize frameSize)
{
    if (!frameSize.isValid()) {
        frameSize = sourceDimensions.size();
    }
 
    // Converter function (constructs ToxAVFrame out of AVFrame*)
    const std::function<ToxYUVFrame(AVFrame * const)> converter = [&](AVFrame* const frame) {
        ToxYUVFrame ret{static_cast<std::uint16_t>(frameSize.width()),
                        static_cast<std::uint16_t>(frameSize.height()), frame->data[0],
                        frame->data[1], frame->data[2]};
 
        return ret;
    };
 
    return toGenericObject(frameSize, AV_PIX_FMT_YUV420P, true, converter,
                           ToxYUVFrame{0, 0, nullptr, nullptr, nullptr});
}
 
VideoFrame::IDType VideoFrame::getFrameID() const
{
    return frameID;
}
 
VideoFrame::IDType VideoFrame::getSourceID() const
{
    return sourceID;
}
 
QRect VideoFrame::getSourceDimensions() const
{
    return sourceDimensions;
}
 
int VideoFrame::getSourcePixelFormat() const
{
    return sourcePixelFormat;
}
 
 
VideoFrame::FrameBufferKey::FrameBufferKey(const int width, const int height, const int pixFmt,
                                           const bool lineAligned)
    : frameWidth(width)
    , frameHeight(height)
    , pixelFormat(pixFmt)
    , linesizeAligned(lineAligned)
{
}
 
bool VideoFrame::FrameBufferKey::operator==(const FrameBufferKey& other) const
{
    return pixelFormat == other.pixelFormat && frameWidth == other.frameWidth
           && frameHeight == other.frameHeight && linesizeAligned == other.linesizeAligned;
}
 
bool VideoFrame::FrameBufferKey::operator!=(const FrameBufferKey& other) const
{
    return !operator==(other);
}
 
size_t VideoFrame::FrameBufferKey::hash(const FrameBufferKey& key)
{
    std::hash<int> intHasher;
    std::hash<bool> boolHasher;
 
    // Use java-style hash function to combine fields
    // See: https://en.wikipedia.org/wiki/Java_hashCode%28%29#hashCode.28.29_in_general
 
    size_t ret = 47;
 
    ret = 37 * ret + intHasher(key.frameWidth);
    ret = 37 * ret + intHasher(key.frameHeight);
    ret = 37 * ret + intHasher(key.pixelFormat);
    ret = 37 * ret + boolHasher(key.linesizeAligned);
 
    return ret;
}
 
VideoFrame::FrameBufferKey VideoFrame::getFrameKey(const QSize& frameSize, const int pixFmt,
                                                   const int linesize)
{
    return getFrameKey(frameSize, pixFmt, frameSize.width() == linesize);
}
 
VideoFrame::FrameBufferKey VideoFrame::getFrameKey(const QSize& frameSize, const int pixFmt,
                                                   const bool frameAligned)
{
    return {frameSize.width(), frameSize.height(), pixFmt, frameAligned};
}
 
AVFrame* VideoFrame::retrieveAVFrame(const QSize& dimensions, const int pixelFormat,
                                     const bool requireAligned)
{
    if (!requireAligned) {
        /*
         * We attempt to obtain a unaligned frame first because an unaligned linesize corresponds
         * to a data aligned frame.
         */
        FrameBufferKey frameKey = getFrameKey(dimensions, pixelFormat, false);
 
        if (frameBuffer.count(frameKey) > 0) {
            return frameBuffer[frameKey];
        }
    }
 
    FrameBufferKey frameKey = getFrameKey(dimensions, pixelFormat, true);
 
    if (frameBuffer.count(frameKey) > 0) {
        return frameBuffer[frameKey];
    } else {
        return nullptr;
    }
}
 
AVFrame* VideoFrame::generateAVFrame(const QSize& dimensions, const int pixelFormat,
                                     const bool requireAligned)
{
    AVFrame* ret = av_frame_alloc();
 
    if (!ret) {
        return nullptr;
    }
 
    // Populate AVFrame fields
    ret->width = dimensions.width();
    ret->height = dimensions.height();
    ret->format = pixelFormat;
 
    /*
     * We generate a frame under data alignment only if the dimensions allow us to be frame aligned
     * or if the caller doesn't require frame alignment
     */
 
    int bufSize;
 
    const bool alreadyAligned = dimensions.width() % dataAlignment == 0 && dimensions.height() % dataAlignment == 0;
 
    if (!requireAligned || alreadyAligned) {
        bufSize = av_image_alloc(ret->data, ret->linesize, dimensions.width(), dimensions.height(),
                                 static_cast<AVPixelFormat>(pixelFormat), dataAlignment);
    } else {
        bufSize = av_image_alloc(ret->data, ret->linesize, dimensions.width(), dimensions.height(),
                                 static_cast<AVPixelFormat>(pixelFormat), 1);
    }
 
    if (bufSize < 0) {
        av_frame_free(&ret);
        return nullptr;
    }
 
    // Bilinear is better for shrinking, bicubic better for upscaling
    int resizeAlgo = sourceDimensions.width() > dimensions.width() ? SWS_BILINEAR : SWS_BICUBIC;
 
    SwsContext* swsCtx =
        sws_getContext(sourceDimensions.width(), sourceDimensions.height(),
                       static_cast<AVPixelFormat>(sourcePixelFormat), dimensions.width(),
                       dimensions.height(), static_cast<AVPixelFormat>(pixelFormat), resizeAlgo,
                       nullptr, nullptr, nullptr);
 
    if (!swsCtx) {
        av_freep(&ret->data[0]);
#if LIBAVCODEC_VERSION_INT < 3747941
        av_frame_unref(ret);
#endif
        av_frame_free(&ret);
        return nullptr;
    }
 
    AVFrame* source = frameBuffer[sourceFrameKey];
 
    sws_scale(swsCtx, source->data, source->linesize, 0, sourceDimensions.height(), ret->data,
              ret->linesize);
    sws_freeContext(swsCtx);
 
    return ret;
}
 
AVFrame* VideoFrame::storeAVFrame(AVFrame* frame, const QSize& dimensions, const int pixelFormat)
{
    FrameBufferKey frameKey = getFrameKey(dimensions, pixelFormat, frame->linesize[0]);
 
    // We check the prescence of the frame in case of double-computation
    if (frameBuffer.count(frameKey) > 0) {
        AVFrame* old_ret = frameBuffer[frameKey];
 
        // Free new frame
        av_freep(&frame->data[0]);
#if LIBAVCODEC_VERSION_INT < 3747941
        av_frame_unref(frame);
#endif
        av_frame_free(&frame);
 
        return old_ret;
    } else {
        frameBuffer[frameKey] = frame;
 
        return frame;
    }
}
 
void VideoFrame::deleteFrameBuffer()
{
    // An empty framebuffer represents a frame that's already been freed
    if (frameBuffer.empty()) {
        return;
    }
 
    for (const auto& frameIterator : frameBuffer) {
        AVFrame* frame = frameIterator.second;
 
        // Treat source frame and derived frames separately
        if (sourceFrameKey == frameIterator.first) {
            if (freeSourceFrame) {
                av_freep(&frame->data[0]);
            }
#if LIBAVCODEC_VERSION_INT < 3747941
            av_frame_unref(frame);
#endif
            av_frame_free(&frame);
        } else {
            av_freep(&frame->data[0]);
#if LIBAVCODEC_VERSION_INT < 3747941
            av_frame_unref(frame);
#endif
            av_frame_free(&frame);
        }
    }
 
    frameBuffer.clear();
}
 
template <typename T>
T VideoFrame::toGenericObject(const QSize& dimensions, const int pixelFormat, const bool requireAligned,
                              const std::function<T(AVFrame* const)>& objectConstructor,
                              const T& nullObject)
{
    frameLock.lockForRead();
 
    // We return nullObject if the VideoFrame is no longer valid
    if (frameBuffer.size() == 0) {
        frameLock.unlock();
        return nullObject;
    }
 
    AVFrame* frame = retrieveAVFrame(dimensions, static_cast<int>(pixelFormat), requireAligned);
 
    if (frame) {
        T ret = objectConstructor(frame);
 
        frameLock.unlock();
        return ret;
    }
 
    // VideoFrame does not contain an AVFrame to spec, generate one here
    frame = generateAVFrame(dimensions, static_cast<int>(pixelFormat), requireAligned);
 
    /*
     * We need to "upgrade" the lock to a write lock so we can update our frameBuffer map.
     *
     * It doesn't matter if another thread obtains the write lock before we finish since it is
     * likely writing to somewhere else. Worst-case scenario, we merely perform the generation
     * process twice, and discard the old result.
     */
    frameLock.unlock();
    frameLock.lockForWrite();
 
    frame = storeAVFrame(frame, dimensions, static_cast<int>(pixelFormat));
 
    T ret = objectConstructor(frame);
 
    frameLock.unlock();
    return ret;
}
 
// Explicitly specialize VideoFrame::toGenericObject() function
template QImage VideoFrame::toGenericObject<QImage>(
    const QSize& dimensions, const int pixelFormat, const bool requireAligned,
    const std::function<QImage(AVFrame* const)> &objectConstructor, const QImage& nullObject);
template ToxYUVFrame VideoFrame::toGenericObject<ToxYUVFrame>(
    const QSize& dimensions, const int pixelFormat, const bool requireAligned,
    const std::function<ToxYUVFrame(AVFrame* const)> &objectConstructor, const ToxYUVFrame& nullObject);
 
bool ToxYUVFrame::isValid() const
{
    return width > 0 && height > 0;
}
 
ToxYUVFrame::operator bool() const
{
    return isValid();
}