#include "AudioProcessor.h" #include "SpeexDSP.h" #include "Timer.h" #include "WebRTCDSP.h" #include #include namespace SpeexWebRTCTest { namespace { Q_LOGGING_CATEGORY(processor, "processor") } QVector calculateAudioLevels(const QAudioBuffer& buffer); QByteArray popFront(QByteArray& buffer, const std::size_t size) { QByteArray result; result.append(buffer.data(), size); buffer.remove(0, size); return result; } AudioProcessor::AudioProcessor(const QAudioFormat& format, const QAudioFormat& monitorFormat, QBuffer& monitorDevice, QObject* parent) : QIODevice(parent), bufferSize_(1024), format_(format), monitorFormat_(monitorFormat), monitorDevice_(monitorDevice) { qRegisterMetaType>(); switchBackend(Backend::Speex); connect(&monitorDevice_, &QIODevice::readyRead, [this] { monitorDevice_.seek(0); { std::unique_lock lock(monitorMutex_); monitorBuffer_.append(monitorDevice_.readAll()); } monitorDevice_.buffer().clear(); monitorDevice_.seek(0); }); doWork_ = true; worker_ = std::thread([this] { process(); }); } AudioProcessor::~AudioProcessor() { doWork_ = false; worker_.join(); } qint64 AudioProcessor::readData(char* data, qint64 maxlen) { std::unique_lock lock(outputMutex_); int len = std::min((qint64)outputBuffer_.size(), maxlen); QByteArray buffer = popFront(outputBuffer_, len); memcpy(data, buffer.data(), len); return len; } qint64 AudioProcessor::writeData(const char* data, qint64 len) { std::unique_lock lock1(inputMutex_); std::unique_lock lock2(inputEventMutex_); inputBuffer_.append(data, len); inputEvent_.notify_all(); return len; } void AudioProcessor::process() { while (doWork_) { std::unique_lock processLock(processMutex_); const std::size_t bytesToRead = bufferSize_ * format_.sampleSize() / 8 * format_.channelCount(); const std::size_t monitorToRead = bufferSize_ * monitorFormat_.sampleSize() / 8 * monitorFormat_.channelCount(); QByteArray buffer; QByteArray monitorBuffer; { std::unique_lock lock(inputMutex_); if (inputBuffer_.size() >= bytesToRead) buffer = popFront(inputBuffer_, bytesToRead); } if (!buffer.isEmpty()) { { std::unique_lock lock(monitorMutex_); if (monitorBuffer_.size() >= (monitorToRead)) monitorBuffer = popFront(monitorBuffer_, monitorToRead); else monitorBuffer = QByteArray(monitorToRead, 0); } QAudioBuffer buf(buffer, format_); QAudioBuffer monitorBuf(monitorBuffer, monitorFormat_); if (sourceEncoder_->isOpen()) sourceEncoder_->write(buf.constData(), buf.byteCount()); processBuffer(buf, monitorBuf); if (processedEncoder_->isOpen()) processedEncoder_->write(buf.constData(), buf.byteCount()); { std::unique_lock lock(outputMutex_); outputBuffer_.append(buf.constData(), buf.byteCount()); emit readyRead(); } } else { processLock.unlock(); std::unique_lock lock(inputEventMutex_); inputEvent_.wait(lock); } } } void AudioProcessor::processBuffer(QAudioBuffer& inputBuffer, const QAudioBuffer& monitorBuffer) { TIMER(qDebug(processor)) QVector inputLevels = calculateAudioLevels(inputBuffer); if (dsp_) dsp_->processFrame(inputBuffer, monitorBuffer); QVector outputLevels = calculateAudioLevels(inputBuffer); emit inputLevelsChanged(inputLevels); emit outputLevelsChanged(outputLevels); } void AudioProcessor::clearBuffers() { { std::unique_lock lock(inputMutex_); inputBuffer_.clear(); } { std::unique_lock lock(outputMutex_); outputBuffer_.clear(); } { std::unique_lock lock(monitorMutex_); monitorBuffer_.clear(); } } bool AudioProcessor::isSequential() const { return true; } bool AudioProcessor::canReadLine() const { return false; } qint64 AudioProcessor::bytesAvailable() const { std::unique_lock lock(outputMutex_); return outputBuffer_.size(); } bool AudioProcessor::open(QIODevice::OpenMode mode) { std::unique_lock lock(processMutex_); clearBuffers(); sourceEncoder_.reset(new WavFileWriter("source.wav", format_)); sourceEncoder_->open(); processedEncoder_.reset(new WavFileWriter("processed.wav", format_)); processedEncoder_->open(); return QIODevice::open(mode); } void AudioProcessor::close() { if (sourceEncoder_) { sourceEncoder_->close(); sourceEncoder_.reset(); } if (processedEncoder_) { processedEncoder_->close(); processedEncoder_.reset(); } QIODevice::close(); } Backend AudioProcessor::getCurrentBackend() const { if (dynamic_cast(dsp_.get())) return Backend::Speex; else return Backend::WebRTC; } void AudioProcessor::switchBackend(Backend backend) { std::unique_lock lock(processMutex_); clearBuffers(); if (backend == Backend::Speex) dsp_.reset(new SpeexDSP(format_, monitorFormat_)); else dsp_.reset(new WebRTCDSP(format_, monitorFormat_)); bufferSize_ = dsp_->getFrameSize(); connect(dsp_.get(), &AudioEffect::voiceActivityChanged, this, &AudioProcessor::voiceActivityChanged); } void AudioProcessor::setEffectParam(const QString& param, const QVariant& value) { dsp_->setParameter(param, value); } //////////////////////////////////////////////////////////// // This function returns the maximum possible sample value for a given audio format qreal getPeakValue(const QAudioFormat& format) { // Note: Only the most common sample formats are supported if (!format.isValid()) return qreal(0); if (format.codec() != "audio/pcm") return qreal(0); switch (format.sampleType()) { case QAudioFormat::Unknown: break; case QAudioFormat::Float: if (format.sampleSize() != 32) // other sample formats are not supported return qreal(0); return qreal(1.00003); case QAudioFormat::SignedInt: if (format.sampleSize() == 32) return qreal(INT_MAX); if (format.sampleSize() == 16) return qreal(SHRT_MAX); if (format.sampleSize() == 8) return qreal(CHAR_MAX); break; case QAudioFormat::UnSignedInt: if (format.sampleSize() == 32) return qreal(UINT_MAX); if (format.sampleSize() == 16) return qreal(USHRT_MAX); if (format.sampleSize() == 8) return qreal(UCHAR_MAX); break; } return qreal(0); } template QVector getBufferLevels(const T* buffer, int frames, int channels) { QVector max_values; max_values.fill(0, channels); for (int i = 0; i < frames; ++i) { for (int j = 0; j < channels; ++j) { qreal value = qAbs(qreal(buffer[i * channels + j])); if (value > max_values.at(j)) max_values.replace(j, value); } } return max_values; } QVector calculateAudioLevels(const QAudioBuffer& buffer) { QVector values; if (!buffer.format().isValid() || buffer.format().byteOrder() != QAudioFormat::LittleEndian) return values; if (buffer.format().codec() != "audio/pcm") return values; int channelCount = buffer.format().channelCount(); values.fill(0, channelCount); qreal peak_value = getPeakValue(buffer.format()); if (qFuzzyCompare(peak_value, qreal(0))) return values; switch (buffer.format().sampleType()) { case QAudioFormat::Unknown: case QAudioFormat::UnSignedInt: if (buffer.format().sampleSize() == 32) values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); if (buffer.format().sampleSize() == 16) values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); if (buffer.format().sampleSize() == 8) values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); for (int i = 0; i < values.size(); ++i) values[i] = qAbs(values.at(i) - peak_value / 2) / (peak_value / 2); break; case QAudioFormat::Float: if (buffer.format().sampleSize() == 32) { values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); for (int i = 0; i < values.size(); ++i) values[i] /= peak_value; } break; case QAudioFormat::SignedInt: if (buffer.format().sampleSize() == 32) values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); if (buffer.format().sampleSize() == 16) values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); if (buffer.format().sampleSize() == 8) values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); for (int i = 0; i < values.size(); ++i) values[i] /= peak_value; break; } // Convert to dBFS for (auto& level : values) level = QAudio::convertVolume(level, QAudio::LinearVolumeScale, QAudio::DecibelVolumeScale); return values; } } // namespace SpeexWebRTCTest