From 3fd40888aac30324794630fb3afd10737346b5f4 Mon Sep 17 00:00:00 2001 From: Kirill Kirilenko Date: Thu, 12 Nov 2020 21:18:47 +0300 Subject: [PATCH] Remove sleep_until from processing loop Fix DSP parameters control in MainWindow Move QAudioInput and QAudioOutput to different threads --- src/AudioProcessor.cpp | 42 +++++++++++++++++++++------------------- src/AudioProcessor.h | 3 +++ src/MainWindow.cpp | 44 ++++++++++++++++++++++++++++++++++++------ src/MainWindow.h | 4 ++++ src/WebRTCDSP.cpp | 3 +++ 5 files changed, 70 insertions(+), 26 deletions(-) diff --git a/src/AudioProcessor.cpp b/src/AudioProcessor.cpp index f13dcaf..a817bec 100644 --- a/src/AudioProcessor.cpp +++ b/src/AudioProcessor.cpp @@ -79,7 +79,7 @@ void AudioProcessor::process() { while (doWork_) { - auto waitUntil = std::chrono::high_resolution_clock::now() + std::chrono::milliseconds(5); + std::unique_lock processLock(processMutex_); const std::size_t bytesToRead = bufferSize_ * format_.sampleSize() / 8 * format_.channelCount(); @@ -119,13 +119,9 @@ void AudioProcessor::process() { std::unique_lock lock(outputMutex_); outputBuffer_.append(buf.constData(), buf.byteCount()); + emit readyRead(); } - - - emit readyRead(); } - - std::this_thread::sleep_until(waitUntil); } } @@ -144,6 +140,22 @@ void AudioProcessor::processBuffer(QAudioBuffer& inputBuffer, const QAudioBuffer emit outputLevelsChanged(outputLevels); } +void AudioProcessor::clearBuffers() +{ + { + std::unique_lock lock(inputMutex_); + inputBuffer_.clear(); + } + { + std::unique_lock lock(outputMutex_); + outputBuffer_.clear(); + } + { + std::unique_lock lock(monitorMutex_); + monitorBuffer_.clear(); + } +} + bool AudioProcessor::isSequential() const { return true; @@ -162,13 +174,8 @@ qint64 AudioProcessor::bytesAvailable() const bool AudioProcessor::open(QIODevice::OpenMode mode) { - std::unique_lock lock1(inputMutex_); - std::unique_lock lock2(outputMutex_); - std::unique_lock lock3(monitorMutex_); - - inputBuffer_.clear(); - outputBuffer_.clear(); - monitorBuffer_.clear(); + std::unique_lock lock(processMutex_); + clearBuffers(); sourceEncoder_.reset(new WavFileWriter("source.wav", format_)); sourceEncoder_->open(); @@ -204,13 +211,8 @@ Backend AudioProcessor::getCurrentBackend() const void AudioProcessor::switchBackend(Backend backend) { - std::unique_lock lock1(inputMutex_); - std::unique_lock lock2(outputMutex_); - std::unique_lock lock3(monitorMutex_); - - inputBuffer_.clear(); - outputBuffer_.clear(); - monitorBuffer_.clear(); + std::unique_lock lock(processMutex_); + clearBuffers(); if (backend == Backend::Speex) dsp_.reset(new SpeexDSP(format_, monitorFormat_)); diff --git a/src/AudioProcessor.h b/src/AudioProcessor.h index be63783..ccbf5e9 100644 --- a/src/AudioProcessor.h +++ b/src/AudioProcessor.h @@ -57,11 +57,14 @@ protected: private: void process(); void processBuffer(QAudioBuffer& inputBuffer, const QAudioBuffer& monitorBuffer); + void clearBuffers(); mutable std::mutex inputMutex_; mutable std::mutex outputMutex_; mutable std::mutex monitorMutex_; + std::mutex processMutex_; + std::size_t bufferSize_; const QAudioFormat format_; const QAudioFormat monitorFormat_; diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp index 1043388..f0d68d5 100644 --- a/src/MainWindow.cpp +++ b/src/MainWindow.cpp @@ -44,6 +44,9 @@ MainWindow::MainWindow() : ui(new Ui::MainWindow) { ui->setupUi(this); + audioInputThread_.start(); + audioOutputThread_.start(); + qDebug(Gui) << "Enumerating audio devices..."; for (auto& deviceInfo : QAudioDeviceInfo::availableDevices(QAudio::AudioInput)) { @@ -83,6 +86,8 @@ MainWindow::MainWindow() : ui(new Ui::MainWindow) MainWindow::~MainWindow() { stopRecording(); + audioInputThread_.exit(0); + audioOutputThread_.exit(0); } void fixFormatForDevice(QAudioFormat& format, const QAudioDeviceInfo& info) @@ -114,6 +119,10 @@ void MainWindow::initializeAudio(const QAudioDeviceInfo& inputDeviceInfo, audioOutput_.reset(new QAudioOutput(outputDeviceInfo, outputFormat)); monitorInput_.reset(new QAudioInput(monitorDeviceInfo, monitorFormat)); + audioInput_->moveToThread(&audioInputThread_); + audioOutput_->moveToThread(&audioOutputThread_); + monitorInput_->moveToThread(&audioInputThread_); + processor_.reset(new AudioProcessor(captureFormat, monitorFormat, monitorBuffer_)); connect(processor_.get(), &AudioProcessor::voiceActivityChanged, this, &MainWindow::updateVoiceActivity); @@ -132,6 +141,9 @@ void MainWindow::startRecording() audioInput_->start(processor_.get()); audioOutput_->start(processor_.get()); monitorInput_->start(&monitorBuffer_); + + qInfo(Gui) << "input buffer size:" << audioInput_->bufferSize(); + qInfo(Gui) << "output buffer size:" << audioOutput_->bufferSize(); } void MainWindow::stopRecording() @@ -191,11 +203,21 @@ QString levelFromCode(int value) void MainWindow::setupDials(Backend backend) { ui->noiseGroupBox->setChecked(false); - ui->agcGroupBox->setChecked(false); - ui->aecGroupBox->setChecked(false); ui->noiseSuppressionDial->setValue(0); + + ui->agcGroupBox->setChecked(false); ui->agcLevelDial->setValue(0); ui->agcLevelValue->setText("0 dBFS"); + ui->agcMaxGainDial->setValue(0); + ui->agcMaxGainValue->setText("0 dB"); + ui->agcMaxIncrementDial->setValue(0); + ui->agcMaxIncrementValue->setText("0 dB/sec"); + ui->agcMaxDecrementDial->setValue(0); + ui->agcMaxDecrementValue->setText("0 dB/sec"); + + ui->aecGroupBox->setChecked(false); + ui->aecSuppressionDial->setValue(0); + updateVoiceActivity(false); if (backend == Backend::Speex) @@ -234,15 +256,18 @@ void MainWindow::setupDials(Backend backend) void MainWindow::changeNoiseReductionSettings() { - processor_->setEffectParam("noise_reduction_enabled", ui->noiseGroupBox->isChecked()); if (currentBackend() == Backend::Speex) { + std::int32_t enabled = ui->noiseGroupBox->isChecked() ? 1 : 0; + processor_->setEffectParam("noise_reduction_enabled", enabled); + std::int32_t maxAttenuation = -ui->noiseSuppressionDial->value(); ui->noiseSuppressionValue->setText(QString("%1 dB").arg(-maxAttenuation)); processor_->setEffectParam("noise_reduction_max_attenuation", maxAttenuation); } else { + processor_->setEffectParam("noise_reduction_enabled", ui->noiseGroupBox->isChecked()); int suppressionLevel = ui->noiseSuppressionDial->value(); ui->noiseSuppressionValue->setText(levelFromCode(suppressionLevel)); processor_->setEffectParam("noise_reduction_suppression_level", suppressionLevel); @@ -251,10 +276,11 @@ void MainWindow::changeNoiseReductionSettings() void MainWindow::changeAGCSettings() { - processor_->setEffectParam("gain_control_enabled", ui->agcGroupBox->isChecked()); - if (currentBackend() == Backend::Speex) { + std::int32_t enabled = ui->agcGroupBox->isChecked() ? 1 : 0; + processor_->setEffectParam("gain_control_enabled", enabled); + std::int32_t level = QAudio::convertVolume(-ui->agcLevelDial->value(), QAudio::DecibelVolumeScale, QAudio::LinearVolumeScale) * @@ -276,6 +302,8 @@ void MainWindow::changeAGCSettings() } else { + processor_->setEffectParam("gain_control_enabled", ui->agcGroupBox->isChecked()); + int level = ui->agcLevelDial->value(); ui->agcLevelValue->setText(QString("%1 dBFS").arg(-ui->agcLevelDial->value())); processor_->setEffectParam("gain_control_target_level", level); @@ -288,9 +316,11 @@ void MainWindow::changeAGCSettings() void MainWindow::changeAECSettings() { - processor_->setEffectParam("echo_cancellation_enabled", ui->aecGroupBox->isChecked()); if (currentBackend() == Backend::Speex) { + std::int32_t enabled = ui->aecGroupBox->isChecked() ? 1 : 0; + processor_->setEffectParam("echo_cancellation_enabled", enabled); + std::int32_t maxAttenuation = -ui->aecSuppressionDial->value(); ui->aecSuppressionValue->setText(QString("%1 dB").arg(-maxAttenuation)); @@ -298,6 +328,8 @@ void MainWindow::changeAECSettings() } else { + processor_->setEffectParam("echo_cancellation_enabled", ui->aecGroupBox->isChecked()); + int suppressionLevel = ui->aecSuppressionDial->value(); ui->aecSuppressionValue->setText(levelFromCode(suppressionLevel)); diff --git a/src/MainWindow.h b/src/MainWindow.h index 4fdd0de..9966c09 100644 --- a/src/MainWindow.h +++ b/src/MainWindow.h @@ -7,6 +7,7 @@ #include #include #include +#include QT_BEGIN_NAMESPACE namespace Ui { @@ -51,6 +52,9 @@ private: Ui::MainWindow* ui = nullptr; + QThread audioInputThread_; + QThread audioOutputThread_; + QScopedPointer audioInput_; QScopedPointer monitorInput_; QScopedPointer audioOutput_; diff --git a/src/WebRTCDSP.cpp b/src/WebRTCDSP.cpp index 509ae4e..4eafb2f 100644 --- a/src/WebRTCDSP.cpp +++ b/src/WebRTCDSP.cpp @@ -166,7 +166,10 @@ void WebRTCDSP::setParameter(const QString& param, QVariant value) config.noise_suppression.level = static_cast(NoiseSuppressionLevel::kLow + value.toUInt()); else if (param == "echo_cancellation_enabled") + { config.echo_canceller.enabled = value.toBool(); + config.residual_echo_detector.enabled = value.toBool(); + } else if (param == "echo_cancellation_suppression_level") return; // TODO ??? else if (param == "gain_control_enabled")