Skip to content

Enable Opus ML improvements #3487

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Enable Opus ML improvements #3487

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f31a9bd
Enable Opus ML improvements
ann0see Apr 6, 2025
5097306
Undo change to Jamulus.pro
softins May 1, 2025
e320601
Add Deep PLC and DRED files for OPUS
softins Apr 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 120 additions & 9 deletions Jamulus.pro
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,16 @@ contains(CONFIG, "headless") {
QT += multimedia
}

# DRED is a superset of Deep PLC and requires it
contains(CONFIG, "opus_dred") {
message("Building OPUS with Deep Redundancy.")
CONFIG += "opus_deep_plc"
}

contains(CONFIG, "opus_deep_plc") {
message("Building OPUS with Deep PLC.")
}

# Do not set LRELEASE_DIR explicitly when using embed_translations.
# It doesn't work with multiple targets or architectures.
TRANSLATIONS = src/translation/translation_de_DE.ts \
Expand All @@ -75,6 +85,10 @@ INCLUDEPATH_OPUS = libs/opus/include \
libs/opus/silk/fixed \
libs/opus

contains(CONFIG, "opus_deep_plc") {
INCLUDEPATH_OPUS += libs/opus/dnn
}

# As JACK is used in multiple OS, we declare it globally
HEADERS_JACK = src/sound/jack/sound.h
SOURCES_JACK = src/sound/jack/sound.cpp
Expand Down Expand Up @@ -490,7 +504,40 @@ HEADERS_OPUS_X86 = libs/opus/celt/x86/celt_lpc_sse.h \
libs/opus/celt/x86/pitch_sse.h \
libs/opus/celt/x86/vq_sse.h \
libs/opus/celt/x86/x86cpu.h \
$$files(libs/opus/silk/x86/*.h)
libs/opus/silk/x86/SigProc_FIX_sse.h \
libs/opus/silk/x86/main_sse.h

HEADERS_OPUS_DEEP_PLC = \
libs/opus/dnn/lpcnet.h \
libs/opus/dnn/burg.h \
libs/opus/dnn/common.h \
libs/opus/dnn/freq.h \
libs/opus/dnn/fargan.h \
libs/opus/dnn/fargan_data.h \
libs/opus/dnn/lpcnet_private.h \
libs/opus/dnn/nnet.h \
libs/opus/dnn/plc_data.h \
libs/opus/dnn/vec.h \
libs/opus/dnn/vec_avx.h \
libs/opus/dnn/vec_neon.h \
libs/opus/dnn/pitchdnn.h \
libs/opus/dnn/pitchdnn_data.h \
libs/opus/dnn/x86/dnn_x86.h \
libs/opus/dnn/nnet_arch.h \
libs/opus/dnn/arm/dnn_arm.h

HEADERS_OPUS_DRED = \
libs/opus/dnn/dred_coding.h \
libs/opus/dnn/dred_config.h \
libs/opus/dnn/dred_decoder.h \
libs/opus/dnn/dred_encoder.h \
libs/opus/dnn/dred_rdovae.h \
libs/opus/dnn/dred_rdovae_constants.h \
libs/opus/dnn/dred_rdovae_enc.h \
libs/opus/dnn/dred_rdovae_enc_data.h \
libs/opus/dnn/dred_rdovae_dec.h \
libs/opus/dnn/dred_rdovae_dec_data.h \
libs/opus/dnn/dred_rdovae_stats_data.h

SOURCES += src/plugins/audioreverb.cpp \
src/buffer.cpp \
Expand Down Expand Up @@ -674,29 +721,93 @@ SOURCES_OPUS_X86_SSE = libs/opus/celt/x86/x86cpu.c \
libs/opus/celt/x86/x86_celt_map.c \
libs/opus/celt/x86/pitch_sse.c
SOURCES_OPUS_X86_SSE2 = libs/opus/celt/x86/pitch_sse2.c \
libs/opus/celt/x86/vq_sse2.c
libs/opus/celt/x86/vq_sse2.c
SOURCES_OPUS_X86_SSE4 = libs/opus/celt/x86/celt_lpc_sse4_1.c \
libs/opus/celt/x86/pitch_sse4_1.c \
libs/opus/silk/x86/NSQ_sse4_1.c \
libs/opus/silk/x86/NSQ_del_dec_sse4_1.c \
libs/opus/silk/x86/x86_silk_map.c \
libs/opus/silk/x86/VAD_sse4_1.c \
libs/opus/silk/x86/VQ_WMat_EC_sse4_1.c
libs/opus/celt/x86/pitch_sse4_1.c \
libs/opus/silk/x86/NSQ_sse4_1.c \
libs/opus/silk/x86/NSQ_del_dec_sse4_1.c \
libs/opus/silk/x86/x86_silk_map.c \
libs/opus/silk/x86/VAD_sse4_1.c \
libs/opus/silk/x86/VQ_WMat_EC_sse4_1.c

SOURCES_OPUS_DEEP_PLC = \
libs/opus/dnn/burg.c \
libs/opus/dnn/freq.c \
libs/opus/dnn/fargan.c \
libs/opus/dnn/fargan_data.c \
libs/opus/dnn/lpcnet_enc.c \
libs/opus/dnn/lpcnet_plc.c \
libs/opus/dnn/lpcnet_tables.c \
libs/opus/dnn/nnet.c \
libs/opus/dnn/nnet_default.c \
libs/opus/dnn/plc_data.c \
libs/opus/dnn/parse_lpcnet_weights.c \
libs/opus/dnn/pitchdnn.c \
libs/opus/dnn/pitchdnn_data.c

SOURCES_OPUS_DRED = \
libs/opus/dnn/dred_rdovae_enc.c \
libs/opus/dnn/dred_rdovae_enc_data.c \
libs/opus/dnn/dred_rdovae_dec.c \
libs/opus/dnn/dred_rdovae_dec_data.c \
libs/opus/dnn/dred_rdovae_stats_data.c \
libs/opus/dnn/dred_encoder.c \
libs/opus/dnn/dred_coding.c \
libs/opus/dnn/dred_decoder.c

SOURCES_OPUS_DNN_X86_RTCD = \
libs/opus/dnn/x86/x86_dnn_map.c
SOURCES_OPUS_DNN_AVX2 = \
libs/opus/dnn/x86/nnet_avx2.c
SOURCES_OPUS_DNN_SSE4_1 = \
libs/opus/dnn/x86/nnet_sse4_1.c
SOURCES_OPUS_DNN_SSE2 = \
libs/opus/dnn/x86/nnet_sse2.c

SOURCES_OPUS_DNN_ARM_RTCD = \
libs/opus/dnn/arm/arm_dnn_map.c
SOURCES_OPUS_DNN_DOTPROD = \
libs/opus/dnn/arm/nnet_dotprod.c
SOURCES_OPUS_DNN_NEON = \
libs/opus/dnn/arm/nnet_neon.c

contains(CONFIG, "opus_deep_plc") {
HEADERS_OPUS += $$HEADERS_OPUS_DEEP_PLC
SOURCES_OPUS += $$SOURCES_OPUS_DEEP_PLC
SOURCES_OPUS_ARM += $$SOURCES_OPUS_DNN_ARM_RTCD $$SOURCES_OPUS_DNN_DOTPROD $$SOURCES_OPUS_DNN_NEON
SOURCES_OPUS_X86 += $$SOURCES_OPUS_DNN_X86_RTCD
SOURCES_OPUS_X86_SSE2 += $$SOURCES_OPUS_DNN_SSE2
SOURCES_OPUS_X86_SSE4 += $$SOURCES_OPUS_DNN_SSE4_1
#SOURCES_OPUS_X86_AVX2 += $$SOURCES_OPUS_DNN_AVX2
}

contains(CONFIG, "opus_dred") {
HEADERS_OPUS += $$HEADERS_OPUS_DRED
SOURCES_OPUS += $$SOURCES_OPUS_DRED
}

contains(QT_ARCH, armeabi-v7a) | contains(QT_ARCH, arm64-v8a) {
HEADERS_OPUS += $$HEADERS_OPUS_ARM
SOURCES_OPUS_ARCH += $$SOURCES_OPUS_ARM
DEFINES_OPUS += OPUS_ARM_PRESUME_NEON=1 OPUS_ARM_PRESUME_NEON_INTR=1
contains(QT_ARCH, arm64-v8a):DEFINES_OPUS += OPUS_ARM_PRESUME_AARCH64_NEON_INTR
contains(QT_ARCH, arm64-v8a):DEFINES_OPUS += OPUS_ARM_PRESUME_AARCH64_NEON_INTR=1
} else:contains(QT_ARCH, x86) | contains(QT_ARCH, x86_64) {
HEADERS_OPUS += $$HEADERS_OPUS_X86
SOURCES_OPUS += $$SOURCES_OPUS_X86
SOURCES_OPUS_ARCH += $$SOURCES_OPUS_X86_SSE $$SOURCES_OPUS_X86_SSE2 $$SOURCES_OPUS_X86_SSE4
DEFINES_OPUS += OPUS_X86_MAY_HAVE_SSE OPUS_X86_MAY_HAVE_SSE2 OPUS_X86_MAY_HAVE_SSE4_1
# x86_64 implies SSE2
contains(QT_ARCH, x86_64):DEFINES_OPUS += OPUS_X86_PRESUME_SSE=1 OPUS_X86_PRESUME_SSE2=1
DEFINES_OPUS += CPU_INFO_BY_C
}
DEFINES_OPUS += OPUS_BUILD=1 USE_ALLOCA=1 OPUS_HAVE_RTCD=1 HAVE_LRINTF=1 HAVE_LRINT=1
contains(CONFIG, "opus_deep_plc") {
DEFINES_OPUS += ENABLE_DEEP_PLC=1
}

contains(CONFIG, "opus_dred") {
DEFINES_OPUS += ENABLE_DRED=1
}

DISTFILES += ChangeLog \
COMPILING.md \
Expand Down
4 changes: 4 additions & 0 deletions src/client.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,10 @@ CClient::CClient ( const quint16 iPortNumber,
opus_custom_encoder_ctl ( OpusEncoderMono, OPUS_SET_COMPLEXITY ( 1 ) );
opus_custom_encoder_ctl ( OpusEncoderStereo, OPUS_SET_COMPLEXITY ( 1 ) );

// enable lowest decoder complexity for Opus64 decoders to enable DRED
opus_custom_decoder_ctl ( Opus64DecoderMono, OPUS_SET_COMPLEXITY ( 5 ) );
opus_custom_decoder_ctl ( Opus64DecoderStereo, OPUS_SET_COMPLEXITY ( 5 ) );
Comment on lines +109 to +111
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it correct that these lines are for decoder when all the lines above them are for encoder?

Also, as this only specifies Opus64, it looks like it only enables complexity 5 for when "Small Network Buffers" is ticked in the client.

I've never been exactly clear on the benefits or not of that setting, from an audio perspective, and usually run without it.

Is there a reason not to set complexity 5 for the original 128 Opus too?

Copy link
Collaborator

@pljones pljones Apr 7, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Small Network Buffers, as I understand it, has no impact on audio -- excepting that the network audio buffer size is 64 frames rather than 128. That, in itself, could affect a lossy compression algorithm (i.e. due to less information being received to reconstruct the compressed waveform in each block) - it's a trade off against "smoother networking" (I hesitate to say "lower latency" as the overall latency isn't really affected).

(Of course, if the OPUS output buffer size is 128 and the network buffer size is 64 - i.e. splitting the compressed buffer in two regardless of audio buffer size - the compression isn't affected at all.)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it correct that these lines are for decoder when all the lines above them are for encoder?

Yes. As far as I understand the documentation this is correct:

When building Opus, using --enable-deep-plc will compile in the deep PLC code at a cost of about 1 MB in binary size. To actually enable it at run time, you will need to set the decoder complexity to 5 or more. Previously, only the encoder had a complexity knob, but the decoder is now getting one too. It can be set with the -dec_complexity option to opus_demo, or OPUS_SET_COMPLEXITY() in the API (like for the encoder). The extra complexity from running PLC at a high loss rate is about 1% of a laptop CPU core. Because deep PLC only affects the decoder, turning it on does not have any compatibility implications.

Also, as this only specifies Opus64, it looks like it only enables complexity 5 for when "Small Network Buffers" is ticked in the client.

For now, this choice was for testing only. But it's still to be discussed/tested.


// Connections -------------------------------------------------------------
// connections for the protocol mechanism
QObject::connect ( &Channel, &CChannel::MessReadyForSending, this, &CClient::OnSendProtMessage );
Expand Down
4 changes: 4 additions & 0 deletions src/server.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,10 @@ CServer::CServer ( const int iNewMaxNumChan,
opus_custom_encoder_ctl ( OpusEncoderMono[i], OPUS_SET_COMPLEXITY ( 1 ) );
opus_custom_encoder_ctl ( OpusEncoderStereo[i], OPUS_SET_COMPLEXITY ( 1 ) );

// enable lowest decoder complexity for Opus64 decoders to enable DRED
opus_custom_decoder_ctl ( Opus64DecoderMono[i], OPUS_SET_COMPLEXITY ( 5 ) );
opus_custom_decoder_ctl ( Opus64DecoderStereo[i], OPUS_SET_COMPLEXITY ( 5 ) );
Comment on lines +119 to +121
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Comment as above


// init double-to-normal frame size conversion buffers -----------------
// use worst case memory initialization to avoid allocating memory in
// the time-critical thread
Expand Down
Loading