Use Oboe to create clean audio and forward to other devices

[sarahachem]

Hi,
is it possible to use oboe to clean audio and forward it to other devices using bluetooth? I hear audio going through but it s noisy and with interruptions.
here is my code so far but i am missing the enhancing part

#include <jni.h>
#include <oboe/Oboe.h>
#include <android/log.h>
#include <vector>

using namespace oboe;

std::shared_ptr<AudioStream> recordingStream;
std::shared_ptr<AudioStream> playbackStream;
JavaVM* javaVM = nullptr;
jobject globalAudioEngine = nullptr;
const int32_t FIXED_BUFFER_SIZE_FRAMES = 2048;

// Data callback for recording stream
class RecordingDataCallback : public AudioStreamDataCallback {
public:
    DataCallbackResult onAudioReady(AudioStream *oboeStream, void *audioData, int32_t numFrames) override {
        __android_log_print(ANDROID_LOG_DEBUG, "testing", "Recording data callback - Frames available: %d", numFrames);

        int16_t* audioBuffer = static_cast<int16_t*>(audioData);
        callSendAudioDataToBluetooth(audioBuffer, numFrames);

        return DataCallbackResult::Continue;
    }

private:
    // Function to send audio data to Kotlin for Bluetooth transmission
    void callSendAudioDataToBluetooth(int16_t* audioBuffer, int32_t numFrames) {
        if (javaVM == nullptr || globalAudioEngine == nullptr) {
            __android_log_print(ANDROID_LOG_ERROR, "testing", "Java VM or AudioEngine reference is null");
            return;
        }

        JNIEnv* env;
        bool isAttached = false;

        // Attach the current thread to the JVM if needed
        if (javaVM->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) == JNI_EDETACHED) {
            if (javaVM->AttachCurrentThread(&env, nullptr) == 0) {
                isAttached = true;
            } else {
                __android_log_print(ANDROID_LOG_ERROR, "testing", "Failed to attach thread to JVM");
                return;
            }
        }

        // Prepare audio data as byte array
        int32_t bufferSize = numFrames * sizeof(int16_t);
        jbyteArray audioDataArray = env->NewByteArray(bufferSize);
        env->SetByteArrayRegion(audioDataArray, 0, bufferSize, reinterpret_cast<jbyte*>(audioBuffer));

        // Find and call the Java method to send the audio data
        jclass audioEngineClass = env->GetObjectClass(globalAudioEngine);
        jmethodID sendAudioDataMethod = env->GetMethodID(audioEngineClass, "sendAudioDataToBluetooth", "([B)V");
        if (sendAudioDataMethod == nullptr) {
            __android_log_print(ANDROID_LOG_ERROR, "testing", "Method sendAudioDataToBluetooth not found");
            env->DeleteLocalRef(audioDataArray);
            if (isAttached) javaVM->DetachCurrentThread();
            return;
        }

        env->CallVoidMethod(globalAudioEngine, sendAudioDataMethod, audioDataArray);
        env->DeleteLocalRef(audioDataArray);

        // Detach the thread if it was attached
        if (isAttached) {
            javaVM->DetachCurrentThread();
        }
    }
};

// Error callback for logging errors
class ErrorCallback : public AudioStreamErrorCallback {
public:
    void onErrorBeforeClose(AudioStream *oboeStream, Result error) override {
        __android_log_print(ANDROID_LOG_ERROR, "testing", "Error before close: %s", convertToText(error));
    }
    void onErrorAfterClose(AudioStream *oboeStream, Result error) override {
        __android_log_print(ANDROID_LOG_ERROR, "testing", "Error after close: %s", convertToText(error));
    }
};

RecordingDataCallback recordingCallback;
ErrorCallback errorCallback;

extern "C" JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* /* reserved */) {
    javaVM = vm;
    return JNI_VERSION_1_6;
}

extern "C" JNIEXPORT void JNICALL
Java_com_kido_scorpionhelmets_models_AudioEngine_init(JNIEnv *env, jobject thiz) {
    globalAudioEngine = env->NewGlobalRef(thiz);
}

extern "C" JNIEXPORT jboolean JNICALL
Java_com_kido_scorpionhelmets_models_AudioEngine_startAudioStream(JNIEnv *env, jobject /* this */) {
    auto playbackBuilder = std::make_shared<AudioStreamBuilder>();
    playbackBuilder->setSampleRate(16000)
            ->setFormat(AudioFormat::I16)
            ->setChannelCount(ChannelCount::Mono)
            ->setDirection(Direction::Output)
            ->setPerformanceMode(PerformanceMode::LowLatency)
            ->setErrorCallback(&errorCallback);

    Result result = playbackBuilder->openStream(playbackStream);
    if (result != Result::OK || !playbackStream) {
        __android_log_print(ANDROID_LOG_ERROR, "testing", "Failed to start playback stream: %s", convertToText(result));
        return JNI_FALSE;
    }

    playbackStream->setBufferSizeInFrames(FIXED_BUFFER_SIZE_FRAMES);
    __android_log_print(ANDROID_LOG_DEBUG, "testing", "Playback stream buffer size set to: %d frames (Capacity: %d frames)",
                        playbackStream->getBufferSizeInFrames(), playbackStream->getBufferCapacityInFrames());

    playbackStream->requestStart();
    return JNI_TRUE;
}

extern "C" JNIEXPORT jboolean JNICALL
Java_com_kido_scorpionhelmets_models_AudioEngine_startRecordingStream(JNIEnv *env, jobject /* this */) {
    auto recordingBuilder = std::make_shared<AudioStreamBuilder>();
    recordingBuilder->setSampleRate(16000)
            ->setChannelCount(ChannelCount::Mono)
            ->setFormat(AudioFormat::I16)
            ->setDirection(Direction::Input)
            ->setPerformanceMode(PerformanceMode::LowLatency)
            ->setInputPreset(InputPreset::VoiceCommunication)
            ->setDataCallback(&recordingCallback)
            ->setErrorCallback(&errorCallback);

    Result result = recordingBuilder->openStream(recordingStream);
    if (result != Result::OK || !recordingStream) {
        __android_log_print(ANDROID_LOG_ERROR, "testing", "Failed to open recording stream: %s", convertToText(result));
        return JNI_FALSE;
    }

    result = recordingStream->requestStart();
    if (result != Result::OK) {
        __android_log_print(ANDROID_LOG_ERROR, "testing", "Failed to start recording stream: %s", convertToText(result));
        return JNI_FALSE;
    }

    StreamState streamState = recordingStream->getState();
    __android_log_print(ANDROID_LOG_DEBUG, "testing", "Recording stream state after start request: %d", static_cast<int>(streamState));
    if (streamState != StreamState::Started) {
        __android_log_print(ANDROID_LOG_ERROR, "testing", "Recording stream is not in Started state.");
        return JNI_FALSE;
    }

    return JNI_TRUE;
}

extern "C" JNIEXPORT void JNICALL
Java_com_kido_scorpionhelmets_models_AudioEngine_stopAudioStream(JNIEnv *env, jobject /* this */) {
    if (playbackStream) {
        playbackStream->stop();
        playbackStream->close();
        playbackStream.reset();
    }
    if (recordingStream) {
        recordingStream->stop();
        recordingStream->close();
        recordingStream.reset();
    }
    if (globalAudioEngine != nullptr) {
        env->DeleteGlobalRef(globalAudioEngine);
        globalAudioEngine = nullptr;
    }
}

extern "C" JNIEXPORT void JNICALL
Java_com_kido_scorpionhelmets_models_AudioEngine_playReceivedAudioData(JNIEnv *env, jobject, jbyteArray data, jint size) {
    if (playbackStream && playbackStream->getState() == StreamState::Started) {
        jbyte *byteData = env->GetByteArrayElements(data, NULL);
        int32_t samplesToWrite = size / sizeof(int16_t);

        auto writeResult = playbackStream->write(byteData, samplesToWrite, 0);
        if (!writeResult) {
            __android_log_print(ANDROID_LOG_ERROR, "testing", "Error writing to playback stream");
        }

        env->ReleaseByteArrayElements(data, byteData, JNI_ABORT);
    }
}

[philburk]

You have a low-latency stream calling Java through reverse JNI.
That can be very slow and can block the real-time audio callback.
In general you should not do blocking operations in a callback. Avoid network I/O, file I/O, calling Java, memory allocation, etc.

I recommend either passing the data from the recording callback to Java through a FIFO.
Or run a thread in Java that calls down to native and does non-blocking reads from the Oboe input stream.

Similarly, use a FIFO or use non-blocking writes.

Is low-latency a requirement?