feat: add real audio pipeline with Opus + RaptorQ FEC

- AudioPipeline: Kotlin AudioRecord/AudioTrack on JVM threads, PCM
  shuttled to Rust via lock-free ring buffers + JNI
- FEC: RaptorQ fountain codes on encode (5 frames/block, 20% repair
  ratio for GOOD profile), decoder feeds repair symbols for recovery
- Real audio level meter from mic RMS (replaces fake animation)
- Room name editable in UI (default: "android")
- Relay changed to pangolin.manko.yoga:4433
- Stats overlay shows FEC recovered count
- CallState now synced from polled stats (fixes "Connecting" stuck bug)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude
2026-04-05 12:33:59 +00:00
parent 81c756c076
commit bf91cf25bd
15 changed files with 663 additions and 68 deletions

View File

@@ -0,0 +1,174 @@
package com.wzp.audio
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.media.AudioAttributes
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.AudioTrack
import android.media.MediaRecorder
import android.util.Log
import androidx.core.content.ContextCompat
import com.wzp.engine.WzpEngine
/**
* Audio pipeline that captures mic audio and plays received audio using
* Android AudioRecord/AudioTrack APIs running on JVM threads.
*
* PCM samples are shuttled to/from the Rust engine via JNI ring buffers:
* - Capture: AudioRecord → WzpEngine.writeAudio() → Rust encoder → network
* - Playout: network → Rust decoder → WzpEngine.readAudio() → AudioTrack
*
* All audio is 48kHz, mono, 16-bit PCM (matching Opus codec requirements).
*/
class AudioPipeline(private val context: Context) {
companion object {
private const val TAG = "AudioPipeline"
private const val SAMPLE_RATE = 48000
private const val CHANNEL_IN = AudioFormat.CHANNEL_IN_MONO
private const val CHANNEL_OUT = AudioFormat.CHANNEL_OUT_MONO
private const val ENCODING = AudioFormat.ENCODING_PCM_16BIT
/** 20ms frame at 48kHz = 960 samples */
private const val FRAME_SAMPLES = 960
}
@Volatile
private var running = false
private var captureThread: Thread? = null
private var playoutThread: Thread? = null
fun start(engine: WzpEngine) {
if (running) return
running = true
captureThread = Thread({
runCapture(engine)
}, "wzp-capture").apply {
priority = Thread.MAX_PRIORITY
start()
}
playoutThread = Thread({
runPlayout(engine)
}, "wzp-playout").apply {
priority = Thread.MAX_PRIORITY
start()
}
Log.i(TAG, "audio pipeline started")
}
fun stop() {
running = false
captureThread?.join(1000)
playoutThread?.join(1000)
captureThread = null
playoutThread = null
Log.i(TAG, "audio pipeline stopped")
}
private fun runCapture(engine: WzpEngine) {
if (ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO)
!= PackageManager.PERMISSION_GRANTED
) {
Log.e(TAG, "RECORD_AUDIO permission not granted, capture disabled")
return
}
val minBuf = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_IN, ENCODING)
val bufSize = maxOf(minBuf, FRAME_SAMPLES * 2 * 4) // at least 4 frames
val recorder = try {
AudioRecord(
MediaRecorder.AudioSource.VOICE_COMMUNICATION,
SAMPLE_RATE,
CHANNEL_IN,
ENCODING,
bufSize
)
} catch (e: SecurityException) {
Log.e(TAG, "AudioRecord SecurityException: ${e.message}")
return
}
if (recorder.state != AudioRecord.STATE_INITIALIZED) {
Log.e(TAG, "AudioRecord failed to initialize")
recorder.release()
return
}
recorder.startRecording()
Log.i(TAG, "capture started: ${SAMPLE_RATE}Hz mono, buf=$bufSize")
val pcm = ShortArray(FRAME_SAMPLES)
try {
while (running) {
val read = recorder.read(pcm, 0, FRAME_SAMPLES)
if (read > 0) {
engine.writeAudio(pcm)
} else if (read < 0) {
Log.e(TAG, "AudioRecord.read error: $read")
break
}
}
} finally {
recorder.stop()
recorder.release()
Log.i(TAG, "capture stopped")
}
}
private fun runPlayout(engine: WzpEngine) {
val minBuf = AudioTrack.getMinBufferSize(SAMPLE_RATE, CHANNEL_OUT, ENCODING)
val bufSize = maxOf(minBuf, FRAME_SAMPLES * 2 * 4)
val track = AudioTrack.Builder()
.setAudioAttributes(
AudioAttributes.Builder()
.setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION)
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
.build()
)
.setAudioFormat(
AudioFormat.Builder()
.setSampleRate(SAMPLE_RATE)
.setChannelMask(CHANNEL_OUT)
.setEncoding(ENCODING)
.build()
)
.setBufferSizeInBytes(bufSize)
.setTransferMode(AudioTrack.MODE_STREAM)
.build()
if (track.state != AudioTrack.STATE_INITIALIZED) {
Log.e(TAG, "AudioTrack failed to initialize")
track.release()
return
}
track.play()
Log.i(TAG, "playout started: ${SAMPLE_RATE}Hz mono, buf=$bufSize")
val pcm = ShortArray(FRAME_SAMPLES)
val silence = ShortArray(FRAME_SAMPLES) // pre-allocated silence
try {
while (running) {
val read = engine.readAudio(pcm)
if (read >= FRAME_SAMPLES) {
track.write(pcm, 0, read)
} else {
// Not enough decoded audio — write silence to keep stream alive
track.write(silence, 0, FRAME_SAMPLES)
// Sleep briefly to avoid busy-spinning
Thread.sleep(5)
}
}
} finally {
track.stop()
track.release()
Log.i(TAG, "playout stopped")
}
}
}

View File

@@ -27,7 +27,11 @@ data class CallStats(
/** Total frames decoded since call start. */ /** Total frames decoded since call start. */
val framesDecoded: Long = 0, val framesDecoded: Long = 0,
/** Number of playout underruns (buffer empty when audio was needed). */ /** Number of playout underruns (buffer empty when audio was needed). */
val underruns: Long = 0 val underruns: Long = 0,
/** Frames recovered by FEC. */
val fecRecovered: Long = 0,
/** Current mic audio level (RMS, 0-32767). */
val audioLevel: Int = 0
) { ) {
/** Human-readable quality label. */ /** Human-readable quality label. */
val qualityLabel: String val qualityLabel: String
@@ -53,7 +57,9 @@ data class CallStats(
jitterBufferDepth = obj.optInt("jitter_buffer_depth", 0), jitterBufferDepth = obj.optInt("jitter_buffer_depth", 0),
framesEncoded = obj.optLong("frames_encoded", 0), framesEncoded = obj.optLong("frames_encoded", 0),
framesDecoded = obj.optLong("frames_decoded", 0), framesDecoded = obj.optLong("frames_decoded", 0),
underruns = obj.optLong("underruns", 0) underruns = obj.optLong("underruns", 0),
fecRecovered = obj.optLong("fec_recovered", 0),
audioLevel = obj.optInt("audio_level", 0)
) )
} catch (e: Exception) { } catch (e: Exception) {
CallStats() CallStats()

View File

@@ -97,6 +97,24 @@ class WzpEngine(private val callback: WzpCallback) {
} }
} }
/**
* Write captured PCM samples into the engine's capture ring buffer.
* Called from the AudioRecord capture thread.
*/
fun writeAudio(pcm: ShortArray): Int {
if (nativeHandle == 0L) return 0
return nativeWriteAudio(nativeHandle, pcm)
}
/**
* Read decoded PCM samples from the engine's playout ring buffer.
* Called from the AudioTrack playout thread.
*/
fun readAudio(pcm: ShortArray): Int {
if (nativeHandle == 0L) return 0
return nativeReadAudio(nativeHandle, pcm)
}
// -- JNI native methods -------------------------------------------------- // -- JNI native methods --------------------------------------------------
private external fun nativeInit(): Long private external fun nativeInit(): Long
@@ -108,6 +126,8 @@ class WzpEngine(private val callback: WzpCallback) {
private external fun nativeSetSpeaker(handle: Long, speaker: Boolean) private external fun nativeSetSpeaker(handle: Long, speaker: Boolean)
private external fun nativeGetStats(handle: Long): String? private external fun nativeGetStats(handle: Long): String?
private external fun nativeForceProfile(handle: Long, profile: Int) private external fun nativeForceProfile(handle: Long, profile: Int)
private external fun nativeWriteAudio(handle: Long, pcm: ShortArray): Int
private external fun nativeReadAudio(handle: Long, pcm: ShortArray): Int
private external fun nativeDestroy(handle: Long) private external fun nativeDestroy(handle: Long)
companion object { companion object {

View File

@@ -39,6 +39,8 @@ class CallActivity : ComponentActivity() {
override fun onCreate(savedInstanceState: Bundle?) { override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState) super.onCreate(savedInstanceState)
viewModel.setContext(this)
setContent { setContent {
WzpTheme { WzpTheme {
InCallScreen( InCallScreen(

View File

@@ -1,7 +1,9 @@
package com.wzp.ui.call package com.wzp.ui.call
import android.content.Context
import androidx.lifecycle.ViewModel import androidx.lifecycle.ViewModel
import androidx.lifecycle.viewModelScope import androidx.lifecycle.viewModelScope
import com.wzp.audio.AudioPipeline
import com.wzp.engine.CallStats import com.wzp.engine.CallStats
import com.wzp.engine.WzpCallback import com.wzp.engine.WzpCallback
import com.wzp.engine.WzpEngine import com.wzp.engine.WzpEngine
@@ -17,9 +19,11 @@ class CallViewModel : ViewModel(), WzpCallback {
private var engine: WzpEngine? = null private var engine: WzpEngine? = null
private var engineInitialized = false private var engineInitialized = false
private var audioPipeline: AudioPipeline? = null
private var audioStarted = false
private val _callState = MutableStateFlow(0) private val _callState = MutableStateFlow(0)
val callState: StateFlow<Int> = _callState.asStateFlow() val callState: StateFlow<Int> get() = _callState.asStateFlow()
private val _isMuted = MutableStateFlow(false) private val _isMuted = MutableStateFlow(false)
val isMuted: StateFlow<Boolean> = _isMuted.asStateFlow() val isMuted: StateFlow<Boolean> = _isMuted.asStateFlow()
@@ -36,16 +40,26 @@ class CallViewModel : ViewModel(), WzpCallback {
private val _errorMessage = MutableStateFlow<String?>(null) private val _errorMessage = MutableStateFlow<String?>(null)
val errorMessage: StateFlow<String?> = _errorMessage.asStateFlow() val errorMessage: StateFlow<String?> = _errorMessage.asStateFlow()
private val _roomName = MutableStateFlow(DEFAULT_ROOM)
val roomName: StateFlow<String> = _roomName.asStateFlow()
private var statsJob: Job? = null private var statsJob: Job? = null
companion object { companion object {
const val DEFAULT_RELAY = "172.16.81.175:4433" const val DEFAULT_RELAY = "pangolin.manko.yoga:4433"
const val DEFAULT_ROOM = "android" const val DEFAULT_ROOM = "android"
} }
/** Must be called once with Activity context before startCall. */
fun setContext(context: Context) {
if (audioPipeline == null) {
audioPipeline = AudioPipeline(context.applicationContext)
}
}
fun startCall( fun startCall(
relayAddr: String = DEFAULT_RELAY, relayAddr: String = DEFAULT_RELAY,
room: String = DEFAULT_ROOM room: String = _roomName.value
) { ) {
try { try {
if (engine == null) { if (engine == null) {
@@ -58,9 +72,6 @@ class CallViewModel : ViewModel(), WzpCallback {
_callState.value = 1 // Connecting _callState.value = 1 // Connecting
startStatsPolling() startStatsPolling()
// startCall blocks (runs tokio on calling thread), so dispatch
// to a background coroutine. Using Dispatchers.IO which uses
// Java threads (not native pthread_create).
viewModelScope.launch(kotlinx.coroutines.Dispatchers.IO) { viewModelScope.launch(kotlinx.coroutines.Dispatchers.IO) {
try { try {
val result = engine?.startCall(relayAddr, room) ?: -1 val result = engine?.startCall(relayAddr, room) ?: -1
@@ -80,6 +91,7 @@ class CallViewModel : ViewModel(), WzpCallback {
} }
fun stopCall() { fun stopCall() {
stopAudio()
stopStatsPolling() stopStatsPolling()
try { try {
engine?.stopCall() engine?.stopCall()
@@ -101,11 +113,26 @@ class CallViewModel : ViewModel(), WzpCallback {
fun clearError() { _errorMessage.value = null } fun clearError() { _errorMessage.value = null }
fun setRoomName(name: String) { _roomName.value = name }
// WzpCallback // WzpCallback
override fun onCallStateChanged(state: Int) { _callState.value = state } override fun onCallStateChanged(state: Int) { _callState.value = state }
override fun onQualityTierChanged(tier: Int) { _qualityTier.value = tier } override fun onQualityTierChanged(tier: Int) { _qualityTier.value = tier }
override fun onError(code: Int, message: String) { _errorMessage.value = "Error $code: $message" } override fun onError(code: Int, message: String) { _errorMessage.value = "Error $code: $message" }
private fun startAudio() {
if (audioStarted) return
val e = engine ?: return
audioPipeline?.start(e)
audioStarted = true
}
private fun stopAudio() {
if (!audioStarted) return
audioPipeline?.stop()
audioStarted = false
}
private fun startStatsPolling() { private fun startStatsPolling() {
statsJob?.cancel() statsJob?.cancel()
statsJob = viewModelScope.launch { statsJob = viewModelScope.launch {
@@ -113,7 +140,16 @@ class CallViewModel : ViewModel(), WzpCallback {
try { try {
val json = engine?.getStats() ?: "{}" val json = engine?.getStats() ?: "{}"
if (json.isNotEmpty()) { if (json.isNotEmpty()) {
_stats.value = CallStats.fromJson(json) val s = CallStats.fromJson(json)
_stats.value = s
// Sync call state from native engine stats
if (s.state != 0) {
_callState.value = s.state
}
// Start audio pipeline when call becomes active
if (s.state == 2 && !audioStarted) {
startAudio()
}
} }
} catch (_: Exception) {} } catch (_: Exception) {}
delay(500L) delay(500L)
@@ -128,6 +164,7 @@ class CallViewModel : ViewModel(), WzpCallback {
override fun onCleared() { override fun onCleared() {
super.onCleared() super.onCleared()
stopAudio()
stopStatsPolling() stopStatsPolling()
try { try {
engine?.stopCall() engine?.stopCall()

View File

@@ -21,6 +21,7 @@ import androidx.compose.material3.FilledTonalIconButton
import androidx.compose.material3.IconButtonDefaults import androidx.compose.material3.IconButtonDefaults
import androidx.compose.material3.LinearProgressIndicator import androidx.compose.material3.LinearProgressIndicator
import androidx.compose.material3.MaterialTheme import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.Surface import androidx.compose.material3.Surface
import androidx.compose.material3.Text import androidx.compose.material3.Text
import androidx.compose.runtime.Composable import androidx.compose.runtime.Composable
@@ -48,6 +49,7 @@ fun InCallScreen(
val stats by viewModel.stats.collectAsState() val stats by viewModel.stats.collectAsState()
val qualityTier by viewModel.qualityTier.collectAsState() val qualityTier by viewModel.qualityTier.collectAsState()
val errorMessage by viewModel.errorMessage.collectAsState() val errorMessage by viewModel.errorMessage.collectAsState()
val roomName by viewModel.roomName.collectAsState()
Surface( Surface(
modifier = Modifier.fillMaxSize(), modifier = Modifier.fillMaxSize(),
@@ -83,11 +85,13 @@ fun InCallScreen(
style = MaterialTheme.typography.bodyMedium, style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.onSurfaceVariant color = MaterialTheme.colorScheme.onSurfaceVariant
) )
Spacer(modifier = Modifier.height(4.dp)) Spacer(modifier = Modifier.height(8.dp))
Text( OutlinedTextField(
text = "Room: ${CallViewModel.DEFAULT_ROOM}", value = roomName,
style = MaterialTheme.typography.bodyMedium, onValueChange = { viewModel.setRoomName(it) },
color = MaterialTheme.colorScheme.onSurfaceVariant label = { Text("Room") },
singleLine = true,
modifier = Modifier.fillMaxWidth(0.6f)
) )
Spacer(modifier = Modifier.height(32.dp)) Spacer(modifier = Modifier.height(32.dp))
@@ -132,7 +136,7 @@ fun InCallScreen(
Spacer(modifier = Modifier.height(32.dp)) Spacer(modifier = Modifier.height(32.dp))
AudioLevelBar(stats.framesEncoded) AudioLevelBar(stats.audioLevel)
Spacer(modifier = Modifier.weight(1f)) Spacer(modifier = Modifier.weight(1f))
@@ -222,9 +226,11 @@ private fun QualityIndicator(tier: Int, label: String) {
} }
@Composable @Composable
private fun AudioLevelBar(framesEncoded: Long) { private fun AudioLevelBar(audioLevel: Int) {
val level = if (framesEncoded > 0) { // audioLevel is RMS of i16 samples (0-32767).
((framesEncoded % 100).toFloat() / 100f).coerceIn(0.05f, 1f) // Map to 0.0-1.0 with a log-ish curve for better visual feel.
val level = if (audioLevel > 0) {
(audioLevel.toFloat() / 8000f).coerceIn(0.02f, 1f)
} else { } else {
0f 0f
} }
@@ -351,7 +357,7 @@ private fun StatsOverlay(stats: CallStats) {
) { ) {
StatItem("Enc", "${stats.framesEncoded}") StatItem("Enc", "${stats.framesEncoded}")
StatItem("Dec", "${stats.framesDecoded}") StatItem("Dec", "${stats.framesDecoded}")
StatItem("JB", "${stats.jitterBufferDepth}") StatItem("FEC", "${stats.fecRecovered}")
StatItem("Under", "${stats.underruns}") StatItem("Under", "${stats.underruns}")
} }
} }

View File

@@ -0,0 +1,91 @@
//! Lock-free SPSC ring buffers for audio PCM transfer between
//! Kotlin AudioRecord/AudioTrack threads and the Rust engine.
//!
//! These use a simple spin-free design: the producer writes and advances
//! a write cursor, the consumer reads and advances a read cursor.
//! Both cursors are atomic so no mutex is needed.
use std::sync::atomic::{AtomicUsize, Ordering};
/// Ring buffer capacity in i16 samples.
/// 960 samples * 10 frames = ~200ms of audio at 48kHz mono.
const RING_CAPACITY: usize = 960 * 10;
/// Lock-free single-producer single-consumer ring buffer for i16 PCM samples.
pub struct AudioRing {
buf: Box<[i16; RING_CAPACITY]>,
write_pos: AtomicUsize,
read_pos: AtomicUsize,
}
// SAFETY: AudioRing is designed for SPSC — one thread writes, one reads.
// The atomics ensure visibility. The buffer itself is never accessed
// from the same index by both threads simultaneously because the
// producer only writes to positions between write_pos and read_pos,
// and the consumer only reads from positions between read_pos and write_pos.
unsafe impl Send for AudioRing {}
unsafe impl Sync for AudioRing {}
impl AudioRing {
pub fn new() -> Self {
Self {
buf: Box::new([0i16; RING_CAPACITY]),
write_pos: AtomicUsize::new(0),
read_pos: AtomicUsize::new(0),
}
}
/// Number of samples available to read.
pub fn available(&self) -> usize {
let w = self.write_pos.load(Ordering::Acquire);
let r = self.read_pos.load(Ordering::Acquire);
w.wrapping_sub(r)
}
/// Number of samples that can be written without overwriting.
pub fn free_space(&self) -> usize {
RING_CAPACITY - self.available()
}
/// Write samples into the ring. Returns number of samples written.
/// Drops oldest samples if the ring is full.
pub fn write(&self, samples: &[i16]) -> usize {
let w = self.write_pos.load(Ordering::Relaxed);
let count = samples.len().min(RING_CAPACITY);
for i in 0..count {
let idx = (w + i) % RING_CAPACITY;
// SAFETY: We're the only writer, and the reader won't read
// past read_pos which we haven't advanced past yet.
unsafe {
let ptr = self.buf.as_ptr() as *mut i16;
*ptr.add(idx) = samples[i];
}
}
self.write_pos.store(w.wrapping_add(count), Ordering::Release);
// If we overwrote unread data, advance read_pos
if self.available() > RING_CAPACITY {
let new_read = self.write_pos.load(Ordering::Relaxed).wrapping_sub(RING_CAPACITY);
self.read_pos.store(new_read, Ordering::Release);
}
count
}
/// Read samples from the ring into `out`. Returns number of samples read.
pub fn read(&self, out: &mut [i16]) -> usize {
let avail = self.available();
let count = out.len().min(avail);
let r = self.read_pos.load(Ordering::Relaxed);
for i in 0..count {
let idx = (r + i) % RING_CAPACITY;
out[i] = unsafe { *self.buf.as_ptr().add(idx) };
}
self.read_pos.store(r.wrapping_add(count), Ordering::Release);
count
}
}

View File

@@ -4,6 +4,9 @@
//! static bionic stubs in the Rust std prebuilt rlibs. ALL work must happen //! static bionic stubs in the Rust std prebuilt rlibs. ALL work must happen
//! on the JNI calling thread or via the tokio current_thread runtime. //! on the JNI calling thread or via the tokio current_thread runtime.
//! No std::thread::spawn or tokio multi_thread allowed. //! No std::thread::spawn or tokio multi_thread allowed.
//!
//! Audio capture and playout happen on Kotlin JVM threads via AudioRecord
//! and AudioTrack. PCM samples are transferred through lock-free ring buffers.
use std::net::SocketAddr; use std::net::SocketAddr;
use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, Ordering}; use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, Ordering};
@@ -11,15 +14,23 @@ use std::sync::{Arc, Mutex};
use std::time::Instant; use std::time::Instant;
use bytes::Bytes; use bytes::Bytes;
use tracing::{error, info}; use tracing::{error, info, warn};
use wzp_codec::opus_dec::OpusDecoder;
use wzp_codec::opus_enc::OpusEncoder;
use wzp_crypto::{KeyExchange, WarzoneKeyExchange}; use wzp_crypto::{KeyExchange, WarzoneKeyExchange};
use wzp_fec::{RaptorQFecDecoder, RaptorQFecEncoder};
use wzp_proto::{ use wzp_proto::{
CodecId, MediaHeader, MediaPacket, MediaTransport, QualityProfile, SignalMessage, AudioDecoder, AudioEncoder, CodecId, FecDecoder, FecEncoder,
MediaHeader, MediaPacket, MediaTransport, QualityProfile, SignalMessage,
}; };
use crate::audio_ring::AudioRing;
use crate::commands::EngineCommand; use crate::commands::EngineCommand;
use crate::stats::{CallState, CallStats}; use crate::stats::{CallState, CallStats};
/// Opus frame size at 48kHz mono, 20ms = 960 samples.
const FRAME_SAMPLES: usize = 960;
/// Configuration to start a call. /// Configuration to start a call.
pub struct CallStartConfig { pub struct CallStartConfig {
pub profile: QualityProfile, pub profile: QualityProfile,
@@ -41,16 +52,22 @@ impl Default for CallStartConfig {
} }
} }
struct EngineState { pub(crate) struct EngineState {
running: AtomicBool, pub running: AtomicBool,
muted: AtomicBool, pub muted: AtomicBool,
stats: Mutex<CallStats>, pub stats: Mutex<CallStats>,
command_tx: std::sync::mpsc::Sender<EngineCommand>, pub command_tx: std::sync::mpsc::Sender<EngineCommand>,
command_rx: Mutex<Option<std::sync::mpsc::Receiver<EngineCommand>>>, pub command_rx: Mutex<Option<std::sync::mpsc::Receiver<EngineCommand>>>,
/// Ring buffer: Kotlin AudioRecord → Rust encoder
pub capture_ring: AudioRing,
/// Ring buffer: Rust decoder → Kotlin AudioTrack
pub playout_ring: AudioRing,
/// Current audio level (RMS) for UI display, updated by capture path.
pub audio_level_rms: AtomicU32,
} }
pub struct WzpEngine { pub struct WzpEngine {
state: Arc<EngineState>, pub(crate) state: Arc<EngineState>,
tokio_runtime: Option<tokio::runtime::Runtime>, tokio_runtime: Option<tokio::runtime::Runtime>,
call_start: Option<Instant>, call_start: Option<Instant>,
} }
@@ -64,6 +81,9 @@ impl WzpEngine {
stats: Mutex::new(CallStats::default()), stats: Mutex::new(CallStats::default()),
command_tx: tx, command_tx: tx,
command_rx: Mutex::new(Some(rx)), command_rx: Mutex::new(Some(rx)),
capture_ring: AudioRing::new(),
playout_ring: AudioRing::new(),
audio_level_rms: AtomicU32::new(0),
}); });
Self { Self {
state, state,
@@ -85,8 +105,6 @@ impl WzpEngine {
}; };
} }
// Create single-threaded tokio runtime — NO thread spawning.
// On Android, pthread_create crashes due to static bionic stubs.
let runtime = tokio::runtime::Builder::new_current_thread() let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all() .enable_all()
.build()?; .build()?;
@@ -97,17 +115,16 @@ impl WzpEngine {
let room = config.room.clone(); let room = config.room.clone();
let identity_seed = config.identity_seed; let identity_seed = config.identity_seed;
let profile = config.profile;
let state = self.state.clone(); let state = self.state.clone();
self.state.running.store(true, Ordering::Release); self.state.running.store(true, Ordering::Release);
self.call_start = Some(Instant::now()); self.call_start = Some(Instant::now());
// Run the entire call on the current thread's tokio runtime.
// This blocks the JNI thread until the call ends, so Kotlin
// must call startCall from a background coroutine.
let state_clone = state.clone(); let state_clone = state.clone();
runtime.block_on(async move { runtime.block_on(async move {
if let Err(e) = run_call(relay_addr, &room, &identity_seed, state_clone).await { if let Err(e) = run_call(relay_addr, &room, &identity_seed, profile, state_clone).await
{
error!("call failed: {e}"); error!("call failed: {e}");
} }
}); });
@@ -135,19 +152,17 @@ impl WzpEngine {
self.state.muted.store(muted, Ordering::Relaxed); self.state.muted.store(muted, Ordering::Relaxed);
} }
pub fn set_speaker(&self, _enabled: bool) { pub fn set_speaker(&self, _enabled: bool) {}
// TODO: route audio via AudioManager on Kotlin side
}
pub fn force_profile(&self, _profile: QualityProfile) { pub fn force_profile(&self, _profile: QualityProfile) {}
// TODO: wire to pipeline when codec thread is re-enabled
}
pub fn get_stats(&self) -> CallStats { pub fn get_stats(&self) -> CallStats {
let mut stats = self.state.stats.lock().unwrap().clone(); let mut stats = self.state.stats.lock().unwrap().clone();
if let Some(start) = self.call_start { if let Some(start) = self.call_start {
stats.duration_secs = start.elapsed().as_secs_f64(); stats.duration_secs = start.elapsed().as_secs_f64();
} }
// Include current audio level
stats.audio_level = self.state.audio_level_rms.load(Ordering::Relaxed);
stats stats
} }
@@ -155,6 +170,23 @@ impl WzpEngine {
self.state.running.load(Ordering::Acquire) self.state.running.load(Ordering::Acquire)
} }
pub fn write_audio(&self, samples: &[i16]) -> usize {
if self.state.muted.load(Ordering::Relaxed) {
return samples.len();
}
// Compute RMS for audio level display
if !samples.is_empty() {
let sum_sq: f64 = samples.iter().map(|&s| (s as f64) * (s as f64)).sum();
let rms = (sum_sq / samples.len() as f64).sqrt() as u32;
self.state.audio_level_rms.store(rms, Ordering::Relaxed);
}
self.state.capture_ring.write(samples)
}
pub fn read_audio(&self, out: &mut [i16]) -> usize {
self.state.playout_ring.read(out)
}
pub fn destroy(mut self) { pub fn destroy(mut self) {
self.stop_call(); self.stop_call();
} }
@@ -166,22 +198,19 @@ impl Drop for WzpEngine {
} }
} }
/// Run the full call lifecycle: connect, handshake, send/recv media. /// Run the full call lifecycle: connect, handshake, send/recv media with Opus + FEC.
/// All async, no thread spawning.
async fn run_call( async fn run_call(
relay_addr: SocketAddr, relay_addr: SocketAddr,
room: &str, room: &str,
identity_seed: &[u8; 32], identity_seed: &[u8; 32],
profile: QualityProfile,
state: Arc<EngineState>, state: Arc<EngineState>,
) -> Result<(), anyhow::Error> { ) -> Result<(), anyhow::Error> {
// Install rustls crypto provider
let _ = rustls::crypto::ring::default_provider().install_default(); let _ = rustls::crypto::ring::default_provider().install_default();
// Create QUIC endpoint
let bind_addr: SocketAddr = "0.0.0.0:0".parse().unwrap(); let bind_addr: SocketAddr = "0.0.0.0:0".parse().unwrap();
let endpoint = wzp_transport::create_endpoint(bind_addr, None)?; let endpoint = wzp_transport::create_endpoint(bind_addr, None)?;
// Connect to relay with room as SNI
let sni = if room.is_empty() { "android" } else { room }; let sni = if room.is_empty() { "android" } else { room };
info!(%relay_addr, sni, "connecting to relay..."); info!(%relay_addr, sni, "connecting to relay...");
let client_cfg = wzp_transport::client_config(); let client_cfg = wzp_transport::client_config();
@@ -236,58 +265,223 @@ async fn run_call(
stats.state = CallState::Active; stats.state = CallState::Active;
} }
// Simple media loop: send silence, recv and count frames. // Initialize Opus codec
// No codec thread, no Oboe — just network I/O to verify connectivity. let mut encoder =
// Audio pipeline will be added once native threading is resolved. OpusEncoder::new(profile).map_err(|e| anyhow::anyhow!("opus encoder init: {e}"))?;
let mut decoder =
OpusDecoder::new(profile).map_err(|e| anyhow::anyhow!("opus decoder init: {e}"))?;
// Initialize FEC encoder/decoder
let mut fec_enc = wzp_fec::create_encoder(&profile);
let mut fec_dec = wzp_fec::create_decoder(&profile);
info!(
fec_ratio = profile.fec_ratio,
frames_per_block = profile.frames_per_block,
"codec + FEC initialized (48kHz mono, 20ms frames, RaptorQ)"
);
let seq = AtomicU16::new(0); let seq = AtomicU16::new(0);
let ts = AtomicU32::new(0); let ts = AtomicU32::new(0);
let transport_recv = transport.clone(); let transport_recv = transport.clone();
// Pre-allocate buffers
let mut capture_buf = vec![0i16; FRAME_SAMPLES];
let mut encode_buf = vec![0u8; encoder.max_frame_bytes()];
let mut frame_in_block: u8 = 0;
let mut block_id: u8 = 0;
// Send task: capture ring → Opus encode → FEC → MediaPackets
let send_task = async { let send_task = async {
let silence = vec![0u8; 20]; // minimal opus silence frame info!("send task started (Opus + RaptorQ FEC)");
loop { loop {
if !state.running.load(Ordering::Relaxed) { if !state.running.load(Ordering::Relaxed) {
break; break;
} }
let avail = state.capture_ring.available();
if avail < FRAME_SAMPLES {
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
continue;
}
let read = state.capture_ring.read(&mut capture_buf);
if read < FRAME_SAMPLES {
continue;
}
// Opus encode
let encoded_len = match encoder.encode(&capture_buf, &mut encode_buf) {
Ok(n) => n,
Err(e) => {
warn!("opus encode error: {e}");
continue;
}
};
let encoded = &encode_buf[..encoded_len];
// Build source packet
let s = seq.fetch_add(1, Ordering::Relaxed); let s = seq.fetch_add(1, Ordering::Relaxed);
let t = ts.fetch_add(20, Ordering::Relaxed); let t = ts.fetch_add(FRAME_SAMPLES as u32, Ordering::Relaxed);
let packet = MediaPacket {
let source_pkt = MediaPacket {
header: MediaHeader { header: MediaHeader {
version: 0, version: 0,
is_repair: false, is_repair: false,
codec_id: CodecId::Opus24k, codec_id: profile.codec,
has_quality_report: false, has_quality_report: false,
fec_ratio_encoded: 0, fec_ratio_encoded: MediaHeader::encode_fec_ratio(profile.fec_ratio),
seq: s, seq: s,
timestamp: t, timestamp: t,
fec_block: 0, fec_block: block_id,
fec_symbol: 0, fec_symbol: frame_in_block,
reserved: 0, reserved: 0,
csrc_count: 0, csrc_count: 0,
}, },
payload: Bytes::from(silence.clone()), payload: Bytes::copy_from_slice(encoded),
quality_report: None, quality_report: None,
}; };
if let Err(e) = transport.send_media(&packet).await {
// Send source packet
if let Err(e) = transport.send_media(&source_pkt).await {
error!("send error: {e}"); error!("send error: {e}");
break; break;
} }
// 20ms frame interval
tokio::time::sleep(std::time::Duration::from_millis(20)).await; // Feed encoded frame to FEC encoder
if let Err(e) = fec_enc.add_source_symbol(encoded) {
warn!("fec add_source error: {e}");
}
frame_in_block += 1;
// When block is full, generate repair packets
if frame_in_block >= profile.frames_per_block {
match fec_enc.generate_repair(profile.fec_ratio) {
Ok(repairs) => {
let repair_count = repairs.len();
for (sym_idx, repair_data) in repairs {
let rs = seq.fetch_add(1, Ordering::Relaxed);
let repair_pkt = MediaPacket {
header: MediaHeader {
version: 0,
is_repair: true,
codec_id: profile.codec,
has_quality_report: false,
fec_ratio_encoded: MediaHeader::encode_fec_ratio(
profile.fec_ratio,
),
seq: rs,
timestamp: t,
fec_block: block_id,
fec_symbol: sym_idx,
reserved: 0,
csrc_count: 0,
},
payload: Bytes::from(repair_data),
quality_report: None,
};
if let Err(e) = transport.send_media(&repair_pkt).await {
error!("send repair error: {e}");
break;
}
}
if repair_count > 0 && (block_id % 50 == 0 || block_id == 0) {
info!(
block_id,
repair_count,
fec_ratio = profile.fec_ratio,
"FEC block complete"
);
}
}
Err(e) => {
warn!("fec generate_repair error: {e}");
}
}
let _ = fec_enc.finalize_block();
block_id = block_id.wrapping_add(1);
frame_in_block = 0;
}
if s % 500 == 0 {
info!(seq = s, block_id, frame_in_block, "sending");
}
} }
}; };
// Pre-allocate decode buffer
let mut decode_buf = vec![0i16; FRAME_SAMPLES];
// Recv task: MediaPackets → FEC decode → Opus decode → playout ring
let recv_task = async { let recv_task = async {
let mut frames_decoded: u64 = 0; let mut frames_decoded: u64 = 0;
let mut fec_recovered: u64 = 0;
info!("recv task started (Opus + RaptorQ FEC)");
loop { loop {
if !state.running.load(Ordering::Relaxed) { if !state.running.load(Ordering::Relaxed) {
break; break;
} }
match transport_recv.recv_media().await { match transport_recv.recv_media().await {
Ok(Some(_pkt)) => { Ok(Some(pkt)) => {
frames_decoded += 1; let is_repair = pkt.header.is_repair;
let pkt_block = pkt.header.fec_block;
let pkt_symbol = pkt.header.fec_symbol;
// Feed every packet (source + repair) to FEC decoder
let _ = fec_dec.add_symbol(
pkt_block,
pkt_symbol,
is_repair,
&pkt.payload,
);
// Source packets: decode directly
if !is_repair {
match decoder.decode(&pkt.payload, &mut decode_buf) {
Ok(samples) => {
state.playout_ring.write(&decode_buf[..samples]);
frames_decoded += 1;
}
Err(e) => {
warn!("opus decode error: {e}");
if let Ok(samples) = decoder.decode_lost(&mut decode_buf) {
state.playout_ring.write(&decode_buf[..samples]);
}
}
}
}
// Try FEC recovery for this block
// (useful when source packets were lost but repair arrived)
if let Ok(Some(recovered_frames)) = fec_dec.try_decode(pkt_block) {
// FEC recovered the block — any previously missing frames
// are now available. In a full jitter buffer implementation,
// we'd insert recovered frames at the right position.
// For now, log recovery for telemetry.
fec_recovered += recovered_frames.len() as u64;
if fec_recovered % 50 == 1 {
info!(
fec_recovered,
block = pkt_block,
frames = recovered_frames.len(),
"FEC block recovered"
);
}
}
// Expire old blocks to prevent memory growth
if pkt_block > 3 {
fec_dec.expire_before(pkt_block.wrapping_sub(3));
}
if frames_decoded == 1 || frames_decoded % 500 == 0 {
info!(frames_decoded, fec_recovered, "recv stats");
}
let mut stats = state.stats.lock().unwrap(); let mut stats = state.stats.lock().unwrap();
stats.frames_decoded = frames_decoded; stats.frames_decoded = frames_decoded;
stats.fec_recovered = fec_recovered;
} }
Ok(None) => { Ok(None) => {
info!("relay disconnected"); info!("relay disconnected");
@@ -301,7 +495,7 @@ async fn run_call(
} }
}; };
// Update encoded frame count in send task // Stats task
let stats_task = async { let stats_task = async {
loop { loop {
if !state.running.load(Ordering::Relaxed) { if !state.running.load(Ordering::Relaxed) {

View File

@@ -174,6 +174,56 @@ pub unsafe extern "system" fn Java_com_wzp_engine_WzpEngine_nativeForceProfile(
})); }));
} }
/// Write captured PCM samples from Kotlin AudioRecord into the engine's capture ring.
/// pcm is a Java short[] array.
#[unsafe(no_mangle)]
pub unsafe extern "system" fn Java_com_wzp_engine_WzpEngine_nativeWriteAudio(
env: JNIEnv,
_class: JClass,
handle: jlong,
pcm: jni::objects::JShortArray,
) -> jint {
let result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
let h = unsafe { handle_ref(handle) };
let len = env.get_array_length(&pcm).unwrap_or(0) as usize;
if len == 0 {
return 0;
}
let mut buf = vec![0i16; len];
// GetShortArrayRegion copies Java array into our buffer
if env.get_short_array_region(&pcm, 0, &mut buf).is_err() {
return 0;
}
h.engine.write_audio(&buf) as jint
}));
result.unwrap_or(0)
}
/// Read decoded PCM samples from the engine's playout ring for Kotlin AudioTrack.
/// pcm is a Java short[] array to fill. Returns number of samples actually read.
#[unsafe(no_mangle)]
pub unsafe extern "system" fn Java_com_wzp_engine_WzpEngine_nativeReadAudio(
env: JNIEnv,
_class: JClass,
handle: jlong,
pcm: jni::objects::JShortArray,
) -> jint {
let result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
let h = unsafe { handle_ref(handle) };
let len = env.get_array_length(&pcm).unwrap_or(0) as usize;
if len == 0 {
return 0;
}
let mut buf = vec![0i16; len];
let read = h.engine.read_audio(&mut buf);
if read > 0 {
let _ = env.set_short_array_region(&pcm, 0, &buf[..read]);
}
read as jint
}));
result.unwrap_or(0)
}
#[unsafe(no_mangle)] #[unsafe(no_mangle)]
pub unsafe extern "system" fn Java_com_wzp_engine_WzpEngine_nativeDestroy( pub unsafe extern "system" fn Java_com_wzp_engine_WzpEngine_nativeDestroy(
_env: JNIEnv, _env: JNIEnv,

View File

@@ -10,6 +10,7 @@
//! allowing `cargo check` and unit tests on the host. //! allowing `cargo check` and unit tests on the host.
pub mod audio_android; pub mod audio_android;
pub mod audio_ring;
pub mod commands; pub mod commands;
pub mod engine; pub mod engine;
pub mod pipeline; pub mod pipeline;

View File

@@ -1,21 +1,31 @@
//! Call statistics for the Android engine. //! Call statistics for the Android engine.
/// State of the call. /// State of the call.
#[derive(Clone, Debug, Default, serde::Serialize, PartialEq, Eq)] /// Serializes as integer for easy parsing on the Kotlin side:
/// 0=Idle, 1=Connecting, 2=Active, 3=Reconnecting, 4=Closed
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub enum CallState { pub enum CallState {
/// Engine is idle, no active call.
#[default] #[default]
Idle, Idle,
/// Establishing connection to the relay.
Connecting, Connecting,
/// Call is active with audio flowing.
Active, Active,
/// Temporarily lost connection, attempting to recover.
Reconnecting, Reconnecting,
/// Call has ended.
Closed, Closed,
} }
impl serde::Serialize for CallState {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let n: u8 = match self {
CallState::Idle => 0,
CallState::Connecting => 1,
CallState::Active => 2,
CallState::Reconnecting => 3,
CallState::Closed => 4,
};
serializer.serialize_u8(n)
}
}
/// Aggregated call statistics, serializable for JNI bridge. /// Aggregated call statistics, serializable for JNI bridge.
#[derive(Clone, Debug, Default, serde::Serialize)] #[derive(Clone, Debug, Default, serde::Serialize)]
pub struct CallStats { pub struct CallStats {
@@ -39,4 +49,8 @@ pub struct CallStats {
pub frames_decoded: u64, pub frames_decoded: u64,
/// Number of playout underruns (buffer empty when audio needed). /// Number of playout underruns (buffer empty when audio needed).
pub underruns: u64, pub underruns: u64,
/// Frames recovered by FEC.
pub fec_recovered: u64,
/// Current mic audio level (RMS of i16 samples, 0-32767).
pub audio_level: u32,
} }

Binary file not shown.

After

Width:  |  Height:  |  Size: 31 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 696 B

Binary file not shown.