feat: add real audio pipeline with Opus + RaptorQ FEC
- AudioPipeline: Kotlin AudioRecord/AudioTrack on JVM threads, PCM shuttled to Rust via lock-free ring buffers + JNI - FEC: RaptorQ fountain codes on encode (5 frames/block, 20% repair ratio for GOOD profile), decoder feeds repair symbols for recovery - Real audio level meter from mic RMS (replaces fake animation) - Room name editable in UI (default: "android") - Relay changed to pangolin.manko.yoga:4433 - Stats overlay shows FEC recovered count - CallState now synced from polled stats (fixes "Connecting" stuck bug) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Binary file not shown.
174
android/app/src/main/java/com/wzp/audio/AudioPipeline.kt
Normal file
174
android/app/src/main/java/com/wzp/audio/AudioPipeline.kt
Normal file
@@ -0,0 +1,174 @@
|
|||||||
|
package com.wzp.audio
|
||||||
|
|
||||||
|
import android.Manifest
|
||||||
|
import android.content.Context
|
||||||
|
import android.content.pm.PackageManager
|
||||||
|
import android.media.AudioAttributes
|
||||||
|
import android.media.AudioFormat
|
||||||
|
import android.media.AudioRecord
|
||||||
|
import android.media.AudioTrack
|
||||||
|
import android.media.MediaRecorder
|
||||||
|
import android.util.Log
|
||||||
|
import androidx.core.content.ContextCompat
|
||||||
|
import com.wzp.engine.WzpEngine
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Audio pipeline that captures mic audio and plays received audio using
|
||||||
|
* Android AudioRecord/AudioTrack APIs running on JVM threads.
|
||||||
|
*
|
||||||
|
* PCM samples are shuttled to/from the Rust engine via JNI ring buffers:
|
||||||
|
* - Capture: AudioRecord → WzpEngine.writeAudio() → Rust encoder → network
|
||||||
|
* - Playout: network → Rust decoder → WzpEngine.readAudio() → AudioTrack
|
||||||
|
*
|
||||||
|
* All audio is 48kHz, mono, 16-bit PCM (matching Opus codec requirements).
|
||||||
|
*/
|
||||||
|
class AudioPipeline(private val context: Context) {
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private const val TAG = "AudioPipeline"
|
||||||
|
private const val SAMPLE_RATE = 48000
|
||||||
|
private const val CHANNEL_IN = AudioFormat.CHANNEL_IN_MONO
|
||||||
|
private const val CHANNEL_OUT = AudioFormat.CHANNEL_OUT_MONO
|
||||||
|
private const val ENCODING = AudioFormat.ENCODING_PCM_16BIT
|
||||||
|
/** 20ms frame at 48kHz = 960 samples */
|
||||||
|
private const val FRAME_SAMPLES = 960
|
||||||
|
}
|
||||||
|
|
||||||
|
@Volatile
|
||||||
|
private var running = false
|
||||||
|
private var captureThread: Thread? = null
|
||||||
|
private var playoutThread: Thread? = null
|
||||||
|
|
||||||
|
fun start(engine: WzpEngine) {
|
||||||
|
if (running) return
|
||||||
|
running = true
|
||||||
|
|
||||||
|
captureThread = Thread({
|
||||||
|
runCapture(engine)
|
||||||
|
}, "wzp-capture").apply {
|
||||||
|
priority = Thread.MAX_PRIORITY
|
||||||
|
start()
|
||||||
|
}
|
||||||
|
|
||||||
|
playoutThread = Thread({
|
||||||
|
runPlayout(engine)
|
||||||
|
}, "wzp-playout").apply {
|
||||||
|
priority = Thread.MAX_PRIORITY
|
||||||
|
start()
|
||||||
|
}
|
||||||
|
|
||||||
|
Log.i(TAG, "audio pipeline started")
|
||||||
|
}
|
||||||
|
|
||||||
|
fun stop() {
|
||||||
|
running = false
|
||||||
|
captureThread?.join(1000)
|
||||||
|
playoutThread?.join(1000)
|
||||||
|
captureThread = null
|
||||||
|
playoutThread = null
|
||||||
|
Log.i(TAG, "audio pipeline stopped")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun runCapture(engine: WzpEngine) {
|
||||||
|
if (ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO)
|
||||||
|
!= PackageManager.PERMISSION_GRANTED
|
||||||
|
) {
|
||||||
|
Log.e(TAG, "RECORD_AUDIO permission not granted, capture disabled")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
val minBuf = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_IN, ENCODING)
|
||||||
|
val bufSize = maxOf(minBuf, FRAME_SAMPLES * 2 * 4) // at least 4 frames
|
||||||
|
|
||||||
|
val recorder = try {
|
||||||
|
AudioRecord(
|
||||||
|
MediaRecorder.AudioSource.VOICE_COMMUNICATION,
|
||||||
|
SAMPLE_RATE,
|
||||||
|
CHANNEL_IN,
|
||||||
|
ENCODING,
|
||||||
|
bufSize
|
||||||
|
)
|
||||||
|
} catch (e: SecurityException) {
|
||||||
|
Log.e(TAG, "AudioRecord SecurityException: ${e.message}")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (recorder.state != AudioRecord.STATE_INITIALIZED) {
|
||||||
|
Log.e(TAG, "AudioRecord failed to initialize")
|
||||||
|
recorder.release()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
recorder.startRecording()
|
||||||
|
Log.i(TAG, "capture started: ${SAMPLE_RATE}Hz mono, buf=$bufSize")
|
||||||
|
|
||||||
|
val pcm = ShortArray(FRAME_SAMPLES)
|
||||||
|
try {
|
||||||
|
while (running) {
|
||||||
|
val read = recorder.read(pcm, 0, FRAME_SAMPLES)
|
||||||
|
if (read > 0) {
|
||||||
|
engine.writeAudio(pcm)
|
||||||
|
} else if (read < 0) {
|
||||||
|
Log.e(TAG, "AudioRecord.read error: $read")
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
recorder.stop()
|
||||||
|
recorder.release()
|
||||||
|
Log.i(TAG, "capture stopped")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun runPlayout(engine: WzpEngine) {
|
||||||
|
val minBuf = AudioTrack.getMinBufferSize(SAMPLE_RATE, CHANNEL_OUT, ENCODING)
|
||||||
|
val bufSize = maxOf(minBuf, FRAME_SAMPLES * 2 * 4)
|
||||||
|
|
||||||
|
val track = AudioTrack.Builder()
|
||||||
|
.setAudioAttributes(
|
||||||
|
AudioAttributes.Builder()
|
||||||
|
.setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION)
|
||||||
|
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
||||||
|
.build()
|
||||||
|
)
|
||||||
|
.setAudioFormat(
|
||||||
|
AudioFormat.Builder()
|
||||||
|
.setSampleRate(SAMPLE_RATE)
|
||||||
|
.setChannelMask(CHANNEL_OUT)
|
||||||
|
.setEncoding(ENCODING)
|
||||||
|
.build()
|
||||||
|
)
|
||||||
|
.setBufferSizeInBytes(bufSize)
|
||||||
|
.setTransferMode(AudioTrack.MODE_STREAM)
|
||||||
|
.build()
|
||||||
|
|
||||||
|
if (track.state != AudioTrack.STATE_INITIALIZED) {
|
||||||
|
Log.e(TAG, "AudioTrack failed to initialize")
|
||||||
|
track.release()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
track.play()
|
||||||
|
Log.i(TAG, "playout started: ${SAMPLE_RATE}Hz mono, buf=$bufSize")
|
||||||
|
|
||||||
|
val pcm = ShortArray(FRAME_SAMPLES)
|
||||||
|
val silence = ShortArray(FRAME_SAMPLES) // pre-allocated silence
|
||||||
|
try {
|
||||||
|
while (running) {
|
||||||
|
val read = engine.readAudio(pcm)
|
||||||
|
if (read >= FRAME_SAMPLES) {
|
||||||
|
track.write(pcm, 0, read)
|
||||||
|
} else {
|
||||||
|
// Not enough decoded audio — write silence to keep stream alive
|
||||||
|
track.write(silence, 0, FRAME_SAMPLES)
|
||||||
|
// Sleep briefly to avoid busy-spinning
|
||||||
|
Thread.sleep(5)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
track.stop()
|
||||||
|
track.release()
|
||||||
|
Log.i(TAG, "playout stopped")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -27,7 +27,11 @@ data class CallStats(
|
|||||||
/** Total frames decoded since call start. */
|
/** Total frames decoded since call start. */
|
||||||
val framesDecoded: Long = 0,
|
val framesDecoded: Long = 0,
|
||||||
/** Number of playout underruns (buffer empty when audio was needed). */
|
/** Number of playout underruns (buffer empty when audio was needed). */
|
||||||
val underruns: Long = 0
|
val underruns: Long = 0,
|
||||||
|
/** Frames recovered by FEC. */
|
||||||
|
val fecRecovered: Long = 0,
|
||||||
|
/** Current mic audio level (RMS, 0-32767). */
|
||||||
|
val audioLevel: Int = 0
|
||||||
) {
|
) {
|
||||||
/** Human-readable quality label. */
|
/** Human-readable quality label. */
|
||||||
val qualityLabel: String
|
val qualityLabel: String
|
||||||
@@ -53,7 +57,9 @@ data class CallStats(
|
|||||||
jitterBufferDepth = obj.optInt("jitter_buffer_depth", 0),
|
jitterBufferDepth = obj.optInt("jitter_buffer_depth", 0),
|
||||||
framesEncoded = obj.optLong("frames_encoded", 0),
|
framesEncoded = obj.optLong("frames_encoded", 0),
|
||||||
framesDecoded = obj.optLong("frames_decoded", 0),
|
framesDecoded = obj.optLong("frames_decoded", 0),
|
||||||
underruns = obj.optLong("underruns", 0)
|
underruns = obj.optLong("underruns", 0),
|
||||||
|
fecRecovered = obj.optLong("fec_recovered", 0),
|
||||||
|
audioLevel = obj.optInt("audio_level", 0)
|
||||||
)
|
)
|
||||||
} catch (e: Exception) {
|
} catch (e: Exception) {
|
||||||
CallStats()
|
CallStats()
|
||||||
|
|||||||
@@ -97,6 +97,24 @@ class WzpEngine(private val callback: WzpCallback) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write captured PCM samples into the engine's capture ring buffer.
|
||||||
|
* Called from the AudioRecord capture thread.
|
||||||
|
*/
|
||||||
|
fun writeAudio(pcm: ShortArray): Int {
|
||||||
|
if (nativeHandle == 0L) return 0
|
||||||
|
return nativeWriteAudio(nativeHandle, pcm)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read decoded PCM samples from the engine's playout ring buffer.
|
||||||
|
* Called from the AudioTrack playout thread.
|
||||||
|
*/
|
||||||
|
fun readAudio(pcm: ShortArray): Int {
|
||||||
|
if (nativeHandle == 0L) return 0
|
||||||
|
return nativeReadAudio(nativeHandle, pcm)
|
||||||
|
}
|
||||||
|
|
||||||
// -- JNI native methods --------------------------------------------------
|
// -- JNI native methods --------------------------------------------------
|
||||||
|
|
||||||
private external fun nativeInit(): Long
|
private external fun nativeInit(): Long
|
||||||
@@ -108,6 +126,8 @@ class WzpEngine(private val callback: WzpCallback) {
|
|||||||
private external fun nativeSetSpeaker(handle: Long, speaker: Boolean)
|
private external fun nativeSetSpeaker(handle: Long, speaker: Boolean)
|
||||||
private external fun nativeGetStats(handle: Long): String?
|
private external fun nativeGetStats(handle: Long): String?
|
||||||
private external fun nativeForceProfile(handle: Long, profile: Int)
|
private external fun nativeForceProfile(handle: Long, profile: Int)
|
||||||
|
private external fun nativeWriteAudio(handle: Long, pcm: ShortArray): Int
|
||||||
|
private external fun nativeReadAudio(handle: Long, pcm: ShortArray): Int
|
||||||
private external fun nativeDestroy(handle: Long)
|
private external fun nativeDestroy(handle: Long)
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
|
|||||||
@@ -39,6 +39,8 @@ class CallActivity : ComponentActivity() {
|
|||||||
override fun onCreate(savedInstanceState: Bundle?) {
|
override fun onCreate(savedInstanceState: Bundle?) {
|
||||||
super.onCreate(savedInstanceState)
|
super.onCreate(savedInstanceState)
|
||||||
|
|
||||||
|
viewModel.setContext(this)
|
||||||
|
|
||||||
setContent {
|
setContent {
|
||||||
WzpTheme {
|
WzpTheme {
|
||||||
InCallScreen(
|
InCallScreen(
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
package com.wzp.ui.call
|
package com.wzp.ui.call
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
import androidx.lifecycle.ViewModel
|
import androidx.lifecycle.ViewModel
|
||||||
import androidx.lifecycle.viewModelScope
|
import androidx.lifecycle.viewModelScope
|
||||||
|
import com.wzp.audio.AudioPipeline
|
||||||
import com.wzp.engine.CallStats
|
import com.wzp.engine.CallStats
|
||||||
import com.wzp.engine.WzpCallback
|
import com.wzp.engine.WzpCallback
|
||||||
import com.wzp.engine.WzpEngine
|
import com.wzp.engine.WzpEngine
|
||||||
@@ -17,9 +19,11 @@ class CallViewModel : ViewModel(), WzpCallback {
|
|||||||
|
|
||||||
private var engine: WzpEngine? = null
|
private var engine: WzpEngine? = null
|
||||||
private var engineInitialized = false
|
private var engineInitialized = false
|
||||||
|
private var audioPipeline: AudioPipeline? = null
|
||||||
|
private var audioStarted = false
|
||||||
|
|
||||||
private val _callState = MutableStateFlow(0)
|
private val _callState = MutableStateFlow(0)
|
||||||
val callState: StateFlow<Int> = _callState.asStateFlow()
|
val callState: StateFlow<Int> get() = _callState.asStateFlow()
|
||||||
|
|
||||||
private val _isMuted = MutableStateFlow(false)
|
private val _isMuted = MutableStateFlow(false)
|
||||||
val isMuted: StateFlow<Boolean> = _isMuted.asStateFlow()
|
val isMuted: StateFlow<Boolean> = _isMuted.asStateFlow()
|
||||||
@@ -36,16 +40,26 @@ class CallViewModel : ViewModel(), WzpCallback {
|
|||||||
private val _errorMessage = MutableStateFlow<String?>(null)
|
private val _errorMessage = MutableStateFlow<String?>(null)
|
||||||
val errorMessage: StateFlow<String?> = _errorMessage.asStateFlow()
|
val errorMessage: StateFlow<String?> = _errorMessage.asStateFlow()
|
||||||
|
|
||||||
|
private val _roomName = MutableStateFlow(DEFAULT_ROOM)
|
||||||
|
val roomName: StateFlow<String> = _roomName.asStateFlow()
|
||||||
|
|
||||||
private var statsJob: Job? = null
|
private var statsJob: Job? = null
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
const val DEFAULT_RELAY = "172.16.81.175:4433"
|
const val DEFAULT_RELAY = "pangolin.manko.yoga:4433"
|
||||||
const val DEFAULT_ROOM = "android"
|
const val DEFAULT_ROOM = "android"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Must be called once with Activity context before startCall. */
|
||||||
|
fun setContext(context: Context) {
|
||||||
|
if (audioPipeline == null) {
|
||||||
|
audioPipeline = AudioPipeline(context.applicationContext)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fun startCall(
|
fun startCall(
|
||||||
relayAddr: String = DEFAULT_RELAY,
|
relayAddr: String = DEFAULT_RELAY,
|
||||||
room: String = DEFAULT_ROOM
|
room: String = _roomName.value
|
||||||
) {
|
) {
|
||||||
try {
|
try {
|
||||||
if (engine == null) {
|
if (engine == null) {
|
||||||
@@ -58,9 +72,6 @@ class CallViewModel : ViewModel(), WzpCallback {
|
|||||||
_callState.value = 1 // Connecting
|
_callState.value = 1 // Connecting
|
||||||
startStatsPolling()
|
startStatsPolling()
|
||||||
|
|
||||||
// startCall blocks (runs tokio on calling thread), so dispatch
|
|
||||||
// to a background coroutine. Using Dispatchers.IO which uses
|
|
||||||
// Java threads (not native pthread_create).
|
|
||||||
viewModelScope.launch(kotlinx.coroutines.Dispatchers.IO) {
|
viewModelScope.launch(kotlinx.coroutines.Dispatchers.IO) {
|
||||||
try {
|
try {
|
||||||
val result = engine?.startCall(relayAddr, room) ?: -1
|
val result = engine?.startCall(relayAddr, room) ?: -1
|
||||||
@@ -80,6 +91,7 @@ class CallViewModel : ViewModel(), WzpCallback {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fun stopCall() {
|
fun stopCall() {
|
||||||
|
stopAudio()
|
||||||
stopStatsPolling()
|
stopStatsPolling()
|
||||||
try {
|
try {
|
||||||
engine?.stopCall()
|
engine?.stopCall()
|
||||||
@@ -101,11 +113,26 @@ class CallViewModel : ViewModel(), WzpCallback {
|
|||||||
|
|
||||||
fun clearError() { _errorMessage.value = null }
|
fun clearError() { _errorMessage.value = null }
|
||||||
|
|
||||||
|
fun setRoomName(name: String) { _roomName.value = name }
|
||||||
|
|
||||||
// WzpCallback
|
// WzpCallback
|
||||||
override fun onCallStateChanged(state: Int) { _callState.value = state }
|
override fun onCallStateChanged(state: Int) { _callState.value = state }
|
||||||
override fun onQualityTierChanged(tier: Int) { _qualityTier.value = tier }
|
override fun onQualityTierChanged(tier: Int) { _qualityTier.value = tier }
|
||||||
override fun onError(code: Int, message: String) { _errorMessage.value = "Error $code: $message" }
|
override fun onError(code: Int, message: String) { _errorMessage.value = "Error $code: $message" }
|
||||||
|
|
||||||
|
private fun startAudio() {
|
||||||
|
if (audioStarted) return
|
||||||
|
val e = engine ?: return
|
||||||
|
audioPipeline?.start(e)
|
||||||
|
audioStarted = true
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun stopAudio() {
|
||||||
|
if (!audioStarted) return
|
||||||
|
audioPipeline?.stop()
|
||||||
|
audioStarted = false
|
||||||
|
}
|
||||||
|
|
||||||
private fun startStatsPolling() {
|
private fun startStatsPolling() {
|
||||||
statsJob?.cancel()
|
statsJob?.cancel()
|
||||||
statsJob = viewModelScope.launch {
|
statsJob = viewModelScope.launch {
|
||||||
@@ -113,7 +140,16 @@ class CallViewModel : ViewModel(), WzpCallback {
|
|||||||
try {
|
try {
|
||||||
val json = engine?.getStats() ?: "{}"
|
val json = engine?.getStats() ?: "{}"
|
||||||
if (json.isNotEmpty()) {
|
if (json.isNotEmpty()) {
|
||||||
_stats.value = CallStats.fromJson(json)
|
val s = CallStats.fromJson(json)
|
||||||
|
_stats.value = s
|
||||||
|
// Sync call state from native engine stats
|
||||||
|
if (s.state != 0) {
|
||||||
|
_callState.value = s.state
|
||||||
|
}
|
||||||
|
// Start audio pipeline when call becomes active
|
||||||
|
if (s.state == 2 && !audioStarted) {
|
||||||
|
startAudio()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} catch (_: Exception) {}
|
} catch (_: Exception) {}
|
||||||
delay(500L)
|
delay(500L)
|
||||||
@@ -128,6 +164,7 @@ class CallViewModel : ViewModel(), WzpCallback {
|
|||||||
|
|
||||||
override fun onCleared() {
|
override fun onCleared() {
|
||||||
super.onCleared()
|
super.onCleared()
|
||||||
|
stopAudio()
|
||||||
stopStatsPolling()
|
stopStatsPolling()
|
||||||
try {
|
try {
|
||||||
engine?.stopCall()
|
engine?.stopCall()
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import androidx.compose.material3.FilledTonalIconButton
|
|||||||
import androidx.compose.material3.IconButtonDefaults
|
import androidx.compose.material3.IconButtonDefaults
|
||||||
import androidx.compose.material3.LinearProgressIndicator
|
import androidx.compose.material3.LinearProgressIndicator
|
||||||
import androidx.compose.material3.MaterialTheme
|
import androidx.compose.material3.MaterialTheme
|
||||||
|
import androidx.compose.material3.OutlinedTextField
|
||||||
import androidx.compose.material3.Surface
|
import androidx.compose.material3.Surface
|
||||||
import androidx.compose.material3.Text
|
import androidx.compose.material3.Text
|
||||||
import androidx.compose.runtime.Composable
|
import androidx.compose.runtime.Composable
|
||||||
@@ -48,6 +49,7 @@ fun InCallScreen(
|
|||||||
val stats by viewModel.stats.collectAsState()
|
val stats by viewModel.stats.collectAsState()
|
||||||
val qualityTier by viewModel.qualityTier.collectAsState()
|
val qualityTier by viewModel.qualityTier.collectAsState()
|
||||||
val errorMessage by viewModel.errorMessage.collectAsState()
|
val errorMessage by viewModel.errorMessage.collectAsState()
|
||||||
|
val roomName by viewModel.roomName.collectAsState()
|
||||||
|
|
||||||
Surface(
|
Surface(
|
||||||
modifier = Modifier.fillMaxSize(),
|
modifier = Modifier.fillMaxSize(),
|
||||||
@@ -83,11 +85,13 @@ fun InCallScreen(
|
|||||||
style = MaterialTheme.typography.bodyMedium,
|
style = MaterialTheme.typography.bodyMedium,
|
||||||
color = MaterialTheme.colorScheme.onSurfaceVariant
|
color = MaterialTheme.colorScheme.onSurfaceVariant
|
||||||
)
|
)
|
||||||
Spacer(modifier = Modifier.height(4.dp))
|
Spacer(modifier = Modifier.height(8.dp))
|
||||||
Text(
|
OutlinedTextField(
|
||||||
text = "Room: ${CallViewModel.DEFAULT_ROOM}",
|
value = roomName,
|
||||||
style = MaterialTheme.typography.bodyMedium,
|
onValueChange = { viewModel.setRoomName(it) },
|
||||||
color = MaterialTheme.colorScheme.onSurfaceVariant
|
label = { Text("Room") },
|
||||||
|
singleLine = true,
|
||||||
|
modifier = Modifier.fillMaxWidth(0.6f)
|
||||||
)
|
)
|
||||||
|
|
||||||
Spacer(modifier = Modifier.height(32.dp))
|
Spacer(modifier = Modifier.height(32.dp))
|
||||||
@@ -132,7 +136,7 @@ fun InCallScreen(
|
|||||||
|
|
||||||
Spacer(modifier = Modifier.height(32.dp))
|
Spacer(modifier = Modifier.height(32.dp))
|
||||||
|
|
||||||
AudioLevelBar(stats.framesEncoded)
|
AudioLevelBar(stats.audioLevel)
|
||||||
|
|
||||||
Spacer(modifier = Modifier.weight(1f))
|
Spacer(modifier = Modifier.weight(1f))
|
||||||
|
|
||||||
@@ -222,9 +226,11 @@ private fun QualityIndicator(tier: Int, label: String) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Composable
|
@Composable
|
||||||
private fun AudioLevelBar(framesEncoded: Long) {
|
private fun AudioLevelBar(audioLevel: Int) {
|
||||||
val level = if (framesEncoded > 0) {
|
// audioLevel is RMS of i16 samples (0-32767).
|
||||||
((framesEncoded % 100).toFloat() / 100f).coerceIn(0.05f, 1f)
|
// Map to 0.0-1.0 with a log-ish curve for better visual feel.
|
||||||
|
val level = if (audioLevel > 0) {
|
||||||
|
(audioLevel.toFloat() / 8000f).coerceIn(0.02f, 1f)
|
||||||
} else {
|
} else {
|
||||||
0f
|
0f
|
||||||
}
|
}
|
||||||
@@ -351,7 +357,7 @@ private fun StatsOverlay(stats: CallStats) {
|
|||||||
) {
|
) {
|
||||||
StatItem("Enc", "${stats.framesEncoded}")
|
StatItem("Enc", "${stats.framesEncoded}")
|
||||||
StatItem("Dec", "${stats.framesDecoded}")
|
StatItem("Dec", "${stats.framesDecoded}")
|
||||||
StatItem("JB", "${stats.jitterBufferDepth}")
|
StatItem("FEC", "${stats.fecRecovered}")
|
||||||
StatItem("Under", "${stats.underruns}")
|
StatItem("Under", "${stats.underruns}")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
91
crates/wzp-android/src/audio_ring.rs
Normal file
91
crates/wzp-android/src/audio_ring.rs
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
//! Lock-free SPSC ring buffers for audio PCM transfer between
|
||||||
|
//! Kotlin AudioRecord/AudioTrack threads and the Rust engine.
|
||||||
|
//!
|
||||||
|
//! These use a simple spin-free design: the producer writes and advances
|
||||||
|
//! a write cursor, the consumer reads and advances a read cursor.
|
||||||
|
//! Both cursors are atomic so no mutex is needed.
|
||||||
|
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
/// Ring buffer capacity in i16 samples.
|
||||||
|
/// 960 samples * 10 frames = ~200ms of audio at 48kHz mono.
|
||||||
|
const RING_CAPACITY: usize = 960 * 10;
|
||||||
|
|
||||||
|
/// Lock-free single-producer single-consumer ring buffer for i16 PCM samples.
|
||||||
|
pub struct AudioRing {
|
||||||
|
buf: Box<[i16; RING_CAPACITY]>,
|
||||||
|
write_pos: AtomicUsize,
|
||||||
|
read_pos: AtomicUsize,
|
||||||
|
}
|
||||||
|
|
||||||
|
// SAFETY: AudioRing is designed for SPSC — one thread writes, one reads.
|
||||||
|
// The atomics ensure visibility. The buffer itself is never accessed
|
||||||
|
// from the same index by both threads simultaneously because the
|
||||||
|
// producer only writes to positions between write_pos and read_pos,
|
||||||
|
// and the consumer only reads from positions between read_pos and write_pos.
|
||||||
|
unsafe impl Send for AudioRing {}
|
||||||
|
unsafe impl Sync for AudioRing {}
|
||||||
|
|
||||||
|
impl AudioRing {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
buf: Box::new([0i16; RING_CAPACITY]),
|
||||||
|
write_pos: AtomicUsize::new(0),
|
||||||
|
read_pos: AtomicUsize::new(0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Number of samples available to read.
|
||||||
|
pub fn available(&self) -> usize {
|
||||||
|
let w = self.write_pos.load(Ordering::Acquire);
|
||||||
|
let r = self.read_pos.load(Ordering::Acquire);
|
||||||
|
w.wrapping_sub(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Number of samples that can be written without overwriting.
|
||||||
|
pub fn free_space(&self) -> usize {
|
||||||
|
RING_CAPACITY - self.available()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write samples into the ring. Returns number of samples written.
|
||||||
|
/// Drops oldest samples if the ring is full.
|
||||||
|
pub fn write(&self, samples: &[i16]) -> usize {
|
||||||
|
let w = self.write_pos.load(Ordering::Relaxed);
|
||||||
|
let count = samples.len().min(RING_CAPACITY);
|
||||||
|
|
||||||
|
for i in 0..count {
|
||||||
|
let idx = (w + i) % RING_CAPACITY;
|
||||||
|
// SAFETY: We're the only writer, and the reader won't read
|
||||||
|
// past read_pos which we haven't advanced past yet.
|
||||||
|
unsafe {
|
||||||
|
let ptr = self.buf.as_ptr() as *mut i16;
|
||||||
|
*ptr.add(idx) = samples[i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.write_pos.store(w.wrapping_add(count), Ordering::Release);
|
||||||
|
|
||||||
|
// If we overwrote unread data, advance read_pos
|
||||||
|
if self.available() > RING_CAPACITY {
|
||||||
|
let new_read = self.write_pos.load(Ordering::Relaxed).wrapping_sub(RING_CAPACITY);
|
||||||
|
self.read_pos.store(new_read, Ordering::Release);
|
||||||
|
}
|
||||||
|
|
||||||
|
count
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read samples from the ring into `out`. Returns number of samples read.
|
||||||
|
pub fn read(&self, out: &mut [i16]) -> usize {
|
||||||
|
let avail = self.available();
|
||||||
|
let count = out.len().min(avail);
|
||||||
|
|
||||||
|
let r = self.read_pos.load(Ordering::Relaxed);
|
||||||
|
for i in 0..count {
|
||||||
|
let idx = (r + i) % RING_CAPACITY;
|
||||||
|
out[i] = unsafe { *self.buf.as_ptr().add(idx) };
|
||||||
|
}
|
||||||
|
|
||||||
|
self.read_pos.store(r.wrapping_add(count), Ordering::Release);
|
||||||
|
count
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,6 +4,9 @@
|
|||||||
//! static bionic stubs in the Rust std prebuilt rlibs. ALL work must happen
|
//! static bionic stubs in the Rust std prebuilt rlibs. ALL work must happen
|
||||||
//! on the JNI calling thread or via the tokio current_thread runtime.
|
//! on the JNI calling thread or via the tokio current_thread runtime.
|
||||||
//! No std::thread::spawn or tokio multi_thread allowed.
|
//! No std::thread::spawn or tokio multi_thread allowed.
|
||||||
|
//!
|
||||||
|
//! Audio capture and playout happen on Kotlin JVM threads via AudioRecord
|
||||||
|
//! and AudioTrack. PCM samples are transferred through lock-free ring buffers.
|
||||||
|
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, Ordering};
|
||||||
@@ -11,15 +14,23 @@ use std::sync::{Arc, Mutex};
|
|||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use tracing::{error, info};
|
use tracing::{error, info, warn};
|
||||||
|
use wzp_codec::opus_dec::OpusDecoder;
|
||||||
|
use wzp_codec::opus_enc::OpusEncoder;
|
||||||
use wzp_crypto::{KeyExchange, WarzoneKeyExchange};
|
use wzp_crypto::{KeyExchange, WarzoneKeyExchange};
|
||||||
|
use wzp_fec::{RaptorQFecDecoder, RaptorQFecEncoder};
|
||||||
use wzp_proto::{
|
use wzp_proto::{
|
||||||
CodecId, MediaHeader, MediaPacket, MediaTransport, QualityProfile, SignalMessage,
|
AudioDecoder, AudioEncoder, CodecId, FecDecoder, FecEncoder,
|
||||||
|
MediaHeader, MediaPacket, MediaTransport, QualityProfile, SignalMessage,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use crate::audio_ring::AudioRing;
|
||||||
use crate::commands::EngineCommand;
|
use crate::commands::EngineCommand;
|
||||||
use crate::stats::{CallState, CallStats};
|
use crate::stats::{CallState, CallStats};
|
||||||
|
|
||||||
|
/// Opus frame size at 48kHz mono, 20ms = 960 samples.
|
||||||
|
const FRAME_SAMPLES: usize = 960;
|
||||||
|
|
||||||
/// Configuration to start a call.
|
/// Configuration to start a call.
|
||||||
pub struct CallStartConfig {
|
pub struct CallStartConfig {
|
||||||
pub profile: QualityProfile,
|
pub profile: QualityProfile,
|
||||||
@@ -41,16 +52,22 @@ impl Default for CallStartConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct EngineState {
|
pub(crate) struct EngineState {
|
||||||
running: AtomicBool,
|
pub running: AtomicBool,
|
||||||
muted: AtomicBool,
|
pub muted: AtomicBool,
|
||||||
stats: Mutex<CallStats>,
|
pub stats: Mutex<CallStats>,
|
||||||
command_tx: std::sync::mpsc::Sender<EngineCommand>,
|
pub command_tx: std::sync::mpsc::Sender<EngineCommand>,
|
||||||
command_rx: Mutex<Option<std::sync::mpsc::Receiver<EngineCommand>>>,
|
pub command_rx: Mutex<Option<std::sync::mpsc::Receiver<EngineCommand>>>,
|
||||||
|
/// Ring buffer: Kotlin AudioRecord → Rust encoder
|
||||||
|
pub capture_ring: AudioRing,
|
||||||
|
/// Ring buffer: Rust decoder → Kotlin AudioTrack
|
||||||
|
pub playout_ring: AudioRing,
|
||||||
|
/// Current audio level (RMS) for UI display, updated by capture path.
|
||||||
|
pub audio_level_rms: AtomicU32,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct WzpEngine {
|
pub struct WzpEngine {
|
||||||
state: Arc<EngineState>,
|
pub(crate) state: Arc<EngineState>,
|
||||||
tokio_runtime: Option<tokio::runtime::Runtime>,
|
tokio_runtime: Option<tokio::runtime::Runtime>,
|
||||||
call_start: Option<Instant>,
|
call_start: Option<Instant>,
|
||||||
}
|
}
|
||||||
@@ -64,6 +81,9 @@ impl WzpEngine {
|
|||||||
stats: Mutex::new(CallStats::default()),
|
stats: Mutex::new(CallStats::default()),
|
||||||
command_tx: tx,
|
command_tx: tx,
|
||||||
command_rx: Mutex::new(Some(rx)),
|
command_rx: Mutex::new(Some(rx)),
|
||||||
|
capture_ring: AudioRing::new(),
|
||||||
|
playout_ring: AudioRing::new(),
|
||||||
|
audio_level_rms: AtomicU32::new(0),
|
||||||
});
|
});
|
||||||
Self {
|
Self {
|
||||||
state,
|
state,
|
||||||
@@ -85,8 +105,6 @@ impl WzpEngine {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create single-threaded tokio runtime — NO thread spawning.
|
|
||||||
// On Android, pthread_create crashes due to static bionic stubs.
|
|
||||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||||
.enable_all()
|
.enable_all()
|
||||||
.build()?;
|
.build()?;
|
||||||
@@ -97,17 +115,16 @@ impl WzpEngine {
|
|||||||
|
|
||||||
let room = config.room.clone();
|
let room = config.room.clone();
|
||||||
let identity_seed = config.identity_seed;
|
let identity_seed = config.identity_seed;
|
||||||
|
let profile = config.profile;
|
||||||
let state = self.state.clone();
|
let state = self.state.clone();
|
||||||
|
|
||||||
self.state.running.store(true, Ordering::Release);
|
self.state.running.store(true, Ordering::Release);
|
||||||
self.call_start = Some(Instant::now());
|
self.call_start = Some(Instant::now());
|
||||||
|
|
||||||
// Run the entire call on the current thread's tokio runtime.
|
|
||||||
// This blocks the JNI thread until the call ends, so Kotlin
|
|
||||||
// must call startCall from a background coroutine.
|
|
||||||
let state_clone = state.clone();
|
let state_clone = state.clone();
|
||||||
runtime.block_on(async move {
|
runtime.block_on(async move {
|
||||||
if let Err(e) = run_call(relay_addr, &room, &identity_seed, state_clone).await {
|
if let Err(e) = run_call(relay_addr, &room, &identity_seed, profile, state_clone).await
|
||||||
|
{
|
||||||
error!("call failed: {e}");
|
error!("call failed: {e}");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -135,19 +152,17 @@ impl WzpEngine {
|
|||||||
self.state.muted.store(muted, Ordering::Relaxed);
|
self.state.muted.store(muted, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_speaker(&self, _enabled: bool) {
|
pub fn set_speaker(&self, _enabled: bool) {}
|
||||||
// TODO: route audio via AudioManager on Kotlin side
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn force_profile(&self, _profile: QualityProfile) {
|
pub fn force_profile(&self, _profile: QualityProfile) {}
|
||||||
// TODO: wire to pipeline when codec thread is re-enabled
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_stats(&self) -> CallStats {
|
pub fn get_stats(&self) -> CallStats {
|
||||||
let mut stats = self.state.stats.lock().unwrap().clone();
|
let mut stats = self.state.stats.lock().unwrap().clone();
|
||||||
if let Some(start) = self.call_start {
|
if let Some(start) = self.call_start {
|
||||||
stats.duration_secs = start.elapsed().as_secs_f64();
|
stats.duration_secs = start.elapsed().as_secs_f64();
|
||||||
}
|
}
|
||||||
|
// Include current audio level
|
||||||
|
stats.audio_level = self.state.audio_level_rms.load(Ordering::Relaxed);
|
||||||
stats
|
stats
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -155,6 +170,23 @@ impl WzpEngine {
|
|||||||
self.state.running.load(Ordering::Acquire)
|
self.state.running.load(Ordering::Acquire)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn write_audio(&self, samples: &[i16]) -> usize {
|
||||||
|
if self.state.muted.load(Ordering::Relaxed) {
|
||||||
|
return samples.len();
|
||||||
|
}
|
||||||
|
// Compute RMS for audio level display
|
||||||
|
if !samples.is_empty() {
|
||||||
|
let sum_sq: f64 = samples.iter().map(|&s| (s as f64) * (s as f64)).sum();
|
||||||
|
let rms = (sum_sq / samples.len() as f64).sqrt() as u32;
|
||||||
|
self.state.audio_level_rms.store(rms, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
self.state.capture_ring.write(samples)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn read_audio(&self, out: &mut [i16]) -> usize {
|
||||||
|
self.state.playout_ring.read(out)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn destroy(mut self) {
|
pub fn destroy(mut self) {
|
||||||
self.stop_call();
|
self.stop_call();
|
||||||
}
|
}
|
||||||
@@ -166,22 +198,19 @@ impl Drop for WzpEngine {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run the full call lifecycle: connect, handshake, send/recv media.
|
/// Run the full call lifecycle: connect, handshake, send/recv media with Opus + FEC.
|
||||||
/// All async, no thread spawning.
|
|
||||||
async fn run_call(
|
async fn run_call(
|
||||||
relay_addr: SocketAddr,
|
relay_addr: SocketAddr,
|
||||||
room: &str,
|
room: &str,
|
||||||
identity_seed: &[u8; 32],
|
identity_seed: &[u8; 32],
|
||||||
|
profile: QualityProfile,
|
||||||
state: Arc<EngineState>,
|
state: Arc<EngineState>,
|
||||||
) -> Result<(), anyhow::Error> {
|
) -> Result<(), anyhow::Error> {
|
||||||
// Install rustls crypto provider
|
|
||||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||||
|
|
||||||
// Create QUIC endpoint
|
|
||||||
let bind_addr: SocketAddr = "0.0.0.0:0".parse().unwrap();
|
let bind_addr: SocketAddr = "0.0.0.0:0".parse().unwrap();
|
||||||
let endpoint = wzp_transport::create_endpoint(bind_addr, None)?;
|
let endpoint = wzp_transport::create_endpoint(bind_addr, None)?;
|
||||||
|
|
||||||
// Connect to relay with room as SNI
|
|
||||||
let sni = if room.is_empty() { "android" } else { room };
|
let sni = if room.is_empty() { "android" } else { room };
|
||||||
info!(%relay_addr, sni, "connecting to relay...");
|
info!(%relay_addr, sni, "connecting to relay...");
|
||||||
let client_cfg = wzp_transport::client_config();
|
let client_cfg = wzp_transport::client_config();
|
||||||
@@ -236,58 +265,223 @@ async fn run_call(
|
|||||||
stats.state = CallState::Active;
|
stats.state = CallState::Active;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Simple media loop: send silence, recv and count frames.
|
// Initialize Opus codec
|
||||||
// No codec thread, no Oboe — just network I/O to verify connectivity.
|
let mut encoder =
|
||||||
// Audio pipeline will be added once native threading is resolved.
|
OpusEncoder::new(profile).map_err(|e| anyhow::anyhow!("opus encoder init: {e}"))?;
|
||||||
|
let mut decoder =
|
||||||
|
OpusDecoder::new(profile).map_err(|e| anyhow::anyhow!("opus decoder init: {e}"))?;
|
||||||
|
|
||||||
|
// Initialize FEC encoder/decoder
|
||||||
|
let mut fec_enc = wzp_fec::create_encoder(&profile);
|
||||||
|
let mut fec_dec = wzp_fec::create_decoder(&profile);
|
||||||
|
|
||||||
|
info!(
|
||||||
|
fec_ratio = profile.fec_ratio,
|
||||||
|
frames_per_block = profile.frames_per_block,
|
||||||
|
"codec + FEC initialized (48kHz mono, 20ms frames, RaptorQ)"
|
||||||
|
);
|
||||||
|
|
||||||
let seq = AtomicU16::new(0);
|
let seq = AtomicU16::new(0);
|
||||||
let ts = AtomicU32::new(0);
|
let ts = AtomicU32::new(0);
|
||||||
let transport_recv = transport.clone();
|
let transport_recv = transport.clone();
|
||||||
|
|
||||||
|
// Pre-allocate buffers
|
||||||
|
let mut capture_buf = vec![0i16; FRAME_SAMPLES];
|
||||||
|
let mut encode_buf = vec![0u8; encoder.max_frame_bytes()];
|
||||||
|
let mut frame_in_block: u8 = 0;
|
||||||
|
let mut block_id: u8 = 0;
|
||||||
|
|
||||||
|
// Send task: capture ring → Opus encode → FEC → MediaPackets
|
||||||
let send_task = async {
|
let send_task = async {
|
||||||
let silence = vec![0u8; 20]; // minimal opus silence frame
|
info!("send task started (Opus + RaptorQ FEC)");
|
||||||
loop {
|
loop {
|
||||||
if !state.running.load(Ordering::Relaxed) {
|
if !state.running.load(Ordering::Relaxed) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let avail = state.capture_ring.available();
|
||||||
|
if avail < FRAME_SAMPLES {
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let read = state.capture_ring.read(&mut capture_buf);
|
||||||
|
if read < FRAME_SAMPLES {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Opus encode
|
||||||
|
let encoded_len = match encoder.encode(&capture_buf, &mut encode_buf) {
|
||||||
|
Ok(n) => n,
|
||||||
|
Err(e) => {
|
||||||
|
warn!("opus encode error: {e}");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let encoded = &encode_buf[..encoded_len];
|
||||||
|
|
||||||
|
// Build source packet
|
||||||
let s = seq.fetch_add(1, Ordering::Relaxed);
|
let s = seq.fetch_add(1, Ordering::Relaxed);
|
||||||
let t = ts.fetch_add(20, Ordering::Relaxed);
|
let t = ts.fetch_add(FRAME_SAMPLES as u32, Ordering::Relaxed);
|
||||||
let packet = MediaPacket {
|
|
||||||
|
let source_pkt = MediaPacket {
|
||||||
header: MediaHeader {
|
header: MediaHeader {
|
||||||
version: 0,
|
version: 0,
|
||||||
is_repair: false,
|
is_repair: false,
|
||||||
codec_id: CodecId::Opus24k,
|
codec_id: profile.codec,
|
||||||
has_quality_report: false,
|
has_quality_report: false,
|
||||||
fec_ratio_encoded: 0,
|
fec_ratio_encoded: MediaHeader::encode_fec_ratio(profile.fec_ratio),
|
||||||
seq: s,
|
seq: s,
|
||||||
timestamp: t,
|
timestamp: t,
|
||||||
fec_block: 0,
|
fec_block: block_id,
|
||||||
fec_symbol: 0,
|
fec_symbol: frame_in_block,
|
||||||
reserved: 0,
|
reserved: 0,
|
||||||
csrc_count: 0,
|
csrc_count: 0,
|
||||||
},
|
},
|
||||||
payload: Bytes::from(silence.clone()),
|
payload: Bytes::copy_from_slice(encoded),
|
||||||
quality_report: None,
|
quality_report: None,
|
||||||
};
|
};
|
||||||
if let Err(e) = transport.send_media(&packet).await {
|
|
||||||
|
// Send source packet
|
||||||
|
if let Err(e) = transport.send_media(&source_pkt).await {
|
||||||
error!("send error: {e}");
|
error!("send error: {e}");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
// 20ms frame interval
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
// Feed encoded frame to FEC encoder
|
||||||
|
if let Err(e) = fec_enc.add_source_symbol(encoded) {
|
||||||
|
warn!("fec add_source error: {e}");
|
||||||
|
}
|
||||||
|
frame_in_block += 1;
|
||||||
|
|
||||||
|
// When block is full, generate repair packets
|
||||||
|
if frame_in_block >= profile.frames_per_block {
|
||||||
|
match fec_enc.generate_repair(profile.fec_ratio) {
|
||||||
|
Ok(repairs) => {
|
||||||
|
let repair_count = repairs.len();
|
||||||
|
for (sym_idx, repair_data) in repairs {
|
||||||
|
let rs = seq.fetch_add(1, Ordering::Relaxed);
|
||||||
|
let repair_pkt = MediaPacket {
|
||||||
|
header: MediaHeader {
|
||||||
|
version: 0,
|
||||||
|
is_repair: true,
|
||||||
|
codec_id: profile.codec,
|
||||||
|
has_quality_report: false,
|
||||||
|
fec_ratio_encoded: MediaHeader::encode_fec_ratio(
|
||||||
|
profile.fec_ratio,
|
||||||
|
),
|
||||||
|
seq: rs,
|
||||||
|
timestamp: t,
|
||||||
|
fec_block: block_id,
|
||||||
|
fec_symbol: sym_idx,
|
||||||
|
reserved: 0,
|
||||||
|
csrc_count: 0,
|
||||||
|
},
|
||||||
|
payload: Bytes::from(repair_data),
|
||||||
|
quality_report: None,
|
||||||
|
};
|
||||||
|
if let Err(e) = transport.send_media(&repair_pkt).await {
|
||||||
|
error!("send repair error: {e}");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if repair_count > 0 && (block_id % 50 == 0 || block_id == 0) {
|
||||||
|
info!(
|
||||||
|
block_id,
|
||||||
|
repair_count,
|
||||||
|
fec_ratio = profile.fec_ratio,
|
||||||
|
"FEC block complete"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
warn!("fec generate_repair error: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let _ = fec_enc.finalize_block();
|
||||||
|
block_id = block_id.wrapping_add(1);
|
||||||
|
frame_in_block = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if s % 500 == 0 {
|
||||||
|
info!(seq = s, block_id, frame_in_block, "sending");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Pre-allocate decode buffer
|
||||||
|
let mut decode_buf = vec![0i16; FRAME_SAMPLES];
|
||||||
|
|
||||||
|
// Recv task: MediaPackets → FEC decode → Opus decode → playout ring
|
||||||
let recv_task = async {
|
let recv_task = async {
|
||||||
let mut frames_decoded: u64 = 0;
|
let mut frames_decoded: u64 = 0;
|
||||||
|
let mut fec_recovered: u64 = 0;
|
||||||
|
info!("recv task started (Opus + RaptorQ FEC)");
|
||||||
loop {
|
loop {
|
||||||
if !state.running.load(Ordering::Relaxed) {
|
if !state.running.load(Ordering::Relaxed) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
match transport_recv.recv_media().await {
|
match transport_recv.recv_media().await {
|
||||||
Ok(Some(_pkt)) => {
|
Ok(Some(pkt)) => {
|
||||||
|
let is_repair = pkt.header.is_repair;
|
||||||
|
let pkt_block = pkt.header.fec_block;
|
||||||
|
let pkt_symbol = pkt.header.fec_symbol;
|
||||||
|
|
||||||
|
// Feed every packet (source + repair) to FEC decoder
|
||||||
|
let _ = fec_dec.add_symbol(
|
||||||
|
pkt_block,
|
||||||
|
pkt_symbol,
|
||||||
|
is_repair,
|
||||||
|
&pkt.payload,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Source packets: decode directly
|
||||||
|
if !is_repair {
|
||||||
|
match decoder.decode(&pkt.payload, &mut decode_buf) {
|
||||||
|
Ok(samples) => {
|
||||||
|
state.playout_ring.write(&decode_buf[..samples]);
|
||||||
frames_decoded += 1;
|
frames_decoded += 1;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
warn!("opus decode error: {e}");
|
||||||
|
if let Ok(samples) = decoder.decode_lost(&mut decode_buf) {
|
||||||
|
state.playout_ring.write(&decode_buf[..samples]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try FEC recovery for this block
|
||||||
|
// (useful when source packets were lost but repair arrived)
|
||||||
|
if let Ok(Some(recovered_frames)) = fec_dec.try_decode(pkt_block) {
|
||||||
|
// FEC recovered the block — any previously missing frames
|
||||||
|
// are now available. In a full jitter buffer implementation,
|
||||||
|
// we'd insert recovered frames at the right position.
|
||||||
|
// For now, log recovery for telemetry.
|
||||||
|
fec_recovered += recovered_frames.len() as u64;
|
||||||
|
if fec_recovered % 50 == 1 {
|
||||||
|
info!(
|
||||||
|
fec_recovered,
|
||||||
|
block = pkt_block,
|
||||||
|
frames = recovered_frames.len(),
|
||||||
|
"FEC block recovered"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expire old blocks to prevent memory growth
|
||||||
|
if pkt_block > 3 {
|
||||||
|
fec_dec.expire_before(pkt_block.wrapping_sub(3));
|
||||||
|
}
|
||||||
|
|
||||||
|
if frames_decoded == 1 || frames_decoded % 500 == 0 {
|
||||||
|
info!(frames_decoded, fec_recovered, "recv stats");
|
||||||
|
}
|
||||||
|
|
||||||
let mut stats = state.stats.lock().unwrap();
|
let mut stats = state.stats.lock().unwrap();
|
||||||
stats.frames_decoded = frames_decoded;
|
stats.frames_decoded = frames_decoded;
|
||||||
|
stats.fec_recovered = fec_recovered;
|
||||||
}
|
}
|
||||||
Ok(None) => {
|
Ok(None) => {
|
||||||
info!("relay disconnected");
|
info!("relay disconnected");
|
||||||
@@ -301,7 +495,7 @@ async fn run_call(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Update encoded frame count in send task
|
// Stats task
|
||||||
let stats_task = async {
|
let stats_task = async {
|
||||||
loop {
|
loop {
|
||||||
if !state.running.load(Ordering::Relaxed) {
|
if !state.running.load(Ordering::Relaxed) {
|
||||||
|
|||||||
@@ -174,6 +174,56 @@ pub unsafe extern "system" fn Java_com_wzp_engine_WzpEngine_nativeForceProfile(
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Write captured PCM samples from Kotlin AudioRecord into the engine's capture ring.
|
||||||
|
/// pcm is a Java short[] array.
|
||||||
|
#[unsafe(no_mangle)]
|
||||||
|
pub unsafe extern "system" fn Java_com_wzp_engine_WzpEngine_nativeWriteAudio(
|
||||||
|
env: JNIEnv,
|
||||||
|
_class: JClass,
|
||||||
|
handle: jlong,
|
||||||
|
pcm: jni::objects::JShortArray,
|
||||||
|
) -> jint {
|
||||||
|
let result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
|
||||||
|
let h = unsafe { handle_ref(handle) };
|
||||||
|
let len = env.get_array_length(&pcm).unwrap_or(0) as usize;
|
||||||
|
if len == 0 {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let mut buf = vec![0i16; len];
|
||||||
|
// GetShortArrayRegion copies Java array into our buffer
|
||||||
|
if env.get_short_array_region(&pcm, 0, &mut buf).is_err() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
h.engine.write_audio(&buf) as jint
|
||||||
|
}));
|
||||||
|
result.unwrap_or(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read decoded PCM samples from the engine's playout ring for Kotlin AudioTrack.
|
||||||
|
/// pcm is a Java short[] array to fill. Returns number of samples actually read.
|
||||||
|
#[unsafe(no_mangle)]
|
||||||
|
pub unsafe extern "system" fn Java_com_wzp_engine_WzpEngine_nativeReadAudio(
|
||||||
|
env: JNIEnv,
|
||||||
|
_class: JClass,
|
||||||
|
handle: jlong,
|
||||||
|
pcm: jni::objects::JShortArray,
|
||||||
|
) -> jint {
|
||||||
|
let result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
|
||||||
|
let h = unsafe { handle_ref(handle) };
|
||||||
|
let len = env.get_array_length(&pcm).unwrap_or(0) as usize;
|
||||||
|
if len == 0 {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let mut buf = vec![0i16; len];
|
||||||
|
let read = h.engine.read_audio(&mut buf);
|
||||||
|
if read > 0 {
|
||||||
|
let _ = env.set_short_array_region(&pcm, 0, &buf[..read]);
|
||||||
|
}
|
||||||
|
read as jint
|
||||||
|
}));
|
||||||
|
result.unwrap_or(0)
|
||||||
|
}
|
||||||
|
|
||||||
#[unsafe(no_mangle)]
|
#[unsafe(no_mangle)]
|
||||||
pub unsafe extern "system" fn Java_com_wzp_engine_WzpEngine_nativeDestroy(
|
pub unsafe extern "system" fn Java_com_wzp_engine_WzpEngine_nativeDestroy(
|
||||||
_env: JNIEnv,
|
_env: JNIEnv,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@
|
|||||||
//! allowing `cargo check` and unit tests on the host.
|
//! allowing `cargo check` and unit tests on the host.
|
||||||
|
|
||||||
pub mod audio_android;
|
pub mod audio_android;
|
||||||
|
pub mod audio_ring;
|
||||||
pub mod commands;
|
pub mod commands;
|
||||||
pub mod engine;
|
pub mod engine;
|
||||||
pub mod pipeline;
|
pub mod pipeline;
|
||||||
|
|||||||
@@ -1,21 +1,31 @@
|
|||||||
//! Call statistics for the Android engine.
|
//! Call statistics for the Android engine.
|
||||||
|
|
||||||
/// State of the call.
|
/// State of the call.
|
||||||
#[derive(Clone, Debug, Default, serde::Serialize, PartialEq, Eq)]
|
/// Serializes as integer for easy parsing on the Kotlin side:
|
||||||
|
/// 0=Idle, 1=Connecting, 2=Active, 3=Reconnecting, 4=Closed
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||||
pub enum CallState {
|
pub enum CallState {
|
||||||
/// Engine is idle, no active call.
|
|
||||||
#[default]
|
#[default]
|
||||||
Idle,
|
Idle,
|
||||||
/// Establishing connection to the relay.
|
|
||||||
Connecting,
|
Connecting,
|
||||||
/// Call is active with audio flowing.
|
|
||||||
Active,
|
Active,
|
||||||
/// Temporarily lost connection, attempting to recover.
|
|
||||||
Reconnecting,
|
Reconnecting,
|
||||||
/// Call has ended.
|
|
||||||
Closed,
|
Closed,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl serde::Serialize for CallState {
|
||||||
|
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||||
|
let n: u8 = match self {
|
||||||
|
CallState::Idle => 0,
|
||||||
|
CallState::Connecting => 1,
|
||||||
|
CallState::Active => 2,
|
||||||
|
CallState::Reconnecting => 3,
|
||||||
|
CallState::Closed => 4,
|
||||||
|
};
|
||||||
|
serializer.serialize_u8(n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Aggregated call statistics, serializable for JNI bridge.
|
/// Aggregated call statistics, serializable for JNI bridge.
|
||||||
#[derive(Clone, Debug, Default, serde::Serialize)]
|
#[derive(Clone, Debug, Default, serde::Serialize)]
|
||||||
pub struct CallStats {
|
pub struct CallStats {
|
||||||
@@ -39,4 +49,8 @@ pub struct CallStats {
|
|||||||
pub frames_decoded: u64,
|
pub frames_decoded: u64,
|
||||||
/// Number of playout underruns (buffer empty when audio needed).
|
/// Number of playout underruns (buffer empty when audio needed).
|
||||||
pub underruns: u64,
|
pub underruns: u64,
|
||||||
|
/// Frames recovered by FEC.
|
||||||
|
pub fec_recovered: u64,
|
||||||
|
/// Current mic audio level (RMS of i16 samples, 0-32767).
|
||||||
|
pub audio_level: u32,
|
||||||
}
|
}
|
||||||
|
|||||||
BIN
images/photo_2026-04-05_16-03-40.jpg
Normal file
BIN
images/photo_2026-04-05_16-03-40.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 31 KiB |
BIN
qr-download.png
BIN
qr-download.png
Binary file not shown.
|
Before Width: | Height: | Size: 696 B |
BIN
wzp-release.apk
BIN
wzp-release.apk
Binary file not shown.
Reference in New Issue
Block a user