From b329402faf25b2200c9c5a0735232e17d4c82aed Mon Sep 17 00:00:00 2001 From: jubb Date: Thu, 1 Jul 2021 17:06:42 +1000 Subject: [PATCH] fix: audio waveforms decoded on the attachment download --- app/build.gradle | 4 +-- .../attachments/DatabaseAttachmentProvider.kt | 8 +++++ .../v2/messages/VoiceMessageView.kt | 30 +++++++++------- .../api/PrepareAttachmentAudioExtrasJob.kt | 34 ++----------------- .../securesms/loki/views/WaveformSeekBar.kt | 2 +- build.gradle | 1 + libsession/build.gradle | 4 +++ .../database/MessageDataProvider.kt | 1 + .../messaging/jobs/AttachmentDownloadJob.kt | 16 +++++++++ .../libsession}/utilities/DecodedAudio.kt | 33 +++++++++++++++++- 10 files changed, 84 insertions(+), 49 deletions(-) rename {app/src/main/java/org/thoughtcrime/securesms/loki => libsession/src/main/java/org/session/libsession}/utilities/DecodedAudio.kt (94%) diff --git a/app/build.gradle b/app/build.gradle index cf772c0a7..6554e34fb 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -194,8 +194,8 @@ android { versionCode canonicalVersionCode * postFixSize versionName canonicalVersionName - minSdkVersion 23 - targetSdkVersion 30 + minSdkVersion androidMinimumSdkVersion + targetSdkVersion androidCompileSdkVersion multiDexEnabled = true diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachmentProvider.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachmentProvider.kt index d017e770f..18b5ff78f 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachmentProvider.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachmentProvider.kt @@ -97,6 +97,14 @@ class DatabaseAttachmentProvider(context: Context, helper: SQLCipherOpenHelper) attachmentDatabase.insertAttachmentsForPlaceholder(messageId, attachmentId, stream) } + override fun updateAudioAttachmentDuration(attachmentId: AttachmentId, durationMs: Long) { + DatabaseFactory.getAttachmentDatabase(context).setAttachmentAudioExtras(DatabaseAttachmentAudioExtras( + attachmentId = attachmentId, + visualSamples = byteArrayOf(), + durationMs = durationMs + )) + } + override fun isOutgoingMessage(timestamp: Long): Boolean { val smsDatabase = DatabaseFactory.getSmsDatabase(context) val mmsDatabase = DatabaseFactory.getMmsDatabase(context) diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VoiceMessageView.kt b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VoiceMessageView.kt index b957b0a16..22e6a33ad 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VoiceMessageView.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VoiceMessageView.kt @@ -10,9 +10,11 @@ import android.widget.RelativeLayout import androidx.core.view.isVisible import kotlinx.android.synthetic.main.view_voice_message.view.* import network.loki.messenger.R +import org.session.libsession.messaging.sending_receiving.attachments.DatabaseAttachment import org.thoughtcrime.securesms.audio.AudioSlidePlayer import org.thoughtcrime.securesms.components.CornerMask import org.thoughtcrime.securesms.conversation.v2.utilities.MessageBubbleUtilities +import org.thoughtcrime.securesms.database.DatabaseFactory import org.thoughtcrime.securesms.database.model.MmsMessageRecord import java.util.concurrent.TimeUnit import kotlin.math.roundToInt @@ -44,27 +46,29 @@ class VoiceMessageView : LinearLayout, AudioSlidePlayer.Listener { val audio = message.slideDeck.audioSlide!! val player = AudioSlidePlayer.createFor(context, audio, this) this.player = player - isPreparing = true - if (!audio.isPendingDownload && !audio.isInProgress) { - player.play(0.0) - } voiceMessageViewLoader.isVisible = audio.isPendingDownload val cornerRadii = MessageBubbleUtilities.calculateRadii(context, isStartOfMessageCluster, isEndOfMessageCluster, message.isOutgoing) cornerMask.setTopLeftRadius(cornerRadii[0]) cornerMask.setTopRightRadius(cornerRadii[1]) cornerMask.setBottomRightRadius(cornerRadii[2]) cornerMask.setBottomLeftRadius(cornerRadii[3]) + + // only process audio if downloaded + if (audio.isPendingDownload || audio.isInProgress) return + + (audio.asAttachment() as? DatabaseAttachment)?.let { attachment -> + DatabaseFactory.getAttachmentDatabase(context).getAttachmentAudioExtras(attachment.attachmentId)?.let { audioExtras -> + if (audioExtras.durationMs > 0) { + voiceMessageViewDurationTextView.visibility = View.VISIBLE + voiceMessageViewDurationTextView.text = String.format("%02d:%02d", + TimeUnit.MILLISECONDS.toMinutes(audioExtras.durationMs), + TimeUnit.MILLISECONDS.toSeconds(audioExtras.durationMs)) + } + } + } } - override fun onPlayerStart(player: AudioSlidePlayer) { - if (!isPreparing) { return } - isPreparing = false - duration = player.duration - voiceMessageViewDurationTextView.text = String.format("%01d:%02d", - TimeUnit.MILLISECONDS.toMinutes(duration), - TimeUnit.MILLISECONDS.toSeconds(duration)) - player.stop() - } + override fun onPlayerStart(player: AudioSlidePlayer) {} override fun onPlayerProgress(player: AudioSlidePlayer, progress: Double, unused: Long) { if (progress == 1.0) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt b/app/src/main/java/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt index 6c1a96d1d..fa92ba3d2 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt @@ -9,10 +9,11 @@ import org.session.libsession.messaging.utilities.Data import org.session.libsession.messaging.sending_receiving.attachments.Attachment import org.session.libsession.messaging.sending_receiving.attachments.AttachmentId import org.session.libsession.messaging.sending_receiving.attachments.DatabaseAttachmentAudioExtras +import org.session.libsession.utilities.DecodedAudio +import org.session.libsession.utilities.InputStreamMediaDataSource import org.thoughtcrime.securesms.database.DatabaseFactory import org.thoughtcrime.securesms.jobmanager.Job import org.thoughtcrime.securesms.jobs.BaseJob -import org.thoughtcrime.securesms.loki.utilities.DecodedAudio import org.thoughtcrime.securesms.mms.PartAuthority import java.io.InputStream import java.lang.IllegalStateException @@ -133,35 +134,4 @@ class PrepareAttachmentAudioExtrasJob : BaseJob { /** Gets dispatched once the audio extras have been updated. */ data class AudioExtrasUpdatedEvent(val attachmentId: AttachmentId) - - @RequiresApi(Build.VERSION_CODES.M) - private class InputStreamMediaDataSource: MediaDataSource { - - private val data: ByteArray - - constructor(inputStream: InputStream): super() { - this.data = inputStream.readBytes() - } - - override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int { - val length: Int = data.size - if (position >= length) { - return -1 // -1 indicates EOF - } - var actualSize = size - if (position + size > length) { - actualSize -= (position + size - length).toInt() - } - System.arraycopy(data, position.toInt(), buffer, offset, actualSize) - return actualSize - } - - override fun getSize(): Long { - return data.size.toLong() - } - - override fun close() { - // We don't need to close the wrapped stream. - } - } } \ No newline at end of file diff --git a/app/src/main/java/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt b/app/src/main/java/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index 86028521f..df74bd9ca 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -14,7 +14,7 @@ import android.view.ViewConfiguration import android.view.animation.DecelerateInterpolator import androidx.core.math.MathUtils import network.loki.messenger.R -import org.thoughtcrime.securesms.loki.utilities.byteToNormalizedFloat +import org.session.libsession.utilities.byteToNormalizedFloat import kotlin.math.abs import kotlin.math.max import kotlin.math.min diff --git a/build.gradle b/build.gradle index 3343145c6..884e6ed08 100644 --- a/build.gradle +++ b/build.gradle @@ -50,6 +50,7 @@ allprojects { } project.ext { + androidMinimumSdkVersion = 23 androidCompileSdkVersion = 30 } } \ No newline at end of file diff --git a/libsession/build.gradle b/libsession/build.gradle index 6763e0cff..8bd46532e 100644 --- a/libsession/build.gradle +++ b/libsession/build.gradle @@ -6,6 +6,10 @@ plugins { android { compileSdkVersion androidCompileSdkVersion + defaultConfig { + minSdkVersion androidMinimumSdkVersion + } + compileOptions { sourceCompatibility JavaVersion.VERSION_1_8 targetCompatibility JavaVersion.VERSION_1_8 diff --git a/libsession/src/main/java/org/session/libsession/database/MessageDataProvider.kt b/libsession/src/main/java/org/session/libsession/database/MessageDataProvider.kt index 4cf93a429..37d6d44e4 100644 --- a/libsession/src/main/java/org/session/libsession/database/MessageDataProvider.kt +++ b/libsession/src/main/java/org/session/libsession/database/MessageDataProvider.kt @@ -20,6 +20,7 @@ interface MessageDataProvider { fun getSignalAttachmentPointer(attachmentId: Long): SignalServiceAttachmentPointer? fun setAttachmentState(attachmentState: AttachmentState, attachmentId: Long, messageID: Long) fun insertAttachment(messageId: Long, attachmentId: AttachmentId, stream : InputStream) + fun updateAudioAttachmentDuration(attachmentId: AttachmentId, durationMs: Long) fun isOutgoingMessage(timestamp: Long): Boolean fun handleSuccessfulAttachmentUpload(attachmentId: Long, attachmentStream: SignalServiceAttachmentStream, attachmentKey: ByteArray, uploadResult: UploadResult) fun handleFailedAttachmentUpload(attachmentId: Long) diff --git a/libsession/src/main/java/org/session/libsession/messaging/jobs/AttachmentDownloadJob.kt b/libsession/src/main/java/org/session/libsession/messaging/jobs/AttachmentDownloadJob.kt index a5aaeaa42..c2c43b9a3 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/jobs/AttachmentDownloadJob.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/jobs/AttachmentDownloadJob.kt @@ -1,15 +1,22 @@ package org.session.libsession.messaging.jobs +import android.content.ContentResolver +import android.media.MediaDataSource +import android.media.MediaExtractor import okhttp3.HttpUrl import org.session.libsession.messaging.MessagingModuleConfiguration import org.session.libsession.messaging.open_groups.OpenGroupAPIV2 import org.session.libsession.messaging.sending_receiving.attachments.AttachmentState import org.session.libsession.messaging.utilities.Data +import org.session.libsession.utilities.DecodedAudio import org.session.libsession.utilities.DownloadUtilities +import org.session.libsession.utilities.FileUtils +import org.session.libsession.utilities.InputStreamMediaDataSource import org.session.libsignal.streams.AttachmentCipherInputStream import org.session.libsignal.utilities.Base64 import org.session.libsignal.utilities.Log import java.io.File +import java.io.FileDescriptor import java.io.FileInputStream class AttachmentDownloadJob(val attachmentID: Long, val databaseMessageID: Long) : Job { @@ -67,6 +74,15 @@ class AttachmentDownloadJob(val attachmentID: Long, val databaseMessageID: Long) } FileInputStream(tempFile) } + + if (attachment.contentType.startsWith("audio/")) { + // process the duration + InputStreamMediaDataSource(inputStream).use { mediaDataSource -> + val durationMs = (DecodedAudio.create(mediaDataSource).totalDuration / 1000.0).toLong() + messageDataProvider.updateAudioAttachmentDuration(attachment.attachmentId, durationMs) + } + } + messageDataProvider.insertAttachment(databaseMessageID, attachment.attachmentId, inputStream) tempFile.delete() handleSuccess() diff --git a/app/src/main/java/org/thoughtcrime/securesms/loki/utilities/DecodedAudio.kt b/libsession/src/main/java/org/session/libsession/utilities/DecodedAudio.kt similarity index 94% rename from app/src/main/java/org/thoughtcrime/securesms/loki/utilities/DecodedAudio.kt rename to libsession/src/main/java/org/session/libsession/utilities/DecodedAudio.kt index 77f7f8898..ef21abe4c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/loki/utilities/DecodedAudio.kt +++ b/libsession/src/main/java/org/session/libsession/utilities/DecodedAudio.kt @@ -1,4 +1,4 @@ -package org.thoughtcrime.securesms.loki.utilities +package org.session.libsession.utilities import android.media.AudioFormat import android.media.MediaCodec @@ -11,6 +11,7 @@ import androidx.annotation.RequiresApi import java.io.FileDescriptor import java.io.IOException +import java.io.InputStream import java.nio.ByteBuffer import java.nio.ByteOrder import java.nio.ShortBuffer @@ -365,4 +366,34 @@ inline fun byteToNormalizedFloat(value: Byte): Float { /** Turns a [0..1] float into a signed byte. */ inline fun normalizedFloatToByte(value: Float): Byte { return (255f * value - 128f).roundToInt().toByte() +} + +class InputStreamMediaDataSource: MediaDataSource { + + private val data: ByteArray + + constructor(inputStream: InputStream): super() { + this.data = inputStream.readBytes() + } + + override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int { + val length: Int = data.size + if (position >= length) { + return -1 // -1 indicates EOF + } + var actualSize = size + if (position + size > length) { + actualSize -= (position + size - length).toInt() + } + System.arraycopy(data, position.toInt(), buffer, offset, actualSize) + return actualSize + } + + override fun getSize(): Long { + return data.size.toLong() + } + + override fun close() { + // We don't need to close the wrapped stream. + } } \ No newline at end of file