oxen-io
/
session-android
mirror of https://github.com/oxen-io/session-android.git
1
0
Fork 0
Code Issues 1 Projects Releases Wiki Activity

Merge pull request #524 from hjubb/file_server_v2 

New File Server V2 URL handling and open group v2 fixes / features
This commit is contained in:
Niels Andriesse 2021-05-06 14:03:31 +10:00 committed by GitHub
parent 54d64192fb a4b7b34f2e
commit bfa75a3db9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 222 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
app/src/main/java/org/thoughtcrime/securesms/database/Storage.kt
View File

@ -2,6 +2,7 @@ package org.thoughtcrime.securesms.database
import android.content.Context
import android.net.Uri
import okhttp3.HttpUrl
import org.session.libsession.messaging.StorageProtocol
import org.session.libsession.messaging.jobs.AttachmentUploadJob
import org.session.libsession.messaging.jobs.Job
@ -378,9 +379,9 @@ class Storage(context: Context, helper: SQLCipherOpenHelper) : Database(context,
SessionMetaProtocol.addTimestamp(timestamp)
}
// override fun removeReceivedMessageTimestamps(timestamps: Set<Long>) {
// TODO("Not yet implemented")
// }
override fun removeReceivedMessageTimestamps(timestamps: Set<Long>) {
SessionMetaProtocol.removeTimestamps(timestamps)
}
override fun getMessageIdInDatabase(timestamp: Long, author: String): Long? {
val database = DatabaseFactory.getMmsSmsDatabase(context)
@ -543,8 +544,23 @@ class Storage(context: Context, helper: SQLCipherOpenHelper) : Database(context,
return DatabaseFactory.getLokiThreadDatabase(context).getAllV2OpenGroups()
}
override fun addOpenGroup(server: String, channel: Long) {
OpenGroupUtilities.addGroup(context, server, channel)
override fun addOpenGroup(serverUrl: String, channel: Long) {
val httpUrl = HttpUrl.parse(serverUrl) ?: return
if (httpUrl.queryParameterNames().contains("public_key")) {
// open group v2
val server = HttpUrl.Builder().scheme(httpUrl.scheme()).host(httpUrl.host()).apply {
if (httpUrl.port() != 80 || httpUrl.port() != 443) {
// non-standard port, add to server
this.port(httpUrl.port())
}
}.build()
val room = httpUrl.pathSegments().firstOrNull() ?: return
val publicKey = httpUrl.queryParameter("public_key") ?: return
OpenGroupUtilities.addGroup(context, server.toString().removeSuffix("/"), room, publicKey)
} else {
OpenGroupUtilities.addGroup(context, serverUrl, channel)
}
}
override fun getAllGroups(): List<GroupRecord> {

2
app/src/main/java/org/thoughtcrime/securesms/database/ThreadDatabase.java
View File

@ -49,6 +49,7 @@ import org.thoughtcrime.securesms.database.model.MediaMmsMessageRecord;
import org.thoughtcrime.securesms.database.model.MessageRecord;
import org.thoughtcrime.securesms.database.model.MmsMessageRecord;
import org.thoughtcrime.securesms.database.model.ThreadRecord;
import org.thoughtcrime.securesms.loki.protocol.SessionMetaProtocol;
import org.thoughtcrime.securesms.mms.Slide;
import org.thoughtcrime.securesms.mms.SlideDeck;
@ -410,6 +411,7 @@ public class ThreadDatabase extends Database {
deleteThread(threadId);
notifyConversationListeners(threadId);
notifyConversationListListeners();
SessionMetaProtocol.clearReceivedMessages();
}
public boolean hasThread(long threadId) {

9
app/src/main/java/org/thoughtcrime/securesms/loki/protocol/SessionMetaProtocol.kt
View File

@ -24,6 +24,15 @@ object SessionMetaProtocol {
timestamps.add(timestamp)
}
@JvmStatic
fun clearReceivedMessages() {
timestamps.clear()
}
fun removeTimestamps(timestamps: Set<Long>) {
this.timestamps.removeAll(timestamps)
}
@JvmStatic
fun shouldIgnoreMessage(timestamp: Long): Boolean {
val shouldIgnoreMessage = timestamps.contains(timestamp)

4
libsession/src/main/java/org/session/libsession/messaging/StorageProtocol.kt
View File

@ -63,7 +63,7 @@ interface StorageProtocol {
// Open Groups
fun getThreadID(openGroupID: String): String?
fun addOpenGroup(server: String, channel: Long)
fun addOpenGroup(serverUrl: String, channel: Long)
fun setOpenGroupServerMessageID(messageID: Long, serverID: Long, threadID: Long, isSms: Boolean)
fun getQuoteServerID(quoteID: Long, publicKey: String): Long?
@ -95,7 +95,7 @@ interface StorageProtocol {
fun isMessageDuplicated(timestamp: Long, sender: String): Boolean
fun getReceivedMessageTimestamps(): Set<Long>
fun addReceivedMessageTimestamp(timestamp: Long)
// fun removeReceivedMessageTimestamps(timestamps: Set<Long>)
fun removeReceivedMessageTimestamps(timestamps: Set<Long>)
// Returns the IDs of the saved attachments.
fun persistAttachments(messageId: Long, attachments: List<Attachment>): List<Long>
fun getAttachmentsForMessage(messageId: Long): List<DatabaseAttachment>

106
libsession/src/main/java/org/session/libsession/messaging/file_server/FileServerAPIV2.kt Normal file
View File

@ -0,0 +1,106 @@
package org.session.libsession.messaging.file_server
import nl.komponents.kovenant.Promise
import nl.komponents.kovenant.functional.bind
import nl.komponents.kovenant.functional.map
import okhttp3.Headers
import okhttp3.HttpUrl
import okhttp3.MediaType
import okhttp3.RequestBody
import org.session.libsession.messaging.MessagingModuleConfiguration
import org.session.libsession.messaging.open_groups.OpenGroupAPIV2
import org.session.libsession.snode.OnionRequestAPI
import org.session.libsignal.service.loki.HTTP
import org.session.libsignal.service.loki.utilities.retryIfNeeded
import org.session.libsignal.utilities.Base64
import org.session.libsignal.utilities.JsonUtil
import org.session.libsignal.utilities.logging.Log
object FileServerAPIV2 {
const val DEFAULT_SERVER = "http://88.99.175.227"
private const val DEFAULT_SERVER_PUBLIC_KEY = "7cb31905b55cd5580c686911debf672577b3fb0bff81df4ce2d5c4cb3a7aaa69"
sealed class Error : Exception() {
object PARSING_FAILED : Error()
object INVALID_URL : Error()
fun errorDescription() = when (this) {
PARSING_FAILED -> "Invalid response."
INVALID_URL -> "Invalid URL."
}
}
data class Request(
val verb: HTTP.Verb,
val endpoint: String,
val queryParameters: Map<String, String> = mapOf(),
val parameters: Any? = null,
val headers: Map<String, String> = mapOf(),
// Always `true` under normal circumstances. You might want to disable
// this when running over Lokinet.
val useOnionRouting: Boolean = true
)
private fun createBody(parameters: Any?): RequestBody? {
if (parameters == null) return null
val parametersAsJSON = JsonUtil.toJson(parameters)
return RequestBody.create(MediaType.get("application/json"), parametersAsJSON)
}
private fun send(request: Request): Promise<Map<*, *>, Exception> {
val parsed = HttpUrl.parse(DEFAULT_SERVER) ?: return Promise.ofFail(OpenGroupAPIV2.Error.INVALID_URL)
val urlBuilder = HttpUrl.Builder()
.scheme(parsed.scheme())
.host(parsed.host())
.port(parsed.port())
.addPathSegments(request.endpoint)
if (request.verb == HTTP.Verb.GET) {
for ((key, value) in request.queryParameters) {
urlBuilder.addQueryParameter(key, value)
}
}
val requestBuilder = okhttp3.Request.Builder()
.url(urlBuilder.build())
.headers(Headers.of(request.headers))
when (request.verb) {
HTTP.Verb.GET -> requestBuilder.get()
HTTP.Verb.PUT -> requestBuilder.put(createBody(request.parameters)!!)
HTTP.Verb.POST -> requestBuilder.post(createBody(request.parameters)!!)
HTTP.Verb.DELETE -> requestBuilder.delete(createBody(request.parameters))
}
if (request.useOnionRouting) {
return OnionRequestAPI.sendOnionRequest(requestBuilder.build(), DEFAULT_SERVER, DEFAULT_SERVER_PUBLIC_KEY)
.fail { e ->
Log.e("Loki", "FileServerV2 failed with error",e)
}
} else {
return Promise.ofFail(IllegalStateException("It's currently not allowed to send non onion routed requests."))
}
}
// region Sending
fun upload(file: ByteArray): Promise<Long, Exception> {
val base64EncodedFile = Base64.encodeBytes(file)
val parameters = mapOf("file" to base64EncodedFile)
val request = Request(verb = HTTP.Verb.POST, endpoint = "files", parameters = parameters)
return send(request).map { json ->
json["result"] as? Long ?: throw OpenGroupAPIV2.Error.PARSING_FAILED
}
}
fun download(file: Long): Promise<ByteArray, Exception> {
val request = Request(verb = HTTP.Verb.GET, endpoint = "files/$file")
return send(request).map { json ->
val base64EncodedFile = json["result"] as? String ?: throw Error.PARSING_FAILED
Base64.decode(base64EncodedFile) ?: throw Error.PARSING_FAILED
}
}
}

7
libsession/src/main/java/org/session/libsession/messaging/jobs/AttachmentDownloadJob.kt
View File

@ -3,9 +3,11 @@ package org.session.libsession.messaging.jobs
import okhttp3.HttpUrl
import org.session.libsession.messaging.MessagingModuleConfiguration
import org.session.libsession.messaging.file_server.FileServerAPI
import org.session.libsession.messaging.file_server.FileServerAPIV2
import org.session.libsession.messaging.open_groups.OpenGroupAPIV2
import org.session.libsession.messaging.sending_receiving.attachments.AttachmentState
import org.session.libsession.messaging.utilities.DotNetAPI
import org.session.libsession.utilities.DownloadUtilities
import org.session.libsignal.service.api.crypto.AttachmentCipherInputStream
import org.session.libsignal.utilities.Base64
import org.session.libsignal.utilities.logging.Log
@ -58,10 +60,7 @@ class AttachmentDownloadJob(val attachmentID: Long, val databaseMessageID: Long)
val openGroupV2 = MessagingModuleConfiguration.shared.storage.getV2OpenGroup(threadId.toString())
val stream = if (openGroupV2 == null) {
FileServerAPI.shared.downloadFile(tempFile, attachment.url, null)
DownloadUtilities.downloadFile(tempFile, attachment.url, FileServerAPI.maxFileSize, null)
// Assume we're retrieving an attachment for an open group server if the digest is not set
if (attachment.digest?.size ?: 0 == 0 || attachment.key.isNullOrEmpty()) FileInputStream(tempFile)
else AttachmentCipherInputStream.createForAttachment(tempFile, attachment.size, Base64.decode(attachment.key), attachment.digest)

7
libsession/src/main/java/org/session/libsession/messaging/messages/control/ConfigurationMessage.kt
View File

@ -113,8 +113,11 @@ class ConfigurationMessage(var closedGroups: List<ClosedGroup>, var openGroups:
}
if (groupRecord.isOpenGroup) {
val threadID = storage.getThreadID(groupRecord.encodedId) ?: continue
val openGroup = storage.getOpenGroup(threadID) ?: continue
openGroups.add(openGroup.server)
val openGroup = storage.getOpenGroup(threadID)
val openGroupV2 = storage.getV2OpenGroup(threadID)
val shareUrl = openGroup?.server ?: openGroupV2?.toJoinUrl() ?: continue
openGroups.add(shareUrl)
}
}

19
libsession/src/main/java/org/session/libsession/messaging/open_groups/OpenGroupAPIV2.kt
View File

@ -45,6 +45,16 @@ object OpenGroupAPIV2 {
object SIGNING_FAILED : Error()
object INVALID_URL : Error()
object NO_PUBLIC_KEY : Error()
fun errorDescription() = when (this) {
Error.GENERIC -> "An error occurred."
Error.PARSING_FAILED -> "Invalid response."
Error.DECRYPTION_FAILED -> "Couldn't decrypt response."
Error.SIGNING_FAILED -> "Couldn't sign message."
Error.INVALID_URL -> "Invalid URL."
Error.NO_PUBLIC_KEY -> "Couldn't find server public key."
}
}
data class DefaultGroup(val id: String,
@ -493,13 +503,4 @@ object OpenGroupAPIV2 {
}
// endregion
}
fun Error.errorDescription() = when (this) {
Error.GENERIC -> "An error occurred."
Error.PARSING_FAILED -> "Invalid response."
Error.DECRYPTION_FAILED -> "Couldn't decrypt response."
Error.SIGNING_FAILED -> "Couldn't sign message."
Error.INVALID_URL -> "Invalid URL."
Error.NO_PUBLIC_KEY -> "Couldn't find server public key."
}

2
libsession/src/main/java/org/session/libsession/messaging/open_groups/OpenGroupV2.kt
View File

@ -39,6 +39,8 @@ data class OpenGroupV2(
}
fun toJoinUrl(): String = "$server/$room?public_key=$publicKey"
fun toJson(): Map<String,String> = mapOf(
"room" to room,
"server" to server,

6
libsession/src/main/java/org/session/libsession/messaging/sending_receiving/ReceivedMessageHandler.kt
View File

@ -1,6 +1,7 @@
package org.session.libsession.messaging.sending_receiving
import android.text.TextUtils
import okhttp3.HttpUrl
import org.session.libsession.messaging.MessagingModuleConfiguration
import org.session.libsession.messaging.jobs.AttachmentDownloadJob
import org.session.libsession.messaging.jobs.JobQueue
@ -125,10 +126,9 @@ private fun handleConfigurationMessage(message: ConfigurationMessage) {
handleNewClosedGroup(message.sender!!, message.sentTimestamp!!, closeGroup.publicKey, closeGroup.name, closeGroup.encryptionKeyPair!!, closeGroup.members, closeGroup.admins, message.sentTimestamp!!)
}
val allOpenGroups = storage.getAllOpenGroups().map { it.value.server }
val allV2OpenGroups = storage.getAllV2OpenGroups().map { it.value.server }
val allV2OpenGroups = storage.getAllV2OpenGroups().map { it.value.toJoinUrl() }
for (openGroup in message.openGroups) {
if (allOpenGroups.contains(openGroup)) continue
// TODO: add in v2
if (allOpenGroups.contains(openGroup) || allV2OpenGroups.contains(openGroup)) continue
storage.addOpenGroup(openGroup, 1)
}
if (message.displayName.isNotEmpty()) {

73
libsession/src/main/java/org/session/libsession/messaging/utilities/DotNetAPI.kt
View File

@ -178,80 +178,9 @@ open class DotNetAPI {
return execute(HTTPVerb.PATCH, server, "users/me", parameters = parameters)
}
// DOWNLOAD
/**
* Blocks the calling thread.
*/
fun downloadFile(destination: File, url: String, listener: SignalServiceAttachment.ProgressListener?) {
val outputStream = FileOutputStream(destination) // Throws
var remainingAttempts = 4
var exception: Exception? = null
while (remainingAttempts > 0) {
remainingAttempts -= 1
try {
downloadFile(outputStream, url, listener)
exception = null
break
} catch (e: Exception) {
exception = e
}
}
if (exception != null) { throw exception }
}
/**
* Blocks the calling thread.
*/
fun downloadFile(outputStream: OutputStream, url: String, listener: SignalServiceAttachment.ProgressListener?) {
// We need to throw a PushNetworkException or NonSuccessfulResponseCodeException
// because the underlying Signal logic requires these to work correctly
val oldPrefixedHost = "https://" + HttpUrl.get(url).host()
var newPrefixedHost = oldPrefixedHost
if (oldPrefixedHost.contains(FileServerAPI.fileStorageBucketURL)) {
newPrefixedHost = FileServerAPI.shared.server
}
// Edge case that needs to work: https://file-static.lokinet.org/i1pNmpInq3w9gF3TP8TFCa1rSo38J6UM
// → https://file.getsession.org/loki/v1/f/XLxogNXVEIWHk14NVCDeppzTujPHxu35
val fileID = url.substringAfter(oldPrefixedHost).substringAfter("/f/")
val sanitizedURL = "$newPrefixedHost/loki/v1/f/$fileID"
val request = Request.Builder().url(sanitizedURL).get()
try {
val serverPublicKey = if (newPrefixedHost.contains(FileServerAPI.shared.server)) FileServerAPI.fileServerPublicKey
else FileServerAPI.shared.getPublicKeyForOpenGroupServer(newPrefixedHost).get()
val json = OnionRequestAPI.sendOnionRequest(request.build(), newPrefixedHost, serverPublicKey, isJSONRequired = false).get()
val result = json["result"] as? String
if (result == null) {
Log.d("Loki", "Couldn't parse attachment from: $json.")
throw PushNetworkException("Missing response body.")
}
val body = Base64.decode(result)
if (body.size > FileServerAPI.maxFileSize) {
Log.d("Loki", "Attachment size limit exceeded.")
throw PushNetworkException("Max response size exceeded.")
}
val input = body.inputStream()
val buffer = ByteArray(32768)
var count = 0
var bytes = input.read(buffer)
while (bytes >= 0) {
outputStream.write(buffer, 0, bytes)
count += bytes
if (count > FileServerAPI.maxFileSize) {
Log.d("Loki", "Attachment size limit exceeded.")
throw PushNetworkException("Max response size exceeded.")
}
listener?.onAttachmentProgress(body.size.toLong(), count.toLong())
bytes = input.read(buffer)
}
} catch (e: Exception) {
Log.d("Loki", "Couldn't download attachment due to error: $e.")
throw if (e is NonSuccessfulResponseCodeException) e else PushNetworkException(e)
}
}
// UPLOAD
// TODO: migrate to v2 file server
@Throws(PushNetworkException::class, NonSuccessfulResponseCodeException::class)
fun uploadAttachment(server: String, attachment: PushAttachmentData): UploadResult {
// This function mimics what Signal does in PushServiceSocket

100
libsession/src/main/java/org/session/libsession/utilities/DownloadUtilities.kt
View File

@ -3,7 +3,9 @@ package org.session.libsession.utilities
import okhttp3.HttpUrl
import okhttp3.Request
import org.session.libsession.messaging.file_server.FileServerAPI
import org.session.libsession.messaging.file_server.FileServerAPIV2
import org.session.libsession.snode.OnionRequestAPI
import org.session.libsignal.service.api.crypto.AttachmentCipherInputStream
import org.session.libsignal.utilities.logging.Log
import org.session.libsignal.service.api.messages.SignalServiceAttachment
import org.session.libsignal.service.api.push.exceptions.NonSuccessfulResponseCodeException
@ -39,50 +41,64 @@ object DownloadUtilities {
*/
@JvmStatic
fun downloadFile(outputStream: OutputStream, url: String, maxSize: Int, listener: SignalServiceAttachment.ProgressListener?) {
// We need to throw a PushNetworkException or NonSuccessfulResponseCodeException
// because the underlying Signal logic requires these to work correctly
val oldPrefixedHost = "https://" + HttpUrl.get(url).host()
var newPrefixedHost = oldPrefixedHost
if (oldPrefixedHost.contains(FileServerAPI.fileStorageBucketURL)) {
newPrefixedHost = FileServerAPI.shared.server
}
// Edge case that needs to work: https://file-static.lokinet.org/i1pNmpInq3w9gF3TP8TFCa1rSo38J6UM
// → https://file.getsession.org/loki/v1/f/XLxogNXVEIWHk14NVCDeppzTujPHxu35
val fileID = url.substringAfter(oldPrefixedHost).substringAfter("/f/")
val sanitizedURL = "$newPrefixedHost/loki/v1/f/$fileID"
val request = Request.Builder().url(sanitizedURL).get()
try {
val serverPublicKey = if (newPrefixedHost.contains(FileServerAPI.shared.server)) FileServerAPI.fileServerPublicKey
else FileServerAPI.shared.getPublicKeyForOpenGroupServer(newPrefixedHost).get()
val json = OnionRequestAPI.sendOnionRequest(request.build(), newPrefixedHost, serverPublicKey, isJSONRequired = false).get()
val result = json["result"] as? String
if (result == null) {
Log.d("Loki", "Couldn't parse attachment from: $json.")
throw PushNetworkException("Missing response body.")
}
val body = Base64.decode(result)
if (body.size > maxSize) {
Log.d("Loki", "Attachment size limit exceeded.")
throw PushNetworkException("Max response size exceeded.")
}
body.inputStream().use { input ->
val buffer = ByteArray(32768)
var count = 0
var bytes = input.read(buffer)
while (bytes >= 0) {
outputStream.write(buffer, 0, bytes)
count += bytes
if (count > maxSize) {
Log.d("Loki", "Attachment size limit exceeded.")
throw PushNetworkException("Max response size exceeded.")
}
listener?.onAttachmentProgress(body.size.toLong(), count.toLong())
bytes = input.read(buffer)
if (url.contains(FileServerAPIV2.DEFAULT_SERVER)) {
val httpUrl = HttpUrl.parse(url)!!
val fileId = httpUrl.pathSegments().last()
try {
FileServerAPIV2.download(fileId.toLong()).get().let {
outputStream.write(it)
}
} catch (e: Exception) {
Log.e("Loki", "Couln't download attachment due to error",e)
throw e
}
} else {
// We need to throw a PushNetworkException or NonSuccessfulResponseCodeException
// because the underlying Signal logic requires these to work correctly
val oldPrefixedHost = "https://" + HttpUrl.get(url).host()
var newPrefixedHost = oldPrefixedHost
if (oldPrefixedHost.contains(FileServerAPI.fileStorageBucketURL)) {
newPrefixedHost = FileServerAPI.shared.server
}
// Edge case that needs to work: https://file-static.lokinet.org/i1pNmpInq3w9gF3TP8TFCa1rSo38J6UM
// → https://file.getsession.org/loki/v1/f/XLxogNXVEIWHk14NVCDeppzTujPHxu35
val fileID = url.substringAfter(oldPrefixedHost).substringAfter("/f/")
val sanitizedURL = "$newPrefixedHost/loki/v1/f/$fileID"
val request = Request.Builder().url(sanitizedURL).get()
try {
val serverPublicKey = if (newPrefixedHost.contains(FileServerAPI.shared.server)) FileServerAPI.fileServerPublicKey
else FileServerAPI.shared.getPublicKeyForOpenGroupServer(newPrefixedHost).get()
val json = OnionRequestAPI.sendOnionRequest(request.build(), newPrefixedHost, serverPublicKey, isJSONRequired = false).get()
val result = json["result"] as? String
if (result == null) {
Log.d("Loki", "Couldn't parse attachment from: $json.")
throw PushNetworkException("Missing response body.")
}
val body = Base64.decode(result)
if (body.size > maxSize) {
Log.d("Loki", "Attachment size limit exceeded.")
throw PushNetworkException("Max response size exceeded.")
}
body.inputStream().use { input ->
val buffer = ByteArray(32768)
var count = 0
var bytes = input.read(buffer)
while (bytes >= 0) {
outputStream.write(buffer, 0, bytes)
count += bytes
if (count > maxSize) {
Log.d("Loki", "Attachment size limit exceeded.")
throw PushNetworkException("Max response size exceeded.")
}
listener?.onAttachmentProgress(body.size.toLong(), count.toLong())
bytes = input.read(buffer)
}
}
} catch (e: Exception) {
Log.e("Loki", "Couldn't download attachment due to error", e)
throw if (e is NonSuccessfulResponseCodeException) e else PushNetworkException(e)
}
} catch (e: Exception) {
Log.d("Loki", "Couldn't download attachment due to error: $e.")
throw if (e is NonSuccessfulResponseCodeException) e else PushNetworkException(e)
}
}
}