feat: update libsession-util and get signCallbacks working for authenticated retrieve, handling incoming messages in new closed groups

app/src/main/java/org/thoughtcrime/securesms/database/Storage.kt

@@ -1209,25 +1209,26 @@ open class Storage(
         configFactory.getGroupMemberConfig(SessionId.from(groupPublicKey))?.use { it.all() }?.toList() ?: emptyList()
 
     override fun acceptClosedGroupInvite(groupId: SessionId, name: String, authData: ByteArray, invitingAdmin: SessionId) {
-        val recipient = Recipient.from(context, Address.fromSerialized(groupId.hexString()), false)
+        val recipient = Recipient.from(context, fromSerialized(groupId.hexString()), false)
         val profileManager = SSKEnvironment.shared.profileManager
         val groups = configFactory.userGroups ?: return
         val closedGroupInfo = GroupInfo.ClosedGroupInfo(
             groupId, byteArrayOf(), authData, PRIORITY_VISIBLE
         )
         groups.set(closedGroupInfo)
+        configFactory.persist(groups, SnodeAPI.nowWithOffset)
         profileManager.setName(context, recipient, name)
         setRecipientApprovedMe(recipient, true)
         setRecipientApproved(recipient, true)
         getOrCreateThreadIdFor(recipient.address)
         pollerFactory.pollerFor(groupId)?.start()
-        val invitingAdminAddress = Address.fromSerialized(invitingAdmin.hexString())
         val inviteResponse = GroupUpdateInviteResponseMessage.newBuilder()
             .setIsApproved(true)
         val responseData = DataMessage.GroupUpdateMessage.newBuilder()
             .setInviteResponse(inviteResponse)
         val responseMessage = GroupUpdated(responseData.build())
-        MessageSender.send(responseMessage, invitingAdminAddress)
+        // this will fail the first couple of times :)
+        MessageSender.send(responseMessage, fromSerialized(groupId.hexString()))
     }
 
     override fun setGroupInviteComplete(approved: Boolean, invitee: String, closedGroup: SessionId) {

app/src/main/res/xml/network_security_configuration.xml

@@ -21,4 +21,7 @@
             <certificates src="@raw/seed3"/>
         </trust-anchors>
     </domain-config>
+    <domain-config cleartextTrafficPermitted="true">
+        <domain includeSubdomains="false">public.loki.foundation</domain>
+    </domain-config>
 </network-security-config>

libsession-util/libsession-util

@@ -1 +1 @@
-Subproject commit e4b0358a50a65796ac5e2da4938505bc8ebf0313
+Subproject commit 986caaa28305fa6481238d98e6f7dd096c562336

libsession/src/main/java/org/session/libsession/messaging/jobs/InviteContactJob.kt

@@ -1,9 +1,25 @@
 package org.session.libsession.messaging.jobs
 
+import com.google.protobuf.ByteString
+import org.session.libsession.messaging.MessagingModuleConfiguration
+import org.session.libsession.messaging.messages.Destination
+import org.session.libsession.messaging.messages.control.GroupUpdated
+import org.session.libsession.messaging.sending_receiving.MessageSender
 import org.session.libsession.messaging.utilities.Data
+import org.session.libsession.messaging.utilities.SodiumUtilities
+import org.session.libsession.snode.SnodeAPI
+import org.session.libsignal.protos.SignalServiceProtos.DataMessage.GroupUpdateInviteMessage
+import org.session.libsignal.protos.SignalServiceProtos.DataMessage.GroupUpdateMessage
+import org.session.libsignal.protos.SignalServiceProtos.DataMessage.LokiProfile
+import org.session.libsignal.utilities.Log
+import org.session.libsignal.utilities.SessionId
 
 class InviteContactJob(val groupSessionId: String, val memberSessionId: String): Job {
 
+    sealed class InviteError(message: String): Exception(message) {
+        object NO_GROUP_KEYS: InviteError("No group keys config for this group")
+    }
+
     companion object {
         const val KEY = "InviteContactJob"
         private const val GROUP = "group"
@@ -16,7 +32,47 @@ class InviteContactJob(val groupSessionId: String, val memberSessionId: String):
     override val maxFailureCount: Int = 1
 
     override suspend fun execute(dispatcherName: String) {
-        TODO("Not yet implemented")
+        val delegate = delegate ?: return
+        val configs = MessagingModuleConfiguration.shared.configFactory
+        val storage = MessagingModuleConfiguration.shared.storage
+        val adminKey = configs.userGroups?.getClosedGroup(groupSessionId)?.adminKey
+                ?: return delegate.handleJobFailedPermanently(this, dispatcherName, NullPointerException("No admin key"))
+        val subAccount = configs.getGroupKeysConfig(SessionId.from(groupSessionId))?.use { keys ->
+            keys.makeSubAccount(SessionId.from(memberSessionId))
+        } ?: return delegate.handleJobFailedPermanently(this, dispatcherName, InviteError.NO_GROUP_KEYS)
+        val timestamp = SnodeAPI.nowWithOffset
+        val messageToSign = "INVITE$memberSessionId$timestamp"
+        val signature = SodiumUtilities.sign(messageToSign.toByteArray(), adminKey)
+        val userProfile = storage.getUserProfile()
+        val lokiProfile = LokiProfile.newBuilder()
+                .setDisplayName(userProfile.displayName)
+        if (userProfile.profilePictureURL?.isNotEmpty() == true) {
+            lokiProfile.profilePicture = userProfile.profilePictureURL
+        }
+        val groupInvite = GroupUpdateInviteMessage.newBuilder()
+                .setGroupSessionId(groupSessionId)
+                .setMemberAuthData(ByteString.copyFrom(subAccount))
+                .setAdminSignature(ByteString.copyFrom(signature))
+                .setName(userProfile.displayName)
+                .setProfile(lokiProfile.build())
+        if (userProfile.profileKey?.isNotEmpty() == true) {
+            groupInvite.profileKey = ByteString.copyFrom(userProfile.profileKey)
+        }
+        val message = GroupUpdateMessage.newBuilder()
+                .setInviteMessage(groupInvite)
+                .build()
+        val update = GroupUpdated(message).apply {
+            sentTimestamp = timestamp
+        }
+        try {
+            MessageSender.send(update, Destination.Contact(memberSessionId), false).get()
+            Log.d("InviteContactJob", "Sent invite message successfully")
+            delegate.handleJobSucceeded(this, dispatcherName)
+        } catch (e: Exception) {
+            Log.e("InviteContactJob", e)
+            delegate.handleJobFailed(this, dispatcherName, e)
+        }
+
     }
 
     override fun serialize(): Data =

libsession/src/main/java/org/session/libsession/messaging/jobs/JobQueue.kt

@@ -122,7 +122,11 @@ class JobQueue : JobDelegate {
 
             while (isActive) {
                 when (val job = queue.receive()) {
-                    is NotifyPNServerJob, is AttachmentUploadJob, is MessageSendJob, is ConfigurationSyncJob -> {
+                    is NotifyPNServerJob,
+                    is AttachmentUploadJob,
+                    is MessageSendJob,
+                    is ConfigurationSyncJob,
+                    is InviteContactJob -> {
                         txQueue.send(job)
                     }
                     is RetrieveProfileAvatarJob,

libsession/src/main/java/org/session/libsession/messaging/messages/control/GroupUpdated.kt

@@ -6,6 +6,10 @@ import org.session.libsignal.protos.SignalServiceProtos.DataMessage.GroupUpdateM
 
 class GroupUpdated(val inner: GroupUpdateMessage): ControlMessage() {
 
+    override fun isValid(): Boolean {
+        return true // TODO: add the validation here
+    }
+
     companion object {
         fun fromProto(message: Content): GroupUpdated? =
             if (message.hasDataMessage() && message.dataMessage.hasGroupUpdateMessage())

libsession/src/main/java/org/session/libsession/messaging/messages/signal/IncomingTextMessage.java

@@ -8,11 +8,12 @@ import androidx.annotation.Nullable;
 import org.session.libsession.messaging.calls.CallMessageType;
 import org.session.libsession.messaging.messages.visible.OpenGroupInvitation;
 import org.session.libsession.messaging.messages.visible.VisibleMessage;
-import org.session.libsession.utilities.Address;
 import org.session.libsession.messaging.utilities.UpdateMessageData;
+import org.session.libsession.utilities.Address;
 import org.session.libsession.utilities.GroupUtil;
-import org.session.libsignal.utilities.guava.Optional;
 import org.session.libsignal.messages.SignalServiceGroup;
+import org.session.libsignal.utilities.Hex;
+import org.session.libsignal.utilities.guava.Optional;
 
 public class IncomingTextMessage implements Parcelable {
 
@@ -78,7 +79,14 @@ public class IncomingTextMessage implements Parcelable {
     this.hasMention           = hasMention;
 
     if (group.isPresent()) {
-      this.groupId = Address.fromSerialized(GroupUtil.getEncodedId(group.get()));
+      SignalServiceGroup groupObject = group.get();
+      if (groupObject.isNewClosedGroup()) {
+        // new closed group 03..etc..
+        this.groupId = Address.fromSerialized(Hex.toStringCondensed(groupObject.getGroupId()));
+      } else {
+        // old closed group or open group
+        this.groupId = Address.fromSerialized(GroupUtil.getEncodedId(group.get()));
+      }
     } else {
       this.groupId = null;
     }

libsession/src/main/java/org/session/libsession/messaging/sending_receiving/MessageReceiver.kt

@@ -154,8 +154,8 @@ object MessageReceiver {
             MessageRequestResponse.fromProto(proto) ?:
             CallMessage.fromProto(proto) ?:
             SharedConfigurationMessage.fromProto(proto) ?:
-            VisibleMessage.fromProto(proto) ?:
+            GroupUpdated.fromProto(proto) ?:
-            GroupUpdated.fromProto(proto) ?: run {
+            VisibleMessage.fromProto(proto) ?: run {
             throw Error.UnknownMessage
         }
         val isUserBlindedSender = sender == openGroupPublicKey?.let { SodiumUtilities.blindedKeyPair(it, MessagingModuleConfiguration.shared.getUserED25519KeyPair()!!) }?.let { SessionId(IdPrefix.BLINDED, it.publicKey.asBytes).hexString() }

libsession/src/main/java/org/session/libsession/messaging/sending_receiving/MessageSender.kt

@@ -169,6 +169,9 @@ object MessageSender {
                 val groupKeys = configFactory.getGroupKeysConfig(SessionId.from(destination.publicKey)) ?: throw Error.NoKeyPair
                 val envelope = MessageWrapper.createEnvelope(kind, message.sentTimestamp!!, senderPublicKey, proto.build().toByteArray())
                 groupKeys.use { keys ->
+                    if (keys.keys().isEmpty()) {
+                        throw Error.EncryptionFailed
+                    }
                     keys.encrypt(envelope.toByteArray())
                 }
             }
@@ -199,7 +202,6 @@ object MessageSender {
         val storage = MessagingModuleConfiguration.shared.storage
         val configFactory = MessagingModuleConfiguration.shared.configFactory
         val userPublicKey = storage.getUserPublicKey()
-        val ourProfile = storage.getUserProfile()
 
         // recipient will be set later, so initialize it as a function here
         val isSelfSend = { message.recipient == userPublicKey }

libsession/src/main/java/org/session/libsession/messaging/sending_receiving/ReceivedMessageHandler.kt

@@ -70,7 +70,7 @@ fun MessageReceiver.handle(message: Message, proto: SignalServiceProtos.Content,
         is ReadReceipt -> handleReadReceipt(message)
         is TypingIndicator -> handleTypingIndicator(message)
         is ClosedGroupControlMessage -> handleClosedGroupControlMessage(message)
-        is GroupUpdated -> handleGroupUpdated(message, closedGroup!!)
+        is GroupUpdated -> handleGroupUpdated(message, closedGroup)
         is ExpirationTimerUpdate -> handleExpirationTimerUpdate(message)
         is DataExtractionNotification -> handleDataExtractionNotification(message)
         is ConfigurationMessage -> handleConfigurationMessage(message)
@@ -519,10 +519,13 @@ private fun ClosedGroupControlMessage.getPublicKey(): String = kind!!.let { when
     is ClosedGroupControlMessage.Kind.NameChange -> groupPublicKey!!
 }}
 
-private fun MessageReceiver.handleGroupUpdated(message: GroupUpdated, closedGroup: SessionId) {
+private fun MessageReceiver.handleGroupUpdated(message: GroupUpdated, closedGroup: SessionId?) {
+    if (closedGroup == null && !message.inner.hasInviteMessage()) { // TODO: add all the cases for this
+        throw NullPointerException("Message wasn't polled from a closed group!")
+    }
     when {
         message.inner.hasInviteMessage() -> handleNewLibSessionClosedGroupMessage(message)
-        message.inner.hasInviteResponse() -> handleInviteResponse(message, closedGroup)
+        message.inner.hasInviteResponse() -> handleInviteResponse(message, closedGroup!!)
     }
 }
 

libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/ClosedGroupPoller.kt

@@ -10,6 +10,7 @@ import network.loki.messenger.libsession_util.GroupInfoConfig
 import network.loki.messenger.libsession_util.GroupKeysConfig
 import network.loki.messenger.libsession_util.GroupMembersConfig
 import network.loki.messenger.libsession_util.util.GroupInfo
+import network.loki.messenger.libsession_util.util.GroupInfo.ClosedGroupInfo.Companion.isAuthData
 import org.session.libsession.messaging.MessagingModuleConfiguration
 import org.session.libsession.messaging.jobs.BatchMessageReceiveJob
 import org.session.libsession.messaging.jobs.JobQueue
@@ -30,9 +31,9 @@ class ClosedGroupPoller(private val executor: CoroutineScope,
                         private val configFactoryProtocol: ConfigFactoryProtocol) {
 
     data class ParsedRawMessage(
-        val data: ByteArray,
+            val data: ByteArray,
-        val hash: String,
+            val hash: String,
-        val timestamp: Long
+            val timestamp: Long
     ) {
         override fun equals(other: Any?): Boolean {
             if (this === other) return true
@@ -69,7 +70,7 @@ class ClosedGroupPoller(private val executor: CoroutineScope,
         if (ENABLE_LOGGING) Log.d("ClosedGroupPoller", "Starting closed group poller for ${closedGroupSessionId.hexString().take(4)}")
         job?.cancel()
         job = executor.launch(Dispatchers.IO) {
-            val closedGroups = configFactoryProtocol.userGroups?: return@launch
+            val closedGroups = configFactoryProtocol.userGroups ?: return@launch
             isRunning = true
             while (isActive && isRunning) {
                 val group = closedGroups.getClosedGroup(closedGroupSessionId.hexString()) ?: break
@@ -95,7 +96,8 @@ class ClosedGroupPoller(private val executor: CoroutineScope,
         try {
             val snode = SnodeAPI.getSingleTargetSnode(closedGroupSessionId.hexString()).get()
             val info = configFactoryProtocol.getGroupInfoConfig(closedGroupSessionId) ?: return null
-            val members = configFactoryProtocol.getGroupMemberConfig(closedGroupSessionId) ?: return null
+            val members = configFactoryProtocol.getGroupMemberConfig(closedGroupSessionId)
+                    ?: return null
             val keys = configFactoryProtocol.getGroupKeysConfig(closedGroupSessionId) ?: return null
 
             val hashesToExtend = mutableSetOf<String>()
@@ -109,53 +111,58 @@ class ClosedGroupPoller(private val executor: CoroutineScope,
             val membersIndex = 2
             val messageIndex = 3
 
+            val authData = group.signingKey()
+            val signCallback = if (isAuthData(authData)) {
+                SnodeAPI.subkeyCallback(authData, keys, false)
+            } else SnodeAPI.signingKeyCallback(authData)
+
             val messagePoll = SnodeAPI.buildAuthenticatedRetrieveBatchRequest(
-                snode,
+                    snode,
-                closedGroupSessionId.hexString(),
+                    closedGroupSessionId.hexString(),
-                Namespace.CLOSED_GROUP_MESSAGES(),
+                    Namespace.CLOSED_GROUP_MESSAGES(),
-                maxSize = null,
+                    maxSize = null,
-                group.signingKey()
+                    signCallback
             ) ?: return null
             val infoPoll = SnodeAPI.buildAuthenticatedRetrieveBatchRequest(
-                snode,
+                    snode,
-                closedGroupSessionId.hexString(),
+                    closedGroupSessionId.hexString(),
-                info.namespace(),
+                    info.namespace(),
-                maxSize = null,
+                    maxSize = null,
-                group.signingKey()
+                    signCallback
             ) ?: return null
             val membersPoll = SnodeAPI.buildAuthenticatedRetrieveBatchRequest(
-                snode,
+                    snode,
-                closedGroupSessionId.hexString(),
+                    closedGroupSessionId.hexString(),
-                members.namespace(),
+                    members.namespace(),
-                maxSize = null,
+                    maxSize = null,
-                group.signingKey()
+                    signCallback
             ) ?: return null
             val keysPoll = SnodeAPI.buildAuthenticatedRetrieveBatchRequest(
-                snode,
+                    snode,
-                closedGroupSessionId.hexString(),
+                    closedGroupSessionId.hexString(),
-                keys.namespace(),
+                    keys.namespace(),
-                maxSize = null,
+                    maxSize = null,
-                group.signingKey()
+                    signCallback
             ) ?: return null
 
             val requests = mutableListOf(keysPoll, infoPoll, membersPoll, messagePoll)
 
             if (hashesToExtend.isNotEmpty()) {
                 SnodeAPI.buildAuthenticatedAlterTtlBatchRequest(
-                    messageHashes = hashesToExtend.toList(),
+                        messageHashes = hashesToExtend.toList(),
-                    publicKey = closedGroupSessionId.hexString(),
+                        publicKey = closedGroupSessionId.hexString(),
-                    signingKey = group.signingKey(),
+                        signingKey = group.signingKey(),
-                    newExpiry = SnodeAPI.nowWithOffset + 14.days.inWholeMilliseconds,
+                        newExpiry = SnodeAPI.nowWithOffset + 14.days.inWholeMilliseconds,
-                    extend = true
+                        extend = true
                 )?.let { extensionRequest ->
                     requests += extensionRequest
                 }
             }
 
             val pollResult = SnodeAPI.getRawBatchResponse(
-                snode,
+                    snode,
-                closedGroupSessionId.hexString(),
+                    closedGroupSessionId.hexString(),
-                requests
+                    requests
             ).get()
 
             // if poll result body is null here we don't have any things ig

libsession/src/main/java/org/session/libsession/messaging/utilities/SodiumUtilities.kt

@@ -5,6 +5,7 @@ import com.goterl.lazysodium.SodiumAndroid
 import com.goterl.lazysodium.interfaces.AEAD
 import com.goterl.lazysodium.interfaces.GenericHash
 import com.goterl.lazysodium.interfaces.Hash
+import com.goterl.lazysodium.interfaces.Sign
 import com.goterl.lazysodium.utils.Key
 import com.goterl.lazysodium.utils.KeyPair
 import org.session.libsignal.utilities.Hex
@@ -237,4 +238,15 @@ object SodiumUtilities {
         return sodium.cryptoSignVerifyDetached(signature, messageToVerify, messageToVerify.size, publicKey)
     }
 
+    /**
+     * For signing
+     */
+    fun sign(message: ByteArray, signingKey: ByteArray): ByteArray {
+        val signature = ByteArray(Sign.BYTES)
+
+        if (!sodium.cryptoSignDetached(signature, message, message.size.toLong(), signingKey)) {
+            throw SecurityException("Couldn't sign the message with the signing key")
+        }
+        return signature
+    }
 }

libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt

@@ -90,7 +90,7 @@ object SnodeAPI {
     private const val snodeFailureThreshold = 3
     private const val useOnionRequests = true
 
-    const val useTestnet = false
+    const val useTestnet = true
 
     // Error
     internal sealed class Error(val description: String) : Exception(description) {
@@ -465,7 +465,7 @@ object SnodeAPI {
                                                publicKey: String,
                                                namespace: Int,
                                                maxSize: Int? = null,
-                                               signingKey: ByteArray,
+                                               signCallback: SignCallback,
                                                ed25519PublicKey: Key? = null): SnodeBatchRequestInfo? {
         val lastHashValue = database.getLastMessageHashValue(snode, publicKey, namespace) ?: ""
         val params = mutableMapOf<String, Any>(
@@ -476,25 +476,10 @@ object SnodeAPI {
             params["pubkey_ed25519"] = ed25519PublicKey.asHexString
         }
         val timestamp = nowWithOffset
-        val signature = ByteArray(Sign.BYTES)
+        val verificationData = if (namespace == 0) "retrieve$timestamp"
-        val verificationData = if (namespace == 0) "retrieve$timestamp".toByteArray()
+            else "retrieve$namespace$timestamp"
-        else "retrieve$namespace$timestamp".toByteArray()
+        val signParameters = signCallback(verificationData, timestamp, namespace)
-        try {
+        params += signParameters
-            sodium.cryptoSignDetached(
-                signature,
-                verificationData,
-                verificationData.size.toLong(),
-                signingKey
-            )
-        } catch (e: Exception) {
-            Log.e("BatchRetrieve", "Signing data failed with provided signing key", e)
-            return null
-        }
-        params["timestamp"] = timestamp
-        params["signature"] = Base64.encodeBytes(signature)
-        if (namespace != 0) {
-            params["namespace"] = namespace
-        }
         if (maxSize != null) {
             params["max_size"] = maxSize
         }
@@ -518,7 +503,7 @@ object SnodeAPI {
             publicKey,
             namespace,
             maxSize,
-            secretKey,
+            signingKeyCallback(secretKey),
             ed25519PublicKey
         )
     }
@@ -701,23 +686,30 @@ object SnodeAPI {
         } catch (exception: Exception) {
             throw Error.SigningFailed
         }
-        mapOf(
+        val params = mutableMapOf<String,Any>(
-            "timestamp" to timestamp,
+                "timestamp" to timestamp,
-            "signature" to Base64.encodeBytes(signature),
+                "signature" to Base64.encodeBytes(signature),
-            "namespace" to namespace
         )
+        if (namespace != Namespace.DEFAULT()) {
+            params += "namespace" to namespace
+        }
+        params
     }
 
-    fun subkeyCallback(authData: ByteArray, groupKeysConfig: GroupKeysConfig): SignCallback = { message, timestamp, namespace ->
+    fun subkeyCallback(authData: ByteArray, groupKeysConfig: GroupKeysConfig, freeAfter: Boolean = true): SignCallback = { message, timestamp, namespace ->
         val (subaccount, subaccountSig, sig) = groupKeysConfig.subAccountSign(message.toByteArray(),authData)
-        val params = mapOf(
+        val params = mutableMapOf(
             "subaccount" to subaccount,
             "subaccount_sig" to subaccountSig,
             "signature" to sig,
             "timestamp" to timestamp,
-            "namespace" to namespace
         )
-        groupKeysConfig.free()
+        if (namespace != Namespace.DEFAULT()) {
+            params += "namespace" to namespace
+        }
+        if (freeAfter) {
+            groupKeysConfig.free()
+        }
         params
     }
 

libsignal/src/main/java/org/session/libsignal/messages/SignalServiceGroup.java

@@ -86,6 +86,10 @@ public class SignalServiceGroup {
     return groupId;
   }
 
+  public boolean isNewClosedGroup() {
+    return groupId.length == 33 && groupId[0] == 0x03;
+  }
+
   public GroupType getGroupType() { return  groupType; }
 
   public Type getType() {