2020-12-02 06:39:02 +01:00
|
|
|
package org.session.libsession.messaging.jobs
|
|
|
|
|
2021-03-12 07:15:33 +01:00
|
|
|
import kotlinx.coroutines.*
|
|
|
|
import kotlinx.coroutines.channels.Channel
|
|
|
|
import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED
|
2021-04-26 03:14:45 +02:00
|
|
|
import org.session.libsession.messaging.MessagingModuleConfiguration
|
2021-05-18 01:12:33 +02:00
|
|
|
import org.session.libsignal.utilities.Log
|
2021-03-12 07:15:33 +01:00
|
|
|
import java.util.*
|
2021-04-09 06:19:48 +02:00
|
|
|
import java.util.concurrent.ConcurrentHashMap
|
2021-03-12 07:15:33 +01:00
|
|
|
import java.util.concurrent.Executors
|
2021-04-09 06:19:48 +02:00
|
|
|
import java.util.concurrent.atomic.AtomicInteger
|
2020-12-02 06:39:02 +01:00
|
|
|
import kotlin.concurrent.schedule
|
2021-03-12 07:15:33 +01:00
|
|
|
import kotlin.math.min
|
|
|
|
import kotlin.math.pow
|
2020-12-02 06:39:02 +01:00
|
|
|
import kotlin.math.roundToLong
|
|
|
|
|
2020-11-25 02:06:41 +01:00
|
|
|
class JobQueue : JobDelegate {
|
2020-12-02 06:39:02 +01:00
|
|
|
private var hasResumedPendingJobs = false // Just for debugging
|
2021-04-09 06:19:48 +02:00
|
|
|
private val jobTimestampMap = ConcurrentHashMap<Long, AtomicInteger>()
|
2021-05-12 08:48:18 +02:00
|
|
|
private val rxDispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
|
|
|
|
private val txDispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
|
2021-07-09 08:04:06 +02:00
|
|
|
private val attachmentDispatcher = Executors.newFixedThreadPool(4).asCoroutineDispatcher()
|
2021-03-12 07:15:33 +01:00
|
|
|
private val scope = GlobalScope + SupervisorJob()
|
|
|
|
private val queue = Channel<Job>(UNLIMITED)
|
2021-07-05 07:00:32 +02:00
|
|
|
private val pendingJobIds = mutableSetOf<String>()
|
2021-04-26 02:58:48 +02:00
|
|
|
|
2021-04-06 09:09:21 +02:00
|
|
|
val timer = Timer()
|
2021-03-12 07:15:33 +01:00
|
|
|
|
2021-05-12 08:48:18 +02:00
|
|
|
private fun CoroutineScope.processWithDispatcher(channel: Channel<Job>, dispatcher: CoroutineDispatcher) = launch(dispatcher) {
|
|
|
|
for (job in channel) {
|
|
|
|
if (!isActive) break
|
|
|
|
job.delegate = this@JobQueue
|
|
|
|
job.execute()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-03-12 07:15:33 +01:00
|
|
|
init {
|
2021-04-26 02:58:48 +02:00
|
|
|
// Process jobs
|
2021-05-12 08:48:18 +02:00
|
|
|
scope.launch {
|
2021-05-26 07:22:19 +02:00
|
|
|
val rxQueue = Channel<Job>(capacity = 4096)
|
|
|
|
val txQueue = Channel<Job>(capacity = 4096)
|
|
|
|
val attachmentQueue = Channel<Job>(capacity = 4096)
|
2021-05-12 08:48:18 +02:00
|
|
|
|
|
|
|
val receiveJob = processWithDispatcher(rxQueue, rxDispatcher)
|
|
|
|
val txJob = processWithDispatcher(txQueue, txDispatcher)
|
|
|
|
val attachmentJob = processWithDispatcher(attachmentQueue, attachmentDispatcher)
|
|
|
|
|
2021-03-12 07:15:33 +01:00
|
|
|
while (isActive) {
|
2021-05-12 08:48:18 +02:00
|
|
|
for (job in queue) {
|
|
|
|
when (job) {
|
2021-05-13 01:38:39 +02:00
|
|
|
is NotifyPNServerJob, is AttachmentUploadJob, is MessageSendJob -> txQueue.send(job)
|
2021-05-12 08:48:18 +02:00
|
|
|
is AttachmentDownloadJob -> attachmentQueue.send(job)
|
2021-09-29 07:29:24 +02:00
|
|
|
is MessageReceiveJob, is BatchMessageReceiveJob, is TrimThreadJob -> rxQueue.send(job)
|
2021-05-13 01:38:39 +02:00
|
|
|
else -> throw IllegalStateException("Unexpected job type.")
|
2021-04-27 09:29:37 +02:00
|
|
|
}
|
2021-03-12 07:15:33 +01:00
|
|
|
}
|
|
|
|
}
|
2021-05-12 08:48:18 +02:00
|
|
|
|
2021-05-13 01:38:39 +02:00
|
|
|
// The job has been cancelled
|
2021-05-12 08:48:18 +02:00
|
|
|
receiveJob.cancel()
|
|
|
|
txJob.cancel()
|
|
|
|
attachmentJob.cancel()
|
|
|
|
|
2021-03-12 07:15:33 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-12-02 06:39:02 +01:00
|
|
|
companion object {
|
2021-05-12 08:17:25 +02:00
|
|
|
|
2021-03-18 03:36:56 +01:00
|
|
|
@JvmStatic
|
2020-12-02 06:39:02 +01:00
|
|
|
val shared: JobQueue by lazy { JobQueue() }
|
|
|
|
}
|
|
|
|
|
|
|
|
fun add(job: Job) {
|
|
|
|
addWithoutExecuting(job)
|
2021-03-12 07:15:33 +01:00
|
|
|
queue.offer(job) // offer always called on unlimited capacity
|
2020-12-02 06:39:02 +01:00
|
|
|
}
|
|
|
|
|
2021-03-30 07:23:12 +02:00
|
|
|
private fun addWithoutExecuting(job: Job) {
|
2021-04-09 06:19:48 +02:00
|
|
|
// When adding multiple jobs in rapid succession, timestamps might not be good enough as a unique ID. To
|
2021-04-26 02:58:48 +02:00
|
|
|
// deal with this we keep track of the number of jobs with a given timestamp and add that to the end of the
|
2021-04-09 06:19:48 +02:00
|
|
|
// timestamp to make it a unique ID. We can't use a random number because we do still want to keep track
|
|
|
|
// of the order in which the jobs were added.
|
|
|
|
val currentTime = System.currentTimeMillis()
|
|
|
|
jobTimestampMap.putIfAbsent(currentTime, AtomicInteger())
|
2021-04-14 04:26:34 +02:00
|
|
|
job.id = currentTime.toString() + jobTimestampMap[currentTime]!!.getAndIncrement().toString()
|
2021-04-26 03:14:45 +02:00
|
|
|
MessagingModuleConfiguration.shared.storage.persistJob(job)
|
2020-12-02 06:39:02 +01:00
|
|
|
}
|
|
|
|
|
2021-07-05 07:00:32 +02:00
|
|
|
fun resumePendingSendMessage(job: Job) {
|
|
|
|
val id = job.id ?: run {
|
|
|
|
Log.e("Loki", "tried to resume pending send job with no ID")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if (!pendingJobIds.add(id)) {
|
|
|
|
Log.e("Loki","tried to re-queue pending/in-progress job")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
queue.offer(job)
|
|
|
|
Log.d("Loki", "resumed pending send message $id")
|
|
|
|
}
|
|
|
|
|
2021-07-09 07:13:43 +02:00
|
|
|
fun resumePendingJobs(typeKey: String) {
|
|
|
|
val allPendingJobs = MessagingModuleConfiguration.shared.storage.getAllPendingJobs(typeKey)
|
|
|
|
val pendingJobs = mutableListOf<Job>()
|
|
|
|
for ((id, job) in allPendingJobs) {
|
|
|
|
if (job == null) {
|
|
|
|
// Job failed to deserialize, remove it from the DB
|
|
|
|
handleJobFailedPermanently(id)
|
|
|
|
} else {
|
|
|
|
pendingJobs.add(job)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
pendingJobs.sortedBy { it.id }.forEach { job ->
|
|
|
|
Log.i("Loki", "Resuming pending job of type: ${job::class.simpleName}.")
|
|
|
|
queue.offer(job) // Offer always called on unlimited capacity
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-12-02 06:39:02 +01:00
|
|
|
fun resumePendingJobs() {
|
|
|
|
if (hasResumedPendingJobs) {
|
|
|
|
Log.d("Loki", "resumePendingJobs() should only be called once.")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
hasResumedPendingJobs = true
|
2021-05-12 08:17:25 +02:00
|
|
|
val allJobTypes = listOf(
|
|
|
|
AttachmentUploadJob.KEY,
|
|
|
|
AttachmentDownloadJob.KEY,
|
|
|
|
MessageReceiveJob.KEY,
|
|
|
|
MessageSendJob.KEY,
|
2021-09-29 07:29:24 +02:00
|
|
|
NotifyPNServerJob.KEY,
|
|
|
|
BatchMessageReceiveJob.KEY
|
2021-05-07 03:48:03 +02:00
|
|
|
)
|
2020-12-02 06:39:02 +01:00
|
|
|
allJobTypes.forEach { type ->
|
2021-07-09 07:13:43 +02:00
|
|
|
resumePendingJobs(type)
|
2020-12-02 06:39:02 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
override fun handleJobSucceeded(job: Job) {
|
2021-05-07 03:48:03 +02:00
|
|
|
val jobId = job.id ?: return
|
|
|
|
MessagingModuleConfiguration.shared.storage.markJobAsSucceeded(jobId)
|
2021-07-05 07:00:32 +02:00
|
|
|
pendingJobIds.remove(jobId)
|
2020-12-02 06:39:02 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
override fun handleJobFailed(job: Job, error: Exception) {
|
2021-05-13 01:38:39 +02:00
|
|
|
// Canceled
|
2021-04-26 03:14:45 +02:00
|
|
|
val storage = MessagingModuleConfiguration.shared.storage
|
2021-05-12 08:48:18 +02:00
|
|
|
if (storage.isJobCanceled(job)) {
|
|
|
|
return Log.i("Loki", "${job::class.simpleName} canceled.")
|
|
|
|
}
|
2021-05-13 01:38:39 +02:00
|
|
|
// Message send jobs waiting for the attachment to upload
|
|
|
|
if (job is MessageSendJob && error is MessageSendJob.AwaitingAttachmentUploadException) {
|
|
|
|
Log.i("Loki", "Message send job waiting for attachment upload to finish.")
|
|
|
|
return
|
|
|
|
}
|
2021-09-29 07:29:24 +02:00
|
|
|
// Batch message receive job, re-queue non-permanently failed jobs
|
|
|
|
if (job is BatchMessageReceiveJob) {
|
|
|
|
val replacementParameters = job.failures
|
|
|
|
}
|
|
|
|
|
2021-05-13 01:38:39 +02:00
|
|
|
// Regular job failure
|
|
|
|
job.failureCount += 1
|
2021-05-12 08:48:18 +02:00
|
|
|
if (job.failureCount >= job.maxFailureCount) {
|
2021-05-07 03:48:03 +02:00
|
|
|
handleJobFailedPermanently(job, error)
|
2020-12-02 06:39:02 +01:00
|
|
|
} else {
|
2021-05-07 03:48:03 +02:00
|
|
|
storage.persistJob(job)
|
2020-12-02 06:39:02 +01:00
|
|
|
val retryInterval = getRetryInterval(job)
|
2021-05-12 08:17:25 +02:00
|
|
|
Log.i("Loki", "${job::class.simpleName} failed; scheduling retry (failure count is ${job.failureCount}).")
|
2021-04-06 09:09:21 +02:00
|
|
|
timer.schedule(delay = retryInterval) {
|
2021-05-12 08:17:25 +02:00
|
|
|
Log.i("Loki", "Retrying ${job::class.simpleName}.")
|
2021-03-15 23:44:55 +01:00
|
|
|
queue.offer(job)
|
2020-12-02 06:39:02 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
override fun handleJobFailedPermanently(job: Job, error: Exception) {
|
2021-05-07 03:48:03 +02:00
|
|
|
val jobId = job.id ?: return
|
|
|
|
handleJobFailedPermanently(jobId)
|
|
|
|
}
|
|
|
|
|
|
|
|
private fun handleJobFailedPermanently(jobId: String) {
|
2021-04-26 03:14:45 +02:00
|
|
|
val storage = MessagingModuleConfiguration.shared.storage
|
2021-05-13 01:38:39 +02:00
|
|
|
storage.markJobAsFailedPermanently(jobId)
|
2020-12-02 06:39:02 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
private fun getRetryInterval(job: Job): Long {
|
|
|
|
// Arbitrary backoff factor...
|
2021-02-10 06:48:03 +01:00
|
|
|
// try 1 delay: 0.5s
|
|
|
|
// try 2 delay: 1s
|
2020-12-02 06:39:02 +01:00
|
|
|
// ...
|
2021-02-10 06:48:03 +01:00
|
|
|
// try 5 delay: 16s
|
2020-12-02 06:39:02 +01:00
|
|
|
// ...
|
2021-02-10 06:48:03 +01:00
|
|
|
// try 11 delay: 512s
|
|
|
|
val maxBackoff = (10 * 60).toDouble() // 10 minutes
|
|
|
|
return (1000 * 0.25 * min(maxBackoff, (2.0).pow(job.failureCount))).roundToLong()
|
2020-12-02 06:39:02 +01:00
|
|
|
}
|
2021-07-05 07:00:32 +02:00
|
|
|
|
|
|
|
private fun Job.isSend() = this is MessageSendJob || this is AttachmentUploadJob
|
|
|
|
|
2020-11-25 02:06:41 +01:00
|
|
|
}
|