Modify CN state machine params for prod to improve throughput (#3855)

* Enable recoverOrphanedDataQueue (heavily throttled)

* Increase maxRecurringRequestSyncJobConcurrency on prod from 30 to 50

* Increase maxUpdateReplicaSetJobConcurrency prod from 15 to 25

* 10x prod recurringSyncQueue.maxWaitingJobs from 10k to 100k; 10x prod updateReplicaSetQueue.maxWaitingJobs from 1k to 10k

Author: SidSethi

creator-node/src/config.js

@@ -488,7 +488,7 @@ const config = convict({
     doc: 'number of recover-orphaned-data jobs that can run in each interval (0 to pause queue)',
     format: 'nat',
     env: 'recoverOrphanedDataQueueRateLimitJobsPerInterval',
-    default: 0
+    default: 1
   },
   debounceTime: {
     doc: 'sync debounce time in ms',
@@ -554,13 +554,13 @@ const config = convict({
     doc: 'Max bull queue concurrency for recurring sync request jobs',
     format: 'nat',
     env: 'maxRecurringRequestSyncJobConcurrency',
-    default: 30
+    default: 50
   },
   maxUpdateReplicaSetJobConcurrency: {
     doc: 'Max bull queue concurrency for update replica set jobs',
     format: 'nat',
     env: 'maxUpdateReplicaSetJobConcurrency',
-    default: 15
+    default: 25
   },
   peerHealthCheckRequestTimeout: {
     doc: 'Timeout [ms] for checking health check route',

creator-node/src/services/stateMachineManager/index.js

@@ -71,11 +71,11 @@ class StateMachineManager {
       },
       [QUEUE_NAMES.RECURRING_SYNC]: {
         queue: recurringSyncQueue,
-        maxWaitingJobs: 10000
+        maxWaitingJobs: 100000
       },
       [QUEUE_NAMES.UPDATE_REPLICA_SET]: {
         queue: updateReplicaSetQueue,
-        maxWaitingJobs: 1000
+        maxWaitingJobs: 10000
       },
       [QUEUE_NAMES.RECOVER_ORPHANED_DATA]: {
         queue: recoverOrphanedDataQueue,