Skip to content

Commit b22b148

Browse files
dfarrdfarr
authored
Add loop in background coroutines to clear backlog (#955)
1 parent 10e327f commit b22b148

6 files changed

Lines changed: 571 additions & 489 deletions

File tree

‎cmd/test/config_test.go‎

Lines changed: 125 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -53,14 +53,17 @@ func TestConfig(t *testing.T) {
5353
cmds: []cmd{serveCmd, devCmd},
5454
expected: &config.Config{
5555
System: system.Config{
56-
Url: "",
57-
CoroutineMaxSize: 1000,
58-
SubmissionBatchSize: 1000,
59-
CompletionBatchSize: 1000,
60-
PromiseBatchSize: 100,
61-
ScheduleBatchSize: 100,
62-
TaskBatchSize: 100,
63-
SignalTimeout: 1 * time.Second,
56+
Url: "",
57+
CoroutineMaxSize: 1000,
58+
SubmissionBatchSize: 1000,
59+
CompletionBatchSize: 1000,
60+
PromiseBatchSize: 100,
61+
PromiseMaxIterations: 1000,
62+
ScheduleBatchSize: 100,
63+
ScheduleMaxIterations: 1000,
64+
TaskBatchSize: 100,
65+
TaskMaxIterations: 1000,
66+
SignalTimeout: 1 * time.Second,
6467
},
6568
API: config.API{
6669
Size: 1000,
@@ -90,8 +93,11 @@ system:
9093
submissionBatchSize: 2
9194
completionBatchSize: 3
9295
promiseBatchSize: 4
96+
promiseMaxIterations: 40
9397
scheduleBatchSize: 5
98+
scheduleMaxIterations: 50
9499
taskBatchSize: 6
100+
taskMaxIterations: 60
95101
signalTimeout: 7s
96102
api:
97103
size: 1
@@ -103,14 +109,17 @@ metricsAddr: "localhost:8080"
103109
logLevel: "warn"`,
104110
expected: &config.Config{
105111
System: system.Config{
106-
Url: "https://resonatehq.io:9000",
107-
CoroutineMaxSize: 1,
108-
SubmissionBatchSize: 2,
109-
CompletionBatchSize: 3,
110-
PromiseBatchSize: 4,
111-
ScheduleBatchSize: 5,
112-
TaskBatchSize: 6,
113-
SignalTimeout: 7 * time.Second,
112+
Url: "https://resonatehq.io:9000",
113+
CoroutineMaxSize: 1,
114+
SubmissionBatchSize: 2,
115+
CompletionBatchSize: 3,
116+
PromiseBatchSize: 4,
117+
PromiseMaxIterations: 40,
118+
ScheduleBatchSize: 5,
119+
ScheduleMaxIterations: 50,
120+
TaskBatchSize: 6,
121+
TaskMaxIterations: 60,
122+
SignalTimeout: 7 * time.Second,
114123
},
115124
API: config.API{
116125
Size: 1,
@@ -134,8 +143,11 @@ logLevel: "warn"`,
134143
"--system-submission-batch-size", "3",
135144
"--system-completion-batch-size", "4",
136145
"--system-promise-batch-size", "5",
146+
"--system-promise-max-iterations", "50",
137147
"--system-schedule-batch-size", "6",
148+
"--system-schedule-max-iterations", "60",
138149
"--system-task-batch-size", "7",
150+
"--system-task-max-iterations", "70",
139151
"--system-signal-timeout", "8s",
140152
"--api-size", "2",
141153
"--api-auth-public-key", "key2.pub",
@@ -145,14 +157,17 @@ logLevel: "warn"`,
145157
},
146158
expected: &config.Config{
147159
System: system.Config{
148-
Url: "https://resonatehq.io:9001",
149-
CoroutineMaxSize: 2,
150-
SubmissionBatchSize: 3,
151-
CompletionBatchSize: 4,
152-
PromiseBatchSize: 5,
153-
ScheduleBatchSize: 6,
154-
TaskBatchSize: 7,
155-
SignalTimeout: 8 * time.Second,
160+
Url: "https://resonatehq.io:9001",
161+
CoroutineMaxSize: 2,
162+
SubmissionBatchSize: 3,
163+
CompletionBatchSize: 4,
164+
PromiseBatchSize: 5,
165+
PromiseMaxIterations: 50,
166+
ScheduleBatchSize: 6,
167+
ScheduleMaxIterations: 60,
168+
TaskBatchSize: 7,
169+
TaskMaxIterations: 70,
170+
SignalTimeout: 8 * time.Second,
156171
},
157172
API: config.API{
158173
Size: 2,
@@ -177,8 +192,11 @@ system:
177192
submissionBatchSize: 2
178193
completionBatchSize: 3
179194
promiseBatchSize: 4
195+
promiseMaxIterations: 40
180196
scheduleBatchSize: 5
197+
scheduleMaxIterations: 50
181198
taskBatchSize: 6
199+
taskMaxIterations: 60
182200
signalTimeout: 7s
183201
api:
184202
size: 1
@@ -199,14 +217,17 @@ logLevel: "warn"`,
199217
},
200218
expected: &config.Config{
201219
System: system.Config{
202-
Url: "https://resonatehq.io:9001",
203-
CoroutineMaxSize: 1,
204-
SubmissionBatchSize: 3,
205-
CompletionBatchSize: 3,
206-
PromiseBatchSize: 5,
207-
ScheduleBatchSize: 5,
208-
TaskBatchSize: 7,
209-
SignalTimeout: 7 * time.Second,
220+
Url: "https://resonatehq.io:9001",
221+
CoroutineMaxSize: 1,
222+
SubmissionBatchSize: 3,
223+
CompletionBatchSize: 3,
224+
PromiseBatchSize: 5,
225+
PromiseMaxIterations: 40,
226+
ScheduleBatchSize: 5,
227+
ScheduleMaxIterations: 50,
228+
TaskBatchSize: 7,
229+
TaskMaxIterations: 60,
230+
SignalTimeout: 7 * time.Second,
210231
},
211232
API: config.API{
212233
Size: 2,
@@ -236,14 +257,17 @@ logLevel: "warn"`,
236257
},
237258
expected: &config.Config{
238259
System: system.Config{
239-
Url: "",
240-
CoroutineMaxSize: 1000,
241-
SubmissionBatchSize: 1000,
242-
CompletionBatchSize: 1000,
243-
PromiseBatchSize: 100,
244-
ScheduleBatchSize: 100,
245-
TaskBatchSize: 100,
246-
SignalTimeout: 1 * time.Second,
260+
Url: "",
261+
CoroutineMaxSize: 1000,
262+
SubmissionBatchSize: 1000,
263+
CompletionBatchSize: 1000,
264+
PromiseBatchSize: 100,
265+
PromiseMaxIterations: 1000,
266+
ScheduleBatchSize: 100,
267+
ScheduleMaxIterations: 1000,
268+
TaskBatchSize: 100,
269+
TaskMaxIterations: 1000,
270+
SignalTimeout: 1 * time.Second,
247271
},
248272
API: withAPISubsystems(
249273
config.API{
@@ -369,14 +393,17 @@ logLevel: "warn"`,
369393
},
370394
expected: &config.Config{
371395
System: system.Config{
372-
Url: "",
373-
CoroutineMaxSize: 1000,
374-
SubmissionBatchSize: 1000,
375-
CompletionBatchSize: 1000,
376-
PromiseBatchSize: 100,
377-
ScheduleBatchSize: 100,
378-
TaskBatchSize: 100,
379-
SignalTimeout: 1 * time.Second,
396+
Url: "",
397+
CoroutineMaxSize: 1000,
398+
SubmissionBatchSize: 1000,
399+
CompletionBatchSize: 1000,
400+
PromiseBatchSize: 100,
401+
PromiseMaxIterations: 1000,
402+
ScheduleBatchSize: 100,
403+
ScheduleMaxIterations: 1000,
404+
TaskBatchSize: 100,
405+
TaskMaxIterations: 1000,
406+
SignalTimeout: 1 * time.Second,
380407
},
381408
API: withAPISubsystems(
382409
config.API{
@@ -550,14 +577,17 @@ logLevel: "warn"`,
550577
},
551578
expected: &config.Config{
552579
System: system.Config{
553-
Url: "",
554-
CoroutineMaxSize: 1000,
555-
SubmissionBatchSize: 1000,
556-
CompletionBatchSize: 1000,
557-
PromiseBatchSize: 100,
558-
ScheduleBatchSize: 100,
559-
TaskBatchSize: 100,
560-
SignalTimeout: 1 * time.Second,
580+
Url: "",
581+
CoroutineMaxSize: 1000,
582+
SubmissionBatchSize: 1000,
583+
CompletionBatchSize: 1000,
584+
PromiseBatchSize: 100,
585+
PromiseMaxIterations: 1000,
586+
ScheduleBatchSize: 100,
587+
ScheduleMaxIterations: 1000,
588+
TaskBatchSize: 100,
589+
TaskMaxIterations: 1000,
590+
SignalTimeout: 1 * time.Second,
561591
},
562592
API: withAPISubsystems(
563593
config.API{
@@ -674,14 +704,17 @@ logLevel: "warn"`,
674704
},
675705
expected: &config.Config{
676706
System: system.Config{
677-
Url: "",
678-
CoroutineMaxSize: 1000,
679-
SubmissionBatchSize: 1000,
680-
CompletionBatchSize: 1000,
681-
PromiseBatchSize: 100,
682-
ScheduleBatchSize: 100,
683-
TaskBatchSize: 100,
684-
SignalTimeout: 1 * time.Second,
707+
Url: "",
708+
CoroutineMaxSize: 1000,
709+
SubmissionBatchSize: 1000,
710+
CompletionBatchSize: 1000,
711+
PromiseBatchSize: 100,
712+
PromiseMaxIterations: 1000,
713+
ScheduleBatchSize: 100,
714+
ScheduleMaxIterations: 1000,
715+
TaskBatchSize: 100,
716+
TaskMaxIterations: 1000,
717+
SignalTimeout: 1 * time.Second,
685718
},
686719
API: withAPISubsystems(
687720
config.API{
@@ -833,8 +866,11 @@ system:
833866
submissionBatchSize: 2
834867
completionBatchSize: 3
835868
promiseBatchSize: 4
869+
promiseMaxIterations: 40
836870
scheduleBatchSize: 5
871+
scheduleMaxIterations: 50
837872
taskBatchSize: 6
873+
taskMaxIterations: 60
838874
signalTimeout: 7s
839875
api:
840876
size: 1
@@ -919,14 +955,17 @@ logLevel: "warn"`,
919955
},
920956
expected: &config.Config{
921957
System: system.Config{
922-
Url: "https://resonatehq.io:9000",
923-
CoroutineMaxSize: 1,
924-
SubmissionBatchSize: 2,
925-
CompletionBatchSize: 3,
926-
PromiseBatchSize: 4,
927-
ScheduleBatchSize: 5,
928-
TaskBatchSize: 6,
929-
SignalTimeout: 7 * time.Second,
958+
Url: "https://resonatehq.io:9000",
959+
CoroutineMaxSize: 1,
960+
SubmissionBatchSize: 2,
961+
CompletionBatchSize: 3,
962+
PromiseBatchSize: 4,
963+
PromiseMaxIterations: 40,
964+
ScheduleBatchSize: 5,
965+
ScheduleMaxIterations: 50,
966+
TaskBatchSize: 6,
967+
TaskMaxIterations: 60,
968+
SignalTimeout: 7 * time.Second,
930969
},
931970
API: withAPISubsystems(
932971
config.API{
@@ -1040,8 +1079,11 @@ system:
10401079
submissionBatchSize: 2
10411080
completionBatchSize: 3
10421081
promiseBatchSize: 4
1082+
promiseMaxIterations: 40
10431083
scheduleBatchSize: 5
1084+
scheduleMaxIterations: 50
10441085
taskBatchSize: 6
1086+
taskMaxIterations: 60
10451087
signalTimeout: 7s
10461088
api:
10471089
size: 1
@@ -1148,14 +1190,17 @@ logLevel: "warn"`,
11481190
},
11491191
expected: &config.Config{
11501192
System: system.Config{
1151-
Url: "https://resonatehq.io:9000",
1152-
CoroutineMaxSize: 1,
1153-
SubmissionBatchSize: 2,
1154-
CompletionBatchSize: 3,
1155-
PromiseBatchSize: 4,
1156-
ScheduleBatchSize: 5,
1157-
TaskBatchSize: 6,
1158-
SignalTimeout: 7 * time.Second,
1193+
Url: "https://resonatehq.io:9000",
1194+
CoroutineMaxSize: 1,
1195+
SubmissionBatchSize: 2,
1196+
CompletionBatchSize: 3,
1197+
PromiseBatchSize: 4,
1198+
PromiseMaxIterations: 40,
1199+
ScheduleBatchSize: 5,
1200+
ScheduleMaxIterations: 50,
1201+
TaskBatchSize: 6,
1202+
TaskMaxIterations: 60,
1203+
SignalTimeout: 7 * time.Second,
11591204
},
11601205
API: withAPISubsystems(
11611206
config.API{

0 commit comments

Comments
 (0)