Add API tests for max jobs

And fix a bug found by the tests

2 files changed, 39 insertions, 2 deletions

diff --git a/src/mem3/src/mem3_reshard.erl b/src/mem3/src/mem3_reshard.erl
index 3ed9e97e1..46c013d58 100644
--- a/src/mem3/src/mem3_reshard.erl
+++ b/src/mem3/src/mem3_reshard.erl
@@ -218,7 +218,7 @@ handle_call({start_job, #job{id = Id, source = Source} = Job}, _From, State) ->
     couch_log:notice("~p start_job call ~p", [?MODULE, jobfmt(Job)]),
     Total = ets:info(?MODULE, size),
     SourceOk = mem3_reshard_validate:source(Source),
-    case {job_by_id(Id), Total =<  get_max_jobs(), SourceOk} of
+    case {job_by_id(Id), Total + 1 =<  get_max_jobs(), SourceOk} of
         {not_found, true, ok} ->
             case State#state.state of
                 running ->
diff --git a/src/mem3/test/mem3_reshard_api_test.erl b/src/mem3/test/mem3_reshard_api_test.erl
index 1822b0db1..48acd7124 100644
--- a/src/mem3/test/mem3_reshard_api_test.erl
+++ b/src/mem3/test/mem3_reshard_api_test.erl
@@ -100,7 +100,8 @@ mem3_reshard_api_test_() ->
                     fun recover_in_update_shard_map/1,
                     fun recover_in_wait_source_close/1,
                     fun recover_in_topoff3/1,
-                    fun recover_in_source_delete/1
+                    fun recover_in_source_delete/1,
+                    fun check_max_jobs/1
                 ]
             }
         }
@@ -642,6 +643,42 @@ recover_in_source_delete(Top) ->
     end).
 
 
+check_max_jobs(Top) ->
+    ?_test(begin
+        Jobs = Top ++ ?JOBS,
+
+        config:set("mem3_reshard", "max_jobs", "0", _Persist=false),
+        {C1, R1} = req(post, Jobs, #{type => split, db => <<?DB1>>}),
+        ?assertMatch({500, [#{<<"error">> := <<"max_jobs_exceeded">>}]}, {C1, R1}),
+
+        config:set("mem3_reshard", "max_jobs", "1", _Persist=false),
+        {C2, R2} = req(post, Jobs,  #{type => split, db => <<?DB1>>}),
+        wait_to_complete(Top, R2),
+
+        % Stop clustering so jobs are not started anymore and ensure max jobs is
+        % is enforced even if jobs are stopped
+        ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => stopped})),
+
+        {C3, R3} = req(post, Jobs,  #{type => split, db => <<?DB2>>}),
+        ?assertMatch({500, [#{<<"error">> := <<"max_jobs_exceeded">>}]}, {C3, R3}),
+
+        % Allow the job to be created by raising max_jobs
+        config:set("mem3_reshard", "max_jobs", "2", _Persist=false),
+
+        {C4, R4} = req(post, Jobs,  #{type => split, db => <<?DB2>>}),
+        ?assertEqual(201, C4),
+
+        % Lower max_jobs after job is created but it's not running
+        config:set("mem3_reshard", "max_jobs", "1", _Persist=false),
+
+        % Start resharding again
+        ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => running})),
+
+        % Jobs that have been created already are not removed if max jobs is lowered
+        % so make sure the job completes
+        wait_to_complete(Top, R4)
+    end).
+
 
 % Test help functions