MarkLogic 7 spawn-function

Question

I have a REST endpoint and it needs to proces a long list of codes. Because this may trigger time-outs I try to use spawn-function and do the magic in the background. But it looks like the spawn-function is holding the 200 OK response from my REST endpoint, so it's not really spawning.

I've added the log lines to check where it strands. All log lines pop up in the debug log.

With small amounts of data, this works fine. With a larger set (60k codes) it fails.

After changing the code to spawn the function for each item in $text, so 60k spawns, I get this error:

2015-07-28 10:20:02.326 Debug: Forest::insert: STRLF3-content-001-1 XDMP-INMMFULL: In-memory storage full; list: table=5%, wordsused=3%, wordsfree=95%, overhead=1%; tree: table=8%, wordsused=3%, wordsfree=97%, overhead=0%

Inserted data:

{   
    ProjectID: 102124,
    Text: "2311\n2253\n2312\n6626\n2253\n1234"
}

Calling the spawn proces:

(: ======================================================================= :) 
(: ! Load Transactions into seperate XML files                             :)
(: ======================================================================= :) 

declare 
%roxy:params("")
function strlf:post(
    $context as map:map,
    $params  as map:map,
    $input   as document-node()*
) as document-node()?
{
    map:put($context, "output-types", "application/json"),
    xdmp:set-response-code(200, "OK"),
    document { 

                  (: Get project ID :)
                  let $_ := xdmp:log('TransTest - stap1', 'debug')
                  let $project  := json:transform-from-json($input)/ns:ProjectID

                  let $_ := xdmp:log('TransTest - stap2', 'debug')
                  let $codes    := json:transform-from-json($input)/ns:Text

                  (: Clean current project :)
                  let $_ := xdmp:log('TransTest - stap3', 'debug')
                  let $uridir   := fn:concat('/app/transactie/', $project/text(), '/', '*')

                  let $_ := xdmp:log('TransTest - stap4', 'debug')
                  let $kill     := xdmp:document-delete(cts:uri-match($uridir))

                  (: Spawn the trannies :)
                  let $_ := xdmp:log('TransTest - stap5', 'debug')

                  (: return 'ja' :)
                  let $_ := xdmp:spawn-function(strlf:spawner($project, $codes, $uridir),
                    <options xmlns="xdmp:eval">
                      <transaction-mode>update-auto-commit</transaction-mode>
                    </options>)

                  return 'done'

    }
};

Function strlf:spawner:

declare private function strlf:spawner(
  $project,
  $codes,
  $uridir
)
{ 
  (: Tokenize on lines :)
  let $text     := fn:tokenize($codes, fn:codepoints-to-string(10))

  let $loop     :=
    for $regel in $text

      let $tokregel := fn:tokenize($regel, ",")           

      let $intvalue := 
       if (fn:contains($regel, ","))
       then fn:substring-after($regel, "€")
       else 1 

      let $code :=
       if (fn:contains($regel, ","))
       then $tokregel[1]
       else $regel 

      (: Build map of maps, p4 should be postcode :)
      let $map      := map:map()
      let $_        := map:put($map, 'code', $code)
      let $_        := map:put($map, 'p4', fn:substring($code[1], 1, 4))
      let $_        := map:put($map, 'value', $intvalue)
      let $_        := map:put($map, 'projectid', $project/text())

      (: Create unverified random doc id :)
      let $docid    := fn:string(xdmp:random(1000000000000))

      (: Build URI :)
      let $uridoc   := fn:concat('/app/transactie/', $project/text(), '/', $docid, '.xml')

    (: Save transaction document and skip header :)
    return 
    (if (map:get($map, 'code') != 'CODE')
    then xdmp:document-insert 
            (
            $uridoc,
              <transaction xmlns='http://www.dikw.nl/transactions' projectid='{map:get($map, 'projectid')}' code='{map:get($map, 'code')}' p4='{map:get($map, 'p4')}'>
                <value>{map:get($map, 'value')}</value>
              </transaction>
            )
    else ()) 

  (: Empty return :)
  return $loop        
};

Answer 1

Correct, you have strlf:spawner($project, $codes, $uridir) as first argument to xdmp:spawn-function, causing it to get executed, and the result being passed into xdmp:spawn-function. And since the spawner function returns an empty sequence, no error is being thrown by spawn-function.

The fix is pretty simple, wrap your spawner call in an anonymous function:

let $_ := xdmp:spawn-function(function () { strlf:spawner($project, $codes, $uridir) },
                <options xmlns="xdmp:eval">
                  <transaction-mode>update-auto-commit</transaction-mode>
                </options>)

HTH!