I should emphasize that this is a reduction of a problem and yes, it's silly to put things in a table just to remove them. In the real system, the table is a cache for subsequent requests. I wanted to make the code demonstrating the problem as small as possible.
Also, I'm running with Nim 2.2.2 which has a slower/smaller leak than Nim 1.6.18. The leak is present on both macOS and Linux.
import std/asyncdispatch
import std/asyncfutures
import std/strutils
import std/tables
import std/times
var workMap = newTable[string, Future[string]]()
const CACHE_LENGTH = 1.seconds
const DELETE_IN_TIMER = not defined(notimer)
proc generateData(): Future[string] {.async.} =
await sleepAsync(100)
return "a".repeat(100_000)
proc queue_generateData(key: string): Future[string] {.async.} =
workMap[key] = generateData()
let data = await workMap[key]
# res.complete(data)
when DELETE_IN_TIMER:
addTimer(CACHE_LENGTH.seconds.int * 1000, true, proc(fd: AsyncFD): bool =
{.gcsafe.}:
echo "(timer) delete: " & $key
workMap.del(key)
)
else:
echo "(sync) delete: " & key
workMap.del(key)
return data
proc main() {.async.} =
echo "starting..."
const chunk = 20
var i = 0
while true:
let data = await queue_generateData($i)
echo "got data: ", $i, " ", $data.len
inc i
when isMainModule:
waitFor main()Amazing what a night's sleep will do... (I prepared my post last night)
Moving the addTimer call to a separate proc fixes both the memory leak and the file descriptor leak. So I guess the memory leak was in keeping the closure context for that timer callback proc?
Why does it fix the file descriptor leak, though?
import std/asyncdispatch
import std/asyncfutures
import std/strutils
import std/tables
import std/times
var workMap = newTable[string, Future[string]]()
const CACHE_LENGTH = 1.seconds
const DELETE_IN_TIMER = not defined(notimer)
proc reclaimLater(key: string) =
addTimer(CACHE_LENGTH.seconds.int * 1000, true, proc(fd: AsyncFD): bool =
{.gcsafe.}:
echo "(timer) delete: " & $key
workMap.del(key)
return true
)
proc generateData(): Future[string] {.async.} =
await sleepAsync(100)
return "a".repeat(100_000)
proc queue_generateData(key: string): Future[string] {.async.} =
workMap[key] = generateData()
let data = await workMap[key]
# res.complete(data)
when DELETE_IN_TIMER:
reclaimLater(key)
else:
echo "(sync) delete: " & key
workMap.del(key)
return data
proc main() {.async.} =
echo "starting..."
const chunk = 20
var i = 0
while true:
let data = await queue_generateData($i)
echo "got data: ", $i, " ", $data.len
inc i
when isMainModule:
waitFor main()Ah! I added return true to the addTimer callback. That's what fixed the file descriptor leak 🤦 I guess oneshot = true is not enough to cause timers to clean up.
Well, thank you Nim Forum, you have been very helpful :)