Mirror of forum.nim-lang.org

4478 :: How to parse JSON at compile time?

• moigagoo (orginal) [2018-12-17T14:55:24+01:00] view original

  Hi!

  I need to generate functions from JSON declarations. So

  import json, macros
  
  macro parseFuncs*(path: string): untyped =
    path.expectKind(nnkStrLit)
    
    echo parseFile(path.strVal)
  
  when isMainModule:
    parseFuncs "funcs.json"
  Run

  I'm getting this weird error: VM is only allowed to 'cast' between integers of same size.

  What am I doing wrong? Tried parsing TOML, got the same error.

---

• nepeckman (orginal) [2018-12-17T21:05:27+01:00] view original

  I'm not sure its possible to parse JSON at compile time. I've tried reading the file with staticRead and passing the resulting const string to the JSON module, but it doesn't seem like the VM had the capability to parse JSON. If you do figure out a way to do it, I'd love to hear it.

---

• alexsad (orginal) [2018-12-18T09:42:08+01:00] view original

  Hi!

  It's a little bit tricky :)

  macro parseSwagger*(iApi: untyped, fn: string): untyped {.gensym.} =
    var
      inputJsonStr = slurp($fn).multiReplace([($'\n', ""), ($'\r', ""), ($'\c', ""), ($'\t', "")])
      parseJsnMcr = ident(nskMacro.genSym("parseJsn").repr)
      parseJsn = parseStmt("macro " & $parseJsnMcr & "(iApi: untyped): untyped {.gensym.} =\n  var jsnData = %*(%tmp%)".replace("%tmp%", inputJsonStr))
    result = quote do:
      `parseJsn`
      `parseJsnMcr`(`iApi`)
  Run

---

• moigagoo (orginal) [2018-12-18T19:16:15+01:00] view original

  Thanks! Indeed, that's tricky a lot 😂

---

• juancarlospaco (orginal) [2018-12-18T19:29:43+01:00] view original

  This would be nice as a Nimble module.

---

• Libman (orginal) [2018-12-20T02:47:53+01:00] view original

  I'm glad that a solution was found for this specific macros case, but it's indeed disappointing that all Nim JSON libraries fail if used in static / compile time context...

  The jsmn module was somewhat promising, but failed for another reason - your mileage may vary.

  In case anyone finds this useful, I'm pasting my test code / notes below. Let me know if there are other json libraries I've missed.

  import os, json, jsmn, packedjson, osproc, macros, strutils
  
  proc readNimbleModuleNames_execCmd(nimbleJsonFile: string): seq[string] =
    # You can do anything in any language - if you can use an external tool...
    let execCmd = "jq -r '.[].name' " & nimbleJsonFile
    return staticExec(execCmd).splitLines
  
  proc readNimbleModuleNames_json(nimbleJsonFile: string): seq[string] =
    # Nim stdlib ``json`` doesn't at compile time / static context: {{{
    # .choosenim/toolchains/nim-#devel/lib/system/sysio.nim(351, 20)
    # Error: cannot 'importc' variable at compile time}}}
    let jsonObj = json.parseFile(nimbleJsonFile)
    for rec in jsonObj:
      result.add rec["name"].str
  
  proc readNimbleModuleNames_packedjson(nimbleJsonFile: string): seq[string] =
    # Same problem for https://github.com/Araq/packedjson ...
    let jsonObj = packedjson.parseFile(nimbleJsonFile)
    for rec in jsonObj:
      result.add rec["name"].getStr
  
  proc readNimbleModuleNames_jsmn(nimbleJsonFile: string): seq[string] =
    # The nimble ``jsmn`` module doesn't work at compile time either: {{{
    # .nimble/pkgs/jsmn-0.1.5/jsmn.nim(223, 11) Error: interpretation
    # requires too many iterations; if you are sure this is not a bug in your
    # code edit compiler/vmdef.MaxLoopIterations and rebuild the compiler}}}
    # NOTE: jsmn code mentions evil Apache license, though nimble says MIT...
    let jsonStr = readFile(nimbleJsonFile)
    var tokensArr: array[100000, JsmnToken]
    let tokenCount = jsmn.parseJson(jsonStr, tokensArr)
    var lastTokenWasName = false
    for tokenNum in 1..tokenCount:
      var token = addr tokensArr[tokenNum]
      if token.kind != JSMN_STRING: continue
      let value = jsonStr[token.start..<token.stop]
      if lastTokenWasName: result.add value
      lastTokenWasName = (value == "name")
  
  when isMainModule:
    const nimbleJsonFile = getHomeDir() / ".nimble/packages_official.json"
    const nimbles = readNimbleModuleNames_execCmd(nimbleJsonFile)
    echo nimbles
    echo nimbles.len
  Run

---