Hi.
I tried to re-write my "type register" to user SharedTable instead. I did not find a documentation page specifying the SharedTable API, so I read the code, but it's not all that obvious to me (in particular, the withXXX pattern). Mainly, atm, it seems my main problem is that SharedTable doesn't seem to have a "hasKey" or a "len" proc, and idk how to work around that. I tried to create a (very) simplified example of what I'm trying to do, but the compiler crashes on it, so I hope you at least get what I'm trying to do, even if this doesn't compile.
import locks
import typetraits
import sharedtables
type
TypeRegister = object
table: SharedTable[char, int]
lock: Lock
proc register*(reg: ptr TypeRegister, T: typedesc): int =
let tk = T.name[0] # In real code I have a "shared string" kind of thing as key
if reg.table.hasKey(tk): # No "hasKey()" :(
result = reg.table.mget(tk) # I only need read access, but no "get(k)" or "[k]" AFAIK
else:
acquire(reg.lock)
try:
if reg.table.hasKey(tk):
result = reg.table.mget(tk)
else:
let value: int = reg.table.len # No len() either. :(
reg.table[tk] = value
result = value
finally:
release(reg.lock)
when isMainModule:
echo("Starting ...")
let tr = createShared(TypeRegister)
tr.table = initSharedTable[char, int]()
initLock tr.lock
echo("Registering bool ...")
let id = register(tr, bool)
echo("Done: " & $id)
On Windows 10 x64 I get a:
SIGSEGV: Illegal storage access. (Attempt to read from nil?)
Possible (ugly) work-around for missing:
I have been trying to work around the missing hasKey() using mget() and a try/finally, but this fails to compile as well.
This is basically what I'm doing:
import sharedtables
import options
import typetraits
type
TypeKey* = int
TypeInfo*[M] = object
myinfo: M
TypeRegister[M] = object
table: SharedTable[TypeKey, TypeInfo[M]]
proc createTypeRegister*(M: typedesc): TypeRegister[M] =
result.table = initSharedTable[TypeKey, TypeInfo[M]](64)
proc find[A,B](table: var SharedTable[A,B], k: A): Option[B] {.inline, noSideEffect.} =
## Ugly work-around for missing SharedTable.hasKey()
try:
some(table.mget(k))
except:
none(B)
proc find*[M](reg: var TypeRegister[M], tk: TypeKey): Option[TypeInfo[M]] {.inline, noSideEffect.} =
reg.table.find(tk)
proc `[]=`*[M](reg: var TypeRegister[M], tk: TypeKey, tv: TypeInfo[M]) {.inline.} =
reg.table[tk] = tv
when isMainModule:
echo("TESTING...")
var t = createTypeRegister(bool)
var tv: TypeInfo[bool]
tv.myinfo = true
t[TypeKey(4)] = tv
let find2 = t.find(TypeKey(2))
let find4 = t.find(TypeKey(4))
assert(isNone(find2))
assert(isSome(find4))
echo("DONE")
This compiles and work fine. In my real code, the two "find()" procs are exactly the same as in this example. But calling "table.mget(k)" there results in this compilation error:
lib\pure\collections\sharedtables.nim(114, 10) Error: undeclared identifier: 'hasKey'
The code of SharedTable.mget() looks like this:
proc mget*[A, B](t: var SharedTable[A, B], key: A): var B =
## retrieves the value at ``t[key]``. The value can be modified.
## If `key` is not in `t`, the ``KeyError`` exception is raised.
withLock t:
var hc: Hash
var index = rawGet(t, key, hc)
let hasKey = index >= 0
if hasKey: result = t.data[index].val
if not hasKey: # THIS IS LINE 114
when compiles($key):
raise newException(KeyError, "key not found: " & $key)
else:
raise newException(KeyError, "key not found")
I totally fail to see how this would produce a different result, if called by basically the same code.
This is the actual code, with one module (sharedarray) inserted "inline" for simplicity:
# A type-register maps types to IDs and back, at runtime.
# No attempt is made to return the same IDs on every run.
import locks
import typetraits
import hashes
import sharedtables
import options
import algorithm
import math
#import tables
#import sharedarray
#############################################################################
# This should be in imported module: sharedarray
#############################################################################
type
# TODO Find a way to make the type restriction be recursive
SharedArray*[T: not (ref|seq|string)] = ptr object
## Simple variable-length shared-heap array, with a pre-allocated capacity.
## Particularly useful because still compatible with open-array.
## It is *not* guarded by a lock, and therefore not thread-safe by default.
size: int
## "first int" size, so we are compatible with openarray.
mycapacity: int
## "second int" is required for seq-to-openarray compatibility, and used as "capacity" in our case.
data: UncheckedArray[T]
## The data itself
proc len*[T](list: SharedArray[T]): int {.inline, noSideEffect.} =
## Returns the current length/size of the shared array. 0 if nil.
if list != nil:
list.size
else:
0
proc isNilOrEmpty*[T](list: SharedArray[T]): bool {.inline, noSideEffect.} =
## Returns true if the current length/size of the shared array is 0 or if it is nil.
(list == nil) or (list.size == 0)
proc clear*[T](list: var SharedArray[T]): void =
## Sets the current length/size of the shared array to 0.
if list != nil:
list.size = 0
proc capacity*[T](list: SharedArray[T]): int {.inline, noSideEffect.} =
## Returns the fixed capacity of the shared array. 0 if nil.
if list != nil:
list.mycapacity
else:
0
iterator items*[T](it: var SharedArray[T]): T =
## Iterates over all array items. it cannot be nil.
assert(it != nil)
for i in 0..it.size-1:
yield it.data[i]
proc initSharedArray*[T](capacity: int = 64): SharedArray[T] =
## Returns a new shared-heap array with a given maximum capacity, which must be power-of-two.
assert(capacity >= 0)
assert(isPowerOfTwo(capacity))
result = cast[SharedArray[T]](allocShared0(2*sizeof(int)+capacity*sizeof(T)))
result.mycapacity = capacity
proc deinitSharedArray*[T](list: var SharedArray[T]): void =
## Destroys shared-heap array, if not nil.
if list != nil:
deallocShared list
list = nil
proc `[]`*[T](list: SharedArray[T]; i: int): var T {.inline, noSideEffect.} =
## Gets an array item.
assert(list != nil)
if (i < 0) or (i >= list.size):
raise newException(Exception, "bad index: " & $i & " must be in [0, " & $list.size & "[")
list.data[i]
proc `[]=`*[T](list: SharedArray[T]; i: int, x: T) {.inline.} =
## Sets an array item.
assert(list != nil)
if (i < 0) or (i >= list.size):
raise newException(Exception, "bad index: " & $i & " must be in [0, " & $list.size & "[")
list.data[i] = x
proc add*[T](list: var SharedArray[T]; x: T): void {.inline.} =
## Appends an array item.
assert(list != nil)
let size = list.size
list.setLen(size+1)
list.data[size] = x
proc toSharedArray*[T](src: openarray[T], capacity: int = -1): SharedArray[T] =
## Copies an open-array into a new SharedArray.
## capacity is used, if bigger than src.len
let size = len(src)
assert(size >= 0)
result = initSharedArray[T](max(size, capacity))
for i in 0..size-1:
result.data[i] = src[i]
result.size = size
proc toSeq*[T](list: SharedArray[T]): seq[T] =
## Copies a SharedArray into a Seq
assert(list != nil)
result = newSeq[T](list.size)
for i in 0..list.size-1:
result[i] = list.data[i]
template asOpenArray*[T](list: SharedArray[T]): openarray[T] =
## Casts a SharedArray to an open array.
cast[seq[T]](list)
proc setLen*[T](list: var SharedArray[T], newLen: int): void =
## Sets the current length/size of the shared array. It cannot be nil.
## If newLen is greater than the capacity, the SharedArray is copied into
## a new SharedArray with the required capacity, list is set to the new
## SharedArray, and the old SharedArray is destroyed.
assert(list != nil)
if newLen < 0:
raise newException(Exception, "newLen " & $newLen & " too small")
if newLen > list.mycapacity:
var old = list
list = toSharedArray(asOpenArray(old), nextPowerOfTwo(newLen))
deinitSharedArray(old)
list.size = newLen
#############################################################################
# End of imported module: sharedarray
#############################################################################
type
TypeKey* = object
## Represents a "key" for the type register.
## Types are automatically converted to it with a converter.
myname: cstring
myhash: Hash
TypeInfo*[M] = object
## The info about a type: it's name, ID, it's size, and whatever meta-info the user also needed.
myname: cstring
myid: uint32
mysize: uint32
myinfo: M
TypeRegister[M] = object
## A type register containing type T meta-info.
table: SharedTable[TypeKey, TypeInfo[M]]
list: SharedArray[TypeInfo[M]] ## Allows quickly accessing TypeInfo[M] by ID
myfrozen: bool ## TypeRegister cannot be modified anymore.
lock: Lock
template withLock(t, x: untyped) =
acquire(t.lock)
try:
x
finally:
release(t.lock)
proc copyCString(s: cstring): cstring =
## Creates a copy of a (temporary) cstring into the shared-heap
let len = s.len
result = cast[cstring](allocShared(len+1))
copyMem(cast[pointer](result), cast[pointer](s), len+1)
converter toTypeKey*(T: typedesc): TypeKey {.inline, noSideEffect.} =
## Automatically converts a type to a TypeKey. The type name is *not* cloned.
result.myname = T.name
result.myhash = hash(result.myname)
proc hash*(tk: TypeKey): Hash {.inline, noSideEffect.} =
## Returns the hash of the TypeKey
result = tk.myhash
proc `$`*(tk: TypeKey): string {.inline.} =
## Returns the string representation of the TypeKey
result = $tk.myname
proc `==`*(a, b: TypeKey): bool {.inline, noSideEffect.} =
## Compares TypeKeys
(a.myhash == b.myhash) and (cmp(a.myname, b.myname) == 0)
proc `$`*[M](ti: TypeInfo[M]): string {.inline.} =
## Returns the string representation of the TypeKey
result = $ti.myname
proc name*[M](ti: TypeInfo[M]): cstring {.inline, noSideEffect.} =
## Returns the id of the type
result = ti.myname
proc id*[M](ti: TypeInfo[M]): uint32 {.inline, noSideEffect.} =
## Returns the id of the type
result = ti.myid
proc size*[M](ti: TypeInfo[M]): uint32 {.inline, noSideEffect.} =
## Returns the size of the type
result = ti.mysize
proc info*[M](ti: TypeInfo[M]): M {.inline, noSideEffect.} =
## Returns the meta-info of the type
result = ti.myinfo
proc initTypeInfo*[M](name: cstring, id: uint32, size: uint32, info: M): TypeInfo[M] {.inline, noSideEffect.} =
## Initialises and return a TypeInfo[M]. name is *not* cloned.
result.myname = name
result.myid = id
result.mysize = size
result.myinfo = info
# NOT CURRENTLY THREAD-SAFE:
#iterator items*[M](reg: TypeRegister[M]): TypeInfo[M] =
# ## iterates over any TypeInfo[M] in the TypeRegister `reg`.
# reg.list.items
proc len*[M](reg: TypeRegister[M]): int {.inline, noSideEffect.} =
## Returns the current size of the TypeRegister.
reg.list.len
proc createTypeRegister*(M: typedesc, capacity: int = 64): TypeRegister[M] =
## Creates a new type register.
assert(capacity >= 0)
assert(isPowerOfTwo(capacity))
result.table = initSharedTable[TypeKey, TypeInfo[M]](capacity)
result.list = initSharedArray[TypeInfo[M]](capacity)
initLock result.lock
proc createTypeRegister*(M: typedesc, src: openArray[TypeInfo[M]]): TypeRegister[M] =
## Creates a new type register, and initialise it from src.
## Note: src will be *sorted* by ID during creation!
# First, validate the hell out of the input.
proc ti_id_cmp(a,b: TypeInfo[M]): int =
int(a.myid) - int(b.myid)
sort(src, ti_id_cmp)
var nextID: uint32 = 0
for ti in src:
if ti.myname == nil:
raise newException(Exception, "Type (ID: " & $ti.myid & "): name cannot be nil!")
if ti.myname.len == 0:
raise newException(Exception, "Type (ID: " & $ti.myid & "): name cannot be empty!")
if ti.myname.split.len != 1:
raise newException(Exception, "Type (ID: " & $ti.myid & ", name: " & ti.myname & "): name cannot contain whitespace!")
if nextID != ti.myid:
raise newException(Exception, "Type (ID: " & $ti.myid & ", name: " & ti.myname & "): ID should be " & $nextID)
if ti.mysize == 0:
raise newException(Exception, "Type (ID: " & $ti.myid & ", name: " & ti.myname & "): cannot have size 0")
nextID.inc
# XXX TODO!
#[
var keys = initTable[TypeKey, bool](nextPowerOfTwo(src.len))
for ti in src:
var tk : TypeKey
tk.myname = ti.myname
tk.myhash = hash(tk.myname)
if keys.hasKeyOrPut(tk):
raise newException(Exception, "Duplicate type name: " & $ti.myname)
]#
# Input seems OK... so create it.
result = createTypeRegister(M, src.len)
result.withLock:
for ti in src:
var tk : TypeKey
tk.myname = copyCString(ti.myname)
tk.myhash = hash(tk.myname)
let ti = initTypeInfo(tk.myname, ti.myid, ti.mysize, ti.myinfo)
result.table[tk] = ti
result.list.add(ti)
proc createTypeRegister*(M: typedesc, src: iterator : TypeInfo[M]): TypeRegister[M] =
## Creates a new type register, and initialise it from src.
var infos = newSeq[TypeInfo[M]]
for ti in src:
infos.add(ti)
createTypeRegister(M, infos)
proc destroyTypeRegister*[M](reg: var TypeRegister[M]): void =
## Destroys a new type register.
reg.withLock:
deinitSharedTable(reg.table)
deinitSharedArray(reg.list)
deinitLock(reg.lock)
proc freeze*[M](reg: var TypeRegister[M]): void =
## Freezes the type register, so that no new type can be registered.
reg.withLock:
reg.myfrozen = true
proc frozen*[M](reg: var TypeRegister[M]): bool =
## Returns true if no new type can be registered.
reg.withLock:
result = reg.myfrozen
proc find[A,B](table: var SharedTable[A,B], k: A): Option[B] {.inline, noSideEffect.} =
## Ugly work-around for missing SharedTable.hasKey()
try:
#############################################################################
# Start of non-compiling code:
#############################################################################
some(table.mget(k))
#############################################################################
# End of non-compiling code.
#############################################################################
except:
none(B)
proc find*[M](reg: var TypeRegister[M], tk: TypeKey): Option[TypeInfo[M]] {.inline, noSideEffect.} =
## Returns a type's type-info, if present.
reg.table.find(tk)
proc register*[M](reg: var TypeRegister[M], T: typedesc, info: M): TypeInfo[M] =
## Register a new type, and returns it's type-info.
let tk = toTypeKey(T)
var opt = reg.find(tk)
if isSome(opt):
result = unsafeGet(opt)
else:
reg.withLock:
var opt = reg.table.find(tk)
if isSome(opt):
result = unsafeGet(opt)
else:
let id = uint32(reg.len)
if id == high(uint32):
raise newException(Exception, "Too many types!")
if reg.myfrozen:
raise newException(Exception, "Type register is frozen!")
var tk2 : TypeKey
tk2.myname = copyCString(tk.myname)
tk2.myhash = tk.myhash
result = initTypeInfo(tk2.myname, id, uint32(sizeof(T)), info)
reg.table[tk] = result
reg.list.add(result)
proc get*[M](reg: var TypeRegister[M], T: typedesc): TypeInfo[M] {.inline.} =
## Returns a type's type-info, if present.
## Otherwise, register the type with a default meta-info!
var m: M
register(reg, T, m)
when isMainModule:
import threadpool
var reg = createTypeRegister(bool)
proc typeIDTestHelper(): uint32 = reg.get(uint8).id
proc testTypeID(): void =
echo "STARTING testTypeID()"
let t0 = reg.get(uint8).id
let t1 = reg.get(bool).id
let fv = spawn typeIDTestHelper()
let t3 = ^fv
assert(t0 != t1)
assert(t0 == t3)
echo "DONE testTypeID()"
proc testTypeName(): void =
echo "STARTING testTypeName()"
let t0 = reg.get(uint8).name
let t1 = reg.get(bool).name
assert($t0 == "uint8")
assert($t1 == "bool")
echo "DONE testTypeName()"
testTypeID()
testTypeName()
destroyTypeRegister(reg)
You have your own withLock implementation that the instantiation of mget prefers which introduces a new scope and so hasKey is outside of the scope. Probably sharedtables should be changed so that withLock is not used as a mixin. In the meantime, name yours withL and it compiles if you also change your find to
proc find[A,B](table: var SharedTable[A,B], k: A): Option[B] {.inline, noSideEffect.} =
## Ugly work-around for missing SharedTable.hasKey()
result = try:
some(table.mget(k))
except:
none(B)
That len and hasKey are missing is no oversight btw, it's almost impossible to use these without introducing subtle races. Yes, yes, I know, you only insert at program startup and then only read from it. But still. Why not use the variant of withValue that takes 2 pieces of code.
@Araq Thanks! This seems to work. :)
it's almost impossible to use these without introducing subtle races
I assume you mean races in the user code, rahter than the SharedTable code, I guess.
But I must still ask: I've seen the "withLock" pattern used im multiple place (OK, mine was slightly changed). As long as it's not "public", I thought they could not interfere with each other? How is that possible? Is it because it's a template?