• There is NO official Otland's Discord server and NO official Otland's server list. The Otland's Staff does not manage any Discord server or server list. Moderators or administrator of any Discord server or server lists have NO connection to the Otland's Staff. Do not get scammed!

[Lua] Array serialization/ deserialization into/from single storage value.

tarjei

Necronian Engineer
Joined
May 25, 2008
Messages
505
Reaction score
126
Location
Poland
Hello, it's been some time since I made anything for public. Evans thread inspired me to improve my MultiSttorage value lib, and here I come with Array serialization.

We need BinaryStream first. I added some modification so I post it again.
Lua:
--[[
    Binary Stream
Allows you to read, edit and create new binary files
Developed by Colex (Alexandre Santos)
]]--
 
require("bit")
 
_BINARY_STREAM = {
    Position = 0
}
function _BINARY_STREAM:setBuffer(buff)
	local _buff = string.gsub(buff,"__ZERO", "\0")
	self.Buffer = _buff
end
BinaryStream = function(filename)
    local obj = {}
	local file = nil
	
	if filename then
     file = io.open(filename, "rb")
	end 
	
    if not (file) then
       -- print("Warrning: file wasnt opened")
    end
 
    setmetatable(obj, _BINARY_STREAM)
    _BINARY_STREAM.__index = _BINARY_STREAM
 
    obj["Buffer"] = file~= nil and file:read("*all") or ""
    obj["Length"] = obj.Buffer:len()
    obj["FileName"] = filename
    if file~= nil then file:close() end
    return obj
end
 
---Read Functions
 
_BINARY_STREAM.ReadByte = function(self)
    self.Position = self.Position + 1
    return self.Buffer:byte(self.Position)
end
_BINARY_STREAM.Check = function(self, ...)
    for i = 1, table.maxv(arg) do --to get rid of embedded zeroes
		if arg[i] == 0 then arg[i] = "__ZERO" end
	end
	return unpack(arg)
end

_BINARY_STREAM.ReadInt16 = function(self, reversed)
 
	local l1,l2 = 0
	if not(reversed) then
     l1 = self:ReadByte()
     l2 = bit.lshift(self:ReadByte(), 8)

	else
     l1 = bit.lshift(self:ReadByte(), 8)
     l2 = self:ReadByte()	 
	end
 
	
    return   bit.bor(l1,l2)
end
 

_BINARY_STREAM.ReadInt32 = function(self, reversed)
	local l1,l2,l3,l4 = 0
	if not(reversed) then
     l1 = self:ReadByte()
     l2 = bit.lshift(self:ReadByte(), 8)
     l3 = bit.lshift(self:ReadByte(), 16)
     l4 = bit.lshift(self:ReadByte(), 24)
	else
     l1 = bit.lshift(self:ReadByte(),24)
     l2 = bit.lshift(self:ReadByte(),16)
     l3 = bit.lshift(self:ReadByte(), 8)
     l4 = self:ReadByte()	 
	end
	
 
    return  bit.bor(l1,l2,l3,l4)
end
 _BINARY_STREAM.ReadString = function(self)
    local len = self:ReadInt16(true)
	local ret = ""
	if len == 0 then
	 return ret
	end
	for i = 0, len - 1 do
		local char = self:ReadByte()
		if char == nil then break end
		ret = ret ..string.char(char)
	end
 
	--print("Reading Streang| Len| "..len)
    return ret
end

--Write Functions
 
_BINARY_STREAM.WriteByte = function(self, byte, insert)
    self.Position = self.Position + 1
	if type(byte) == "number" then
	   byte = string.char(byte)
	end
    if insert then
        self.Buffer = self.Buffer:sub(0,self.Position-1)..byte..self.Buffer:sub(self.Position+1)
    else
        self.Buffer = self.Buffer..byte
    end
end
  _BINARY_STREAM.WriteString = function(self, str)
    local len = str:len()
	--print("[WriteStringg] str:"..str.." Len: "..len)
	self:WriteInt16(len)
	self.Buffer = self.Buffer..str
 
	

end
_BINARY_STREAM.WriteInt16 = function(self, int16, insert)
		local l1 = bit.rshift(int16, 8)
		local l2 = int16 - bit.lshift(l1,8)
	l1,l2 = self:Check(l1,l2)
	
    self:WriteByte(l1, insert)
    self:WriteByte(l2, insert)
end
 
_BINARY_STREAM.WriteInt32 = function(self, int32, insert)
		local l1 = bit.rshift(int32, 24)
		local l2 = bit.rshift(int32, 16) - bit.lshift(l1,8)
		local l3 = bit.rshift(int32, 8) - bit.lshift(l1,16) - bit.lshift(l2,8)
		local l4 = int32 - bit.lshift(l1,24) - bit.lshift(l2,16) - bit.lshift(l3,8)
	
	l1,l2,l3,l4 = self:Check(l1,l2,l3,l4)
	
    self:WriteByte(l4, insert)
    self:WriteByte(l3, insert)
    self:WriteByte(l2, insert)
    self:WriteByte(l1, insert)
end
 
--Insert Function
 
_BINARY_STREAM.InsertByte = function(self, byte)
    self:WriteByte(byte, true)
end
 
_BINARY_STREAM.InsertInt16 = function(self, int16)
    self:WriteInt16(int16, true)
end
 
_BINARY_STREAM.InsertInt32 = function(self, int32)
    self:WriteInt32(int32, true)
end
 
 
 
--Position Functions
 
_BINARY_STREAM.Skip = function(self, bytes)
    self.Position = self.Position + bytes
end
 
_BINARY_STREAM.Seek = function(self, pos, origin)
    if (origin == 0) then --Begin
        self.Position = pos
    elseif (origin == 1) then --Current Position
        self.Position = self.Position + pos
    elseif (origin == 2) then --End
        self.Position = self.length + pos
    end
end
 function _BINARY_STREAM.Tell(self)
	return self.Position
 end
 
 _BINARY_STREAM.resize = function(self, lastpos)
	local len = self.Buffer:len()
	if lastpos < len then
	  return
	end
	self:Seek(len,0)
	for i = 0, lastpos - len do
		
		self:WriteByte(0)
	end
end
_BINARY_STREAM.paste = function(self, buff)
	self.Buffer = self.Buffer:sub(0,self.Position-1)..buff..self.Buffer:sub(self.Position+buff:len())
end
_BINARY_STREAM.cut = function(self, from, to)
	return self.Buffer:sub(from, to)
end
--Save function
 
_BINARY_STREAM.Save = function(self, filename)
    filename = filename or self.FileName
	--print("Saving..")
    local file = io.open(filename, "wb")
    file:write(self.Buffer)
    file:close()
end


And now the good stuff.

Array.lua
Lua:
Array = {
		types = {
				--bit codes for encoding types.
				["string"]= 1,
				["number"]= 2,
				["table"]= 3,
		}

}
function Array:new(arr)
	local obj = {}
	self.__index = self
	obj.array = arr
	obj.output = {}
	setmetatable(obj,self)
	
	return obj
end
function Array:serializeArray(stream, arr)
	stream:WriteInt32(table.maxv(arr))
	for k ,v in pairs(arr) do
		stream:WriteByte(self.types[type(k)])
		if( type(k) == "string" or type(k) == "number") then
			stream:WriteString(tostring(k))
		end
		if( type(k) == "table") then
			error("Array cant be an index")
		end
		stream:WriteByte(self.types[type(v)])
		if( type(v) == "string" or type(v) == "number") then
			stream:WriteString(tostring(v))
		end
		if( type(v) == "table" )then
		
			self:serializeArray(stream, v)
		end	
	end
end
function Array:serialize(cid, storage)
	local multistorage = MultiStorage:new(storage)
	local stream = BinaryStream()
	--How many records this array has
	self:serializeArray(stream, self.array)	
	doCreatureSetStorage(cid,storage, stream["Buffer"])
end

function Array:deserializeArray(stream, arr)
	local size = stream:ReadInt32()
	print(size)
	for i = 1, size do
		local keyType = stream:ReadByte()
		
		local key = stream:ReadString()
		if keyType == 2 then
		  key = tonumber(key)
		end
		local valueType = stream:ReadByte()
		if valueType ~= 3 then
			local value = stream:ReadString()
			if valueType == 2 then
				value = tonumber(value)
			end
			arr[key] = value
		elseif valueType == 3 then
			arr[key] = {}
			self:deserializeArray(stream, arr[key])
		end
	end
end
function Array:deserialize(cid, storage)
	local buffer = getCreatureStorage(cid, storage)
	local stream = BinaryStream()
	stream:setBuffer( buffer)
	self:deserializeArray(stream, self.output)
	return self.output
end

How do we use it?
Its deadly simple :

Lets define a table first.
Lua:
local data = {
	["name"] = "Tarjei",
	["age"] = 21,
	["killCounter"] = 0,
	["features"] = {
		"programmer", "c++", "lua", "OpenTibia"
	},
	[2] = "Well, this is sample index",

}

Then we create new array instance, and simply use it.
Lua:
local test = Array:new(data)
test:serialize(cid,4000)
local output = test:deserialize(cid, 4000)
for k, v in pairs(output["features"]) do
	doCreatureSay(cid,v,1)
end

Code above first saved whole array into storage value, then it read it back and printed saved values.
I find this trick preety nice, thats why I wrote those both code and thread XD

Advantages:
you can pack hell a lot of values into signle storage value.

Disadvatages:
If anyone finds them post it XD

@edit:
I came up with practical use of it. This thing reminded me of counter how many specific monsters player has killed. If I am not wrong ppl were using single storage to keep value for each mob kind counter.
Here is how it might look like with this lib.

It's untested, though it will hold all counters for every creature player will kill in ONE storage value :D
Lua:
g_counters = {
}

local counterStorage = 4001
function onLogin(cid)
	--Preparing countertable on player login.
	local d = Array:new()
	local buffer = getPlayerStorage(cid,counterStorage)
	if type(buffer) ~= "string" then
	 g_counters[cid] = {}
	else
	  g_counters[cid] = d:deserialize(cid,counterStorage)
	end
	return true
end
function onLogout(cid)
	--Preparing countertable on player login.
	local d = Array:new(g_counters[cid])
	d:serialize(cid,counterStorage)
	return true
end
function onKill(cid, target)
	if g_counters[cid][getCreatureName(target)]) then
		g_counters[cid][getCreatureName(target)]) = g_counters[cid][getCreatureName(target)]) + 1
	else
		g_counters[cid][getCreatureName(target)]) = 0
	end
	return true
end
 
Last edited:
No, but I dont really know what for you want to save metatable xD

- - - Updated - - -

Hmm, well you can always try that for metatable : P
 
Well it was a way for me to save data between different script types (actions,creaturescripts,spells etc..) I made a metatable for each player containing the data for each player, but I need to save that data to a storagevalue.
 
Well and how you did that? I mean
Code:
local obj = {}
local meta = {}
meta.__self = obj
setmetatable(obj,meta)
Something like that?
 
Well and how that helps you to comunicate between lua states? XD
This lib serves only to save many using single storage. By the way, if I were you I would get rid of all those lua states and replace it with single one using OTC framework :p
 
I haven't looked into OTC framework, how would I do something like that?
 
Well its preety nasty and time consuming. You should start reading how lua states are implemented and later on OTC framework in order to do that xD
 
Hello, it's been some time since I made anything for public. Evans thread inspired me to improve my MultiSttorage value lib, and here I come with Array serialization.

We need BinaryStream first. I added some modification so I post it again.
Lua:
--[[
    Binary Stream
Allows you to read, edit and create new binary files
Developed by Colex (Alexandre Santos)
]]--

require("bit")

_BINARY_STREAM = {
    Position = 0
}
function _BINARY_STREAM:setBuffer(buff)
    local _buff = string.gsub(buff,"__ZERO", "\0")
    self.Buffer = _buff
end
BinaryStream = function(filename)
    local obj = {}
    local file = nil
   
    if filename then
     file = io.open(filename, "rb")
    end
   
    if not (file) then
       -- print("Warrning: file wasnt opened")
    end

    setmetatable(obj, _BINARY_STREAM)
    _BINARY_STREAM.__index = _BINARY_STREAM

    obj["Buffer"] = file~= nil and file:read("*all") or ""
    obj["Length"] = obj.Buffer:len()
    obj["FileName"] = filename
    if file~= nil then file:close() end
    return obj
end

---Read Functions

_BINARY_STREAM.ReadByte = function(self)
    self.Position = self.Position + 1
    return self.Buffer:byte(self.Position)
end
_BINARY_STREAM.Check = function(self, ...)
    for i = 1, table.maxv(arg) do --to get rid of embedded zeroes
        if arg[I] == 0 then arg[I] = "__ZERO" end
    end
    return unpack(arg)
end

_BINARY_STREAM.ReadInt16 = function(self, reversed)

    local l1,l2 = 0
    if not(reversed) then
     l1 = self:ReadByte()
     l2 = bit.lshift(self:ReadByte(), 8)

    else
     l1 = bit.lshift(self:ReadByte(), 8)
     l2 = self:ReadByte()     
    end

    
    return   bit.bor(l1,l2)
end


_BINARY_STREAM.ReadInt32 = function(self, reversed)
    local l1,l2,l3,l4 = 0
    if not(reversed) then
     l1 = self:ReadByte()
     l2 = bit.lshift(self:ReadByte(), 8)
     l3 = bit.lshift(self:ReadByte(), 16)
     l4 = bit.lshift(self:ReadByte(), 24)
    else
     l1 = bit.lshift(self:ReadByte(),24)
     l2 = bit.lshift(self:ReadByte(),16)
     l3 = bit.lshift(self:ReadByte(), 8)
     l4 = self:ReadByte()     
    end
    

    return  bit.bor(l1,l2,l3,l4)
end
_BINARY_STREAM.ReadString = function(self)
    local len = self:ReadInt16(true)
    local ret = ""
    if len == 0 then
     return ret
    end
    for i = 0, len - 1 do
        local char = self:ReadByte()
        if char == nil then break end
        ret = ret ..string.char(char)
    end

    --print("Reading Streang| Len| "..len)
    return ret
end

--Write Functions

_BINARY_STREAM.WriteByte = function(self, byte, insert)
    self.Position = self.Position + 1
    if type(byte) == "number" then
       byte = string.char(byte)
    end
    if insert then
        self.Buffer = self.Buffer:sub(0,self.Position-1)..byte..self.Buffer:sub(self.Position+1)
    else
        self.Buffer = self.Buffer..byte
    end
end
  _BINARY_STREAM.WriteString = function(self, str)
    local len = str:len()
    --print("[WriteStringg] str:"..str.." Len: "..len)
    self:WriteInt16(len)
    self.Buffer = self.Buffer..str

    

end
_BINARY_STREAM.WriteInt16 = function(self, int16, insert)
        local l1 = bit.rshift(int16, 8)
        local l2 = int16 - bit.lshift(l1,8)
    l1,l2 = self:Check(l1,l2)
    
    self:WriteByte(l1, insert)
    self:WriteByte(l2, insert)
end

_BINARY_STREAM.WriteInt32 = function(self, int32, insert)
        local l1 = bit.rshift(int32, 24)
        local l2 = bit.rshift(int32, 16) - bit.lshift(l1,8)
        local l3 = bit.rshift(int32, 8) - bit.lshift(l1,16) - bit.lshift(l2,8)
        local l4 = int32 - bit.lshift(l1,24) - bit.lshift(l2,16) - bit.lshift(l3,8)
    
    l1,l2,l3,l4 = self:Check(l1,l2,l3,l4)
    
    self:WriteByte(l4, insert)
    self:WriteByte(l3, insert)
    self:WriteByte(l2, insert)
    self:WriteByte(l1, insert)
end

--Insert Function

_BINARY_STREAM.InsertByte = function(self, byte)
    self:WriteByte(byte, true)
end

_BINARY_STREAM.InsertInt16 = function(self, int16)
    self:WriteInt16(int16, true)
end

_BINARY_STREAM.InsertInt32 = function(self, int32)
    self:WriteInt32(int32, true)
end



--Position Functions

_BINARY_STREAM.Skip = function(self, bytes)
    self.Position = self.Position + bytes
end

_BINARY_STREAM.Seek = function(self, pos, origin)
    if (origin == 0) then --Begin
        self.Position = pos
    elseif (origin == 1) then --Current Position
        self.Position = self.Position + pos
    elseif (origin == 2) then --End
        self.Position = self.length + pos
    end
end
function _BINARY_STREAM.Tell(self)
    return self.Position
end

_BINARY_STREAM.resize = function(self, lastpos)
    local len = self.Buffer:len()
    if lastpos < len then
      return
    end
    self:Seek(len,0)
    for i = 0, lastpos - len do
        
        self:WriteByte(0)
    end
end
_BINARY_STREAM.paste = function(self, buff)
    self.Buffer = self.Buffer:sub(0,self.Position-1)..buff..self.Buffer:sub(self.Position+buff:len())
end
_BINARY_STREAM.cut = function(self, from, to)
    return self.Buffer:sub(from, to)
end
--Save function

_BINARY_STREAM.Save = function(self, filename)
    filename = filename or self.FileName
    --print("Saving..")
    local file = io.open(filename, "wb")
    file:write(self.Buffer)
    file:close()
end


And now the good stuff.

Array.lua
Lua:
Array = {
        types = {
                --bit codes for encoding types.
                ["string"]= 1,
                ["number"]= 2,
                ["table"]= 3,
        }

}
function Array:new(arr)
    local obj = {}
    self.__index = self
    obj.array = arr
    obj.output = {}
    setmetatable(obj,self)
    
    return obj
end
function Array:serializeArray(stream, arr)
    stream:WriteInt32(table.maxv(arr))
    for k ,v in pairs(arr) do
        stream:WriteByte(self.types[type(k)])
        if( type(k) == "string" or type(k) == "number") then
            stream:WriteString(tostring(k))
        end
        if( type(k) == "table") then
            error("Array cant be an index")
        end
        stream:WriteByte(self.types[type(v)])
        if( type(v) == "string" or type(v) == "number") then
            stream:WriteString(tostring(v))
        end
        if( type(v) == "table" )then
        
            self:serializeArray(stream, v)
        end    
    end
end
function Array:serialize(cid, storage)
    local multistorage = MultiStorage:new(storage)
    local stream = BinaryStream()
    --How many records this array has
    self:serializeArray(stream, self.array)    
    doCreatureSetStorage(cid,storage, stream["Buffer"])
end

function Array:deserializeArray(stream, arr)
    local size = stream:ReadInt32()
    print(size)
    for i = 1, size do
        local keyType = stream:ReadByte()
        
        local key = stream:ReadString()
        if keyType == 2 then
          key = tonumber(key)
        end
        local valueType = stream:ReadByte()
        if valueType ~= 3 then
            local value = stream:ReadString()
            if valueType == 2 then
                value = tonumber(value)
            end
            arr[key] = value
        elseif valueType == 3 then
            arr[key] = {}
            self:deserializeArray(stream, arr[key])
        end
    end
end
function Array:deserialize(cid, storage)
    local buffer = getCreatureStorage(cid, storage)
    local stream = BinaryStream()
    stream:setBuffer( buffer)
    self:deserializeArray(stream, self.output)
    return self.output
end

How do we use it?
Its deadly simple :

Lets define a table first.
Lua:
local data = {
    ["name"] = "Tarjei",
    ["age"] = 21,
    ["killCounter"] = 0,
    ["features"] = {
        "programmer", "c++", "lua", "OpenTibia"
    },
    [2] = "Well, this is sample index",

}

Then we create new array instance, and simply use it.
Lua:
local test = Array:new(data)
test:serialize(cid,4000)
local output = test:deserialize(cid, 4000)
for k, v in pairs(output["features"]) do
    doCreatureSay(cid,v,1)
end

Code above first saved whole array into storage value, then it read it back and printed saved values.
I find this trick preety nice, thats why I wrote those both code and thread XD

Advantages:
you can pack hell a lot of values into signle storage value.

Disadvatages:
If anyone finds them post it XD

@edit:
I came up with practical use of it. This thing reminded me of counter how many specific monsters player has killed. If I am not wrong ppl were using single storage to keep value for each mob kind counter.
Here is how it might look like with this lib.

It's untested, though it will hold all counters for every creature player will kill in ONE storage value :D
Lua:
g_counters = {
}

local counterStorage = 4001
function onLogin(cid)
    --Preparing countertable on player login.
    local d = Array:new()
    local buffer = getPlayerStorage(cid,counterStorage)
    if type(buffer) ~= "string" then
     g_counters[cid] = {}
    else
      g_counters[cid] = d:deserialize(cid,counterStorage)
    end
    return true
end
function onLogout(cid)
    --Preparing countertable on player login.
    local d = Array:new(g_counters[cid])
    d:serialize(cid,counterStorage)
    return true
end
function onKill(cid, target)
    if g_counters[cid][getCreatureName(target)]) then
        g_counters[cid][getCreatureName(target)]) = g_counters[cid][getCreatureName(target)]) + 1
    else
        g_counters[cid][getCreatureName(target)]) = 0
    end
    return true
end
[/I][/I]

For me, I have a problem with:

local multistorage = MultiStorage:new(storage)

returning nil value
 
@up I believe you are trying it with TFS 1.0? Im not sure if that distro can store strings or binary stuff.
 
Back
Top