• There is NO official Otland's Discord server and NO official Otland's server list. The Otland's Staff does not manage any Discord server or server list. Moderators or administrator of any Discord server or server lists have NO connection to the Otland's Staff. Do not get scammed!

Protod3 - Tool to extract .proto files from the Tibia client

kokokoko

Veteran OT User
Joined
Feb 4, 2008
Messages
921
Reaction score
256
Hello!

I had some trouble generating .proto files from the Tibia client using sysdream/Protod. I ended up making some changes myself to make it work and thought I'd share those here. I was also able to get the running time down from ~20 seconds to 0.4 seconds on my machine which is nice šŸ˜„

My version is here: giuinktse7/Protod3

To run it, you need Python 3 and the google protobuf library for python. You can get the library for example from pip (protobuf (https://pypi.org/project/protobuf/)) by running
Code:
pip install protobuf

.proto files from the test server client:

appearances.proto
Code:
syntax = "proto2";

package tibia.protobuf.appearances;

import "shared.proto";

enum FIXED_FRAME_GROUP {
  FIXED_FRAME_GROUP_OUTFIT_IDLE = 0;
  FIXED_FRAME_GROUP_OUTFIT_MOVING = 1;
  FIXED_FRAME_GROUP_OBJECT_INITIAL = 2;
}

message Appearances {
  repeated Appearance object = 1;
  repeated Appearance outfit = 2;
  repeated Appearance effect = 3;
  repeated Appearance missile = 4;
  optional SpecialMeaningAppearanceIds special_meaning_appearance_ids = 5;
}

message SpritePhase {
  optional uint32 duration_min = 1;
  optional uint32 duration_max = 2;
}

message SpriteAnimation {
  optional uint32 default_start_phase = 1;
  optional bool synchronized = 2;
  optional bool random_start_phase = 3;
  optional shared.ANIMATION_LOOP_TYPE loop_type = 4;
  optional uint32 loop_count = 5;
  repeated SpritePhase sprite_phase = 6;
}

message Box {
  optional uint32 x = 1;
  optional uint32 y = 2;
  optional uint32 width = 3;
  optional uint32 height = 4;
}

message SpriteInfo {
  optional uint32 pattern_width = 1;
  optional uint32 pattern_height = 2;
  optional uint32 pattern_depth = 3;
  optional uint32 layers = 4;
  repeated uint32 sprite_id = 5;
  optional uint32 bounding_square = 7;
  optional SpriteAnimation animation = 6;
  optional bool is_opaque = 8;
  repeated Box bounding_box_per_direction = 9;
}

message FrameGroup {
  optional FIXED_FRAME_GROUP fixed_frame_group = 1;
  optional uint32 id = 2;
  optional SpriteInfo sprite_info = 3;
}

message Appearance {
  optional uint32 id = 1;
  repeated FrameGroup frame_group = 2;
  optional AppearanceFlags flags = 3;
  optional string name = 4;
  optional string description = 5;
}

message AppearanceFlags {
  optional AppearanceFlagBank bank = 1;
  optional bool clip = 2;
  optional bool bottom = 3;
  optional bool top = 4;
  optional bool container = 5;
  optional bool cumulative = 6;
  optional bool usable = 7;
  optional bool forceuse = 8;
  optional bool multiuse = 9;
  optional AppearanceFlagWrite write = 10;
  optional AppearanceFlagWriteOnce write_once = 11;
  optional bool liquidpool = 12;
  optional bool unpass = 13;
  optional bool unmove = 14;
  optional bool unsight = 15;
  optional bool avoid = 16;
  optional bool no_movement_animation = 17;
  optional bool take = 18;
  optional bool liquidcontainer = 19;
  optional bool hang = 20;
  optional AppearanceFlagHook hook = 21;
  optional bool rotate = 22;
  optional AppearanceFlagLight light = 23;
  optional bool dont_hide = 24;
  optional bool translucent = 25;
  optional AppearanceFlagShift shift = 26;
  optional AppearanceFlagHeight height = 27;
  optional bool lying_object = 28;
  optional bool animate_always = 29;
  optional AppearanceFlagAutomap automap = 30;
  optional AppearanceFlagLenshelp lenshelp = 31;
  optional bool fullbank = 32;
  optional bool ignore_look = 33;
  optional AppearanceFlagClothes clothes = 34;
  optional AppearanceFlagDefaultAction default_action = 35;
  optional AppearanceFlagMarket market = 36;
  optional bool wrap = 37;
  optional bool unwrap = 38;
  optional bool topeffect = 39;
  repeated AppearanceFlagNPC npcsaledata = 40;
  optional AppearanceFlagChangedToExpire changedtoexpire = 41;
  optional bool corpse = 42;
  optional bool player_corpse = 43;
  optional AppearanceFlagCyclopedia cyclopediaitem = 44;
  optional bool ammo = 45;
  optional bool show_off_socket = 46;
  optional bool reportable = 47;
  optional AppearanceFlagUpgradeClassification upgradeclassification = 48;
}

message AppearanceFlagBank {
  optional uint32 waypoints = 1;
}

message AppearanceFlagWrite {
  optional uint32 max_text_length = 1;
}

message AppearanceFlagWriteOnce {
  optional uint32 max_text_length_once = 1;
}

message AppearanceFlagLight {
  optional uint32 brightness = 1;
  optional uint32 color = 2;
}

message AppearanceFlagHeight {
  optional uint32 elevation = 1;
}

message AppearanceFlagShift {
  optional uint32 x = 1;
  optional uint32 y = 2;
}

message AppearanceFlagClothes {
  optional uint32 slot = 1;
}

message AppearanceFlagDefaultAction {
  optional shared.PLAYER_ACTION action = 1;
}

message AppearanceFlagMarket {
  optional shared.ITEM_CATEGORY category = 1;
  optional uint32 trade_as_object_id = 2;
  optional uint32 show_as_object_id = 3;
  repeated shared.VOCATION restrict_to_vocation = 5;
  optional uint32 minimum_level = 6;
}

message AppearanceFlagNPC {
  optional string name = 1;
  optional string location = 2;
  optional uint32 sale_price = 3;
  optional uint32 buy_price = 4;
  optional uint32 currency_object_type_id = 5;
  optional string currency_quest_flag_display_name = 6;
}

message AppearanceFlagAutomap {
  optional uint32 color = 1;
}

message AppearanceFlagHook {
  optional shared.HOOK_TYPE direction = 1;
}

message AppearanceFlagLenshelp {
  optional uint32 id = 1;
}

message AppearanceFlagChangedToExpire {
  optional uint32 former_object_typeid = 1;
}

message AppearanceFlagCyclopedia {
  optional uint32 cyclopedia_type = 1;
}

message AppearanceFlagUpgradeClassification {
  optional uint32 upgrade_classification = 1;
}

message SpecialMeaningAppearanceIds {
  optional uint32 gold_coin_id = 1;
  optional uint32 platinum_coin_id = 2;
  optional uint32 crystal_coin_id = 3;
  optional uint32 tibia_coin_id = 4;
  optional uint32 stamped_letter_id = 5;
  optional uint32 supply_stash_id = 6;
}

map.proto
Code:
syntax = "proto2";

package tibia.protobuf.map;

import "shared.proto";

enum MAP_FILE_TYPE {
  MAP_FILE_TYPE_SUBAREA = 0;
  MAP_FILE_TYPE_SATELLITE = 1;
  MAP_FILE_TYPE_MINIMAP = 2;
}

enum AREA_TYPE {
  AREA_TYPE_NONE = 0;
  AREA_TYPE_AREA = 1;
  AREA_TYPE_SUBAREA = 2;
}

message Map {
  repeated Area areas = 1;
  repeated Npc npcs = 2;
  repeated MapFile resource_files = 3;
  optional shared.Coordinate top_left_tile_coordinate = 4;
  optional shared.Coordinate bottom_right_tile_coordinate = 5;
}

message Area {
  optional uint32 area_id = 1;
  optional string name = 2;
  optional AREA_TYPE area_type = 3;
  repeated uint32 subarea_ids = 4;
  optional shared.Coordinate label_coordinate = 5;
  optional bool reject_donations = 6;
  optional string alias = 7;
}

message Npc {
  optional string name = 1;
  optional shared.Coordinate tile_coordinate = 2;
  optional uint32 subarea_id = 3;
}

message MapFile {
  optional MAP_FILE_TYPE file_type = 1;
  optional shared.Coordinate top_left_coordinate = 2;
  optional string file_name = 3;
  optional uint32 fields_width = 4;
  optional uint32 fields_height = 5;
  optional uint32 area_id = 6;
  optional double scale_factor = 7;
}

shared.proto
Code:
syntax = "proto2";

package tibia.protobuf.shared;

enum PLAYER_ACTION {
  PLAYER_ACTION_NONE = 0;
  PLAYER_ACTION_LOOK = 1;
  PLAYER_ACTION_USE = 2;
  PLAYER_ACTION_OPEN = 3;
  PLAYER_ACTION_AUTOWALK_HIGHLIGHT = 4;
}

enum ITEM_CATEGORY {
  ITEM_CATEGORY_ARMORS = 1;
  ITEM_CATEGORY_AMULETS = 2;
  ITEM_CATEGORY_BOOTS = 3;
  ITEM_CATEGORY_CONTAINERS = 4;
  ITEM_CATEGORY_DECORATION = 5;
  ITEM_CATEGORY_FOOD = 6;
  ITEM_CATEGORY_HELMETS_HATS = 7;
  ITEM_CATEGORY_LEGS = 8;
  ITEM_CATEGORY_OTHERS = 9;
  ITEM_CATEGORY_POTIONS = 10;
  ITEM_CATEGORY_RINGS = 11;
  ITEM_CATEGORY_RUNES = 12;
  ITEM_CATEGORY_SHIELDS = 13;
  ITEM_CATEGORY_TOOLS = 14;
  ITEM_CATEGORY_VALUABLES = 15;
  ITEM_CATEGORY_AMMUNITION = 16;
  ITEM_CATEGORY_AXES = 17;
  ITEM_CATEGORY_CLUBS = 18;
  ITEM_CATEGORY_DISTANCE_WEAPONS = 19;
  ITEM_CATEGORY_SWORDS = 20;
  ITEM_CATEGORY_WANDS_RODS = 21;
  ITEM_CATEGORY_PREMIUM_SCROLLS = 22;
  ITEM_CATEGORY_TIBIA_COINS = 23;
  ITEM_CATEGORY_CREATURE_PRODUCTS = 24;
  ITEM_CATEGORY_QUIVER = 25;
}

enum VOCATION {
  VOCATION_ANY = -1;
  VOCATION_NONE = 0;
  VOCATION_KNIGHT = 1;
  VOCATION_PALADIN = 2;
  VOCATION_SORCERER = 3;
  VOCATION_DRUID = 4;
  VOCATION_PROMOTED = 10;
}

enum ANIMATION_LOOP_TYPE {
  ANIMATION_LOOP_TYPE_PINGPONG = -1;
  ANIMATION_LOOP_TYPE_INFINITE = 0;
  ANIMATION_LOOP_TYPE_COUNTED = 1;
}

enum HOOK_TYPE {
  HOOK_TYPE_SOUTH = 1;
  HOOK_TYPE_EAST = 2;
}

message Coordinate {
  optional uint32 x = 1;
  optional uint32 y = 2;
  optional uint32 z = 3;
}
 
This is really cool, thanks for sharing!
The only advice I'd give is to use tibia's original protos only as reference, not as source of truth, specially in the server side.
Field names and structures can be modified significantly in order to achieve a few things:
  • Clarity on their function
  • Reduced scope for your needs
  • Faster load
  • Memory efficiency

I have a personal source that I use tibia.dat in the server side to load attributes of items, same way that tfs and canary does, but the proto I use is simplified:

Lua:
syntax = "proto2";

package otbr.knary;

message Resources {
  repeated Resource items = 1;
  repeated ResourceId outfits = 2;
  repeated ResourceId effects = 3;
  repeated ResourceId missiles = 4;

  /* 5 - special_meaning_appearance_ids */
  reserved 5;
}

message Resource {
  optional uint32 id = 1;
  optional Flags flags = 3;
  optional bytes name = 4;
  optional bytes description = 5;

  /* 2 - frame_group sprite information */
  reserved 2;
}

message ResourceId {
  optional uint32 id = 1;

  /* Resumed id only Resource */
  reserved 2, 3, 4, 5;
}

message Flags {
  /* Group */
  optional Integer groundSpeed = 1;
  optional bool isContainer = 5;
  optional bool isLiquid = 12;
  optional bool isFluidContainer = 19;

  /* Layering */
  optional bool isDetail = 2;
  optional bool isBottom = 3;
  optional bool isTop = 4;

  /* Capabilities */
  optional bool isStackable = 6;
  optional bool isUsable = 7;
  optional bool forcesUsage = 8;
  optional bool allowsMultiUsages = 9;
  optional bool isWalkable = 13;
  optional bool isImmobile = 14;
  optional bool isTraversable = 15;
  optional bool blocksPath = 16;
  optional bool isPickupable = 18;
  optional bool isHangable = 20;
  optional bool isRotatable = 22;

  /* Writes */
  optional Integer writeSize = 10;
  optional Integer writeOnceSize = 11;
  optional Integer lensesItemId = 31;

  /* Layout */
  optional ResourceLayout layout = 21;

  /* Light */
  optional LightInfo light = 23;

  optional Integer equipmentSlot = 34;
  optional MarketInfo marketInfo = 36;

  /* Corpses */
  optional bool isCorpse = 42;
  optional bool isPlayerCorpse = 43;

  optional Integer upgradeClassification = 48;
  optional bool isAllowedToShowPodium = 46;

  /*
    17 - no_movement_animation
    24 - dont_hide
    25 - translucent
    26 - shift
    27 - height
    28 - lying_object
    29 - animate_always
    30 - automap
    32 - fullbank
    33 - ignore_look
    35 - default_action
    39 - topeffect
    40 - npcsaledata
    41 - changedtoexpire
    44 - cyclopediaitem
    45 - ammo
    47 - reportable
  */
  reserved 17,24,25,26,27,28,29,30,32,33,35,39,40,41,44,45,47;
}

message Integer {
  optional uint32 value = 1;
}

message LightInfo {
  optional uint32 brightness = 1;
  optional uint32 color = 2;
}

enum Placement {
  PLACEMENT_VERTICAL = 1;
  PLACEMENT_HORIZONTAL = 2;
}

message ResourceLayout {
  optional Placement placement = 1;
}

enum ItemType {
  ITEM_TYPE_ARMORS = 1;
  ITEM_TYPE_AMULETS = 2;
  ITEM_TYPE_BOOTS = 3;
  ITEM_TYPE_CONTAINERS = 4;
  ITEM_TYPE_DECORATION = 5;
  ITEM_TYPE_FOOD = 6;
  ITEM_TYPE_HELMETS_HATS = 7;
  ITEM_TYPE_LEGS = 8;
  ITEM_TYPE_OTHERS = 9;
  ITEM_TYPE_POTIONS = 10;
  ITEM_TYPE_RINGS = 11;
  ITEM_TYPE_RUNES = 12;
  ITEM_TYPE_SHIELDS = 13;
  ITEM_TYPE_TOOLS = 14;
  ITEM_TYPE_VALUABLES = 15;
  ITEM_TYPE_AMMUNITION = 16;
  ITEM_TYPE_AXES = 17;
  ITEM_TYPE_CLUBS = 18;
  ITEM_TYPE_DISTANCE_WEAPONS = 19;
  ITEM_TYPE_SWORDS = 20;
  ITEM_TYPE_WANDS_RODS = 21;
  ITEM_TYPE_PREMIUM_SCROLLS = 22;
  ITEM_TYPE_TIBIA_COINS = 23;
  ITEM_TYPE_CREATURE_PRODUCTS = 24;
  ITEM_TYPE_QUIVER = 25;
}

message MarketInfo {
  optional ItemType type = 1;
  optional uint32 tradeObjectId = 2;

  /*
    3 - show_as_object_id
    4 - unknown
    5 - restrict_to_profession
    6 - minimum_level
   */
  reserved 3, 5, 6;
}

Notice that I reserved all the fields that I don't use, that way we don't break the proto contract but still reduce the memory allocated by the auto-generated proto object. I also renamed a lot of fields because Cip is just horrible with their structure and naming šŸ˜….

That allowed me to load all items and effects within 0.1s and using less than 30MB memory:
[INFO] [Resources] 141 'effects' successfully loaded.
[INFO] [Resources] 1118 'outfits' successfully loaded.
[INFO] [Resources] 54 'missiles' successfully loaded.
[INFO] [Resources] 31463 'itemsInfo' successfully loaded.
Loading completed! (163 ms / 26 MB)

But anyway, this tool is great because it actually allow us to see the real proto contract and better based ourselves on that. Thanks again for sharing.
 
Compared to...? 0.2s and 35MB?
What's the point of your comment? You're the one comparing, I was just highlighting some findings.
Have fun with whatever interpretation you wanna take from there!
 
What's the point of your comment? You're the one comparing, I was just highlighting some findings.
Have fun with whatever interpretation you wanna take from there!
Uhh, ok? I'm having hard time trying to figure out how to react to your response šŸ¤£
 
This is really cool, thanks for sharing!
The only advice I'd give is to use tibia's original protos only as reference, not as source of truth, specially in the server side.
Field names and structures can be modified significantly in order to achieve a few things:
  • Clarity on their function
  • Reduced scope for your needs
  • Faster load
  • Memory efficiency

I have a personal source that I use tibia.dat in the server side to load attributes of items, same way that tfs and canary does, but the proto I use is simplified:

Lua:
syntax = "proto2";

package otbr.knary;

message Resources {
  repeated Resource items = 1;
  repeated ResourceId outfits = 2;
  repeated ResourceId effects = 3;
  repeated ResourceId missiles = 4;

  /* 5 - special_meaning_appearance_ids */
  reserved 5;
}

message Resource {
  optional uint32 id = 1;
  optional Flags flags = 3;
  optional bytes name = 4;
  optional bytes description = 5;

  /* 2 - frame_group sprite information */
  reserved 2;
}

message ResourceId {
  optional uint32 id = 1;

  /* Resumed id only Resource */
  reserved 2, 3, 4, 5;
}

message Flags {
  /* Group */
  optional Integer groundSpeed = 1;
  optional bool isContainer = 5;
  optional bool isLiquid = 12;
  optional bool isFluidContainer = 19;

  /* Layering */
  optional bool isDetail = 2;
  optional bool isBottom = 3;
  optional bool isTop = 4;

  /* Capabilities */
  optional bool isStackable = 6;
  optional bool isUsable = 7;
  optional bool forcesUsage = 8;
  optional bool allowsMultiUsages = 9;
  optional bool isWalkable = 13;
  optional bool isImmobile = 14;
  optional bool isTraversable = 15;
  optional bool blocksPath = 16;
  optional bool isPickupable = 18;
  optional bool isHangable = 20;
  optional bool isRotatable = 22;

  /* Writes */
  optional Integer writeSize = 10;
  optional Integer writeOnceSize = 11;
  optional Integer lensesItemId = 31;

  /* Layout */
  optional ResourceLayout layout = 21;

  /* Light */
  optional LightInfo light = 23;

  optional Integer equipmentSlot = 34;
  optional MarketInfo marketInfo = 36;

  /* Corpses */
  optional bool isCorpse = 42;
  optional bool isPlayerCorpse = 43;

  optional Integer upgradeClassification = 48;
  optional bool isAllowedToShowPodium = 46;

  /*
    17 - no_movement_animation
    24 - dont_hide
    25 - translucent
    26 - shift
    27 - height
    28 - lying_object
    29 - animate_always
    30 - automap
    32 - fullbank
    33 - ignore_look
    35 - default_action
    39 - topeffect
    40 - npcsaledata
    41 - changedtoexpire
    44 - cyclopediaitem
    45 - ammo
    47 - reportable
  */
  reserved 17,24,25,26,27,28,29,30,32,33,35,39,40,41,44,45,47;
}

message Integer {
  optional uint32 value = 1;
}

message LightInfo {
  optional uint32 brightness = 1;
  optional uint32 color = 2;
}

enum Placement {
  PLACEMENT_VERTICAL = 1;
  PLACEMENT_HORIZONTAL = 2;
}

message ResourceLayout {
  optional Placement placement = 1;
}

enum ItemType {
  ITEM_TYPE_ARMORS = 1;
  ITEM_TYPE_AMULETS = 2;
  ITEM_TYPE_BOOTS = 3;
  ITEM_TYPE_CONTAINERS = 4;
  ITEM_TYPE_DECORATION = 5;
  ITEM_TYPE_FOOD = 6;
  ITEM_TYPE_HELMETS_HATS = 7;
  ITEM_TYPE_LEGS = 8;
  ITEM_TYPE_OTHERS = 9;
  ITEM_TYPE_POTIONS = 10;
  ITEM_TYPE_RINGS = 11;
  ITEM_TYPE_RUNES = 12;
  ITEM_TYPE_SHIELDS = 13;
  ITEM_TYPE_TOOLS = 14;
  ITEM_TYPE_VALUABLES = 15;
  ITEM_TYPE_AMMUNITION = 16;
  ITEM_TYPE_AXES = 17;
  ITEM_TYPE_CLUBS = 18;
  ITEM_TYPE_DISTANCE_WEAPONS = 19;
  ITEM_TYPE_SWORDS = 20;
  ITEM_TYPE_WANDS_RODS = 21;
  ITEM_TYPE_PREMIUM_SCROLLS = 22;
  ITEM_TYPE_TIBIA_COINS = 23;
  ITEM_TYPE_CREATURE_PRODUCTS = 24;
  ITEM_TYPE_QUIVER = 25;
}

message MarketInfo {
  optional ItemType type = 1;
  optional uint32 tradeObjectId = 2;

  /*
    3 - show_as_object_id
    4 - unknown
    5 - restrict_to_profession
    6 - minimum_level
   */
  reserved 3, 5, 6;
}

Notice that I reserved all the fields that I don't use, that way we don't break the proto contract but still reduce the memory allocated by the auto-generated proto object. I also renamed a lot of fields because Cip is just horrible with their structure and naming šŸ˜….

That allowed me to load all items and effects within 0.1s and using less than 30MB memory:


But anyway, this tool is great because it actually allow us to see the real proto contract and better based ourselves on that. Thanks again for sharing.
You're most welcome!

Oh, interesting! I wasn't aware that you could omit (or rather reserve) fields from the .proto definitions and still parse correctly, could definitely come in handy. Thanks to you too for sharing šŸ‘
 
Does this work on 12.91 client? Can't extract fiels. Tried byte protol proto and proto/x12
 
Last edited:
Back
Top