hytale-server/docs/11-codec-serialization.md

Codec and Serialization System

The Hytale server uses a powerful codec system for data serialization and deserialization. This system is used throughout the codebase for configuration, network packets, asset definitions, and data persistence.

Overview

The codec system provides:

• Type-safe serialization/deserialization
• Support for JSON and BSON formats
• Validation and schema generation
• Composable codecs for complex types
• Builder pattern for object construction

Core Concepts

Codec Interface

public interface Codec<T> {
    T decode(DataInput input);
    void encode(T value, DataOutput output);
}

DataInput/DataOutput

Codecs work with abstract DataInput and DataOutput interfaces that can represent different formats (JSON, BSON, etc.).

Primitive Codecs

The Codec class provides built-in codecs for primitive types:

// String
Codec.STRING              // "hello"

// Numbers
Codec.INTEGER             // 42
Codec.LONG                // 123456789L
Codec.FLOAT               // 3.14f
Codec.DOUBLE              // 3.14159

// Boolean
Codec.BOOLEAN             // true/false

// Byte arrays
Codec.BYTE_ARRAY          // [1, 2, 3]

Collection Codecs

Lists

// List of strings
Codec<List<String>> stringList = Codec.STRING.listOf();

// List of integers
Codec<List<Integer>> intList = Codec.INTEGER.listOf();

// List of custom objects
Codec<List<MyObject>> objectList = MyObject.CODEC.listOf();

Sets

// Set of strings
Codec<Set<String>> stringSet = Codec.STRING.setOf();

Maps

// Map with string keys
Codec<Map<String, Integer>> stringToInt = Codec.mapOf(Codec.STRING, Codec.INTEGER);

// Map with custom key type
Codec<Map<UUID, PlayerData>> playerMap = Codec.mapOf(UUID_CODEC, PlayerData.CODEC);

Arrays

// Array codec
Codec<String[]> stringArray = Codec.arrayOf(Codec.STRING, String[]::new);

BuilderCodec

BuilderCodec is the primary way to create codecs for complex objects:

Basic Usage

public class Person {
    public static final Codec<Person> CODEC = BuilderCodec.of(Person::new)
        .with("name", Codec.STRING, p -> p.name)
        .with("age", Codec.INTEGER, p -> p.age)
        .with("email", Codec.STRING, p -> p.email)
        .build();
    
    private final String name;
    private final int age;
    private final String email;
    
    private Person(String name, int age, String email) {
        this.name = name;
        this.age = age;
        this.email = email;
    }
}

With Default Values

public static final Codec<Settings> CODEC = BuilderCodec.of(Settings::new)
    .with("volume", Codec.FLOAT, s -> s.volume, 1.0f)        // Default: 1.0
    .with("muted", Codec.BOOLEAN, s -> s.muted, false)       // Default: false
    .with("language", Codec.STRING, s -> s.language, "en")   // Default: "en"
    .build();

Optional Fields

public static final Codec<User> CODEC = BuilderCodec.of(User::new)
    .with("username", Codec.STRING, u -> u.username)
    .withOptional("nickname", Codec.STRING, u -> u.nickname)  // May be absent
    .build();

Nested Objects

public class Address {
    public static final Codec<Address> CODEC = BuilderCodec.of(Address::new)
        .with("street", Codec.STRING, a -> a.street)
        .with("city", Codec.STRING, a -> a.city)
        .with("zip", Codec.STRING, a -> a.zip)
        .build();
    // ...
}

public class Customer {
    public static final Codec<Customer> CODEC = BuilderCodec.of(Customer::new)
        .with("name", Codec.STRING, c -> c.name)
        .with("address", Address.CODEC, c -> c.address)
        .build();
    // ...
}

KeyedCodec

For objects that have a key/identifier:

public class ItemType {
    public static final KeyedCodec<String, ItemType> KEYED_CODEC = KeyedCodec.of(
        "id",
        Codec.STRING,
        ItemType::getId,
        ItemType.CODEC
    );
    // ...
}

Codec Transformations

Mapping Values

// Transform between types
Codec<UUID> UUID_CODEC = Codec.STRING.xmap(
    UUID::fromString,    // decode: String -> UUID
    UUID::toString       // encode: UUID -> String
);

// Transform integers to enum
Codec<MyEnum> ENUM_CODEC = Codec.INTEGER.xmap(
    i -> MyEnum.values()[i],
    MyEnum::ordinal
);

Validation

// Validate during decode
Codec<Integer> PORT_CODEC = Codec.INTEGER.validate(
    port -> port > 0 && port < 65536,
    "Port must be between 1 and 65535"
);

// Clamp values
Codec<Float> VOLUME_CODEC = Codec.FLOAT.clamp(0.0f, 1.0f);

Enum Codecs

// Automatic enum codec
Codec<GameMode> GAME_MODE_CODEC = Codec.enumCodec(GameMode.class);

// Custom enum serialization
Codec<Direction> DIRECTION_CODEC = Codec.STRING.xmap(
    Direction::valueOf,
    Direction::name
);

Complex Examples

Polymorphic Types

// For types with multiple implementations
public interface Shape {
    Codec<Shape> CODEC = Codec.dispatch(
        "type",
        Codec.STRING,
        shape -> shape.getType(),
        type -> switch (type) {
            case "circle" -> Circle.CODEC;
            case "rectangle" -> Rectangle.CODEC;
            default -> throw new IllegalArgumentException("Unknown shape: " + type);
        }
    );
    
    String getType();
}

public class Circle implements Shape {
    public static final Codec<Circle> CODEC = BuilderCodec.of(Circle::new)
        .with("radius", Codec.DOUBLE, c -> c.radius)
        .build();
    
    @Override
    public String getType() { return "circle"; }
    // ...
}

Recursive Types

public class TreeNode {
    public static final Codec<TreeNode> CODEC = BuilderCodec.of(TreeNode::new)
        .with("value", Codec.STRING, n -> n.value)
        .with("children", Codec.lazy(() -> TreeNode.CODEC.listOf()), n -> n.children, List.of())
        .build();
    
    private final String value;
    private final List<TreeNode> children;
    // ...
}

Record Types (Java 16+)

public record Point(double x, double y, double z) {
    public static final Codec<Point> CODEC = BuilderCodec.of(Point::new)
        .with("x", Codec.DOUBLE, Point::x)
        .with("y", Codec.DOUBLE, Point::y)
        .with("z", Codec.DOUBLE, Point::z)
        .build();
}

Vector and Math Codecs

The math library provides codecs for common types:

// 3D Vector (double)
Vector3d.CODEC  // {"x": 1.0, "y": 2.0, "z": 3.0}

// 3D Vector (int)
Vector3i.CODEC  // {"x": 1, "y": 2, "z": 3}

// 2D Vector
Vector2d.CODEC  // {"x": 1.0, "y": 2.0}

Working with JSON

Encoding to JSON

// Create JSON output
JsonDataOutput output = new JsonDataOutput();
MyObject.CODEC.encode(myObject, output);
String json = output.toJsonString();

Decoding from JSON

// Parse JSON input
JsonDataInput input = JsonDataInput.fromString(jsonString);
MyObject obj = MyObject.CODEC.decode(input);

Working with BSON

BSON Encoding/Decoding

// BSON output
BsonDataOutput output = new BsonDataOutput();
MyObject.CODEC.encode(myObject, output);
BsonDocument bson = output.toBsonDocument();

// BSON input
BsonDataInput input = new BsonDataInput(bsonDocument);
MyObject obj = MyObject.CODEC.decode(input);

Schema Generation

Codecs can generate JSON schemas for documentation:

// Generate schema
JsonSchema schema = MyObject.CODEC.generateSchema();
String schemaJson = schema.toJson();

Best Practices

1. Make codecs static final - Codecs are immutable and should be reused
2. Use BuilderCodec for objects - It's the most flexible approach
3. Provide defaults - Use sensible defaults for optional fields
4. Validate input - Use validation to catch errors early
5. Keep codecs near their classes - Define codec as a static field in the class
6. Test serialization roundtrips - Ensure encode/decode produces identical objects
7. Use meaningful field names - JSON keys should be clear and consistent
8. Handle null carefully - Use Optional or defaults for nullable fields
9. Consider versioning - Plan for schema evolution
10. Document complex codecs - Add comments for non-obvious serialization

Error Handling

Decode Errors

try {
    MyObject obj = MyObject.CODEC.decode(input);
} catch (CodecException e) {
    // Handle missing fields, wrong types, validation failures
    logger.error("Failed to decode: " + e.getMessage());
}

Validation Errors

Codec<Integer> validated = Codec.INTEGER.validate(
    i -> i > 0,
    "Value must be positive"
);

// Throws CodecException with message "Value must be positive"
validated.decode(input);  // if input is <= 0

Integration with Assets

Asset types use codecs for JSON definitions:

public class BlockType {
    public static final Codec<BlockType> CODEC = BuilderCodec.of(BlockType::new)
        .with("id", Codec.STRING, b -> b.id)
        .with("name", Codec.STRING, b -> b.name)
        .with("hardness", Codec.FLOAT, b -> b.hardness, 1.0f)
        .with("drops", Codec.STRING.listOf(), b -> b.drops, List.of())
        .build();
    // ...
}

Integration with Packets

Network packets use codecs for serialization:

public class PlayerPositionPacket implements Packet {
    public static final Codec<PlayerPositionPacket> CODEC = BuilderCodec.of(PlayerPositionPacket::new)
        .with("playerId", UUID_CODEC, p -> p.playerId)
        .with("position", Vector3d.CODEC, p -> p.position)
        .with("yaw", Codec.FLOAT, p -> p.yaw)
        .with("pitch", Codec.FLOAT, p -> p.pitch)
        .build();
    // ...
}