Change TCP server socket data size indicator from Go/Protobuf specific to Bitcoin-style varints which are easy to implement in every language

client/rpc/utils/utils.go

@@ -19,10 +19,15 @@ import (
 	"encoding/binary"
 	"github.com/golang/protobuf/proto"
 	"io"
+    "math"
 )
 
 const (
 	maxRequestSize = 1048576 // 1MB
+
+    varintTwoBytes = 0xfd
+    varintFourBytes = 0xfe
+    varintEightBytes = 0xff
 )
 
 
@@ -41,7 +46,7 @@ func ReadProtoMessage(msg proto.Message, r io.Reader) error {
 	if !ok {
 		reader = bufio.NewReader(r)
 	}
-	length64, err := binary.ReadVarint(reader)
+	length64, err := ReadVarUintSimple(reader)
 	if err != nil {
 		return err
 	}
@@ -61,17 +66,85 @@ func ReadProtoMessage(msg proto.Message, r io.Reader) error {
 // Apache 2.0 license
 
 func encodeByteSlice(w io.Writer, bz []byte) (err error) {
-	err = encodeVarint(w, int64(len(bz)))
+	err = encodeVarint(w, uint64(len(bz)))
 	if err != nil {
 		return
 	}
 	_, err = w.Write(bz)
 	return
 }
 
-func encodeVarint(w io.Writer, i int64) (err error) {
+func encodeVarint(w io.Writer, i uint64) (err error) {
 	var buf [10]byte
-	n := binary.PutVarint(buf[:], i)
+	n := PutVarUintSimple(buf[:], i)
 	_, err = w.Write(buf[0:n])
 	return
-}
+}
+
+// Bitcoin-style variable uints
+// https://learnmeabitcoin.com/guide/varint
+
+func PutVarUintSimple(buffer []byte, value uint64) uint {
+    if value < varintTwoBytes {
+        buffer[0] = uint8(value)
+        return 1
+    } else if value <= math.MaxUint16 {
+        buffer[0] = uint8(varintTwoBytes)
+        binary.BigEndian.PutUint16(buffer[1:], uint16(value))
+        return 3
+    } else if value <= math.MaxUint32 {
+        buffer[0] = uint8(varintFourBytes)
+        binary.BigEndian.PutUint32(buffer[1:], uint32(value))
+        return 5
+    } else {
+        buffer[0] = uint8(varintEightBytes)
+        binary.BigEndian.PutUint64(buffer[1:], uint64(value))
+        return 9
+    }
+}
+
+func ReadNBytes(reader io.ByteReader, buffer []byte, number_bytes int) error {
+    for i := 0; i < number_bytes; i++ {
+        value, err := reader.ReadByte()
+        if err != nil {
+            return err
+        }
+        buffer[i] = value
+    }
+    return nil
+}
+
+func ReadVarUintSimple(reader io.ByteReader) (uint64, error) {
+    value, err := reader.ReadByte()
+    if err != nil {
+        return 0, err
+    }
+
+    switch value {
+    case varintEightBytes:
+        buffer := make([]byte, 8)
+        err = ReadNBytes(reader, buffer, 8)
+        if err != nil {
+            return 0, err
+        }
+        return binary.BigEndian.Uint64(buffer), nil
+    case varintFourBytes:
+        buffer := make([]byte, 4)
+        err = ReadNBytes(reader, buffer, 4)
+        if err != nil {
+            return 0, err
+        }
+        return uint64(binary.BigEndian.Uint32(buffer)), nil
+    case varintTwoBytes:
+        buffer := make([]byte, 2)
+        err = ReadNBytes(reader, buffer, 2)
+        if err != nil {
+            return 0, err
+        }
+        return uint64(binary.BigEndian.Uint16(buffer)), nil
+    default:
+        return uint64(value), nil
+    }
+    return 0, nil
+}
+

extern/python-tcp-example/proto/__init__.py

@@ -0,0 +1,4 @@
+import proto.structs_pb2 as config
+import proto.types_pb2 as types
+import proto.message_pb2 as message
+

extern/python-tcp-example/proto/message.proto

@@ -0,0 +1,12 @@
+syntax = "proto3";
+package message;
+
+// fields are actually ordered in the order of priority for implementation
+message ChatMessage {
+    bytes Content = 1;
+    bytes SenderPublicKey = 2;
+    bytes SenderProviderPublicKey = 3;
+    int64 MessageNonce = 4;
+    int64 SenderTimestamp = 5; // in unix nano
+    bytes Signature = 6;
+}

extern/python-tcp-example/proto/structs.proto

@@ -0,0 +1,35 @@
+syntax = "proto3";
+package config;
+
+option go_package = "github.com/nymtech/nym-mixnet/config";
+
+message MixConfig {
+    string Id = 1;
+    string Host = 2;
+    string Port = 3;
+    bytes PubKey = 4;
+    uint64 Layer = 5;
+}
+
+message ClientConfig {
+    string Id = 1;
+    string Host = 2;
+    string Port = 3;
+    bytes PubKey = 4;
+    MixConfig Provider = 5;
+}
+
+message GeneralPacket {
+    bytes Flag = 1;
+    bytes Data = 2;
+}
+
+message ProviderResponse {
+    uint64 NumberOfPackets = 1;
+    repeated bytes Packets = 2;
+}
+
+message PullRequest {
+    bytes Token = 1;
+    bytes ClientPublicKey = 2;
+}