From bbe214128b6f7cb4e57fcda7adc3205ec9ec66cb Mon Sep 17 00:00:00 2001 From: Determinant Date: Sun, 30 Jun 2019 18:35:20 -0400 Subject: allow manual memory management for types frequently used in messaging --- bench_network/main.go | 15 ++++-- crypto.go | 18 +++---- msg.go | 27 ++++++++--- stream.go | 123 ++++++++++++++++++++++++++++++------------------ test_msgnet/main.go | 6 +-- test_msgnet_tls/main.go | 8 ++-- test_p2p_stress/main.go | 14 +++--- 7 files changed, 131 insertions(+), 80 deletions(-) diff --git a/bench_network/main.go b/bench_network/main.go index 842fed2..fc46ea0 100644 --- a/bench_network/main.go +++ b/bench_network/main.go @@ -22,12 +22,15 @@ const ( MSG_OPCODE_BYTES salticidae.Opcode = iota ) -func msgBytesSerialize(size int) salticidae.Msg { - serialized := salticidae.NewDataStream() +func msgBytesSerialize(size int) (res salticidae.Msg) { + serialized := salticidae.NewDataStream(false) serialized.PutU32(salticidae.ToLittleEndianU32(uint32(size))) serialized.PutData(make([]byte, size)) - return salticidae.NewMsgMovedFromByteArray( - MSG_OPCODE_BYTES, salticidae.NewByteArrayMovedFromDataStream(serialized)) + ba := salticidae.NewByteArrayMovedFromDataStream(serialized, false) + serialized.Free() + res = salticidae.NewMsgMovedFromByteArray(MSG_OPCODE_BYTES, ba, false) + ba.Free() + return } func checkError(err *salticidae.Error) { @@ -72,7 +75,9 @@ func onTerm(_ C.int, _ unsafe.Pointer) { func onTrigger(_ *C.threadcall_handle_t, userdata unsafe.Pointer) { id := *(*int)(userdata) mynet := &mynets[id] - mynet.net.SendMsg(msgBytesSerialize(256), mynet.conn) + payload := msgBytesSerialize(256) + mynet.net.SendMsg(payload, mynet.conn) + payload.Free() if !mynet.conn.IsTerminated() { mynet.tcall.AsyncCall(salticidae.ThreadCallCallback(C.onTrigger), userdata) } diff --git a/crypto.go b/crypto.go index c0012d7..aba5602 100644 --- a/crypto.go +++ b/crypto.go @@ -47,9 +47,9 @@ func (self X509) GetPubKey() PKey { return res } -func (self X509) GetDer() ByteArray { - res := &byteArray{ inner: C.x509_get_der(self.inner) } - runtime.SetFinalizer(res, func(self ByteArray) { self.free() }) +func (self X509) GetDer(autoFree bool) ByteArray { + res := ByteArrayFromC(C.x509_get_der(self.inner)) + byteArraySetFinalizer(res, autoFree) return res } @@ -85,14 +85,14 @@ func NewPrivKeyFromDer(der ByteArray, err *Error) PKey { } func (self PKey) free() { C.pkey_free(self.inner) } -func (self PKey) GetPubKeyDer() ByteArray { - res := &byteArray{ inner: C.pkey_get_pubkey_der(self.inner) } - runtime.SetFinalizer(res, func(self ByteArray) { self.free() }) +func (self PKey) GetPubKeyDer(autoFree bool) ByteArray { + res := ByteArrayFromC(C.pkey_get_pubkey_der(self.inner)) + byteArraySetFinalizer(res, autoFree) return res } -func (self PKey) GetPrivKeyDer() ByteArray { - res := &byteArray{ inner: C.pkey_get_privkey_der(self.inner) } - runtime.SetFinalizer(res, func(self ByteArray) { self.free() }) +func (self PKey) GetPrivKeyDer(autoFree bool) ByteArray { + res := ByteArrayFromC(C.pkey_get_privkey_der(self.inner)) + byteArraySetFinalizer(res, autoFree) return res } diff --git a/msg.go b/msg.go index d1ee1bb..23ddcc1 100644 --- a/msg.go +++ b/msg.go @@ -7,7 +7,10 @@ import "runtime" // The C pointer type for a Msg object type CMsg = *C.msg_t -type msg struct { inner CMsg } +type msg struct { + inner CMsg + autoFree bool +} // Message sent by MsgNetwork type Msg = *msg @@ -18,16 +21,28 @@ type Msg = *msg // "FromC" functions. func MsgFromC(ptr *C.msg_t) Msg { return &msg{ inner: ptr } } +func msgSetFinalizer(res Msg, autoFree bool) { + res.autoFree = autoFree + if res != nil && autoFree { + runtime.SetFinalizer(res, func(self Msg) { self.Free() }) + } +} + // Create a message by taking out all data from src. Notice this is a zero-copy // operation that consumes and invalidates the data in src ("move" semantics) // so that no more operation should be done to src after this function call. -func NewMsgMovedFromByteArray(opcode Opcode, src ByteArray) Msg { - res := &msg{ inner: C.msg_new_moved_from_bytearray(C._opcode_t(opcode), src.inner) } - runtime.SetFinalizer(res, func(self Msg) { self.free() }) +func NewMsgMovedFromByteArray(opcode Opcode, src ByteArray, autoFree bool) Msg { + res := MsgFromC(C.msg_new_moved_from_bytearray(C._opcode_t(opcode), src.inner)) + msgSetFinalizer(res, autoFree) return res } -func (self Msg) free() { C.msg_free(self.inner) } +func (self Msg) Free() { + C.msg_free(self.inner) + if self.autoFree { + runtime.SetFinalizer(self, nil) + } +} // Get the message payload by taking out all data. Notice this is a zero-copy // operation that consumes and invalidates the data in the payload ("move" @@ -35,7 +50,7 @@ func (self Msg) free() { C.msg_free(self.inner) } // this function call. func (self Msg) GetPayloadByMove() DataStream { res := DataStreamFromC(C.msg_consume_payload(self.inner)) - runtime.SetFinalizer(res, func(self DataStream) { self.free() }) + runtime.SetFinalizer(res, func(self DataStream) { self.Free() }) return res } diff --git a/stream.go b/stream.go index 1cbe8ba..ce4d7d3 100644 --- a/stream.go +++ b/stream.go @@ -23,7 +23,10 @@ import "runtime" type CByteArray = *C.bytearray_t // The C pointer to a ByteArray object. -type byteArray struct { inner CByteArray } +type byteArray struct { + inner CByteArray + autoFree bool +} // Array of binary data. type ByteArray = *byteArray @@ -31,48 +34,56 @@ func ByteArrayFromC(ptr CByteArray) ByteArray { return &byteArray{ inner: ptr } } -func byteArraySetFinalizer(res ByteArray) { - if res != nil { - runtime.SetFinalizer(res, func(self ByteArray) { self.free() }) +func byteArraySetFinalizer(res ByteArray, autoFree bool) { + res.autoFree = autoFree + if res != nil && autoFree { + runtime.SetFinalizer(res, func(self ByteArray) { self.Free() }) } } // Create an empty byte array (with zero contained bytes). -func NewByteArray() ByteArray { +func NewByteArray(autoFree bool) ByteArray { res := ByteArrayFromC(C.bytearray_new()) - byteArraySetFinalizer(res) + byteArraySetFinalizer(res, autoFree) return res } -func (self ByteArray) free() { C.bytearray_free(self.inner) } +// Free the ByteArray manually. If the object is constructed with autoFree = +// true, this will immediately free the object. +func (self ByteArray) Free() { + C.bytearray_free(self.inner) + if self.autoFree { + runtime.SetFinalizer(self, nil) + } +} // Create a byte array by taking out all data from src. Notice this is a // zero-copy operation that consumes and invalidates the data in src ("move" // semantics) so that no more operation should be done to src after this // function call. Also notice unlike copying, the entire DataStream buffer is // moved (including the possibily consumed part). -func NewByteArrayMovedFromDataStream(src DataStream) (res ByteArray) { +func NewByteArrayMovedFromDataStream(src DataStream, autoFree bool) (res ByteArray) { res = ByteArrayFromC(C.bytearray_new_moved_from_datastream(src.inner)) - byteArraySetFinalizer(res) + byteArraySetFinalizer(res, autoFree) return } // Create a byte array by copying the remaining data from src. -func NewByteArrayCopiedFromDataStream(src DataStream) (res ByteArray) { +func NewByteArrayCopiedFromDataStream(src DataStream, autoFree bool) (res ByteArray) { res = ByteArrayFromC(C.bytearray_new_copied_from_datastream(src.inner)) - byteArraySetFinalizer(res) + byteArraySetFinalizer(res, autoFree) return } -func NewByteArrayFromHex(hex string) (res ByteArray) { +func NewByteArrayFromHex(hex string) (res ByteArray, autoFree bool) { c_str := C.CString(hex) res = ByteArrayFromC(C.bytearray_new_from_hex(c_str)) C.free(rawptr_t(c_str)) - byteArraySetFinalizer(res) + byteArraySetFinalizer(res, autoFree) return } -func NewByteArrayFromBytes(bytes []byte) (res ByteArray) { +func NewByteArrayFromBytes(bytes []byte, autoFree bool) (res ByteArray) { size := len(bytes) if size > 0 { base := (*C.uint8_t)(&bytes[0]) @@ -80,7 +91,7 @@ func NewByteArrayFromBytes(bytes []byte) (res ByteArray) { } else { res = ByteArrayFromC(C.bytearray_new()) } - byteArraySetFinalizer(res) + byteArraySetFinalizer(res, autoFree) return } @@ -89,6 +100,7 @@ type CDataStream = *C.datastream_t type dataStream struct { inner CDataStream attached map[uintptr]interface{} + autoFree bool } // Stream of binary data. @@ -101,21 +113,22 @@ func DataStreamFromC(ptr CDataStream) DataStream { } } -func dataStreamSetFinalizer(res DataStream) { - if res != nil { - runtime.SetFinalizer(res, func(self DataStream) { self.free() }) +func dataStreamSetFinalizer(res DataStream, autoFree bool) { + res.autoFree = autoFree + if res != nil && autoFree { + runtime.SetFinalizer(res, func(self DataStream) { self.Free() }) } } // Create an empty DataStream. -func NewDataStream() DataStream { +func NewDataStream(autoFree bool) DataStream { res := DataStreamFromC(C.datastream_new()) - dataStreamSetFinalizer(res) + dataStreamSetFinalizer(res, autoFree) return res } // Create a DataStream with data copied from bytes. -func NewDataStreamFromBytes(bytes []byte) (res DataStream) { +func NewDataStreamFromBytes(bytes []byte, autoFree bool) (res DataStream) { size := len(bytes) if size > 0 { base := (*C.uint8_t)(&bytes[0]) @@ -123,33 +136,38 @@ func NewDataStreamFromBytes(bytes []byte) (res DataStream) { } else { res = DataStreamFromC(C.datastream_new()) } - dataStreamSetFinalizer(res) + dataStreamSetFinalizer(res, autoFree) return } // Create a DataStream with content moved from a ByteArray. -func NewDataStreamMovedFromByteArray(bytes ByteArray) (res DataStream) { +func NewDataStreamMovedFromByteArray(bytes ByteArray, autoFree bool) (res DataStream) { res = DataStreamFromC(C.datastream_new_moved_from_bytearray(bytes.inner)) - dataStreamSetFinalizer(res) + dataStreamSetFinalizer(res, autoFree) return } // Create a DataStream with content copied from a ByteArray. -func NewDataStreamFromByteArray(bytes ByteArray) (res DataStream) { +func NewDataStreamFromByteArray(bytes ByteArray, autoFree bool) (res DataStream) { res = DataStreamFromC(C.datastream_new_from_bytearray(bytes.inner)) - dataStreamSetFinalizer(res) + dataStreamSetFinalizer(res, autoFree) return } -func (self DataStream) free() { C.datastream_free(self.inner) } +func (self DataStream) Free() { + C.datastream_free(self.inner) + if self.autoFree { + runtime.SetFinalizer(self, nil) + } +} func (self DataStream) attach(ptr rawptr_t, obj interface{}) { self.attached[uintptr(ptr)] = obj } func (self DataStream) detach(ptr rawptr_t) { delete(self.attached, uintptr(ptr)) } // Make a copy of the object. -func (self DataStream) Copy() (res DataStream) { +func (self DataStream) Copy(autoFree bool) (res DataStream) { res = DataStreamFromC(C.datastream_copy(self.inner)) - dataStreamSetFinalizer(res) + dataStreamSetFinalizer(res, autoFree) return } @@ -249,7 +267,10 @@ func (self DataStream) GetDataInPlace(length int) DataStreamBytes { // The C pointer to a UInt256 object. type CUInt256 = *C.uint256_t -type uint256 struct { inner CUInt256 } +type uint256 struct { + inner CUInt256 + autoFree bool +} // 256-bit integer. type UInt256 = *uint256 @@ -257,26 +278,32 @@ func UInt256FromC(ptr CUInt256) UInt256 { return &uint256{ inner: ptr } } -func uint256SetFinalizer(res UInt256) { - if res != nil { - runtime.SetFinalizer(res, func(self UInt256) { self.free() }) +func uint256SetFinalizer(res UInt256, autoFree bool) { + res.autoFree = autoFree + if res != nil && autoFree { + runtime.SetFinalizer(res, func(self UInt256) { self.Free() }) } } // Create a 256-bit integer. -func NewUInt256() UInt256 { +func NewUInt256(autoFree bool) UInt256 { res := &uint256{ inner: C.uint256_new() } - uint256SetFinalizer(res) + uint256SetFinalizer(res, autoFree) return res } -func NewUInt256FromByteArray(bytes ByteArray) (res UInt256) { +func NewUInt256FromByteArray(bytes ByteArray, autoFree bool) (res UInt256) { res = &uint256{ inner: C.uint256_new_from_bytearray(bytes.inner) } - uint256SetFinalizer(res) + uint256SetFinalizer(res, autoFree) return } -func (self UInt256) free() { C.uint256_free(self.inner) } +func (self UInt256) Free() { + C.uint256_free(self.inner) + if self.autoFree { + runtime.SetFinalizer(self, nil) + } +} func (self UInt256) IsNull() bool { return bool(C.uint256_is_null(self.inner)) } @@ -291,15 +318,18 @@ func (self UInt256) Unserialize(s DataStream) { C.uint256_unserialize(self.inner // Get the Sha256 hash of the given DataStream content (without consuming the // stream). -func (self DataStream) GetHash() UInt256 { +func (self DataStream) GetHash(autoFree bool) UInt256 { res := &uint256{ inner: C.datastream_get_hash(self.inner) } - runtime.SetFinalizer(res, func(self UInt256) { self.free() }) + uint256SetFinalizer(res, autoFree) return res } // Get the Sha256 hash of the given ByteArray content. -func (self ByteArray) GetHash() UInt256 { - return NewDataStreamFromByteArray(self).GetHash() +func (self ByteArray) GetHash(autoFree bool) (res UInt256) { + s := NewDataStreamFromByteArray(self, false) + res = s.GetHash(autoFree) + s.Free() + return } // Get hexadicemal string representation of the given DataStream content @@ -312,9 +342,10 @@ func (self DataStream) GetHex() string { } // Get hexadicemal string representation of the 256-bit integer. -func (self UInt256) GetHex() string { - s := NewDataStream() +func (self UInt256) GetHex() (res string) { + s := NewDataStream(false) self.Serialize(s) - res := s.GetHex() - return res + res = s.GetHex() + s.Free() + return } diff --git a/test_msgnet/main.go b/test_msgnet/main.go index 292fb8f..5b31c2c 100644 --- a/test_msgnet/main.go +++ b/test_msgnet/main.go @@ -23,12 +23,12 @@ const ( ) func msgHelloSerialize(name string, text string) salticidae.Msg { - serialized := salticidae.NewDataStream() + serialized := salticidae.NewDataStream(true) serialized.PutU32(salticidae.ToLittleEndianU32(uint32(len(name)))) serialized.PutData([]byte(name)) serialized.PutData([]byte(text)) return salticidae.NewMsgMovedFromByteArray( - MSG_OPCODE_HELLO, salticidae.NewByteArrayMovedFromDataStream(serialized)) + MSG_OPCODE_HELLO, salticidae.NewByteArrayMovedFromDataStream(serialized, true), true) } func msgHelloUnserialize(msg salticidae.Msg) (name string, text string) { @@ -41,7 +41,7 @@ func msgHelloUnserialize(msg salticidae.Msg) (name string, text string) { } func msgAckSerialize() salticidae.Msg { - return salticidae.NewMsgMovedFromByteArray(MSG_OPCODE_ACK, salticidae.NewByteArray()) + return salticidae.NewMsgMovedFromByteArray(MSG_OPCODE_ACK, salticidae.NewByteArray(true), true) } func checkError(err *salticidae.Error) { diff --git a/test_msgnet_tls/main.go b/test_msgnet_tls/main.go index 1d613fd..78601a2 100644 --- a/test_msgnet_tls/main.go +++ b/test_msgnet_tls/main.go @@ -23,12 +23,12 @@ const ( ) func msgHelloSerialize(name string, text string) salticidae.Msg { - serialized := salticidae.NewDataStream() + serialized := salticidae.NewDataStream(true) serialized.PutU32(salticidae.ToLittleEndianU32(uint32(len(name)))) serialized.PutData([]byte(name)) serialized.PutData([]byte(text)) return salticidae.NewMsgMovedFromByteArray( - MSG_OPCODE_HELLO, salticidae.NewByteArrayMovedFromDataStream(serialized)) + MSG_OPCODE_HELLO, salticidae.NewByteArrayMovedFromDataStream(serialized, true), true) } func msgHelloUnserialize(msg salticidae.Msg) (name string, text string) { @@ -41,7 +41,7 @@ func msgHelloUnserialize(msg salticidae.Msg) (name string, text string) { } func msgAckSerialize() salticidae.Msg { - return salticidae.NewMsgMovedFromByteArray(MSG_OPCODE_ACK, salticidae.NewByteArray()) + return salticidae.NewMsgMovedFromByteArray(MSG_OPCODE_ACK, salticidae.NewByteArray(true), true) } func checkError(err *salticidae.Error) { @@ -95,7 +95,7 @@ func connHandler(_conn *C.struct_msgnetwork_conn_t, connected C.bool, userdata u if myName == "bob" { n = bob } res := true if connected { - certHash := conn.GetPeerCert().GetDer().GetHash() + certHash := conn.GetPeerCert().GetDer(true).GetHash(true) res = certHash.IsEq(n.peerCert) if conn.GetMode() == salticidae.CONN_MODE_ACTIVE { fmt.Printf("[%s] connected, sending hello.\n", myName) diff --git a/test_p2p_stress/main.go b/test_p2p_stress/main.go index b358d5d..29cfbfa 100644 --- a/test_p2p_stress/main.go +++ b/test_p2p_stress/main.go @@ -42,7 +42,7 @@ const ( ) func msgRandSerialize(view uint32, size int) (salticidae.Msg, salticidae.UInt256) { - serialized := salticidae.NewDataStream() + serialized := salticidae.NewDataStream(true) serialized.PutU32(salticidae.ToLittleEndianU32(view)) buffer := make([]byte, size) _, err := rand.Read(buffer) @@ -50,32 +50,32 @@ func msgRandSerialize(view uint32, size int) (salticidae.Msg, salticidae.UInt256 panic("rand source failed") } serialized.PutData(buffer) - hash := salticidae.NewByteArrayFromBytes(buffer).GetHash() + hash := salticidae.NewByteArrayFromBytes(buffer, true).GetHash(true) return salticidae.NewMsgMovedFromByteArray( MSG_OPCODE_RAND, - salticidae.NewByteArrayMovedFromDataStream(serialized)), hash + salticidae.NewByteArrayMovedFromDataStream(serialized, true), true), hash } func msgRandUnserialize(msg salticidae.Msg) (view uint32, hash salticidae.UInt256) { d := msg.GetPayloadByMove() succ := true view = salticidae.FromLittleEndianU32(d.GetU32(&succ)) - hash = salticidae.NewByteArrayCopiedFromDataStream(d).GetHash() + hash = salticidae.NewByteArrayCopiedFromDataStream(d, true).GetHash(true) return } func msgAckSerialize(view uint32, hash salticidae.UInt256) salticidae.Msg { - serialized := salticidae.NewDataStream() + serialized := salticidae.NewDataStream(true) serialized.PutU32(salticidae.ToLittleEndianU32(view)) hash.Serialize(serialized) return salticidae.NewMsgMovedFromByteArray( MSG_OPCODE_ACK, - salticidae.NewByteArrayMovedFromDataStream(serialized)) + salticidae.NewByteArrayMovedFromDataStream(serialized, true), true) } func msgAckUnserialize(msg salticidae.Msg) (view uint32, hash salticidae.UInt256) { p := msg.GetPayloadByMove() - hash = salticidae.NewUInt256() + hash = salticidae.NewUInt256(true) succ := true view = salticidae.FromLittleEndianU32(p.GetU32(&succ)) hash.Unserialize(p) -- cgit v1.2.3-70-g09d2