ClusterCockpit/cc-metric-store
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-metric-store.git synced 2024-11-10 05:07:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

test de/serialization; fix bugs

Browse Source
This commit is contained in:
Lou Knauer 2022-07-13 14:17:28 +02:00
parent 85fa3ce7bc
commit a0a9952ef7
2 changed files with 176 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
internal/checkpoints/checkpoints.go
View File

@ -2,7 +2,6 @@ package checkpoints
import (
"bufio"
"encoding/binary"
"fmt"
"io"
"reflect"
@ -38,20 +37,16 @@ type Metrics struct {
}
func (c *Checkpoint) Serialize(w io.Writer) error {
// Write magic value (8 byte):
if _, err := w.Write([]byte(magicValue)); err != nil {
return err
}
// The header:
buf := make([]byte, 0, writeBufferSize*2)
buf = append(buf, magicValue...)
buf = encodeBytes(buf, c.Reserved)
buf = encodeUint64(buf, c.From)
buf = encodeUint64(buf, c.To)
// The rest:
buf, err := c.Root.serialize(w, buf)
if err != nil {
var err error
if buf, err = c.Root.serialize(w, buf); err != nil {
return err
}
@ -99,7 +94,7 @@ func (c *CheckpointLevel) serialize(w io.Writer, buf []byte) ([]byte, error) {
}
func (m *Metrics) serialize(w io.Writer, buf []byte) ([]byte, error) {
buf = encodeBytes(m.Reserved, buf) // Reserved
buf = encodeBytes(buf, m.Reserved) // Reserved
buf = encodeUint64(buf, m.Frequency) // Frequency
buf = encodeUint64(buf, m.From) // First measurmenet timestamp
buf = encodeUint32(buf, uint32(len(m.Data))) // Number of elements
@ -114,7 +109,7 @@ func (m *Metrics) serialize(w io.Writer, buf []byte) ([]byte, error) {
}
func (c *Checkpoint) Deserialize(r *bufio.Reader) error {
buf := make([]byte, 8, 16384)
buf := make([]byte, len(magicValue), 16384)
if _, err := io.ReadFull(r, buf); err != nil {
return err
}
@ -151,19 +146,22 @@ func (c *CheckpointLevel) deserialize(buf []byte, r *bufio.Reader) error {
if err != nil {
return err
}
c.Metrics = make(map[string]*Metrics, n)
for i := 0; i < int(n); i++ {
key, err := decodeString(buf, r)
if err != nil {
return err
}
if n != 0 {
c.Metrics = make(map[string]*Metrics, n)
for i := 0; i < int(n); i++ {
key, err := decodeString(buf, r)
if err != nil {
return fmt.Errorf("decoding metric key: %w", err)
}
metrics := &Metrics{}
if err := metrics.deserialize(buf, r); err != nil {
return err
}
// log.Printf("decoding metric %#v...\n", key)
metrics := &Metrics{}
if err := metrics.deserialize(buf, r); err != nil {
return fmt.Errorf("decoding metric for key %#v: %w", key, err)
}
c.Metrics[key] = metrics
c.Metrics[key] = metrics
}
}
// Sublevels:
@ -171,19 +169,22 @@ func (c *CheckpointLevel) deserialize(buf []byte, r *bufio.Reader) error {
if err != nil {
return err
}
c.Sublevels = make(map[string]*CheckpointLevel, n)
for i := 0; i < int(n); i++ {
key, err := decodeString(buf, r)
if err != nil {
return err
}
if n != 0 {
c.Sublevels = make(map[string]*CheckpointLevel, n)
for i := 0; i < int(n); i++ {
key, err := decodeString(buf, r)
if err != nil {
return fmt.Errorf("decoding sublevel key: %w", err)
}
sublevel := &CheckpointLevel{}
if err := sublevel.deserialize(buf, r); err != nil {
return err
}
// log.Printf("decoding sublevel %#v...\n", key)
sublevel := &CheckpointLevel{}
if err := sublevel.deserialize(buf, r); err != nil {
return fmt.Errorf("decoding sublevel for key %#v: %w", key, err)
}
c.Sublevels[key] = sublevel
c.Sublevels[key] = sublevel
}
}
return nil
@ -212,7 +213,7 @@ func (m *Metrics) deserialize(buf []byte, r *bufio.Reader) error {
elmsize := unsafe.Sizeof(x)
bytes = memstore.RequestBytes(int(n) * int(elmsize))
if _, err := io.ReadFull(r, bytes); err != nil {
return err
return fmt.Errorf("reading payload (n=%d, elmsize=%d): %w", n, elmsize, err)
}
sh := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
@ -222,26 +223,32 @@ func (m *Metrics) deserialize(buf []byte, r *bufio.Reader) error {
func encodeBytes(buf []byte, bytes []byte) []byte {
buf = encodeUint32(buf, uint32(len(bytes)))
buf = append(buf, bytes...)
return buf
return append(buf, bytes...)
}
func encodeString(buf []byte, str string) []byte {
buf = encodeUint32(buf, uint32(len(str)))
buf = append(buf, str...)
return buf
return append(buf, str...)
}
func encodeUint64(buf []byte, i uint64) []byte {
buf = append(buf, 0, 0, 0, 0, 0, 0, 0, 0) // 8 bytes
binary.LittleEndian.PutUint64(buf, i)
return buf[8:]
return append(buf,
byte((i>>0)&0xff),
byte((i>>8)&0xff),
byte((i>>16)&0xff),
byte((i>>24)&0xff),
byte((i>>32)&0xff),
byte((i>>40)&0xff),
byte((i>>48)&0xff),
byte((i>>56)&0xff))
}
func encodeUint32(buf []byte, i uint32) []byte {
buf = append(buf, 0, 0, 0, 0) // 4 bytes
binary.LittleEndian.PutUint32(buf, i)
return buf[4:]
return append(buf,
byte((i>>0)&0xff),
byte((i>>8)&0xff),
byte((i>>16)&0xff),
byte((i>>24)&0xff))
}
func decodeBytes(buf []byte, r *bufio.Reader) ([]byte, error) {
@ -250,9 +257,13 @@ func decodeBytes(buf []byte, r *bufio.Reader) ([]byte, error) {
return nil, err
}
if len == 0 {
return nil, nil
}
bytes := make([]byte, len)
if _, err := io.ReadFull(r, bytes); err != nil {
return nil, err
return nil, fmt.Errorf("decoding %d bytes: %w", len, err)
}
return bytes, nil
}
@ -283,7 +294,14 @@ func decodeUint64(buf []byte, r *bufio.Reader) (uint64, error) {
return 0, err
}
return binary.LittleEndian.Uint64(buf), nil
return (uint64(buf[0]) << 0) |
(uint64(buf[1]) << 8) |
(uint64(buf[2]) << 16) |
(uint64(buf[3]) << 24) |
(uint64(buf[4]) << 32) |
(uint64(buf[5]) << 40) |
(uint64(buf[6]) << 48) |
(uint64(buf[7]) << 56), nil
}
func decodeUint32(buf []byte, r *bufio.Reader) (uint32, error) {
@ -293,5 +311,8 @@ func decodeUint32(buf []byte, r *bufio.Reader) (uint32, error) {
return 0, err
}
return binary.LittleEndian.Uint32(buf), nil
return (uint32(buf[0]) << 0) |
(uint32(buf[1]) << 8) |
(uint32(buf[2]) << 16) |
(uint32(buf[3]) << 24), nil
}

109
internal/checkpoints/checkpoints_test.go Normal file
View File

@ -0,0 +1,109 @@
package checkpoints
import (
"bufio"
"bytes"
"encoding/json"
"log"
"reflect"
"testing"
"github.com/ClusterCockpit/cc-metric-store/internal/types"
)
func TestIntEncoding(t *testing.T) {
buf := make([]byte, 0, 100)
x1 := uint64(0x0102030405060708)
buf = encodeUint64(buf, x1)
x2, err := decodeUint64(make([]byte, 0, 10), bufio.NewReader(bytes.NewReader(buf)))
if err != nil {
t.Fatal(err)
}
if x1 != x2 {
t.Fatalf("uint64: x1 != x2: %x != %x", x1, x2)
}
buf = buf[0:0:100]
x3 := uint32(0xabcde)
buf = encodeUint32(buf, x3)
x4, err := decodeUint32(make([]byte, 0, 10), bufio.NewReader(bytes.NewReader(buf)))
if err != nil {
t.Fatal(err)
}
if x3 != x4 {
t.Fatalf("uint32: x3 != x4: %x != %x", x3, x4)
}
}
func TestIdentity(t *testing.T) {
input := &Checkpoint{
Reserved: []byte("Hallo Welt"),
From: 1234,
To: 5678,
Root: &CheckpointLevel{
Reserved: nil,
Sublevels: map[string]*CheckpointLevel{
"host1": {
Reserved: []byte("some text..."),
Metrics: map[string]*Metrics{
"flops": {
Reserved: []byte("blablabla"),
Frequency: 60,
From: 12345,
Data: []types.Float{1.1, 2.2, 3.3, 4.4, 5.5, 6.6},
},
"xyz": {
Frequency: 10,
From: 123,
Data: []types.Float{1.2, 3.4, 5.6, 7.8},
},
},
},
"host2": {
Sublevels: map[string]*CheckpointLevel{
"cpu0": {
Metrics: map[string]*Metrics{
"abc": {
Frequency: 42,
From: 420,
Data: []types.Float{1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15.},
},
},
},
},
},
},
},
}
disk := &bytes.Buffer{}
if err := input.Serialize(disk); err != nil {
t.Fatal("serialization failed:", err)
}
// fmt.Printf("disk.Len() = %d\n", disk.Len())
// fmt.Printf("disk.Bytes() = %#v", disk.Bytes())
output := &Checkpoint{}
if err := output.Deserialize(bufio.NewReader(disk)); err != nil {
t.Fatal("deserialization failed:", err)
}
if !reflect.DeepEqual(input, output) {
input_json, err := json.Marshal(input)
if err != nil {
log.Fatal(err)
}
output_json, err := json.Marshal(output)
if err != nil {
log.Fatal(err)
}
log.Printf("a: %#v", string(input_json))
log.Printf("b: %#v", string(output_json))
t.Fatal("x != deserialize(serialize(x))")
}
}