package messageprocessor import ( "encoding/json" "errors" "fmt" "testing" "time" lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message" cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" ) func generate_message_lists(num_lists, num_entries int) ([][]lp.CCMessage, error) { mlist := make([][]lp.CCMessage, 0) for j := 0; j < num_lists; j++ { out := make([]lp.CCMessage, 0) for i := 0; i < num_entries; i++ { var x lp.CCMessage var err error = nil switch { case i%4 == 0: x, err = lp.NewEvent("myevent", map[string]string{"type": "socket", "type-id": "0"}, map[string]string{}, "nothing happend", time.Now()) case i%4 == 1: x, err = lp.NewMetric("mymetric", map[string]string{"type": "socket", "type-id": "0"}, map[string]string{"unit": "kByte"}, 12.145, time.Now()) case i%4 == 2: x, err = lp.NewLog("mylog", map[string]string{"type": "socket", "type-id": "0"}, map[string]string{}, "disk status: OK", time.Now()) case i%4 == 3: x, err = lp.NewGetControl("mycontrol", map[string]string{"type": "socket", "type-id": "0"}, map[string]string{}, time.Now()) } if err == nil { x.AddTag("hostname", "myhost") out = append(out, x) } else { return nil, errors.New("failed to create message") } } mlist = append(mlist, out) } return mlist, nil } func TestNewMessageProcessor(t *testing.T) { _, err := NewMessageProcessor() if err != nil { t.Error(err.Error()) } } type Configs struct { name string config json.RawMessage drop bool errors bool pre func(msg lp.CCMessage) error check func(msg lp.CCMessage) error } var test_configs = []Configs{ { name: "single_dropif_nomatch", config: json.RawMessage(`{"drop_messages_if": [ "name == 'testname' && tags.type == 'socket' && tags.typeid % 2 == 1"]}`), }, { name: "drop_by_name", config: json.RawMessage(`{"drop_messages": [ "net_bytes_in"]}`), drop: true, }, { name: "drop_by_type_match", config: json.RawMessage(`{"drop_by_message_type": [ "metric"]}`), drop: true, }, { name: "drop_by_type_nomatch", config: json.RawMessage(`{"drop_by_message_type": [ "event"]}`), }, { name: "single_dropif_match", config: json.RawMessage(`{"drop_messages_if": [ "name == 'net_bytes_in' && tags.type == 'node'"]}`), drop: true, }, { name: "double_dropif_match_nomatch", config: json.RawMessage(`{"drop_messages_if": [ "name == 'net_bytes_in' && tags.type == 'node'", "name == 'testname' && tags.type == 'socket' && tags.typeid % 2 == 1"]}`), drop: true, }, { name: "rename_simple", config: json.RawMessage(`{"rename_messages": { "net_bytes_in" : "net_bytes_out", "rapl_power": "cpu_power"}}`), check: func(msg lp.CCMessage) error { if msg.Name() != "net_bytes_out" { return errors.New("expected name net_bytes_out but still have net_bytes_in") } return nil }, }, { name: "rename_match", config: json.RawMessage(`{"rename_messages_if": { "name == 'net_bytes_in'" : "net_bytes_out", "name == 'rapl_power'": "cpu_power"}}`), check: func(msg lp.CCMessage) error { if msg.Name() != "net_bytes_out" { return errors.New("expected name net_bytes_out but still have net_bytes_in") } return nil }, }, { name: "rename_nomatch", config: json.RawMessage(`{"rename_messages_if": { "name == 'net_bytes_out'" : "net_bytes_in", "name == 'rapl_power'": "cpu_power"}}`), check: func(msg lp.CCMessage) error { if msg.Name() != "net_bytes_in" { return errors.New("expected name net_bytes_in but still have net_bytes_out") } return nil }, }, { name: "add_tag", config: json.RawMessage(`{"add_tags_if": [{"if": "name == 'net_bytes_in'", "key" : "cluster", "value" : "mycluster"}]}`), check: func(msg lp.CCMessage) error { if !msg.HasTag("cluster") { return errors.New("expected new tag 'cluster' but not present") } return nil }, }, { name: "del_tag", config: json.RawMessage(`{"delete_tags_if": [{"if": "name == 'net_bytes_in'", "key" : "type"}]}`), check: func(msg lp.CCMessage) error { if msg.HasTag("type") { return errors.New("expected to have no 'type' but still present") } return nil }, }, { name: "add_meta", config: json.RawMessage(`{"add_meta_if": [{"if": "name == 'net_bytes_in'", "key" : "source", "value" : "example"}]}`), check: func(msg lp.CCMessage) error { if !msg.HasMeta("source") { return errors.New("expected new tag 'source' but not present") } return nil }, }, { name: "del_meta", config: json.RawMessage(`{"delete_meta_if": [{"if": "name == 'net_bytes_in'", "key" : "unit"}]}`), check: func(msg lp.CCMessage) error { if msg.HasMeta("unit") { return errors.New("expected to have no 'unit' but still present") } return nil }, }, { name: "add_field", config: json.RawMessage(`{"add_fields_if": [{"if": "name == 'net_bytes_in'", "key" : "myfield", "value" : "example"}]}`), check: func(msg lp.CCMessage) error { if !msg.HasField("myfield") { return errors.New("expected new tag 'source' but not present") } return nil }, }, { name: "delete_fields_if_protected", config: json.RawMessage(`{"delete_fields_if": [{"if": "name == 'net_bytes_in'", "key" : "value"}]}`), errors: true, check: func(msg lp.CCMessage) error { if !msg.HasField("value") { return errors.New("expected to still have 'value' field because it is a protected field key") } return nil }, }, { name: "delete_fields_if_unprotected", config: json.RawMessage(`{"delete_fields_if": [{"if": "name == 'net_bytes_in'", "key" : "testfield"}]}`), check: func(msg lp.CCMessage) error { if msg.HasField("testfield") { return errors.New("expected to still have 'testfield' field but should be deleted") } return nil }, pre: func(msg lp.CCMessage) error { msg.AddField("testfield", 4.123) return nil }, }, { name: "single_change_prefix_match", config: json.RawMessage(`{"change_unit_prefix": {"name == 'net_bytes_in' && tags.type == 'node'": "M"}}`), check: func(msg lp.CCMessage) error { if u, ok := msg.GetMeta("unit"); ok { if u != "MB" { return fmt.Errorf("expected unit MB but have %s", u) } } else if u, ok := msg.GetTag("unit"); ok { if u != "MB" { return fmt.Errorf("expected unit MB but have %s", u) } } return nil }, }, { name: "normalize_units", config: json.RawMessage(`{"normalize_units": true}`), check: func(msg lp.CCMessage) error { if u, ok := msg.GetMeta("unit"); ok { if u != "B" { return fmt.Errorf("expected unit B but have %s", u) } } else if u, ok := msg.GetTag("unit"); ok { if u != "B" { return fmt.Errorf("expected unit B but have %s", u) } } return nil }, }, { name: "move_tag_to_meta", config: json.RawMessage(`{"move_tag_to_meta_if": [{"if": "name == 'net_bytes_in'", "key" : "type-id", "value": "typeid"}]}`), check: func(msg lp.CCMessage) error { if msg.HasTag("type-id") || !msg.HasMeta("typeid") { return errors.New("moving tag 'type-id' to meta 'typeid' failed") } return nil }, pre: func(msg lp.CCMessage) error { msg.AddTag("type-id", "0") return nil }, }, { name: "move_tag_to_field", config: json.RawMessage(`{"move_tag_to_field_if": [{"if": "name == 'net_bytes_in'", "key" : "type-id", "value": "typeid"}]}`), check: func(msg lp.CCMessage) error { if msg.HasTag("type-id") || !msg.HasField("typeid") { return errors.New("moving tag 'type-id' to field 'typeid' failed") } return nil }, pre: func(msg lp.CCMessage) error { msg.AddTag("type-id", "0") return nil }, }, { name: "move_meta_to_tag", config: json.RawMessage(`{"move_meta_to_tag_if": [{"if": "name == 'net_bytes_in'", "key" : "unit", "value": "unit"}]}`), check: func(msg lp.CCMessage) error { if msg.HasMeta("unit") || !msg.HasTag("unit") { return errors.New("moving meta 'unit' to tag 'unit' failed") } return nil }, }, { name: "move_meta_to_field", config: json.RawMessage(`{"move_meta_to_field_if": [{"if": "name == 'net_bytes_in'", "key" : "unit", "value": "unit"}]}`), check: func(msg lp.CCMessage) error { if msg.HasMeta("unit") || !msg.HasField("unit") { return errors.New("moving meta 'unit' to field 'unit' failed") } return nil }, }, { name: "move_field_to_tag", config: json.RawMessage(`{"move_field_to_tag_if": [{"if": "name == 'net_bytes_in'", "key" : "myfield", "value": "field"}]}`), check: func(msg lp.CCMessage) error { if msg.HasField("myfield") || !msg.HasTag("field") { return errors.New("moving meta 'myfield' to tag 'field' failed") } return nil }, pre: func(msg lp.CCMessage) error { msg.AddField("myfield", 12) return nil }, }, { name: "move_field_to_meta", config: json.RawMessage(`{"move_field_to_meta_if": [{"if": "name == 'net_bytes_in'", "key" : "myfield", "value": "field"}]}`), check: func(msg lp.CCMessage) error { if msg.HasField("myfield") || !msg.HasMeta("field") { return errors.New("moving meta 'myfield' to meta 'field' failed") } return nil }, pre: func(msg lp.CCMessage) error { msg.AddField("myfield", 12) return nil }, }, } func TestConfigList(t *testing.T) { for _, c := range test_configs { t.Run(c.name, func(t *testing.T) { m, err := lp.NewMetric("net_bytes_in", map[string]string{"type": "node", "type-id": "0"}, map[string]string{"unit": "Byte"}, float64(1024.0), time.Now()) if err != nil { t.Error(err.Error()) return } if c.pre != nil { if err = c.pre(m); err != nil { t.Errorf("error running pre-test function: %v", err.Error()) return } } mp, err := NewMessageProcessor() if err != nil { t.Error(err.Error()) return } err = mp.FromConfigJSON(c.config) if err != nil { t.Error(err.Error()) return } //t.Log(m.ToLineProtocol(nil)) out, err := mp.ProcessMessage(m) if err != nil && !c.errors { cclog.SetDebug() mp.ProcessMessage(m) t.Error(err.Error()) return } if out == nil && !c.drop { t.Error("fail, message should NOT be dropped but processor signalled dropping") return } else if out != nil && c.drop { t.Error("fail, message should be dropped but processor signalled NO dropping") return } // { // if c.drop { // t.Error("fail, message should be dropped but processor signalled NO dropping") // } else { // t.Error("fail, message should NOT be dropped but processor signalled dropping") // } // cclog.SetDebug() // mp.ProcessMessage(m) // return // } if c.check != nil { if err := c.check(out); err != nil { t.Errorf("check failed with %v", err.Error()) t.Log("Rerun with debugging") cclog.SetDebug() mp.ProcessMessage(m) return } } }) } } func BenchmarkProcessing(b *testing.B) { mlist, err := generate_message_lists(b.N, 1000) if err != nil { b.Error(err.Error()) return } mp, err := NewMessageProcessor() if err != nil { b.Error(err.Error()) return } err = mp.FromConfigJSON(json.RawMessage(`{"move_meta_to_tag_if": [{"if" : "name == 'mymetric'", "key":"unit", "value":"unit"}]}`)) if err != nil { b.Error(err.Error()) return } b.StartTimer() for i := 0; i < b.N; i++ { for _, m := range mlist[i] { if _, err := mp.ProcessMessage(m); err != nil { b.Errorf("failed processing message '%s': %v", m.ToLineProtocol(nil), err.Error()) return } } } b.StopTimer() b.ReportMetric(float64(b.Elapsed())/float64(len(mlist)*b.N), "ns/message") }