From c47ac2ebc31e93bd8e83040cc2849bd7c237169f Mon Sep 17 00:00:00 2001 From: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> Date: Tue, 8 Feb 2022 12:22:56 +0100 Subject: [PATCH 1/3] Cleanup --- internal/ccMetric/ccMetric.go | 96 +++++++++++++++++------------------ 1 file changed, 47 insertions(+), 49 deletions(-) diff --git a/internal/ccMetric/ccMetric.go b/internal/ccMetric/ccMetric.go index 68f1886..1de325a 100644 --- a/internal/ccMetric/ccMetric.go +++ b/internal/ccMetric/ccMetric.go @@ -22,10 +22,10 @@ type ccMetric struct { tm time.Time // timestamp } -// ccmetric access functions +// ccMetric access functions type CCMetric interface { - ToLineProtocol(metaAsTags bool) string // Generate influxDB line protocol for data type ccMetric ToPoint(metaAsTags bool) *write.Point // Generate influxDB point for data type ccMetric + ToLineProtocol(metaAsTags bool) string // Generate influxDB line protocol for data type ccMetric Name() string // Get metric name SetName(name string) // Set metric name @@ -36,13 +36,13 @@ type CCMetric interface { Tags() map[string]string // Map of tags AddTag(key, value string) // Add a tag GetTag(key string) (value string, ok bool) // Get a tag by its key - HasTag(key string) (ok bool) // Check a tag + HasTag(key string) (ok bool) // Check if a tag key is present RemoveTag(key string) // Remove a tag by its key Meta() map[string]string // Map of meta data tags AddMeta(key, value string) // Add a meta data tag GetMeta(key string) (value string, ok bool) // Get a meta data tab addressed by its key - HasMeta(key string) (ok bool) // Check a meta data tag + HasMeta(key string) (ok bool) // Check if a meta data key is present RemoveMeta(key string) // Remove a meta data tag by its key Fields() map[string]interface{} // Map of fields @@ -52,14 +52,12 @@ type CCMetric interface { RemoveField(key string) // Remove a field addressed by its key } -// Meta returns the meta data tags as key-value mapping -func (m *ccMetric) Meta() map[string]string { - return m.meta -} - // String implements the stringer interface for data type ccMetric func (m *ccMetric) String() string { - return fmt.Sprintf("Name: %s, Tags: %+v, Meta: %+v, fields: %+v, Timestamp: %d", m.name, m.tags, m.meta, m.fields, m.tm.UnixNano()) + return fmt.Sprintf( + "Name: %s, Tags: %+v, Meta: %+v, fields: %+v, Timestamp: %d", + m.name, m.tags, m.meta, m.fields, m.tm.UnixNano(), + ) } // ToLineProtocol generates influxDB line protocol for data type ccMetric @@ -94,20 +92,11 @@ func (m *ccMetric) Name() string { return m.name } +// SetName sets the measurement name func (m *ccMetric) SetName(name string) { m.name = name } -// Tags returns the the list of tags as key-value-mapping -func (m *ccMetric) Tags() map[string]string { - return m.tags -} - -// Fields returns the list of fields as key-value-mapping -func (m *ccMetric) Fields() map[string]interface{} { - return m.fields -} - // Time returns timestamp func (m *ccMetric) Time() time.Time { return m.tm @@ -118,10 +107,14 @@ func (m *ccMetric) SetTime(t time.Time) { m.tm = t } -// HasTag checks if a tag with key equal to is present in the list of tags -func (m *ccMetric) HasTag(key string) bool { - _, ok := m.tags[key] - return ok +// Tags returns the the list of tags as key-value-mapping +func (m *ccMetric) Tags() map[string]string { + return m.tags +} + +// AddTag adds a tag (consisting of key and value) to the map of tags +func (m *ccMetric) AddTag(key, value string) { + m.tags[key] = value } // GetTag returns the tag with tag's key equal to @@ -130,22 +123,25 @@ func (m *ccMetric) GetTag(key string) (string, bool) { return value, ok } +// HasTag checks if a tag with key equal to is present in the list of tags +func (m *ccMetric) HasTag(key string) bool { + _, ok := m.tags[key] + return ok +} + // RemoveTag removes the tag with tag's key equal to -// and keeps the tag list ordered by the keys func (m *ccMetric) RemoveTag(key string) { delete(m.tags, key) } -// AddTag adds a tag (consisting of key and value) -// and keeps the tag list ordered by the keys -func (m *ccMetric) AddTag(key, value string) { - m.tags[key] = value +// Meta returns the meta data tags as key-value mapping +func (m *ccMetric) Meta() map[string]string { + return m.meta } -// HasTag checks if a meta data tag with meta data's key equal to is present in the list of meta data tags -func (m *ccMetric) HasMeta(key string) bool { - _, ok := m.meta[key] - return ok +// AddMeta adds a meta data tag (consisting of key and value) to the map of meta data tags +func (m *ccMetric) AddMeta(key, value string) { + m.meta[key] = value } // GetMeta returns the meta data tag with meta data's key equal to @@ -154,19 +150,23 @@ func (m *ccMetric) GetMeta(key string) (string, bool) { return value, ok } +// HasMeta checks if a meta data tag with meta data's key equal to is present in the map of meta data tags +func (m *ccMetric) HasMeta(key string) bool { + _, ok := m.meta[key] + return ok +} + // RemoveMeta removes the meta data tag with tag's key equal to -// and keeps the meta data tag list ordered by the keys func (m *ccMetric) RemoveMeta(key string) { delete(m.meta, key) } -// AddMeta adds a meta data tag (consisting of key and value) -// and keeps the meta data list ordered by the keys -func (m *ccMetric) AddMeta(key, value string) { - m.meta[key] = value +// Fields returns the list of fields as key-value-mapping +func (m *ccMetric) Fields() map[string]interface{} { + return m.fields } -// AddField adds a field (consisting of key and value) to the unordered list of fields +// AddField adds a field (consisting of key and value) to the map of fields func (m *ccMetric) AddField(key string, value interface{}) { m.fields[key] = value } @@ -177,14 +177,14 @@ func (m *ccMetric) GetField(key string) (interface{}, bool) { return v, ok } -// HasField checks if a field with field's key equal to is present in the list of fields +// HasField checks if a field with field's key equal to is present in the map of fields func (m *ccMetric) HasField(key string) bool { _, ok := m.fields[key] return ok } // RemoveField removes the field with field's key equal to -// from the unordered list of fields +// from the map of fields func (m *ccMetric) RemoveField(key string) { delete(m.fields, key) } @@ -205,17 +205,13 @@ func New( tm: tm, } - // deep copy tags + // deep copy tags, meta data tags and fields for k, v := range tags { m.tags[k] = v } - - // deep copy meta data tags for k, v := range meta { m.meta[k] = v } - - // Unsorted list of fields for k, v := range fields { v := convertField(v) if v == nil { @@ -231,12 +227,13 @@ func New( func FromMetric(other ccMetric) CCMetric { m := &ccMetric{ name: other.Name(), - tags: make(map[string]string), - meta: make(map[string]string), - fields: make(map[string]interface{}), + tags: make(map[string]string, len(other.tags)), + meta: make(map[string]string, len(other.meta)), + fields: make(map[string]interface{}, len(other.fields)), tm: other.Time(), } + // deep copy tags, meta data tags and fields for key, value := range other.tags { m.tags[key] = value } @@ -259,6 +256,7 @@ func FromInfluxMetric(other lp.Metric) CCMetric { tm: other.Time(), } + // deep copy tags and fields for _, otherTag := range other.TagList() { m.tags[otherTag.Key] = otherTag.Value } From e1cf68298923708851d761ed3f5ac12476f5e8aa Mon Sep 17 00:00:00 2001 From: Thomas Gruber Date: Tue, 8 Feb 2022 13:22:20 +0100 Subject: [PATCH 2/3] Add other collectors to README --- collectors/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/collectors/README.md b/collectors/README.md index a79fa03..558649e 100644 --- a/collectors/README.md +++ b/collectors/README.md @@ -22,8 +22,9 @@ In contrast to the configuration files for sinks and receivers, the collectors c * [`loadavg`](./loadavgMetric.md) * [`netstat`](./netstatMetric.md) * [`ibstat`](./infinibandMetric.md) +* [`ibstat_perfquery`](./infinibandPerfQueryMetric.md) * [`tempstat`](./tempMetric.md) -* [`lustre`](./lustreMetric.md) +* [`lustrestat`](./lustreMetric.md) * [`likwid`](./likwidMetric.md) * [`nvidia`](./nvidiaMetric.md) * [`customcmd`](./customCmdMetric.md) @@ -31,10 +32,14 @@ In contrast to the configuration files for sinks and receivers, the collectors c * [`topprocs`](./topprocsMetric.md) * [`nfs3stat`](./nfs3Metric.md) * [`nfs4stat`](./nfs4Metric.md) +* [`cpufreq`](./cpufreqMetric.md) +* [`cpufreq_cpuinfo`](./cpufreqCpuinfoMetric.md) +* [`numastat`](./numastatMetric.md) +* [`gpfs`](./gpfsMetric.md) +* [`ipmistat`](./ipmiMetric.md) ## Todos -* [ ] Exclude devices for `diskstat` collector * [ ] Aggreate metrics to higher topology entity (sum hwthread metrics to socket metric, ...). Needs to be configurable # Contributing own collectors From 377f85111adbafc950dba236a4982f0618fb8a24 Mon Sep 17 00:00:00 2001 From: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> Date: Tue, 8 Feb 2022 13:38:18 +0100 Subject: [PATCH 3/3] Refactoring: Use ccmetric function ToLineProtocol() in stdout sink --- sinks/stdoutSink.go | 66 +++++++++++---------------------------------- 1 file changed, 15 insertions(+), 51 deletions(-) diff --git a/sinks/stdoutSink.go b/sinks/stdoutSink.go index c085c13..5d0761a 100644 --- a/sinks/stdoutSink.go +++ b/sinks/stdoutSink.go @@ -3,7 +3,6 @@ package sinks import ( "encoding/json" "fmt" - "math" "os" "strings" @@ -11,15 +10,13 @@ import ( lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" ) -type StdoutSinkConfig struct { - defaultSinkConfig - Output string `json:"output_file,omitempty"` -} - type StdoutSink struct { - sink + sink // meta_as_tags, name output *os.File - config StdoutSinkConfig + config struct { + defaultSinkConfig + Output string `json:"output_file,omitempty"` + } } func (s *StdoutSink) Init(config json.RawMessage) error { @@ -30,13 +27,15 @@ func (s *StdoutSink) Init(config json.RawMessage) error { return err } } + s.output = os.Stdout if len(s.config.Output) > 0 { - if strings.ToLower(s.config.Output) == "stdout" { + switch strings.ToLower(s.config.Output) { + case "stdout": s.output = os.Stdout - } else if strings.ToLower(s.config.Output) == "stderr" { + case "stderr": s.output = os.Stderr - } else { + default: f, err := os.OpenFile(s.config.Output, os.O_CREATE|os.O_WRONLY, os.FileMode(0600)) if err != nil { return err @@ -48,46 +47,11 @@ func (s *StdoutSink) Init(config json.RawMessage) error { return nil } -func (s *StdoutSink) Write(point lp.CCMetric) error { - var tagsstr []string - var fieldstr []string - for key, value := range point.Tags() { - tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", key, value)) - } - if s.meta_as_tags { - for key, value := range point.Meta() { - tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", key, value)) - } - } - for key, v := range point.Fields() { - switch value := v.(type) { - case float64: - if !math.IsNaN(value) { - fieldstr = append(fieldstr, fmt.Sprintf("%s=%v", key, v)) - } else { - fieldstr = append(fieldstr, fmt.Sprintf("%s=0.0", key)) - } - case float32: - if !math.IsNaN(float64(value)) { - fieldstr = append(fieldstr, fmt.Sprintf("%s=%v", key, v)) - } else { - fieldstr = append(fieldstr, fmt.Sprintf("%s=0.0", key)) - } - case int: - fieldstr = append(fieldstr, fmt.Sprintf("%s=%d", key, v)) - case int64: - fieldstr = append(fieldstr, fmt.Sprintf("%s=%d", key, v)) - case string: - fieldstr = append(fieldstr, fmt.Sprintf("%s=%q", key, v)) - default: - fieldstr = append(fieldstr, fmt.Sprintf("%s=%v", key, value)) - } - } - if len(tagsstr) > 0 { - fmt.Printf("%s,%s %s %d\n", point.Name(), strings.Join(tagsstr, ","), strings.Join(fieldstr, ","), point.Time().Unix()) - } else { - fmt.Printf("%s %s %d\n", point.Name(), strings.Join(fieldstr, ","), point.Time().Unix()) - } +func (s *StdoutSink) Write(m lp.CCMetric) error { + fmt.Fprint( + s.output, + m.ToLineProtocol(s.meta_as_tags), + ) return nil }