Use Ganglia configuration (#44)

* Copy all metric configurations from original Ganglia code

* Use metric configurations from Ganglia for some metrics

* Format value string also for known metrics

sinks/gangliaCommon.go

@@ -1,6 +1,7 @@
 package sinks
 
 import (
+	"fmt"
 	"strings"
 
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
@@ -23,11 +24,8 @@ func GangliaMetricName(point lp.CCMetric) string {
 	return name
 }
 
-func GangliaMetricRename(point lp.CCMetric) string {
+func GangliaMetricRename(name string) string {
-	name := point.Name()
+	if name == "net_bytes_in" {
-	if name == "mem_total" || name == "swap_total" {
-		return name
-	} else if name == "net_bytes_in" {
 		return "bytes_in"
 	} else if name == "net_bytes_out" {
 		return "bytes_out"
@@ -48,3 +46,213 @@ func GangliaSlopeType(point lp.CCMetric) uint {
 	}
 	return 3
 }
+
+const DEFAULT_GANGLIA_METRIC_TMAX = 300
+const DEFAULT_GANGLIA_METRIC_SLOPE = "both"
+
+type GangliaMetric struct {
+	Name  string
+	Type  string
+	Slope string
+	Tmax  int
+	Unit  string
+}
+
+type GangliaMetricGroup struct {
+	Name    string
+	Metrics []GangliaMetric
+}
+
+var CommonGangliaMetrics = []GangliaMetricGroup{
+	{
+		Name: "memory",
+		Metrics: []GangliaMetric{
+			{"mem_total", "float", "zero", 1200, "KB"},
+			{"swap_total", "float", "zero", 1200, "KB"},
+			{"mem_free", "float", "both", 180, "KB"},
+			{"mem_shared", "float", "both", 180, "KB"},
+			{"mem_buffers", "float", "both", 180, "KB"},
+			{"mem_cached", "float", "both", 180, "KB"},
+			{"swap_free", "float", "both", 180, "KB"},
+			{"mem_sreclaimable", "float", "both", 180, "KB"},
+			{"mem_slab", "float", "both", 180, "KB"},
+		},
+	},
+	{
+		Name: "cpu",
+		Metrics: []GangliaMetric{
+			{"cpu_num", "uint32", "zero", 1200, "CPUs"},
+			{"cpu_speed", "uint32", "zero", 1200, "MHz"},
+			{"cpu_user", "float", "both", 90, "%"},
+			{"cpu_nice", "float", "both", 90, "%"},
+			{"cpu_system", "float", "both", 90, "%"},
+			{"cpu_idle", "float", "both", 3800, "%"},
+			{"cpu_aidle", "float", "both", 90, "%"},
+			{"cpu_wio", "float", "both", 90, "%"},
+			{"cpu_intr", "float", "both", 90, "%"},
+			{"cpu_sintr", "float", "both", 90, "%"},
+			{"cpu_steal", "float", "both", 90, "%"},
+			{"cpu_guest", "float", "both", 90, "%"},
+			{"cpu_gnice", "float", "both", 90, "%"},
+		},
+	},
+	{
+		Name: "load",
+		Metrics: []GangliaMetric{
+			{"load_one", "float", "both", 70, ""},
+			{"load_five", "float", "both", 325, ""},
+			{"load_fifteen", "float", "both", 950, ""},
+		},
+	},
+	{
+		Name: "disk",
+		Metrics: []GangliaMetric{
+			{"disk_total", "double", "both", 1200, "GB"},
+			{"disk_free", "double", "both", 180, "GB"},
+			{"part_max_used", "float", "both", 180, "%"},
+		},
+	},
+	{
+		Name: "network",
+		Metrics: []GangliaMetric{
+			{"bytes_out", "float", "both", 300, "bytes/sec"},
+			{"bytes_in", "float", "both", 300, "bytes/sec"},
+			{"pkts_in", "float", "both", 300, "packets/sec"},
+			{"pkts_out", "float", "both", 300, "packets/sec"},
+		},
+	},
+	{
+		Name: "process",
+		Metrics: []GangliaMetric{
+			{"proc_run", "uint32", "both", 950, ""},
+			{"proc_total", "uint32", "both", 950, ""},
+		},
+	},
+	{
+		Name: "system",
+		Metrics: []GangliaMetric{
+			{"boottime", "uint32", "zero", 1200, "s"},
+			{"sys_clock", "uint32", "zero", 1200, "s"},
+			{"machine_type", "string", "zero", 1200, ""},
+			{"os_name", "string", "zero", 1200, ""},
+			{"os_release", "string", "zero", 1200, ""},
+			{"mtu", "uint32", "both", 1200, ""},
+		},
+	},
+}
+
+type GangliaMetricConfig struct {
+	Type  string
+	Slope string
+	Tmax  int
+	Unit  string
+	Group string
+	Value string
+}
+
+func GetCommonGangliaConfig(point lp.CCMetric) GangliaMetricConfig {
+	mname := GangliaMetricRename(point.Name())
+	for _, group := range CommonGangliaMetrics {
+		for _, metric := range group.Metrics {
+			if metric.Name == mname {
+				valueStr := ""
+				value, ok := point.GetField("value")
+				if ok {
+					switch real := value.(type) {
+					case float64:
+						valueStr = fmt.Sprintf("%f", real)
+					case float32:
+						valueStr = fmt.Sprintf("%f", real)
+					case int64:
+						valueStr = fmt.Sprintf("%d", real)
+					case int32:
+						valueStr = fmt.Sprintf("%d", real)
+					case int:
+						valueStr = fmt.Sprintf("%d", real)
+					case uint64:
+						valueStr = fmt.Sprintf("%d", real)
+					case uint32:
+						valueStr = fmt.Sprintf("%d", real)
+					case uint:
+						valueStr = fmt.Sprintf("%d", real)
+					case string:
+						valueStr = real
+					default:
+					}
+				}
+				return GangliaMetricConfig{
+					Group: group.Name,
+					Type:  metric.Type,
+					Slope: metric.Slope,
+					Tmax:  metric.Tmax,
+					Unit:  metric.Unit,
+					Value: valueStr,
+				}
+			}
+		}
+	}
+	return GangliaMetricConfig{
+		Group: "",
+		Type:  "",
+		Slope: "",
+		Tmax:  0,
+		Unit:  "",
+		Value: "",
+	}
+}
+
+func GetGangliaConfig(point lp.CCMetric) GangliaMetricConfig {
+	group := ""
+	if g, ok := point.GetMeta("group"); ok {
+		group = g
+	}
+	unit := ""
+	if u, ok := point.GetMeta("unit"); ok {
+		unit = u
+	}
+	valueType := "double"
+	valueStr := ""
+	value, ok := point.GetField("value")
+	if ok {
+		switch real := value.(type) {
+		case float64:
+			valueStr = fmt.Sprintf("%f", real)
+			valueType = "double"
+		case float32:
+			valueStr = fmt.Sprintf("%f", real)
+			valueType = "float"
+		case int64:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "int32"
+		case int32:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "int32"
+		case int:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "int32"
+		case uint64:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "uint32"
+		case uint32:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "uint32"
+		case uint:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "uint32"
+		case string:
+			valueStr = real
+			valueType = "string"
+		default:
+			valueType = "invalid"
+		}
+	}
+
+	return GangliaMetricConfig{
+		Group: group,
+		Type:  valueType,
+		Slope: DEFAULT_GANGLIA_METRIC_SLOPE,
+		Tmax:  DEFAULT_GANGLIA_METRIC_TMAX,
+		Unit:  unit,
+		Value: valueStr,
+	}
+}

sinks/gangliaSink.go

@@ -24,6 +24,7 @@ type GangliaSinkConfig struct {
 	AddTagsAsDesc   bool   `json:"add_tags_as_desc,omitempty"`
 	ClusterName     string `json:"cluster_name,omitempty"`
 	AddTypeToName   bool   `json:"add_type_to_name,omitempty"`
+	AddUnits        bool   `json:"add_units,omitempty"`
 }
 
 type GangliaSink struct {
@@ -35,91 +36,48 @@ type GangliaSink struct {
 
 func (s *GangliaSink) Write(point lp.CCMetric) error {
 	var err error = nil
-	var tagsstr []string
+	//var tagsstr []string
 	var argstr []string
-	if s.config.AddGangliaGroup {
+
-		if point.HasTag("group") {
+	// Get metric name
-			g, _ := point.GetTag("group")
+	metricname := GangliaMetricRename(point.Name())
-			argstr = append(argstr, fmt.Sprintf("--group=%s", g))
+
-		} else if point.HasMeta("group") {
+	// Get metric config (type, value, ... in suitable format)
-			g, _ := point.GetMeta("group")
+	conf := GetCommonGangliaConfig(point)
-			argstr = append(argstr, fmt.Sprintf("--group=%s", g))
+	if len(conf.Type) == 0 {
-		}
+		conf = GetGangliaConfig(point)
+	}
+	if len(conf.Type) == 0 {
+		return fmt.Errorf("metric %s has no 'value' field", metricname)
 	}
 
-	for key, value := range point.Tags() {
+	if s.config.AddGangliaGroup {
-		switch key {
+		argstr = append(argstr, fmt.Sprintf("--group=%s", conf.Group))
-		case "unit":
-			argstr = append(argstr, fmt.Sprintf("--units=%s", value))
-		default:
-			tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", key, value))
-		}
 	}
-	if s.config.MetaAsTags {
+	if s.config.AddUnits && len(conf.Unit) > 0 {
-		for key, value := range point.Meta() {
+		argstr = append(argstr, fmt.Sprintf("--units=%s", conf.Unit))
-			switch key {
-			case "unit":
-				argstr = append(argstr, fmt.Sprintf("--units=%s", value))
-			default:
-				tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", key, value))
-			}
-		}
 	}
+
 	if len(s.config.ClusterName) > 0 {
 		argstr = append(argstr, fmt.Sprintf("--cluster=%s", s.config.ClusterName))
 	}
-	if s.config.AddTagsAsDesc && len(tagsstr) > 0 {
+	// if s.config.AddTagsAsDesc && len(tagsstr) > 0 {
-		argstr = append(argstr, fmt.Sprintf("--desc=%q", strings.Join(tagsstr, ",")))
+	// 	argstr = append(argstr, fmt.Sprintf("--desc=%q", strings.Join(tagsstr, ",")))
-	}
+	// }
 	if len(s.gmetric_config) > 0 {
 		argstr = append(argstr, fmt.Sprintf("--conf=%s", s.gmetric_config))
 	}
-	name := GangliaMetricRename(point)
 	if s.config.AddTypeToName {
 		argstr = append(argstr, fmt.Sprintf("--name=%s", GangliaMetricName(point)))
 	} else {
-		argstr = append(argstr, fmt.Sprintf("--name=%s", name))
+		argstr = append(argstr, fmt.Sprintf("--name=%s", metricname))
 	}
-	slope := GangliaSlopeType(point)
+	argstr = append(argstr, fmt.Sprintf("--slope=%s", conf.Slope))
-	slopeStr := "both"
+	argstr = append(argstr, fmt.Sprintf("--value=%s", conf.Value))
-	if slope == 0 {
+	argstr = append(argstr, fmt.Sprintf("--type=%s", conf.Type))
-		slopeStr = "zero"
+	argstr = append(argstr, fmt.Sprintf("--tmax=%d", conf.Tmax))
-	}
-	argstr = append(argstr, fmt.Sprintf("--slope=%s", slopeStr))
 
-	for k, v := range point.Fields() {
+	cclog.ComponentDebug(s.name, s.gmetric_path, strings.Join(argstr, " "))
-		if k == "value" {
-			switch value := v.(type) {
-			case float64:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%v", value), "--type=double")
-			case float32:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%v", value), "--type=float")
-			case int:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%d", value), "--type=int32")
-			case int32:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%d", value), "--type=int32")
-			case int64:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%d", value), "--type=int32")
-			case uint:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%d", value), "--type=uint32")
-			case uint32:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%d", value), "--type=uint32")
-			case uint64:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%d", value), "--type=uint32")
-			case string:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%q", value), "--type=string")
-			}
-		}
-	}
 	command := exec.Command(s.gmetric_path, argstr...)
 	command.Wait()
 	_, err = command.Output()

sinks/libgangliaSink.go

@@ -82,21 +82,21 @@ const (
 	GMOND_CONFIG_FILE    = `/etc/ganglia/gmond.conf`
 )
 
-type LibgangliaSinkSpecialMetric struct {
+// type LibgangliaSinkSpecialMetric struct {
-	MetricName string `json:"metric_name,omitempty"`
+// 	MetricName string `json:"metric_name,omitempty"`
-	NewName    string `json:"new_name,omitempty"`
+// 	NewName    string `json:"new_name,omitempty"`
-	Slope      string `json:"slope,omitempty"`
+// 	Slope      string `json:"slope,omitempty"`
-}
+// }
 
 type LibgangliaSinkConfig struct {
 	defaultSinkConfig
-	GangliaLib      string                                 `json:"libganglia_path,omitempty"`
+	GangliaLib      string `json:"libganglia_path,omitempty"`
-	GmondConfig     string                                 `json:"gmond_config,omitempty"`
+	GmondConfig     string `json:"gmond_config,omitempty"`
-	AddGangliaGroup bool                                   `json:"add_ganglia_group,omitempty"`
+	AddGangliaGroup bool   `json:"add_ganglia_group,omitempty"`
-	AddTypeToName   bool                                   `json:"add_type_to_name,omitempty"`
+	AddTypeToName   bool   `json:"add_type_to_name,omitempty"`
-	AddUnits        bool                                   `json:"add_units,omitempty"`
+	AddUnits        bool   `json:"add_units,omitempty"`
-	ClusterName     string                                 `json:"cluster_name,omitempty"`
+	ClusterName     string `json:"cluster_name,omitempty"`
-	SpecialMetrics  map[string]LibgangliaSinkSpecialMetric `json:"rename_metrics,omitempty"` // Map to rename metric name from key to value
+	//SpecialMetrics  map[string]LibgangliaSinkSpecialMetric `json:"rename_metrics,omitempty"` // Map to rename metric name from key to value
 	//AddTagsAsDesc   bool              `json:"add_tags_as_desc,omitempty"`
 }
 
@@ -125,81 +125,48 @@ func (s *LibgangliaSink) Write(point lp.CCMetric) error {
 	}
 
 	// Get metric name
-	metricname := GangliaMetricRename(point)
+	metricname := GangliaMetricRename(point.Name())
-	if s.config.AddTypeToName {
-		c_name = lookup(GangliaMetricName(point))
-	} else {
-		c_name = lookup(metricname)
-	}
 
-	// Get the value C string and lookup the type string in the cache
+	conf := GetCommonGangliaConfig(point)
-	value, ok := point.GetField("value")
+	if len(conf.Type) == 0 {
-	if !ok {
+		conf = GetGangliaConfig(point)
+	}
+	if len(conf.Type) == 0 {
 		return fmt.Errorf("metric %s has no 'value' field", metricname)
 	}
-	switch real := value.(type) {
+
-	case float64:
+	if s.config.AddTypeToName {
-		c_value = C.CString(fmt.Sprintf("%f", real))
+		metricname = GangliaMetricName(point)
-		c_type = lookup("double")
-	case float32:
-		c_value = C.CString(fmt.Sprintf("%f", real))
-		c_type = lookup("float")
-	case int64:
-		c_value = C.CString(fmt.Sprintf("%d", real))
-		c_type = lookup("int32")
-	case int32:
-		c_value = C.CString(fmt.Sprintf("%d", real))
-		c_type = lookup("int32")
-	case int:
-		c_value = C.CString(fmt.Sprintf("%d", real))
-		c_type = lookup("int32")
-	case uint64:
-		c_value = C.CString(fmt.Sprintf("%d", real))
-		c_type = lookup("uint32")
-	case uint32:
-		c_value = C.CString(fmt.Sprintf("%d", real))
-		c_type = lookup("uint32")
-	case uint:
-		c_value = C.CString(fmt.Sprintf("%d", real))
-		c_type = lookup("uint32")
-	case string:
-		c_value = C.CString(real)
-		c_type = lookup("string")
-	default:
-		return fmt.Errorf("metric %s has invalid 'value' type for %s", point.Name(), s.name)
 	}
 
+	c_value = C.CString(conf.Value)
+	c_type = lookup(conf.Type)
+	c_name = lookup(metricname)
+
 	// Add unit
+	unit := ""
 	if s.config.AddUnits {
-		if tagunit, tagok := point.GetTag("unit"); tagok {
+		unit = conf.Unit
-			c_unit = lookup(tagunit)
-		} else if metaunit, metaok := point.GetMeta("unit"); metaok {
-			c_unit = lookup(metaunit)
-		} else {
-			c_unit = lookup("")
-		}
-	} else {
-		c_unit = lookup("")
 	}
+	c_unit = lookup(unit)
 
 	// Determine the slope of the metric. Ganglia's own collector mostly use
 	// 'both' but the mem and swap total uses 'zero'.
-	slope := GangliaSlopeType(point)
 	slope_type := C.GANGLIA_SLOPE_BOTH
-	switch slope {
+	switch conf.Slope {
-	case 0:
+	case "zero":
 		slope_type = C.GANGLIA_SLOPE_ZERO
+	case "both":
+		slope_type = C.GANGLIA_SLOPE_BOTH
 	}
 
 	// Create a new Ganglia metric
 	gmetric := C.Ganglia_metric_create(s.global_context)
 	// Set name, value, type and unit in the Ganglia metric
-	// Since we don't have this information from the collectors,
+	// The default slope_type is both directions, so up and down. Some metrics want 'zero' slope, probably constant.
-	// we assume that the metric value can go up and down (slope),
+	// The 'tmax' value is by default 300.
-	// and there is no maximum for 'dmax' and 'tmax'.
-	// Ganglia's collectors set 'tmax' but not 'dmax'
 	rval := C.int(0)
-	rval = C.Ganglia_metric_set(gmetric, c_name, c_value, c_type, c_unit, C.uint(slope_type), 0, 0)
+	rval = C.Ganglia_metric_set(gmetric, c_name, c_value, c_type, c_unit, C.uint(slope_type), C.uint(conf.Tmax), 0)
 	switch rval {
 	case 1:
 		C.free(unsafe.Pointer(c_value))
@@ -209,10 +176,10 @@ func (s *LibgangliaSink) Write(point lp.CCMetric) error {
 		return errors.New("one of your parameters has an invalid character '\"'")
 	case 3:
 		C.free(unsafe.Pointer(c_value))
-		return fmt.Errorf("the type parameter \"%s\" is not a valid type", C.GoString(c_type))
+		return fmt.Errorf("the type parameter \"%s\" is not a valid type", conf.Type)
 	case 4:
 		C.free(unsafe.Pointer(c_value))
-		return fmt.Errorf("the value parameter \"%s\" does not represent a number", C.GoString(c_value))
+		return fmt.Errorf("the value parameter \"%s\" does not represent a number", conf.Value)
 	default:
 	}
 
@@ -221,8 +188,8 @@ func (s *LibgangliaSink) Write(point lp.CCMetric) error {
 		C.Ganglia_metadata_add(gmetric, lookup("CLUSTER"), lookup(s.config.ClusterName))
 	}
 	// Set the group metadata in the Ganglia metric if configured
-	if group, ok := point.GetMeta("group"); ok && s.config.AddGangliaGroup {
+	if s.config.AddGangliaGroup {
-		c_group := lookup(group)
+		c_group := lookup(conf.Group)
 		C.Ganglia_metadata_add(gmetric, lookup("GROUP"), c_group)
 	}