diff --git a/internal/repository/init.go b/internal/repository/init.go index 76973eb..1ea8d17 100644 --- a/internal/repository/init.go +++ b/internal/repository/init.go @@ -366,3 +366,16 @@ func loadJobStat(job *schema.JobMeta, metric string) float64 { return 0.0 } + +func checkJobData(d *schema.JobData) error { + // for name, scopes := range *d { + + // for scope, metric := range scopes { + // // 1. Unit normalisation + + // } + // // 2. Add node scope if missing + + // } + return nil +} diff --git a/pkg/units/README.md b/pkg/units/README.md index b5ac5ac..862239c 100644 --- a/pkg/units/README.md +++ b/pkg/units/README.md @@ -1,6 +1,7 @@ # cc-units - A unit system for ClusterCockpit -When working with metrics, the problem comes up that they may use different unit name but have the same unit in fact. There are a lot of real world examples like 'kB' and 'Kbyte'. In [cc-metric-collector](https://github.com/ClusterCockpit/cc-metric-collector), the collectors read data from different sources which may use different units or the programmer specifies a unit for a metric by hand. The cc-units system is not comparable with the SI unit system. If you are looking for a package for the SI units, see [here](https://pkg.go.dev/github.com/gurre/si). +When working with metrics, the problem comes up that they may use different unit name but have the same unit in fact. + There are a lot of real world examples like 'kB' and 'Kbyte'. In [cc-metric-collector](https://github.com/ClusterCockpit/cc-metric-collector), the collectors read data from different sources which may use different units or the programmer specifies a unit for a metric by hand. The cc-units system is not comparable with the SI unit system. If you are looking for a package for the SI units, see [here](https://pkg.go.dev/github.com/gurre/si). In order to enable unit comparison and conversion, the ccUnits package provides some helpers: ```go diff --git a/pkg/units/unitPrefix.go b/pkg/units/unitPrefix.go index 347578d..014fcc7 100644 --- a/pkg/units/unitPrefix.go +++ b/pkg/units/unitPrefix.go @@ -1,6 +1,7 @@ package units import ( + "math" "regexp" ) @@ -172,3 +173,20 @@ func NewPrefix(prefix string) Prefix { } return InvalidPrefix } + +func getExponent(p float64) int { + count := 0 + + for p > 1.0 { + p = p / 1000.0 + count++ + } + + return count * 3 +} + +func NewPrefixFromFactor(op Prefix, e int) Prefix { + f := float64(op) + exp := math.Pow10(getExponent(f) - e) + return Prefix(exp) +} diff --git a/pkg/units/units.go b/pkg/units/units.go index 9cca4df..2cb6524 100644 --- a/pkg/units/units.go +++ b/pkg/units/units.go @@ -3,6 +3,7 @@ package units import ( "fmt" + "math" "strings" ) @@ -25,7 +26,9 @@ type Unit interface { var INVALID_UNIT = NewUnit("foobar") -// Valid checks whether a unit is a valid unit. A unit is valid if it has at least a prefix and a measure. The unit denominator is optional. +// Valid checks whether a unit is a valid unit. +// A unit is valid if it has at least a prefix and a measure. +// The unit denominator is optional. func (u *unit) Valid() bool { return u.prefix != InvalidPrefix && u.measure != InvalidMeasure } @@ -71,6 +74,63 @@ func (u *unit) getUnitDenominator() Measure { return u.divMeasure } +func ConvertValue(v *float64, from string, to string) { + uf := NewUnit(from) + ut := NewUnit(to) + factor := float64(uf.getPrefix()) / float64(ut.getPrefix()) + *v = math.Ceil(*v * factor) +} + +func ConvertSeries(s []float64, from string, to string) { + uf := NewUnit(from) + ut := NewUnit(to) + factor := float64(uf.getPrefix()) / float64(ut.getPrefix()) + + for i := 0; i < len(s); i++ { + s[i] = math.Ceil(s[i] * factor) + } +} + +func getNormalizationFactor(v float64) (float64, int) { + count := 0 + scale := -3 + + if v > 1000.0 { + for v > 1000.0 { + v *= 1e-3 + count++ + } + } else { + for v < 1.0 { + v *= 1e3 + count++ + } + scale = 3 + } + return math.Pow10(count * scale), count * scale +} + +func NormalizeValue(v *float64, us string, nu *string) { + u := NewUnit(us) + f, e := getNormalizationFactor((*v)) + *v = math.Ceil(*v * f) + u.setPrefix(NewPrefixFromFactor(u.getPrefix(), e)) + *nu = u.Short() +} + +func NormalizeSeries(s []float64, avg float64, us string, nu *string) { + u := NewUnit(us) + f, e := getNormalizationFactor(avg) + + for i := 0; i < len(s); i++ { + s[i] *= f + s[i] = math.Ceil(s[i]) + } + u.setPrefix(NewPrefixFromFactor(u.getPrefix(), e)) + fmt.Printf("Prefix: %e \n", u.getPrefix()) + *nu = u.Short() +} + // GetPrefixPrefixFactor creates the default conversion function between two prefixes. // It returns a conversation function for the value. func GetPrefixPrefixFactor(in Prefix, out Prefix) func(value interface{}) interface{} { diff --git a/pkg/units/units_test.go b/pkg/units/units_test.go index 5045ab1..cf0bce4 100644 --- a/pkg/units/units_test.go +++ b/pkg/units/units_test.go @@ -2,6 +2,7 @@ package units import ( "fmt" + "reflect" "regexp" "testing" ) @@ -199,3 +200,108 @@ func TestPrefixRegex(t *testing.T) { t.Logf("succussfully compiled regex '%s' for prefix %s", data.Regex, data.Long) } } + +func TestConvertValue(t *testing.T) { + v := float64(103456) + ConvertValue(&v, "MB/s", "GB/s") + + if v != 104.00 { + t.Errorf("Failed ConvertValue: Want 103.456, Got %f", v) + } +} + +func TestConvertValueUp(t *testing.T) { + v := float64(10.3456) + ConvertValue(&v, "GB/s", "MB/s") + + if v != 10346.00 { + t.Errorf("Failed ConvertValue: Want 10346.00, Got %f", v) + } +} +func TestConvertSeries(t *testing.T) { + s := []float64{2890031237, 23998994567, 389734042344, 390349424345} + r := []float64{3, 24, 390, 391} + ConvertSeries(s, "F/s", "GF/s") + + if !reflect.DeepEqual(s, r) { + t.Errorf("Failed ConvertValue: Want 3, 24, 390, 391, Got %v", s) + } +} + +func TestNormalizeValue(t *testing.T) { + var s string + v := float64(103456) + + NormalizeValue(&v, "MB/s", &s) + + if v != 104.00 { + t.Errorf("Failed ConvertValue: Want 104.00, Got %f", v) + } + if s != "GB/s" { + t.Errorf("Failed Prefix or unit: Want GB/s, Got %s", s) + } +} + +func TestNormalizeValueNoPrefix(t *testing.T) { + var s string + v := float64(103458596) + + NormalizeValue(&v, "F/s", &s) + + if v != 104.00 { + t.Errorf("Failed ConvertValue: Want 104.00, Got %f", v) + } + if s != "MFlops/s" { + t.Errorf("Failed Prefix or unit: Want GB/s, Got %s", s) + } +} + +func TestNormalizeValueKeep(t *testing.T) { + var s string + v := float64(345) + + NormalizeValue(&v, "MB/s", &s) + + if v != 345.00 { + t.Errorf("Failed ConvertValue: Want 104.00, Got %f", v) + } + if s != "MB/s" { + t.Errorf("Failed Prefix or unit: Want GB/s, Got %s", s) + } +} + +func TestNormalizeValueDown(t *testing.T) { + var s string + v := float64(0.0004578) + + NormalizeValue(&v, "GB/s", &s) + + if v != 458.00 { + t.Errorf("Failed ConvertValue: Want 458.00, Got %f", v) + } + if s != "KB/s" { + t.Errorf("Failed Prefix or unit: Want KB/s, Got %s", s) + } +} + +func TestNormalizeSeries(t *testing.T) { + var us string + s := []float64{2890031237, 23998994567, 389734042344, 390349424345} + r := []float64{3, 24, 390, 391} + + total := 0.0 + for _, number := range s { + total += number + } + avg := total / float64(len(s)) + + fmt.Printf("AVG: %e\n", avg) + NormalizeSeries(s, avg, "KB/s", &us) + + if !reflect.DeepEqual(s, r) { + t.Errorf("Failed ConvertValue: Want 3, 24, 390, 391, Got %v", s) + } + if us != "TB/s" { + t.Errorf("Failed Prefix or unit: Want TB/s, Got %s", us) + } +}