mirror of
https://github.com/ClusterCockpit/cc-metric-store.git
synced 2024-11-10 05:07:25 +01:00
Move files and cleanup
This commit is contained in:
Lou Knauer
parent
4d35a1ed4e
commit
03a5dee97a
63
float.go
63
float.go
@ -1,63 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Go's JSON encoder for floats does not support NaN (https://github.com/golang/go/issues/3480).
|
||||
// This program uses NaN as a signal for missing data.
|
||||
// For the HTTP JSON API to be able to handle NaN values,
|
||||
// we have to use our own type which implements encoding/json.Marshaler itself.
|
||||
type Float float64
|
||||
|
||||
var NaN Float = Float(math.NaN())
|
||||
var nullAsBytes []byte = []byte("null")
|
||||
|
||||
func (f Float) IsNaN() bool {
|
||||
return math.IsNaN(float64(f))
|
||||
}
|
||||
|
||||
func (f Float) MarshalJSON() ([]byte, error) {
|
||||
if math.IsNaN(float64(f)) {
|
||||
return nullAsBytes, nil
|
||||
}
|
||||
|
||||
return strconv.AppendFloat(make([]byte, 0, 10), float64(f), 'f', 3, 64), nil
|
||||
}
|
||||
|
||||
func (f *Float) UnmarshalJSON(input []byte) error {
|
||||
if string(input) == "null" {
|
||||
*f = NaN
|
||||
return nil
|
||||
}
|
||||
|
||||
val, err := strconv.ParseFloat(string(input), 64)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*f = Float(val)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Same as `[]Float`, but can be marshaled to JSON with less allocations.
|
||||
type FloatArray []Float
|
||||
|
||||
func (fa FloatArray) MarshalJSON() ([]byte, error) {
|
||||
buf := make([]byte, 0, 2+len(fa)*8)
|
||||
buf = append(buf, '[')
|
||||
for i := 0; i < len(fa); i++ {
|
||||
if i != 0 {
|
||||
buf = append(buf, ',')
|
||||
}
|
||||
|
||||
if fa[i].IsNaN() {
|
||||
buf = append(buf, `null`...)
|
||||
} else {
|
||||
buf = strconv.AppendFloat(buf, float64(fa[i]), 'f', 3, 64)
|
||||
|
||||
}
|
||||
}
|
||||
buf = append(buf, ']')
|
||||
return buf, nil
|
||||
}
|
||||
12
go.mod
12
go.mod
@ -4,10 +4,18 @@ go 1.18
|
||||
|
||||
require (
|
||||
github.com/golang-jwt/jwt/v4 v4.0.0
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/google/gops v0.3.22
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/influxdata/line-protocol/v2 v2.2.0
|
||||
github.com/nats-io/nats-server/v2 v2.2.6 // indirect
|
||||
github.com/nats-io/nats.go v1.11.0
|
||||
golang.org/x/sys v0.0.0-20210902050250-f475640dd07b
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.2.6 // indirect
|
||||
github.com/nats-io/nkeys v0.3.0 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b // indirect
|
||||
google.golang.org/protobuf v1.26.0 // indirect
|
||||
)
|
||||
|
||||
17
go.sum
17
go.sum
@ -1,15 +1,12 @@
|
||||
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
|
||||
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
|
||||
github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
|
||||
github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
|
||||
github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
|
||||
github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1 h1:4dntyT+x6QTOSCIrgczbQ+ockAEha0cfxD5Wi0iCzjY=
|
||||
github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o=
|
||||
github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
@ -25,24 +22,27 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/gops v0.3.22 h1:lyvhDxfPLHAOR2xIYwjPhN387qHxyU21Sk9sz/GhmhQ=
|
||||
github.com/google/gops v0.3.22/go.mod h1:7diIdLsqpCihPSX3fQagksT/Ku/y4RL9LHTlKyEUDl8=
|
||||
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
|
||||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/influxdata/line-protocol-corpus v0.0.0-20210519164801-ca6fa5da0184/go.mod h1:03nmhxzZ7Xk2pdG+lmMd7mHDfeVOYFyhOgwO61qWU98=
|
||||
github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937 h1:MHJNQ+p99hFATQm6ORoLmpUCF7ovjwEFshs/NHzAbig=
|
||||
github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937/go.mod h1:BKR9c0uHSmRgM/se9JhFHtTT7JTO67X23MtKMHtZcpo=
|
||||
github.com/influxdata/line-protocol/v2 v2.0.0-20210312151457-c52fdecb625a/go.mod h1:6+9Xt5Sq1rWx+glMgxhcg2c0DUaehK+5TDcPZ76GypY=
|
||||
github.com/influxdata/line-protocol/v2 v2.1.0/go.mod h1:QKw43hdUBg3GTk2iC3iyCxksNj7PX9aUSeYOYE/ceHY=
|
||||
github.com/influxdata/line-protocol/v2 v2.2.0 h1:UPmAqE15Hw5zu9E10SYhoXVLWnEJkWnuCbaCiRsA3c0=
|
||||
github.com/influxdata/line-protocol/v2 v2.2.0/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM=
|
||||
github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19 h1:WjT3fLi9n8YWh/Ih8Q1LHAPsTqGddPcHqscN+PJ3i68=
|
||||
github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
|
||||
github.com/klauspost/compress v1.11.12 h1:famVnQVu7QwryBN4jNseQdUKES71ZAOnB6UQQJPZvqk=
|
||||
github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/minio/highwayhash v1.0.1 h1:dZ6IIu8Z14VlC0VpfKofAhCy74wu/Qb5gcn52yWoz/0=
|
||||
github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
|
||||
@ -61,15 +61,11 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg=
|
||||
github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
|
||||
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
|
||||
github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
|
||||
github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
|
||||
github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=
|
||||
github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
@ -81,7 +77,6 @@ golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210902050250-f475640dd07b h1:S7hKs0Flbq0bbc9xgYt4stIEG1zNDFqyrPwAX2Wj/sE=
|
||||
@ -92,6 +87,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
|
||||
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
@ -106,5 +102,4 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
rsc.io/goversion v1.2.0 h1:SPn+NLTiAG7w30IRK/DKp1BjvpWabYgxlLp/+kx5J8w=
|
||||
rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package main
|
||||
package apiv1
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
@ -7,29 +7,38 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ClusterCockpit/cc-metric-store/internal/api"
|
||||
"github.com/ClusterCockpit/cc-metric-store/internal/memstore"
|
||||
"github.com/ClusterCockpit/cc-metric-store/internal/types"
|
||||
"github.com/golang-jwt/jwt/v4"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/influxdata/line-protocol/v2/lineprotocol"
|
||||
)
|
||||
|
||||
type ApiMetricData struct {
|
||||
Error *string `json:"error,omitempty"`
|
||||
From int64 `json:"from"`
|
||||
To int64 `json:"to"`
|
||||
Data FloatArray `json:"data,omitempty"`
|
||||
Avg Float `json:"avg"`
|
||||
Min Float `json:"min"`
|
||||
Max Float `json:"max"`
|
||||
Error *string `json:"error,omitempty"`
|
||||
From int64 `json:"from"`
|
||||
To int64 `json:"to"`
|
||||
Data types.FloatArray `json:"data,omitempty"`
|
||||
Avg types.Float `json:"avg"`
|
||||
Min types.Float `json:"min"`
|
||||
Max types.Float `json:"max"`
|
||||
}
|
||||
|
||||
type HttpApi struct {
|
||||
MemoryStore *memstore.MemoryStore
|
||||
server *http.Server
|
||||
PublicKey string
|
||||
Address string
|
||||
CertFile, KeyFile string
|
||||
}
|
||||
|
||||
// TODO: Optimize this, just like the stats endpoint!
|
||||
@ -49,15 +58,15 @@ func (data *ApiMetricData) AddStats() {
|
||||
|
||||
if n > 0 {
|
||||
avg := sum / float64(n)
|
||||
data.Avg = Float(avg)
|
||||
data.Min = Float(min)
|
||||
data.Max = Float(max)
|
||||
data.Avg = types.Float(avg)
|
||||
data.Min = types.Float(min)
|
||||
data.Max = types.Float(max)
|
||||
} else {
|
||||
data.Avg, data.Min, data.Max = NaN, NaN, NaN
|
||||
data.Avg, data.Min, data.Max = types.NaN, types.NaN, types.NaN
|
||||
}
|
||||
}
|
||||
|
||||
func (data *ApiMetricData) ScaleBy(f Float) {
|
||||
func (data *ApiMetricData) ScaleBy(f types.Float) {
|
||||
if f == 0 || f == 1 {
|
||||
return
|
||||
}
|
||||
@ -70,17 +79,17 @@ func (data *ApiMetricData) ScaleBy(f Float) {
|
||||
}
|
||||
}
|
||||
|
||||
func (data *ApiMetricData) PadDataWithNull(from, to int64, metric string) {
|
||||
minfo, ok := memoryStore.metrics[metric]
|
||||
func (ha *HttpApi) padWithNaNs(data *ApiMetricData, metric string, from, to int64) {
|
||||
mc, ok := ha.MemoryStore.GetMetricConf(metric)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
|
||||
padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
|
||||
ndata := make([]Float, 0, padfront+len(data.Data))
|
||||
if (data.From / mc.Frequency) > (from / mc.Frequency) {
|
||||
padfront := int((data.From / mc.Frequency) - (from / mc.Frequency))
|
||||
ndata := make([]types.Float, 0, padfront+len(data.Data))
|
||||
for i := 0; i < padfront; i++ {
|
||||
ndata = append(ndata, NaN)
|
||||
ndata = append(ndata, types.NaN)
|
||||
}
|
||||
for j := 0; j < len(data.Data); j++ {
|
||||
ndata = append(ndata, data.Data[j])
|
||||
@ -89,55 +98,7 @@ func (data *ApiMetricData) PadDataWithNull(from, to int64, metric string) {
|
||||
}
|
||||
}
|
||||
|
||||
func handleFree(rw http.ResponseWriter, r *http.Request) {
|
||||
rawTo := r.URL.Query().Get("to")
|
||||
if rawTo == "" {
|
||||
http.Error(rw, "'to' is a required query parameter", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
to, err := strconv.ParseInt(rawTo, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(rw, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// TODO: lastCheckpoint might be modified by different go-routines.
|
||||
// Load it using the sync/atomic package?
|
||||
freeUpTo := lastCheckpoint.Unix()
|
||||
if to < freeUpTo {
|
||||
freeUpTo = to
|
||||
}
|
||||
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
bodyDec := json.NewDecoder(r.Body)
|
||||
var selectors [][]string
|
||||
err = bodyDec.Decode(&selectors)
|
||||
if err != nil {
|
||||
http.Error(rw, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
n := 0
|
||||
for _, sel := range selectors {
|
||||
bn, err := memoryStore.Free(sel, freeUpTo)
|
||||
if err != nil {
|
||||
http.Error(rw, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
n += bn
|
||||
}
|
||||
|
||||
rw.WriteHeader(http.StatusOK)
|
||||
rw.Write([]byte(fmt.Sprintf("buffers freed: %d\n", n)))
|
||||
}
|
||||
|
||||
func handleWrite(rw http.ResponseWriter, r *http.Request) {
|
||||
func (ha *HttpApi) handleWrite(rw http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
@ -150,26 +111,8 @@ func handleWrite(rw http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if debugDump != io.Discard {
|
||||
now := time.Now()
|
||||
msg := make([]byte, 0, 512)
|
||||
msg = append(msg, "\n--- local unix time: "...)
|
||||
msg = strconv.AppendInt(msg, now.Unix(), 10)
|
||||
msg = append(msg, " ---\n"...)
|
||||
|
||||
debugDumpLock.Lock()
|
||||
defer debugDumpLock.Unlock()
|
||||
if _, err := debugDump.Write(msg); err != nil {
|
||||
log.Printf("error while writing to debug dump: %s", err.Error())
|
||||
}
|
||||
if _, err := debugDump.Write(bytes); err != nil {
|
||||
log.Printf("error while writing to debug dump: %s", err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
dec := lineprotocol.NewDecoderWithBytes(bytes)
|
||||
if err := decodeLine(dec, r.URL.Query().Get("cluster")); err != nil {
|
||||
if err := api.DecodeLine(ha.MemoryStore, dec, r.URL.Query().Get("cluster")); err != nil {
|
||||
log.Printf("/api/write error: %s", err.Error())
|
||||
http.Error(rw, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
@ -194,17 +137,17 @@ type ApiQueryResponse struct {
|
||||
}
|
||||
|
||||
type ApiQuery struct {
|
||||
Metric string `json:"metric"`
|
||||
Hostname string `json:"host"`
|
||||
Aggregate bool `json:"aggreg"`
|
||||
ScaleFactor Float `json:"scale-by,omitempty"`
|
||||
Type *string `json:"type,omitempty"`
|
||||
TypeIds []string `json:"type-ids,omitempty"`
|
||||
SubType *string `json:"subtype,omitempty"`
|
||||
SubTypeIds []string `json:"subtype-ids,omitempty"`
|
||||
Metric string `json:"metric"`
|
||||
Hostname string `json:"host"`
|
||||
Aggregate bool `json:"aggreg"`
|
||||
ScaleFactor types.Float `json:"scale-by,omitempty"`
|
||||
Type *string `json:"type,omitempty"`
|
||||
TypeIds []string `json:"type-ids,omitempty"`
|
||||
SubType *string `json:"subtype,omitempty"`
|
||||
SubTypeIds []string `json:"subtype-ids,omitempty"`
|
||||
}
|
||||
|
||||
func handleQuery(rw http.ResponseWriter, r *http.Request) {
|
||||
func (ha *HttpApi) handleQuery(rw http.ResponseWriter, r *http.Request) {
|
||||
var err error
|
||||
var req ApiQueryRequest = ApiQueryRequest{WithStats: true, WithData: true, WithPadding: true}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
@ -216,7 +159,7 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
|
||||
Results: make([][]ApiMetricData, 0, len(req.Queries)),
|
||||
}
|
||||
if req.ForAllNodes != nil {
|
||||
nodes := memoryStore.ListChildren([]string{req.Cluster})
|
||||
nodes := ha.MemoryStore.ListChildren([]string{req.Cluster})
|
||||
for _, node := range nodes {
|
||||
for _, metric := range req.ForAllNodes {
|
||||
q := ApiQuery{
|
||||
@ -230,29 +173,29 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
for _, query := range req.Queries {
|
||||
sels := make([]Selector, 0, 1)
|
||||
sels := make([]types.Selector, 0, 1)
|
||||
if query.Aggregate || query.Type == nil {
|
||||
sel := Selector{{String: req.Cluster}, {String: query.Hostname}}
|
||||
sel := types.Selector{{String: req.Cluster}, {String: query.Hostname}}
|
||||
if query.Type != nil {
|
||||
if len(query.TypeIds) == 1 {
|
||||
sel = append(sel, SelectorElement{String: *query.Type + query.TypeIds[0]})
|
||||
sel = append(sel, types.SelectorElement{String: *query.Type + query.TypeIds[0]})
|
||||
} else {
|
||||
ids := make([]string, len(query.TypeIds))
|
||||
for i, id := range query.TypeIds {
|
||||
ids[i] = *query.Type + id
|
||||
}
|
||||
sel = append(sel, SelectorElement{Group: ids})
|
||||
sel = append(sel, types.SelectorElement{Group: ids})
|
||||
}
|
||||
|
||||
if query.SubType != nil {
|
||||
if len(query.SubTypeIds) == 1 {
|
||||
sel = append(sel, SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
|
||||
sel = append(sel, types.SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
|
||||
} else {
|
||||
ids := make([]string, len(query.SubTypeIds))
|
||||
for i, id := range query.SubTypeIds {
|
||||
ids[i] = *query.SubType + id
|
||||
}
|
||||
sel = append(sel, SelectorElement{Group: ids})
|
||||
sel = append(sel, types.SelectorElement{Group: ids})
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -261,13 +204,13 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
|
||||
for _, typeId := range query.TypeIds {
|
||||
if query.SubType != nil {
|
||||
for _, subTypeId := range query.SubTypeIds {
|
||||
sels = append(sels, Selector{
|
||||
sels = append(sels, types.Selector{
|
||||
{String: req.Cluster}, {String: query.Hostname},
|
||||
{String: *query.Type + typeId},
|
||||
{String: *query.SubType + subTypeId}})
|
||||
}
|
||||
} else {
|
||||
sels = append(sels, Selector{
|
||||
sels = append(sels, types.Selector{
|
||||
{String: req.Cluster},
|
||||
{String: query.Hostname},
|
||||
{String: *query.Type + typeId}})
|
||||
@ -281,7 +224,7 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
|
||||
res := make([]ApiMetricData, 0, len(sels))
|
||||
for _, sel := range sels {
|
||||
data := ApiMetricData{}
|
||||
data.Data, data.From, data.To, err = memoryStore.Read(sel, query.Metric, req.From, req.To)
|
||||
data.Data, data.From, data.To, err = ha.MemoryStore.Read(sel, query.Metric, req.From, req.To)
|
||||
// log.Printf("data: %#v, %#v, %#v, %#v", data.Data, data.From, data.To, err)
|
||||
if err != nil {
|
||||
msg := err.Error()
|
||||
@ -297,7 +240,7 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
|
||||
data.ScaleBy(query.ScaleFactor)
|
||||
}
|
||||
if req.WithPadding {
|
||||
data.PadDataWithNull(req.From, req.To, query.Metric)
|
||||
ha.padWithNaNs(&data, query.Metric, req.From, req.To)
|
||||
}
|
||||
if !req.WithData {
|
||||
data.Data = nil
|
||||
@ -316,7 +259,7 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
func authentication(next http.Handler, publicKey ed25519.PublicKey) http.Handler {
|
||||
func (ha *HttpApi) authentication(next http.Handler, publicKey ed25519.PublicKey) http.Handler {
|
||||
cacheLock := sync.RWMutex{}
|
||||
cache := map[string]*jwt.Token{}
|
||||
|
||||
@ -362,12 +305,11 @@ func authentication(next http.Handler, publicKey ed25519.PublicKey) http.Handler
|
||||
})
|
||||
}
|
||||
|
||||
func StartApiServer(ctx context.Context, httpConfig *HttpConfig) error {
|
||||
func (ha *HttpApi) StartServer(ctx context.Context) error {
|
||||
r := mux.NewRouter()
|
||||
|
||||
r.HandleFunc("/api/free", handleFree)
|
||||
r.HandleFunc("/api/write", handleWrite)
|
||||
r.HandleFunc("/api/query", handleQuery)
|
||||
r.HandleFunc("/api/write", ha.handleWrite)
|
||||
r.HandleFunc("/api/query", ha.handleQuery)
|
||||
r.HandleFunc("/api/debug", func(rw http.ResponseWriter, r *http.Request) {
|
||||
raw := r.URL.Query().Get("selector")
|
||||
selector := []string{}
|
||||
@ -375,7 +317,7 @@ func StartApiServer(ctx context.Context, httpConfig *HttpConfig) error {
|
||||
selector = strings.Split(raw, ":")
|
||||
}
|
||||
|
||||
if err := memoryStore.DebugDump(bufio.NewWriter(rw), selector); err != nil {
|
||||
if err := ha.MemoryStore.DebugDump(bufio.NewWriter(rw), selector); err != nil {
|
||||
rw.WriteHeader(http.StatusBadRequest)
|
||||
rw.Write([]byte(err.Error()))
|
||||
}
|
||||
@ -383,29 +325,30 @@ func StartApiServer(ctx context.Context, httpConfig *HttpConfig) error {
|
||||
|
||||
server := &http.Server{
|
||||
Handler: r,
|
||||
Addr: httpConfig.Address,
|
||||
Addr: ha.Address,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
}
|
||||
ha.server = server
|
||||
|
||||
if len(conf.JwtPublicKey) > 0 {
|
||||
buf, err := base64.StdEncoding.DecodeString(conf.JwtPublicKey)
|
||||
if len(ha.PublicKey) > 0 {
|
||||
buf, err := base64.StdEncoding.DecodeString(ha.PublicKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
publicKey := ed25519.PublicKey(buf)
|
||||
server.Handler = authentication(server.Handler, publicKey)
|
||||
server.Handler = ha.authentication(server.Handler, publicKey)
|
||||
}
|
||||
|
||||
go func() {
|
||||
if httpConfig.CertFile != "" && httpConfig.KeyFile != "" {
|
||||
log.Printf("API https endpoint listening on '%s'\n", httpConfig.Address)
|
||||
err := server.ListenAndServeTLS(httpConfig.CertFile, httpConfig.KeyFile)
|
||||
if ha.CertFile != "" && ha.KeyFile != "" {
|
||||
log.Printf("API https endpoint listening on '%s'\n", ha.Address)
|
||||
err := server.ListenAndServeTLS(ha.CertFile, ha.KeyFile)
|
||||
if err != nil && err != http.ErrServerClosed {
|
||||
log.Println(err)
|
||||
}
|
||||
} else {
|
||||
log.Printf("API http endpoint listening on '%s'\n", httpConfig.Address)
|
||||
log.Printf("API http endpoint listening on '%s'\n", ha.Address)
|
||||
err := server.ListenAndServe()
|
||||
if err != nil && err != http.ErrServerClosed {
|
||||
log.Println(err)
|
||||
@ -1,504 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/ClusterCockpit/cc-metric-store/lineprotocol"
|
||||
)
|
||||
|
||||
var testMetrics [][]lineprotocol.Metric = [][]lineprotocol.Metric{
|
||||
{{"flops", 100.5}, {"mem_bw", 2088.67}},
|
||||
{{"flops", 180.5}, {"mem_bw", 4078.32}, {"mem_capacity", 1020}},
|
||||
{{"flops", 980.5}, {"mem_bw", 9078.32}, {"mem_capacity", 5010}},
|
||||
{{"flops", 940.5}, {"mem_bw", 9278.32}, {"mem_capacity", 6010}},
|
||||
{{"flops", 930.5}, {"mem_bw", 9378.32}, {"mem_capacity", 7010}},
|
||||
{{"flops", 980.5}, {"mem_bw", 9478.32}, {"mem_capacity", 8010}},
|
||||
{{"flops", 980.5}, {"mem_bw", 9478.32}, {"mem_capacity", 8010}},
|
||||
{{"flops", 980.5}, {"mem_bw", 9478.32}, {"mem_capacity", 8010}},
|
||||
{{"flops", 970.5}, {"mem_bw", 9178.32}, {"mem_capacity", 2010}},
|
||||
{{"flops", 970.5}, {"mem_bw", 9178.32}, {"mem_capacity", 2010}}}
|
||||
|
||||
var testMetricsAlt [][]lineprotocol.Metric = [][]lineprotocol.Metric{
|
||||
{{"flops", 120.5}, {"mem_bw", 2080.67}},
|
||||
{{"flops", 130.5}, {"mem_bw", 4071.32}, {"mem_capacity", 1120}},
|
||||
{{"flops", 940.5}, {"mem_bw", 9072.32}, {"mem_capacity", 5210}},
|
||||
{{"flops", 950.5}, {"mem_bw", 9273.32}, {"mem_capacity", 6310}},
|
||||
{{"flops", 960.5}, {"mem_bw", 9374.32}, {"mem_capacity", 7410}},
|
||||
{{"flops", 970.5}, {"mem_bw", 9475.32}, {"mem_capacity", 8510}},
|
||||
{{"flops", 990.5}, {"mem_bw", 9476.32}, {"mem_capacity", 8610}},
|
||||
{{"flops", 910.5}, {"mem_bw", 9477.32}, {"mem_capacity", 8710}},
|
||||
{{"flops", 920.5}, {"mem_bw", 9178.32}, {"mem_capacity", 2810}},
|
||||
{{"flops", 930.5}, {"mem_bw", 9179.32}, {"mem_capacity", 2910}}}
|
||||
|
||||
func dumpStoreBuffer(s *storeBuffer) {
|
||||
log.Printf("Start TS %d\n", s.start)
|
||||
ctr := 0
|
||||
|
||||
for _, val := range s.store {
|
||||
fmt.Printf("%f\t", val)
|
||||
ctr++
|
||||
|
||||
if ctr == 10 {
|
||||
fmt.Printf("\n")
|
||||
ctr = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func printMemStore(m *MemoryStore) {
|
||||
log.Println("########################")
|
||||
log.Printf("Frequency %d, Metrics %d Slots %d\n",
|
||||
m.frequency, m.numMetrics, m.numSlots)
|
||||
log.Println("##Offsets")
|
||||
for key, val := range m.offsets {
|
||||
log.Printf("\t%s = %d\n", key, val)
|
||||
}
|
||||
log.Println("##Containers")
|
||||
for key, c := range m.containers {
|
||||
log.Printf("ID %s\n", key)
|
||||
log.Println("###current")
|
||||
dumpStoreBuffer(c.current)
|
||||
log.Println("###next")
|
||||
dumpStoreBuffer(c.next)
|
||||
}
|
||||
log.Println("########################")
|
||||
}
|
||||
|
||||
//############################
|
||||
//#### Whitebox tests ########
|
||||
//############################
|
||||
func TestAddMetricSimple(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
// printMemStore(m)
|
||||
|
||||
m.AddMetrics(key, 1584022800, testMetrics[0])
|
||||
m.AddMetrics(key, 1584022890, testMetrics[1])
|
||||
|
||||
want := testMetrics[0][0].Value
|
||||
got := m.containers[key].current.store[0]
|
||||
if got != want {
|
||||
t.Errorf("Want %f got %f\n", want, got)
|
||||
}
|
||||
|
||||
want = testMetrics[1][2].Value
|
||||
got = m.containers[key].current.store[21]
|
||||
if got != want {
|
||||
t.Errorf("Want %f got %f\n", want, got)
|
||||
}
|
||||
// printMemStore(m)
|
||||
}
|
||||
|
||||
func TestAddMetricReplace(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
// printMemStore(m)
|
||||
|
||||
m.AddMetrics(key, 1584022800, testMetrics[0])
|
||||
m.AddMetrics(key, 1584022800, testMetrics[1])
|
||||
|
||||
want := testMetrics[1][0].Value
|
||||
got := m.containers[key].current.store[0]
|
||||
if got != want {
|
||||
t.Errorf("Want %f got %f\n", want, got)
|
||||
}
|
||||
|
||||
m.AddMetrics(key, 1584022850, testMetrics[0])
|
||||
want = testMetrics[0][0].Value
|
||||
got = m.containers[key].current.store[0]
|
||||
if got != want {
|
||||
t.Errorf("Want %f got %f\n", want, got)
|
||||
}
|
||||
|
||||
m.AddMetrics(key, 1584022860, testMetrics[1])
|
||||
want = testMetrics[0][0].Value
|
||||
got = m.containers[key].current.store[0]
|
||||
if got != want {
|
||||
t.Errorf("Want %f got %f\n", want, got)
|
||||
}
|
||||
// printMemStore(m)
|
||||
}
|
||||
|
||||
func TestAddMetricSwitch(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
// printMemStore(m)
|
||||
|
||||
m.AddMetrics(key, 1584023000, testMetrics[0])
|
||||
m.AddMetrics(key, 1584023580, testMetrics[1])
|
||||
|
||||
want := testMetrics[1][2].Value
|
||||
got := m.containers[key].current.store[29]
|
||||
if got != want {
|
||||
t.Errorf("Want %f got %f\n", want, got)
|
||||
}
|
||||
|
||||
m.AddMetrics(key, 1584023600, testMetrics[2])
|
||||
want = testMetrics[2][2].Value
|
||||
got = m.containers[key].current.store[20]
|
||||
if got != want {
|
||||
t.Errorf("Want %f got %f\n", want, got)
|
||||
}
|
||||
|
||||
// printMemStore(m)
|
||||
}
|
||||
|
||||
//############################
|
||||
//#### Blackbox tests ########
|
||||
//############################
|
||||
|
||||
func TestAddMetricOutOfBounds(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 30, 60)
|
||||
|
||||
err := m.AddMetrics(key, 1584023000, testMetrics[0])
|
||||
if err != nil {
|
||||
t.Errorf("Got error 1584023000\n")
|
||||
}
|
||||
err = m.AddMetrics(key, 1584026600, testMetrics[0])
|
||||
if err == nil {
|
||||
t.Errorf("Got no error 1584026600\n")
|
||||
}
|
||||
err = m.AddMetrics(key, 1584021580, testMetrics[1])
|
||||
if err == nil {
|
||||
t.Errorf("Got no error 1584021580\n")
|
||||
}
|
||||
err = m.AddMetrics(key, 1584024580, testMetrics[1])
|
||||
if err != nil {
|
||||
t.Errorf("Got error 1584024580\n")
|
||||
}
|
||||
err = m.AddMetrics(key, 1584091580, testMetrics[1])
|
||||
if err == nil {
|
||||
t.Errorf("Got no error 1584091580\n")
|
||||
}
|
||||
err = m.AddMetrics(key, 1584024780, testMetrics[0])
|
||||
if err != nil {
|
||||
t.Errorf("Got error 1584024780\n")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMetricPlainCurrent(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
|
||||
}
|
||||
|
||||
// printMemStore(m)
|
||||
val, tsFrom, err := m.GetMetric(key, "flops", 1584023000, 1584023560)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Got error\n")
|
||||
}
|
||||
if tsFrom != 1584023000 {
|
||||
t.Errorf("Start ts differs: %d\n", tsFrom)
|
||||
}
|
||||
if len(val) != 9 {
|
||||
t.Errorf("Want 9. Got %d\n", len(val))
|
||||
}
|
||||
if val[0] != 100.5 {
|
||||
t.Errorf("Want 100.5 Got %f\n", val[0])
|
||||
}
|
||||
if val[8] != 970.5 {
|
||||
t.Errorf("Want 970.5 Got %f\n", val[9])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMetricPlainNext(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
|
||||
}
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
|
||||
}
|
||||
|
||||
// printMemStore(m)
|
||||
val, tsFrom, err := m.GetMetric(key, "flops", 1584023000, 1584023560)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Got error\n")
|
||||
}
|
||||
if tsFrom != 1584023000 {
|
||||
t.Errorf("Start ts differs: %d\n", tsFrom)
|
||||
}
|
||||
if len(val) != 9 {
|
||||
t.Errorf("Want 9. Got %d\n", len(val))
|
||||
}
|
||||
if val[0] != 100.5 {
|
||||
t.Errorf("Want 100.5 Got %f\n", val[0])
|
||||
}
|
||||
if val[8] != 970.5 {
|
||||
t.Errorf("Want 970.5 Got %f\n", val[9])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMetricGap(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023000+i*120), testMetrics[i])
|
||||
}
|
||||
|
||||
val, tsFrom, err := m.GetMetric(key, "flops", 1584023000, 1584023600)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Got error\n")
|
||||
}
|
||||
if tsFrom != 1584023000 {
|
||||
t.Errorf("Start ts differs: %d\n", tsFrom)
|
||||
}
|
||||
if len(val) != 10 {
|
||||
t.Errorf("Want 10. Got %d\n", len(val))
|
||||
}
|
||||
if val[0] != 100.5 {
|
||||
t.Errorf("Want 100.5 Got %f\n", val[0])
|
||||
}
|
||||
if !math.IsNaN(float64(val[1])) {
|
||||
t.Errorf("Want NaN Got %f\n", val[1])
|
||||
}
|
||||
if val[0] != 100.5 {
|
||||
t.Errorf("Want 100.5 Got %f\n", val[0])
|
||||
}
|
||||
|
||||
// fmt.Println(val)
|
||||
}
|
||||
|
||||
func TestGetMetricSplit(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
|
||||
}
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
|
||||
}
|
||||
|
||||
// printMemStore(m)
|
||||
|
||||
val, tsFrom, err := m.GetMetric(key, "flops", 1584023200, 1584023860)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Got error\n")
|
||||
}
|
||||
if tsFrom != 1584023200 {
|
||||
t.Errorf("Start ts differs: %d\n", tsFrom)
|
||||
}
|
||||
if len(val) != 11 {
|
||||
t.Errorf("Want 11. Got %d\n", len(val))
|
||||
}
|
||||
if val[0] != 940.5 {
|
||||
t.Errorf("Want 940.5 Got %f\n", val[0])
|
||||
}
|
||||
if val[10] != 950.5 {
|
||||
t.Errorf("Want 950.5 Got %f\n", val[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMetricExceedNext(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
|
||||
}
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
|
||||
}
|
||||
|
||||
// printMemStore(m)
|
||||
|
||||
val, tsFrom, err := m.GetMetric(key, "flops", 1584022800, 1584023400)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Got error\n")
|
||||
}
|
||||
if tsFrom != 1584023000 {
|
||||
t.Errorf("Start ts differs: %d\n", tsFrom)
|
||||
}
|
||||
if len(val) != 6 {
|
||||
t.Errorf("Want 6. Got %d\n", len(val))
|
||||
}
|
||||
if val[0] != 100.5 {
|
||||
t.Errorf("Want 100.5 Got %f\n", val[0])
|
||||
}
|
||||
if val[5] != 980.5 {
|
||||
t.Errorf("Want 980.5 Got %f\n", val[5])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMetricExceedNextSplit(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
|
||||
}
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
|
||||
}
|
||||
|
||||
// printMemStore(m)
|
||||
|
||||
val, tsFrom, err := m.GetMetric(key, "flops", 1584022800, 1584023900)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Got error\n")
|
||||
}
|
||||
if tsFrom != 1584023000 {
|
||||
t.Errorf("Start ts differs: %d\n", tsFrom)
|
||||
}
|
||||
if len(val) != 15 {
|
||||
t.Errorf("Want 14. Got %d\n", len(val))
|
||||
}
|
||||
if val[0] != 100.5 {
|
||||
t.Errorf("Want 100.5 Got %f\n", val[0])
|
||||
}
|
||||
if val[14] != 960.5 {
|
||||
t.Errorf("Want 960.5 Got %f\n", val[13])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMetricExceedCurrent(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
|
||||
}
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
|
||||
}
|
||||
|
||||
// printMemStore(m)
|
||||
|
||||
val, tsFrom, err := m.GetMetric(key, "flops", 1584023800, 1584027900)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Got error\n")
|
||||
}
|
||||
if tsFrom != 1584023800 {
|
||||
t.Errorf("Start ts differs: %d\n", tsFrom)
|
||||
}
|
||||
if len(val) != 7 {
|
||||
t.Errorf("Want 6. Got %d\n", len(val))
|
||||
}
|
||||
if val[0] != 950.5 {
|
||||
t.Errorf("Want 950.5 Got %f\n", val[0])
|
||||
}
|
||||
if val[6] != 930.5 {
|
||||
t.Errorf("Want 930.5 Got %f\n", val[5])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMetricExceedCurrentSplit(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
|
||||
}
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
|
||||
}
|
||||
|
||||
// printMemStore(m)
|
||||
|
||||
val, tsFrom, err := m.GetMetric(key, "flops", 1584023120, 1584027900)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Got error\n")
|
||||
}
|
||||
if tsFrom != 1584023120 {
|
||||
t.Errorf("Start ts differs: %d\n", tsFrom)
|
||||
}
|
||||
if len(val) != 18 {
|
||||
t.Errorf("Want 18. Got %d\n", len(val))
|
||||
}
|
||||
if val[0] != 980.5 {
|
||||
t.Errorf("Want 950.5 Got %f\n", val[0])
|
||||
}
|
||||
if val[17] != 930.5 {
|
||||
t.Errorf("Want 930.5 Got %f\n", val[17])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMetricExceedBoth(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
|
||||
}
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
|
||||
}
|
||||
|
||||
// printMemStore(m)
|
||||
|
||||
val, tsFrom, err := m.GetMetric(key, "flops", 1584022800, 1584027900)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Got error\n")
|
||||
}
|
||||
if tsFrom != 1584023000 {
|
||||
t.Errorf("Start ts differs: %d\n", tsFrom)
|
||||
}
|
||||
if len(val) != 20 {
|
||||
t.Errorf("Want 20. Got %d\n", len(val))
|
||||
}
|
||||
if val[0] != 100.5 {
|
||||
t.Errorf("Want 950.5 Got %f\n", val[0])
|
||||
}
|
||||
if val[19] != 930.5 {
|
||||
t.Errorf("Want 930.5 Got %f\n", val[17])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMetricOutUpper(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
|
||||
}
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
|
||||
}
|
||||
|
||||
// printMemStore(m)
|
||||
|
||||
_, _, err := m.GetMetric(key, "flops", 1584032800, 1584037900)
|
||||
|
||||
if err == nil {
|
||||
t.Errorf("Got no error\n")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMetricOutLower(t *testing.T) {
|
||||
key := "m1220"
|
||||
m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
|
||||
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
|
||||
}
|
||||
for i := 0; i < len(testMetrics); i++ {
|
||||
m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
|
||||
}
|
||||
|
||||
// printMemStore(m)
|
||||
|
||||
_, _, err := m.GetMetric(key, "flops", 1584002800, 1584007900)
|
||||
|
||||
if err == nil {
|
||||
t.Errorf("Got no error\n")
|
||||
}
|
||||
}
|
||||
118
stats.go
118
stats.go
@ -1,118 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
)
|
||||
|
||||
type Stats struct {
|
||||
Samples int
|
||||
Avg Float
|
||||
Min Float
|
||||
Max Float
|
||||
}
|
||||
|
||||
func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
|
||||
if from < b.start {
|
||||
if b.prev != nil {
|
||||
return b.prev.stats(from, to)
|
||||
}
|
||||
from = b.start
|
||||
}
|
||||
|
||||
// TODO: Check if b.closed and if so and the full buffer is queried,
|
||||
// use b.statistics instead of iterating over the buffer.
|
||||
|
||||
samples := 0
|
||||
sum, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
|
||||
|
||||
var t int64
|
||||
for t = from; t < to; t += b.frequency {
|
||||
idx := int((t - b.start) / b.frequency)
|
||||
if idx >= cap(b.data) {
|
||||
b = b.next
|
||||
if b == nil {
|
||||
break
|
||||
}
|
||||
idx = 0
|
||||
}
|
||||
|
||||
if t < b.start || idx >= len(b.data) {
|
||||
continue
|
||||
}
|
||||
|
||||
xf := float64(b.data[idx])
|
||||
if math.IsNaN(xf) {
|
||||
continue
|
||||
}
|
||||
|
||||
samples += 1
|
||||
sum += xf
|
||||
min = math.Min(min, xf)
|
||||
max = math.Max(max, xf)
|
||||
}
|
||||
|
||||
return Stats{
|
||||
Samples: samples,
|
||||
Avg: Float(sum) / Float(samples),
|
||||
Min: Float(min),
|
||||
Max: Float(max),
|
||||
}, from, t, nil
|
||||
}
|
||||
|
||||
// Returns statistics for the requested metric on the selected node/level.
|
||||
// Data is aggregated to the selected level the same way as in `MemoryStore.Read`.
|
||||
// If `Stats.Samples` is zero, the statistics should not be considered as valid.
|
||||
func (m *MemoryStore) Stats(selector Selector, metric string, from, to int64) (*Stats, int64, int64, error) {
|
||||
if from > to {
|
||||
return nil, 0, 0, errors.New("invalid time range")
|
||||
}
|
||||
|
||||
minfo, ok := m.metrics[metric]
|
||||
if !ok {
|
||||
return nil, 0, 0, errors.New("unkown metric: " + metric)
|
||||
}
|
||||
|
||||
n, samples := 0, 0
|
||||
avg, min, max := Float(0), math.MaxFloat32, -math.MaxFloat32
|
||||
err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
|
||||
stats, cfrom, cto, err := b.stats(from, to)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
from, to = cfrom, cto
|
||||
} else if from != cfrom || to != cto {
|
||||
return ErrDataDoesNotAlign
|
||||
}
|
||||
|
||||
samples += stats.Samples
|
||||
avg += stats.Avg
|
||||
min = math.Min(min, float64(stats.Min))
|
||||
max = math.Max(max, float64(stats.Max))
|
||||
n += 1
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, 0, 0, err
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
return nil, 0, 0, ErrNoData
|
||||
}
|
||||
|
||||
if minfo.Aggregation == AvgAggregation {
|
||||
avg /= Float(n)
|
||||
} else if n > 1 && minfo.Aggregation != SumAggregation {
|
||||
return nil, 0, 0, errors.New("invalid aggregation")
|
||||
}
|
||||
|
||||
return &Stats{
|
||||
Samples: samples,
|
||||
Avg: avg,
|
||||
Min: Float(min),
|
||||
Max: Float(max),
|
||||
}, from, to, nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user