ClusterCockpit/cc-metric-store
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-metric-store.git synced 2025-04-06 13:55:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: ClusterCockpit:main
Branches Tags
ClusterCockpit:main
ClusterCockpit:devel
ClusterCockpit:avro
ClusterCockpit:hotfix
ClusterCockpit:debug-logging
ClusterCockpit:store_unit_test
ClusterCockpit:store_unit_per_buffer
ClusterCockpit:v0.1.1
ClusterCockpit:v0.1.0
ClusterCockpit:v0.0.1
..
compare: ClusterCockpit:v0.1.0
Branches Tags
ClusterCockpit:devel
ClusterCockpit:avro
ClusterCockpit:main
ClusterCockpit:hotfix
ClusterCockpit:debug-logging
ClusterCockpit:store_unit_test
ClusterCockpit:store_unit_per_buffer
ClusterCockpit:v0.1.1
ClusterCockpit:v0.1.0
ClusterCockpit:v0.0.1

No commits in common. "main" and "v0.1.0" have entirely different histories.
main ... v0.1.0

6 changed files with 27 additions and 211 deletions
Show Stats Expand all files Collapse all files

6
.gitignore vendored
View File

@ -4,6 +4,7 @@
*.dll
*.so
*.dylib
/cc-metric-store
# Test binary, built with `go test -c`
*.test
@ -15,16 +16,15 @@
# vendor/
# Project specific ignores
/cc-metric-store
/var
aditya.creds
test.creds
/config.json
./config.json
migrateTimestamps.pl
test_ccms_api.sh
test_ccms_free_api.sh
test_ccms_write_api.sh
sample_alex.txt
sample_fritz.txt
sample_fritz.txt

2
Makefile
View File

@ -1,6 +1,6 @@
TARGET = ./cc-metric-store
VAR = ./var/checkpoints/
VERSION = 0.1.1
VERSION = 0.1.0
GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S")
LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}'

7
ReleaseNotes.md
View File

@ -1,10 +1,10 @@
# `cc-metric-store` version 0.1.1
# `cc-metric-store` version 0.1.0
This is a bugfix release of `cc-metric-store`, the metric timeseries cache
This is a minor release of `cc-metric-store`, the metric timeseries cache
implementation of ClusterCockpit.
For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/).
Notable changes in version 0.1.0:
Notable changes:
- Cleanup of code and restructuring
- Document REST API with Swagger
@ -14,6 +14,7 @@ Notable changes in version 0.1.0:
- Support also ms, ns in line protocol
- Support NATS credentials
## Breaking changes
None

196
configs/config-nats.json
View File

@ -1,196 +0,0 @@
{
"metrics": {
"debug_metric": {
"frequency": 60,
"aggregation": "avg"
},
"clock": {
"frequency": 60,
"aggregation": "avg"
},
"cpu_idle": {
"frequency": 60,
"aggregation": "avg"
},
"cpu_iowait": {
"frequency": 60,
"aggregation": "avg"
},
"cpu_irq": {
"frequency": 60,
"aggregation": "avg"
},
"cpu_system": {
"frequency": 60,
"aggregation": "avg"
},
"cpu_user": {
"frequency": 60,
"aggregation": "avg"
},
"nv_mem_util": {
"frequency": 60,
"aggregation": "avg"
},
"nv_temp": {
"frequency": 60,
"aggregation": "avg"
},
"nv_sm_clock": {
"frequency": 60,
"aggregation": "avg"
},
"acc_utilization": {
"frequency": 60,
"aggregation": "avg"
},
"acc_mem_used": {
"frequency": 60,
"aggregation": "sum"
},
"acc_power": {
"frequency": 60,
"aggregation": "sum"
},
"flops_any": {
"frequency": 60,
"aggregation": "sum"
},
"flops_dp": {
"frequency": 60,
"aggregation": "sum"
},
"flops_sp": {
"frequency": 60,
"aggregation": "sum"
},
"ib_recv": {
"frequency": 60,
"aggregation": "sum"
},
"ib_xmit": {
"frequency": 60,
"aggregation": "sum"
},
"ib_recv_pkts": {
"frequency": 60,
"aggregation": "sum"
},
"ib_xmit_pkts": {
"frequency": 60,
"aggregation": "sum"
},
"cpu_power": {
"frequency": 60,
"aggregation": "sum"
},
"core_power": {
"frequency": 60,
"aggregation": "sum"
},
"mem_power": {
"frequency": 60,
"aggregation": "sum"
},
"ipc": {
"frequency": 60,
"aggregation": "avg"
},
"cpu_load": {
"frequency": 60,
"aggregation": null
},
"lustre_close": {
"frequency": 60,
"aggregation": null
},
"lustre_open": {
"frequency": 60,
"aggregation": null
},
"lustre_statfs": {
"frequency": 60,
"aggregation": null
},
"lustre_read_bytes": {
"frequency": 60,
"aggregation": null
},
"lustre_write_bytes": {
"frequency": 60,
"aggregation": null
},
"net_bw": {
"frequency": 60,
"aggregation": null
},
"file_bw": {
"frequency": 60,
"aggregation": null
},
"mem_bw": {
"frequency": 60,
"aggregation": "sum"
},
"mem_cached": {
"frequency": 60,
"aggregation": null
},
"mem_used": {
"frequency": 60,
"aggregation": null
},
"net_bytes_in": {
"frequency": 60,
"aggregation": null
},
"net_bytes_out": {
"frequency": 60,
"aggregation": null
},
"nfs4_read": {
"frequency": 60,
"aggregation": null
},
"nfs4_total": {
"frequency": 60,
"aggregation": null
},
"nfs4_write": {
"frequency": 60,
"aggregation": null
},
"vectorization_ratio": {
"frequency": 60,
"aggregation": "avg"
}
},
"checkpoints": {
"interval": "12h",
"directory": "./var/checkpoints",
"restore": "48h"
},
"archive": {
"interval": "50h",
"directory": "./var/archive"
},
"http-api": {
"address": "localhost:8082",
"https-cert-file": null,
"https-key-file": null
},
"retention-in-memory": "48h",
"nats": [
{
"address": "nats://localhost:4222",
"creds-file-path": "test.creds",
"subscriptions": [
{
"subscribe-to": "ee-hpc-nats",
"cluster-tag": "fritz2"
}
]
}
],
"jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
}

13
configs/config.json
View File

@ -180,6 +180,17 @@
"https-key-file": null
},
"retention-in-memory": "48h",
"nats": null,
"nats": [
{
"address": "nats://localhost:4222",
"creds-file-path": "test.creds",
"subscriptions": [
{
"subscribe-to": "ee-hpc-nats",
"cluster-tag": "fritz2"
}
]
}
],
"jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
}

14
pkg/resampler/resampler.go
View File

@ -8,20 +8,20 @@ import (
"github.com/ClusterCockpit/cc-metric-store/internal/util"
)
func SimpleResampler(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, int64, error) {
if old_frequency == 0 || new_frequency == 0 || new_frequency <= old_frequency {
return data, old_frequency, nil
func SimpleResampler(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, error) {
if old_frequency == 0 || new_frequency == 0 {
return nil, errors.New("either old or new frequency is set to 0")
}
if new_frequency%old_frequency != 0 {
return nil, 0, errors.New("new sampling frequency should be multiple of the old frequency")
return nil, errors.New("new sampling frequency should be multiple of the old frequency")
}
var step int = int(new_frequency / old_frequency)
var new_data_length = len(data) / step
if new_data_length == 0 || len(data) < 100 || new_data_length >= len(data) {
return data, old_frequency, nil
return data, nil
}
new_data := make([]util.Float, new_data_length)
@ -30,14 +30,14 @@ func SimpleResampler(data []util.Float, old_frequency int64, new_frequency int64
new_data[i] = data[i*step]
}
return new_data, new_frequency, nil
return new_data, nil
}
// Inspired by one of the algorithms from https://skemman.is/bitstream/1946/15343/3/SS_MSthesis.pdf
// Adapted from https://github.com/haoel/downsampling/blob/master/core/lttb.go
func LargestTriangleThreeBucket(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, int64, error) {
if old_frequency == 0 || new_frequency == 0 || new_frequency <= old_frequency {
if old_frequency == 0 || new_frequency == 0 {
return data, old_frequency, nil
}