Merge pull request #37 from ClusterCockpit/devel

Added update to resampler

.gitignore

@@ -20,7 +20,7 @@
 
 aditya.creds
 test.creds
-./config.json
+/config.json
 
 migrateTimestamps.pl
 test_ccms_api.sh

configs/config-nats.json

@@ -0,0 +1,196 @@
+{
+    "metrics": {
+        "debug_metric": {
+            "frequency": 60,
+            "aggregation": "avg"
+        },
+        "clock": {
+            "frequency": 60,
+            "aggregation": "avg"
+        },
+        "cpu_idle": {
+            "frequency": 60,
+            "aggregation": "avg"
+        },
+        "cpu_iowait": {
+            "frequency": 60,
+            "aggregation": "avg"
+        },
+        "cpu_irq": {
+            "frequency": 60,
+            "aggregation": "avg"
+        },
+        "cpu_system": {
+            "frequency": 60,
+            "aggregation": "avg"
+        },
+        "cpu_user": {
+            "frequency": 60,
+            "aggregation": "avg"
+        },
+        "nv_mem_util": {
+            "frequency": 60,
+            "aggregation": "avg"
+        },
+        "nv_temp": {
+            "frequency": 60,
+            "aggregation": "avg"
+        },
+        "nv_sm_clock": {
+            "frequency": 60,
+            "aggregation": "avg"
+        },
+        "acc_utilization": {
+            "frequency": 60,
+            "aggregation": "avg"
+        },
+        "acc_mem_used": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "acc_power": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "flops_any": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "flops_dp": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "flops_sp": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "ib_recv": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "ib_xmit": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "ib_recv_pkts": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "ib_xmit_pkts": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "cpu_power": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "core_power": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "mem_power": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "ipc": {
+            "frequency": 60,
+            "aggregation": "avg"
+        },
+        "cpu_load": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "lustre_close": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "lustre_open": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "lustre_statfs": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "lustre_read_bytes": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "lustre_write_bytes": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "net_bw": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "file_bw": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "mem_bw": {
+            "frequency": 60,
+            "aggregation": "sum"
+        },
+        "mem_cached": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "mem_used": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "net_bytes_in": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "net_bytes_out": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "nfs4_read": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "nfs4_total": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "nfs4_write": {
+            "frequency": 60,
+            "aggregation": null
+        },
+        "vectorization_ratio": {
+            "frequency": 60,
+            "aggregation": "avg"
+        }
+    },
+    "checkpoints": {
+        "interval": "12h",
+        "directory": "./var/checkpoints",
+        "restore": "48h"
+    },
+    "archive": {
+        "interval": "50h",
+        "directory": "./var/archive"
+    },
+    "http-api": {
+        "address": "localhost:8082",
+        "https-cert-file": null,
+        "https-key-file": null
+    },
+    "retention-in-memory": "48h",
+    "nats": [
+        {
+            "address": "nats://localhost:4222",
+            "creds-file-path": "test.creds",
+            "subscriptions": [
+                {
+                    "subscribe-to": "ee-hpc-nats",
+                    "cluster-tag": "fritz2"
+                }
+            ]
+        }
+    ],
+    "jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
+}

configs/config.json

@@ -180,17 +180,6 @@
         "https-key-file": null
     },
     "retention-in-memory": "48h",
-    "nats": [
+    "nats": null,
-        {
-            "address": "nats://localhost:4222",
-            "creds-file-path": "test.creds",
-            "subscriptions": [
-                {
-                    "subscribe-to": "ee-hpc-nats",
-                    "cluster-tag": "fritz2"
-                }
-            ]
-        }
-    ],
     "jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
 }

pkg/resampler/resampler.go

@@ -8,20 +8,20 @@ import (
 	"github.com/ClusterCockpit/cc-metric-store/internal/util"
 )
 
-func SimpleResampler(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, error) {
+func SimpleResampler(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, int64, error) {
-	if old_frequency == 0 || new_frequency == 0 {
+	if old_frequency == 0 || new_frequency == 0 || new_frequency <= old_frequency {
-		return nil, errors.New("either old or new frequency is set to 0")
+		return data, old_frequency, nil
 	}
 
 	if new_frequency%old_frequency != 0 {
-		return nil, errors.New("new sampling frequency should be multiple of the old frequency")
+		return nil, 0, errors.New("new sampling frequency should be multiple of the old frequency")
 	}
 
 	var step int = int(new_frequency / old_frequency)
 	var new_data_length = len(data) / step
 
 	if new_data_length == 0 || len(data) < 100 || new_data_length >= len(data) {
-		return data, nil
+		return data, old_frequency, nil
 	}
 
 	new_data := make([]util.Float, new_data_length)
@@ -30,14 +30,14 @@ func SimpleResampler(data []util.Float, old_frequency int64, new_frequency int64
 		new_data[i] = data[i*step]
 	}
 
-	return new_data, nil
+	return new_data, new_frequency, nil
 }
 
 // Inspired by one of the algorithms from https://skemman.is/bitstream/1946/15343/3/SS_MSthesis.pdf
 // Adapted from https://github.com/haoel/downsampling/blob/master/core/lttb.go
 func LargestTriangleThreeBucket(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, int64, error) {
 
-	if old_frequency == 0 || new_frequency == 0 {
+	if old_frequency == 0 || new_frequency == 0 || new_frequency <= old_frequency {
 		return data, old_frequency, nil
 	}