Schema for metric store

configs/config-demo.json

@@ -34,9 +34,9 @@
     {
       "name": "fritz",
       "metricDataRepository": {
-        "kind": "cc-metric-store",
+        "kind": "cc-metric-store-internal",
         "url": "http://localhost:8082",
-        "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9"
+        "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
       },
       "filterRanges": {
         "numNodes": {
@@ -56,9 +56,9 @@
     {
       "name": "alex",
       "metricDataRepository": {
-        "kind": "cc-metric-store",
+        "kind": "cc-metric-store-internal",
         "url": "http://localhost:8082",
-        "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9"
+        "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
       },
       "filterRanges": {
         "numNodes": {
@@ -78,15 +78,16 @@
   ],
   "metric-store": {
     "checkpoints": {
-      "file-format": "avro",
+      "file-format": "json",
-      "interval": "2h",
+      "interval": "1h",
       "directory": "./var/checkpoints",
       "restore": "48h"
     },
     "archive": {
-      "interval": "2h",
+      "interval": "1h",
       "directory": "./var/archive"
     },
     "retention-in-memory": "48h"
-  }
+  },
+  "ui-file": "./configs/uiConfig.json"
 }

internal/memorystore/buffer.go

@@ -7,6 +7,7 @@ package memorystore
 
 import (
 	"errors"
+	"fmt"
 	"sync"
 
 	"github.com/ClusterCockpit/cc-lib/schema"
@@ -187,6 +188,8 @@ func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int6
 		i++
 	}
 
+	fmt.Printf("Given From : %d, To: %d\n", from, to)
+
 	return data[:i], from, t, nil
 }
 

internal/memorystore/configSchema.go

@@ -7,181 +7,80 @@ package memorystore
 
 const configSchema = `{
     "type": "object",
+    "description": "Configuration specific to built-in metric-store.",
     "properties": {
-    "jobList": {
+        "checkpoints": {
-      "description": "Job list defaults. Applies to user- and jobs views.",
+            "description": "Configuration for checkpointing the metrics within metric-store",
             "type": "object",
             "properties": {
-        "usePaging": {
+                "file-format": {
-          "description": "If classic paging is used instead of continuous scrolling by default.",
+                    "description": "Specify the type of checkpoint file. There are 2 variants: 'avro' and 'json'. If nothing is specified, 'avro' is default.",
-          "type": "boolean"
-        },
-        "showFootprint": {
-          "description": "If footprint bars are shown as first column by default.",
-          "type": "boolean"
-        }
-      }
-    },
-    "nodeList": {
-      "description": "Node list defaults. Applies to node list view.",
-      "type": "object",
-      "properties": {
-        "usePaging": {
-          "description": "If classic paging is used instead of continuous scrolling by default.",
-          "type": "boolean"
-        }
-      }
-    },
-    "jobView": {
-      "description": "Job view defaults.",
-      "type": "object",
-      "properties": {
-        "showPolarPlot": {
-          "description": "If the job metric footprints polar plot is shown by default.",
-          "type": "boolean"
-        },
-        "showFootprint": {
-          "description": "If the annotated job metric footprint bars are shown by default.",
-          "type": "boolean"
-        },
-        "showRoofline": {
-          "description": "If the job roofline plot is shown by default.",
-          "type": "boolean"
-        },
-        "showStatTable": {
-          "description": "If the job metric statistics table is shown by default.",
-          "type": "boolean"
-        }
-      }
-    },
-    "metricConfig": {
-      "description": "Global initial metric selections for primary views of all clusters.",
-      "type": "object",
-      "properties": {
-        "jobListMetrics": {
-          "description": "Initial metrics shown for new users in job lists (User and jobs view).",
-          "type": "array",
-          "items": {
-            "type": "string",
-            "minItems": 1
-          }
-        },
-        "jobViewPlotMetrics": {
-          "description": "Initial metrics shown for new users as job view metric plots.",
-          "type": "array",
-          "items": {
-            "type": "string",
-            "minItems": 1
-          }
-        },
-        "jobViewTableMetrics": {
-          "description": "Initial metrics shown for new users in job view statistics table.",
-          "type": "array",
-          "items": {
-            "type": "string",
-            "minItems": 1
-          }
-        },
-        "clusters": {
-          "description": "Overrides for global defaults by cluster and subcluster.",
-          "type": "array",
-          "items": {
-            "type": "object",
-            "properties": {
-              "name": {
-                "description": "The name of the cluster."
-              },
-              "jobListMetrics": {
-                "description": "Initial metrics shown for new users in job lists (User and jobs view) for subcluster.",
-                "type": "array",
-                "items": {
-                  "type": "string",
-                  "minItems": 1
-                }
-              },
-              "jobViewPlotMetrics": {
-                "description": "Initial metrics shown for new users as job view timeplots for subcluster.",
-                "type": "array",
-                "items": {
-                  "type": "string",
-                  "minItems": 1
-                }
-              },
-              "jobViewTableMetrics": {
-                "description": "Initial metrics shown for new users in job view statistics table for subcluster.",
-                "type": "array",
-                "items": {
-                  "type": "string",
-                  "minItems": 1
-                }
-              },
-              "subClusters": {
-                "description": "The array of overrides per subcluster.",
-                "type": "array",
-                "items": {
-                  "type": "object",
-                  "properties": {
-                    "name": {
-                      "description": "The name of the subcluster.",
                     "type": "string"
                 },
-                    "jobListMetrics": {
+                "interval": {
-                      "description": "Initial metrics shown for new users in job lists (User and jobs view) for subcluster.",
+                    "description": "Interval at which the metrics should be checkpointed.",
-                      "type": "array",
+                    "type": "string"
-                      "items": {
-                        "type": "string",
-                        "minItems": 1
-                      }
                 },
-                    "jobViewPlotMetrics": {
+                "directory": {
-                      "description": "Initial metrics shown for new users as job view timeplots for subcluster.",
+                    "description": "Specify the parent directy in which the checkpointed files should be placed.",
-                      "type": "array",
+                    "type": "string"
-                      "items": {
-                        "type": "string",
-                        "minItems": 1
-                      }
                 },
-                    "jobViewTableMetrics": {
+                "restore": {
-                      "description": "Initial metrics shown for new users in job view statistics table for subcluster.",
+                    "description": "When cc-backend starts up, look for checkpointed files that are less than X hours old and load metrics from these selected checkpoint files.",
-                      "type": "array",
+                    "type": "string"
-                      "items": {
-                        "type": "string",
-                        "minItems": 1
                 }
             }
         },
-                  "required": ["name"],
+        "archive": {
-                  "minItems": 1
+            "description": "Configuration for archiving the already checkpointed files.",
-                }
-              }
-            },
-            "required": ["name", "subClusters"],
-            "minItems": 1
-          }
-        }
-      }
-    },
-    "plotConfiguration": {
-      "description": "Initial settings for plot render options.",
             "type": "object",
             "properties": {
-        "colorBackground": {
+                "interval": {
-          "description": "If the metric plot backgrounds are initially colored by threshold limits.",
+                    "description": "Interval at which the checkpointed files should be archived.",
-          "type": "boolean"
-        },
-        "plotsPerRow": {
-          "description": "How many plots are initially rendered in per row. Applies to job, single node, and analysis views.",
-          "type": "integer"
-        },
-        "lineWidth": {
-          "description": "Initial thickness of rendered plotlines. Applies to metric plot, job compare plot and roofline.",
-          "type": "integer"
-        },
-        "colorScheme": {
-          "description": "Initial colorScheme to be used for metric plots.",
-          "type": "array",
-          "items": {
                     "type": "string"
+                },
+                "directory": {
+                    "description": "Specify the parent directy in which the archived files should be placed.",
+                    "type": "string"
+                }
+            }
+        },
+        "retention-in-memory": {
+            "description": "Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.",
+            "type": "string"
+        },
+        "nats": {
+            "description": "Configuration for accepting published data through NATS.",
+            "type": "object",
+            "properties": {
+                "address": {
+                    "description": "Address of the NATS server.",
+                    "type": "string"
+                },
+                "username": {
+                    "description": "Optional: If configured with username/password method.",
+                    "type": "string"
+                },
+                "password": {
+                    "description": "Optional: If configured with username/password method.",
+                    "type": "string"
+                },
+                "creds-file-path": {
+                    "description": "Optional: If configured with Credential File method. Path to your NATS cred file.",
+                    "type": "string"
+                },
+                "subscriptions": {
+                    "description": "Array of various subscriptions. Allows to subscibe to different subjects and publishers.",
+                    "type": "object",
+                    "properties": {
+                        "subscribe-to": {
+                            "description": "Channel name",
+                            "type": "string"
+                        },
+                        "cluster-tag": {
+                            "description": "Optional: Allow lines without a cluster tag, use this as default",
+                            "type": "string"
+                        }
                     }
                 }
             }

internal/memorystore/memorystore.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"os"
 	"os/signal"
 	"runtime"
@@ -386,6 +387,8 @@ func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, reso
 
 	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
 
+	fmt.Printf("Requested From : %d, To: %d\n", from, to)
+
 	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
 		cdata, cfrom, cto, err := b.read(from, to, data)
 		if err != nil {

startDemo.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+# rm -rf var
+
 if [ -d './var' ]; then
   echo 'Directory ./var already exists! Skipping initialization.'
   ./cc-backend -server -dev
@@ -14,6 +16,19 @@ else
 
   ./cc-backend -migrate-db
   ./cc-backend -dev -init-db -add-user demo:admin,api:demo
+  
+   # Generate JWT and extract only the token value
+  JWT=$(./cc-backend -jwt demo | grep -oP "(?<=JWT: Successfully generated JWT for user 'demo': ).*")
+
+  # Replace the existing JWT in test_ccms_write_api.sh with the new one
+  if [ -n "$JWT" ]; then
+    sed -i "1s|^JWT=.*|JWT=\"$JWT\"|" test_ccms_write_api.sh
+    echo "✅ Updated JWT in test_ccms_write_api.sh"
+  else
+    echo "❌ Failed to generate JWT for demo user"
+    exit 1
+  fi
+
   ./cc-backend -server -dev
 
 fi