ClusterCockpit/cc-docker
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-docker.git synced 2025-07-26 14:46:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Iterate setupscript, remove startscript, update migratetimestamp script

Browse Source
This commit is contained in:
Christoph Kluge
2022-06-23 18:05:08 +02:00
parent f7914b0291
commit 46c1d90aab
3 changed files with 42 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
migrateTimestamps.pl
View File

@@ -15,7 +15,7 @@ use REST::Client;
my $localtime = localtime;
my $epochtime = $localtime->epoch;
my $archiveTarget = './cc-backend/var/job-archive';
my $archiveSrc = './data/job-archive';
my $archiveSrc = './data/job-archive-source';
my @ArchiveClusters;
# Get clusters by job-archive/$subfolder
@@ -29,7 +29,7 @@ while ( readdir $dh ) {
# start for jobarchive
foreach my $cluster ( @ArchiveClusters ) {
print "Starting to update startTime in job-archive for $cluster\n";
print "Starting to update start- and stoptimes in job-archive for $cluster\n";
opendir my $dhLevel1, "$archiveSrc/$cluster" or die "can't open directory: $!";
while ( readdir $dhLevel1 ) {
@@ -55,7 +55,7 @@ foreach my $cluster ( @ArchiveClusters ) {
}
# check if subfolder contains file, else remove source and skip
if ( ! -e "$jobSource/meta.json") {
rmtree $jobOrigin;
# rmtree $jobOrigin;
next;
}
@@ -92,13 +92,13 @@ foreach my $cluster ( @ArchiveClusters ) {
}
}
print "Done for job-archive\n";
sleep(2);
sleep(1);
## CHECKPOINTS
chomp(my $checkpointStart=`date --date 'TZ="Europe/Berlin" 0:00 7 days ago' +%s`);
my $halfday = 43200;
my $checkpTarget = './data/cc-metric-store_new';
my $checkpSource = './data/cc-metric-store';
my $checkpTarget = './data/cc-metric-store/checkpoints';
my $checkpSource = './data/cc-metric-store-source/checkpoints';
my @CheckpClusters;
# Get clusters by cc-metric-store/$subfolder
@@ -112,7 +112,7 @@ while ( readdir $dhc ) {
# start for checkpoints
foreach my $cluster ( @CheckpClusters ) {
print "Starting to update startTime in checkpoint-files for $cluster\n";
print "Starting to update checkpoint filenames and data starttimes for $cluster\n";
opendir my $dhLevel1, "$checkpSource/$cluster" or die "can't open directory: $!";
while ( readdir $dhLevel1 ) {
@@ -169,85 +169,3 @@ foreach my $cluster ( @CheckpClusters ) {
}
}
print "Done for checkpoints\n";
sleep(2);
### INFLUXDB
my $newCheckpoints = './data/cc-metric-store_new';
my $verbose = 1;
my $restClient = REST::Client->new();
$restClient->setHost('http://localhost:8087');
$restClient->addHeader('Authorization', "Token 74008ea2a8dad5e6f856838a90c6392e");
$restClient->addHeader('Content-Type', 'text/plain; charset=utf-8');
$restClient->addHeader('Accept', 'application/json');
$restClient->getUseragent()->ssl_opts(SSL_verify_mode => 0); # Temporary: Disable Cert Check
$restClient->getUseragent()->ssl_opts(verify_hostname => 0); # Temporary: Disable Cert Check
# Get clusters by folder: Reuse from above
# start to read checkpoints for influx
foreach my $cluster ( @CheckpClusters ) {
print "Starting to read checkpoint-files into influx for $cluster\n";
opendir my $dhLevel1, "$newCheckpoints/$cluster" or die "can't open directory: $!";
while ( readdir $dhLevel1 ) {
chomp; next if $_ eq '.' or $_ eq '..';
my $level1 = $_;
if ( -d "$newCheckpoints/$cluster/$level1" ) {
my $nodeSource = "$newCheckpoints/$cluster/$level1/";
my @files = read_dir($nodeSource);
my $length = @files;
if (!@files || $length != 14) { # needs 14 files == 7 days worth of data
next;
}
my @sortedFiles = sort { versioncmp($a,$b) } @files; # sort alphanumerically: _Really_ start with index == 0 == 1609459200.json
my $nodeMeasurement;
foreach my $file (@sortedFiles) {
# print "$file\n";
my $rawstr = read_file("$nodeSource/$file");
my $json = decode_json($rawstr);
my $fileMeasurement;
foreach my $metric (keys %{$json->{metrics}}) {
my $start = $json->{metrics}->{$metric}->{start};
my $timestep = $json->{metrics}->{$metric}->{frequency};
my $data = $json->{metrics}->{$metric}->{data};
my $length = @$data;
my $measurement;
while (my ($index, $value) = each(@$data)) {
if ($value) {
my $timestamp = $start + ($timestep * $index);
$measurement .= "$metric,cluster=$cluster,hostname=$level1,type=node value=".$value." $timestamp"."\n";
}
}
# Use v2 API for Influx2
if ($measurement) {
# print "Adding: #VALUES $length KEY $metric"."\n";
$fileMeasurement .= $measurement;
}
}
if ($fileMeasurement) {
$nodeMeasurement .= $fileMeasurement;
}
}
$restClient->POST("/api/v2/write?org=ClusterCockpit&bucket=ClusterCockpit&precision=s", "$nodeMeasurement");
my $responseCode = $restClient->responseCode();
if ( $responseCode eq '204') {
if ( $verbose ) {
print "INFLUX API WRITE: CLUSTER $cluster HOST $level1"."\n";
};
} else {
if ( $responseCode ne '422' ) { # Exclude High Frequency Error 422 - Temporary!
my $response = $restClient->responseContent();
print "INFLUX API WRITE ERROR CODE ".$responseCode.": ".$response."\n";
};
};
}
}
}
print "Done for influx\n";