Adopt scripts for sync of DB with json.

acSync.pl

@@ -42,7 +42,8 @@ my %attr = (
 );
 
 my $dbh = DBI->connect(
-    "DBI:SQLite:dbname=$database", "", "", \%attr);
+    "DBI:SQLite:dbname=$database", "", "", \%attr)
+    or die "Could not connect to database: $DBI::errstr";
 
 my $sth_select_job = $dbh->prepare(qq{
     SELECT id, user_id, job_id, cluster_id,

extractJobIDs.pl

@@ -0,0 +1,57 @@
+#!/usr/bin/env perl
+# =======================================================================================
+#
+#      Author:   Jan Eitzinger (je), jan.eitzinger@fau.de
+#      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+#
+#      Permission is hereby granted, free of charge, to any person obtaining a copy
+#      of this software and associated documentation files (the "Software"), to deal
+#      in the Software without restriction, including without limitation the rights
+#      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#      copies of the Software, and to permit persons to whom the Software is
+#      furnished to do so, subject to the following conditions:
+#
+#      The above copyright notice and this permission notice shall be included in all
+#      copies or substantial portions of the Software.
+#
+#      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#      SOFTWARE.
+#
+# =======================================================================================
+
+use strict;
+use warnings;
+use utf8;
+
+my $basedir = $ARGV[0];
+open(my $fh, '>:encoding(UTF-8)', 'jobIds.txt')
+    or die "Could not open file  $!";
+
+opendir my $odh, $basedir or die "can't open directory: $!";
+
+while ( readdir $odh ) {
+    chomp;
+    next if $_ eq '.' or $_ eq '..';
+
+    my $jobID1 = $_;
+    print "Open $jobID1\n";
+
+    opendir my $idh, "$basedir/$jobID1" or die "can't open directory: $!";
+
+    while ( readdir $idh ) {
+        chomp;
+        next if $_ eq '.' or $_ eq '..';
+        my $jobID2 = $_;
+
+        print $fh "$jobID1$jobID2.eadm $jobID1 $jobID2\n";
+    }
+
+    closedir $idh or die "can't close directory: $!";
+}
+closedir $odh or die "can't close directory: $!";
+close $fh;

syncDB.pl

@@ -47,7 +47,9 @@ my $dbh = DBI->connect(
 
 my $sth_select_job = $dbh->prepare(qq{
     SELECT id, user_id, job_id, cluster_id,
-    start_time, stop_time, duration, num_nodes
+    start_time, stop_time, duration, num_nodes,
+    has_profile, mem_used_max, flops_any_avg, mem_bw_avg,
+    ib_bw_avg, file_bw_avg
     FROM job
     WHERE job_id=?
     });
@@ -55,25 +57,27 @@ my $sth_select_job = $dbh->prepare(qq{
 my $sth_update_job = $dbh->prepare(qq{
     UPDATE job
     SET has_profile = ?,
-        mem_used = ?,
+        mem_used_max = ?,
-        flops_any = ?,
+        flops_any_avg = ?,
-        mem_bw = ?
+        mem_bw_avg = ?
     WHERE id=?;
     });
 
 my $jobcount = 0;
 my $wrongjobcount = 0;
-my ($user_id, $num_nodes, $start_time, $stop_time, $queue, $duration, $db_id);
+my ($TS, $TE);
+my $counter = 0;
 
-opendir my $dh, $basedir or die "can't open directory: $!";
+open(my $fh, '<:encoding(UTF-8)', './jobIds.txt')
-while ( readdir $dh ) {
+    or die "Could not open file  $!";
-    chomp;
+$TS = time();
-    next if $_ eq '.' or $_ eq '..';
 
-    my $jobID = $_;
+while ( <$fh> ) {
-    my $needsUpdate = 0;
 
-    my $jobmeta_json = read_file("$basedir/$jobID/meta.json");
+    my ($jobID, $path1, $path2) = split ' ', $_;
+    $counter++;
+
+    my $jobmeta_json = read_file("$basedir/$path1/$path2/meta.json");
     my $job = decode_json $jobmeta_json;
     my @row = $dbh->selectrow_array($sth_select_job, undef, $jobID);
     my ($db_id, $db_user_id, $db_job_id, $db_cluster_id, $db_start_time, $db_stop_time, $db_duration, $db_num_nodes);
@@ -90,30 +94,48 @@ while ( readdir $dh ) {
             $db_duration,
             $db_num_nodes) = @row;
 
-        my $footprint = $job->{footprint};
+        my $stats = $job->{statistics};
 
         if ( $job->{user_id} ne $db_user_id ) {
-            print "$jobID $job->{user_id} $db_user_id\n";
+            print "jobID $jobID $job->{user_id} $db_user_id\n";
             $job->{user_id} = $db_user_id;
         }
 
+        # if ( $job->{start_time} != $db_start_time ) {
+        #     print "start $jobID $job->{start_time} $db_start_time\n";
+        # }
+        # if ( $job->{stop_time} != $db_stop_time ) {
+        #     print "stop $jobID $job->{stop_time} $db_stop_time\n";
+        # }
+        if ( $job->{duration} != $db_duration ) {
+            my $difference = $job->{duration} - $db_duration;
+            if ( abs($difference) > 120 ) {
+                print "duration $jobID $job->{duration} $db_duration $difference\n";
+            }
+        }
 
-        # print "$footprint->{mem_used}->{avg}, $footprint->{flops_any}->{avg}, $footprint->{mem_bw}->{avg}\n";
+        $sth_update_job->execute(
-
+            1,
-        # $sth_update_job->execute(
+            $stats->{mem_used}->{max},
-        #     1,
+            $stats->{flops_any}->{avg},
-        #     $footprint->{mem_used}->{avg},
+            $stats->{mem_bw}->{avg},
-        #     $footprint->{flops_any}->{avg},
+            $db_id
-        #     $footprint->{mem_bw}->{avg},
+        );
-        #     $db_id
-        # );
 
         $jobcount++;
     } else {
         print "$jobID NOT in DB!\n";
     }
+
+    if ( $counter == 50 ) {
+        $TE = time();
+        my $rate = $counter/($TE-$TS);
+        $counter = 0;
+        print "Processing $rate jobs per second\n";
+        $TS = $TE;
+    }
 }
-closedir $dh or die "can't close directory: $!";
 $dbh->disconnect;
+close $fh;
 
 print "$wrongjobcount of $jobcount need update\n";