Update sync script.

Add performance metrics to query script

acQuery.pl

@@ -47,22 +47,30 @@ DateTime::Format::Strptime->new(
 
 my $help = 0;
 my $man = 0;
+my $hasprofile = '';
 my $mode = 'count';
 my $user = '';
 my $project = '';
 my @numnodes;
 my @starttime;
 my @duration;
+my @mem_used;
+my @mem_bandwidth;
+my @flops_any;
 
 GetOptions (
     'help'           => \$help,
     'man'            => \$man,
+    'hasprofile=s'   => \$hasprofile,
     'mode=s'         => \$mode,
     'user=s'         => \$user,
     'project=s'      => \$project,
     'numnodes=i{2}'  => \@numnodes,
     'starttime=s{2}' => \@starttime,
-    'duration=s{2}'  => \@duration
+    'duration=s{2}'  => \@duration,
+    'mem_used=i{2}'  => \@mem_used,
+    'mem_bandwidth=i{2}'  => \@mem_bandwidth,
+    'flops_any=i{2}'  => \@flops_any
 ) or pod2usage(2);
 
 my %attr = (
@@ -227,6 +235,34 @@ if ( @duration ) {
     buildCondition('duration');
 }
 
+if ( @mem_used ) {
+    $hasprofile = 'true';
+    ($add, $from, $to) = processRange($mem_used[0], $mem_used[1]);
+    buildCondition('mem_used');
+}
+
+if ( @mem_bandwidth ) {
+    $hasprofile = 'true';
+    ($add, $from, $to) = processRange($mem_bandwidth[0], $mem_bandwidth[1]);
+    buildCondition('mem_bw');
+}
+
+if ( @flops_any ) {
+    $hasprofile = 'true';
+    ($add, $from, $to) = processRange($flops_any[0], $flops_any[1]);
+    buildCondition('flops_any');
+}
+
+if ( $hasprofile ) {
+    if ( $hasprofile eq 'true' ) {
+        push @conditions, "has_profile=1";
+    } elsif ( $hasprofile eq 'false' ) {
+        push @conditions, "has_profile=0";
+    } else {
+        print "Unknown value for option has_profile: $hasprofile. Can be true or false.\n";
+    }
+}
+
 my $query;
 my $conditionstring;
 

acSync.pl

@@ -33,8 +33,8 @@ use Data::Dumper;
 use JSON::MaybeXS qw(encode_json decode_json);
 use DBI;
 
-my $database = 'jobDB';
-my $basedir = './data';
+my $database = $ARGV[0];
+my $basedir = $ARGV[1];
 
 my %attr = (
     PrintError => 1,

syncDB.pl

@@ -0,0 +1,113 @@
+#!/usr/bin/env perl
+# =======================================================================================
+#
+#      Author:   Jan Eitzinger (je), jan.eitzinger@fau.de
+#      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+#
+#      Permission is hereby granted, free of charge, to any person obtaining a copy
+#      of this software and associated documentation files (the "Software"), to deal
+#      in the Software without restriction, including without limitation the rights
+#      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#      copies of the Software, and to permit persons to whom the Software is
+#      furnished to do so, subject to the following conditions:
+#
+#      The above copyright notice and this permission notice shall be included in all
+#      copies or substantial portions of the Software.
+#
+#      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#      SOFTWARE.
+#
+# =======================================================================================
+
+use strict;
+use warnings;
+use utf8;
+
+use File::Slurp;
+use Data::Dumper;
+use JSON::MaybeXS qw(encode_json decode_json);
+use DBI;
+
+my $database = $ARGV[0];
+my $basedir = $ARGV[1];
+
+my %attr = (
+    PrintError => 1,
+    RaiseError => 1
+);
+
+my $dbh = DBI->connect(
+    "DBI:SQLite:dbname=$database", "", "", \%attr)
+    or die "Could not connect to database: $DBI::errstr";
+
+my $sth_select_job = $dbh->prepare(qq{
+    SELECT id, user_id, job_id, cluster_id,
+    start_time, stop_time, duration, num_nodes
+    FROM job
+    WHERE job_id=?
+    });
+
+my $sth_update_job = $dbh->prepare(qq{
+    UPDATE job
+    SET has_profile = ?,
+        mem_used = ?,
+        flops_any = ?,
+        mem_bw = ?
+    WHERE id=?;
+    });
+
+my $jobcount = 0;
+my $wrongjobcount = 0;
+my ($user_id, $num_nodes, $start_time, $stop_time, $queue, $duration, $db_id);
+
+opendir my $dh, $basedir or die "can't open directory: $!";
+while ( readdir $dh ) {
+    chomp;
+    next if $_ eq '.' or $_ eq '..';
+
+    my $jobID = $_;
+    my $needsUpdate = 0;
+
+    my $jobmeta_json = read_file("$basedir/$jobID/meta.json");
+    my $job = decode_json $jobmeta_json;
+    my @row = $dbh->selectrow_array($sth_select_job, undef, $jobID);
+    my ($db_id, $db_user_id, $db_job_id, $db_cluster_id, $db_start_time, $db_stop_time, $db_duration, $db_num_nodes);
+
+    # print Dumper($job);
+
+    if ( @row ) {
+        ($db_id,
+            $db_user_id,
+            $db_job_id,
+            $db_cluster_id,
+            $db_start_time,
+            $db_stop_time,
+            $db_duration,
+            $db_num_nodes) = @row;
+
+        my $footprint = $job->{footprint};
+
+        # print "$footprint->{mem_used}->{avg}, $footprint->{flops_any}->{avg}, $footprint->{mem_bw}->{avg}\n";
+
+        $sth_update_job->execute(
+            1,
+            $footprint->{mem_used}->{avg},
+            $footprint->{flops_any}->{avg},
+            $footprint->{mem_bw}->{avg},
+            $db_id
+        );
+
+        $jobcount++;
+    } else {
+        print "$jobID NOT in DB!\n";
+    }
+}
+closedir $dh or die "can't close directory: $!";
+$dbh->disconnect;
+
+print "$wrongjobcount of $jobcount need update\n";