diff --git a/acQuery.pl b/acQuery.pl index a323e7b..992463e 100755 --- a/acQuery.pl +++ b/acQuery.pl @@ -47,22 +47,30 @@ DateTime::Format::Strptime->new( my $help = 0; my $man = 0; +my $hasprofile = ''; my $mode = 'count'; my $user = ''; my $project = ''; my @numnodes; my @starttime; my @duration; +my @mem_used; +my @mem_bandwidth; +my @flops_any; GetOptions ( 'help' => \$help, 'man' => \$man, + 'hasprofile=s' => \$hasprofile, 'mode=s' => \$mode, 'user=s' => \$user, 'project=s' => \$project, 'numnodes=i{2}' => \@numnodes, 'starttime=s{2}' => \@starttime, - 'duration=s{2}' => \@duration + 'duration=s{2}' => \@duration, + 'mem_used=i{2}' => \@mem_used, + 'mem_bandwidth=i{2}' => \@mem_bandwidth, + 'flops_any=i{2}' => \@flops_any ) or pod2usage(2); my %attr = ( @@ -227,6 +235,34 @@ if ( @duration ) { buildCondition('duration'); } +if ( @mem_used ) { + $hasprofile = 'true'; + ($add, $from, $to) = processRange($mem_used[0], $mem_used[1]); + buildCondition('mem_used'); +} + +if ( @mem_bandwidth ) { + $hasprofile = 'true'; + ($add, $from, $to) = processRange($mem_bandwidth[0], $mem_bandwidth[1]); + buildCondition('mem_bw'); +} + +if ( @flops_any ) { + $hasprofile = 'true'; + ($add, $from, $to) = processRange($flops_any[0], $flops_any[1]); + buildCondition('flops_any'); +} + +if ( $hasprofile ) { + if ( $hasprofile eq 'true' ) { + push @conditions, "has_profile=1"; + } elsif ( $hasprofile eq 'false' ) { + push @conditions, "has_profile=0"; + } else { + print "Unknown value for option has_profile: $hasprofile. Can be true or false.\n"; + } +} + my $query; my $conditionstring; diff --git a/acSync.pl b/acSync.pl index 116bf28..8948411 100755 --- a/acSync.pl +++ b/acSync.pl @@ -33,8 +33,8 @@ use Data::Dumper; use JSON::MaybeXS qw(encode_json decode_json); use DBI; -my $database = 'jobDB'; -my $basedir = './data'; +my $database = $ARGV[0]; +my $basedir = $ARGV[1]; my %attr = ( PrintError => 1, diff --git a/syncDB.pl b/syncDB.pl new file mode 100755 index 0000000..70a11d6 --- /dev/null +++ b/syncDB.pl @@ -0,0 +1,113 @@ +#!/usr/bin/env perl +# ======================================================================================= +# +# Author: Jan Eitzinger (je), jan.eitzinger@fau.de +# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# ======================================================================================= + +use strict; +use warnings; +use utf8; + +use File::Slurp; +use Data::Dumper; +use JSON::MaybeXS qw(encode_json decode_json); +use DBI; + +my $database = $ARGV[0]; +my $basedir = $ARGV[1]; + +my %attr = ( + PrintError => 1, + RaiseError => 1 +); + +my $dbh = DBI->connect( + "DBI:SQLite:dbname=$database", "", "", \%attr) + or die "Could not connect to database: $DBI::errstr"; + +my $sth_select_job = $dbh->prepare(qq{ + SELECT id, user_id, job_id, cluster_id, + start_time, stop_time, duration, num_nodes + FROM job + WHERE job_id=? + }); + +my $sth_update_job = $dbh->prepare(qq{ + UPDATE job + SET has_profile = ?, + mem_used = ?, + flops_any = ?, + mem_bw = ? + WHERE id=?; + }); + +my $jobcount = 0; +my $wrongjobcount = 0; +my ($user_id, $num_nodes, $start_time, $stop_time, $queue, $duration, $db_id); + +opendir my $dh, $basedir or die "can't open directory: $!"; +while ( readdir $dh ) { + chomp; + next if $_ eq '.' or $_ eq '..'; + + my $jobID = $_; + my $needsUpdate = 0; + + my $jobmeta_json = read_file("$basedir/$jobID/meta.json"); + my $job = decode_json $jobmeta_json; + my @row = $dbh->selectrow_array($sth_select_job, undef, $jobID); + my ($db_id, $db_user_id, $db_job_id, $db_cluster_id, $db_start_time, $db_stop_time, $db_duration, $db_num_nodes); + + # print Dumper($job); + + if ( @row ) { + ($db_id, + $db_user_id, + $db_job_id, + $db_cluster_id, + $db_start_time, + $db_stop_time, + $db_duration, + $db_num_nodes) = @row; + + my $footprint = $job->{footprint}; + + # print "$footprint->{mem_used}->{avg}, $footprint->{flops_any}->{avg}, $footprint->{mem_bw}->{avg}\n"; + + $sth_update_job->execute( + 1, + $footprint->{mem_used}->{avg}, + $footprint->{flops_any}->{avg}, + $footprint->{mem_bw}->{avg}, + $db_id + ); + + $jobcount++; + } else { + print "$jobID NOT in DB!\n"; + } +} +closedir $dh or die "can't close directory: $!"; +$dbh->disconnect; + +print "$wrongjobcount of $jobcount need update\n";