From cb0fae0216f4dd8fccff9677441ce654f74e12bd Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 8 Nov 2019 09:07:02 +0100 Subject: [PATCH] Adopt scripts for sync of DB with json. --- acSync.pl | 3 ++- extractJobIDs.pl | 57 +++++++++++++++++++++++++++++++++++++++ syncDB.pl | 70 +++++++++++++++++++++++++++++++----------------- 3 files changed, 105 insertions(+), 25 deletions(-) create mode 100755 extractJobIDs.pl diff --git a/acSync.pl b/acSync.pl index 8948411..e021368 100755 --- a/acSync.pl +++ b/acSync.pl @@ -42,7 +42,8 @@ my %attr = ( ); my $dbh = DBI->connect( - "DBI:SQLite:dbname=$database", "", "", \%attr); + "DBI:SQLite:dbname=$database", "", "", \%attr) + or die "Could not connect to database: $DBI::errstr"; my $sth_select_job = $dbh->prepare(qq{ SELECT id, user_id, job_id, cluster_id, diff --git a/extractJobIDs.pl b/extractJobIDs.pl new file mode 100755 index 0000000..f5586c9 --- /dev/null +++ b/extractJobIDs.pl @@ -0,0 +1,57 @@ +#!/usr/bin/env perl +# ======================================================================================= +# +# Author: Jan Eitzinger (je), jan.eitzinger@fau.de +# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# ======================================================================================= + +use strict; +use warnings; +use utf8; + +my $basedir = $ARGV[0]; +open(my $fh, '>:encoding(UTF-8)', 'jobIds.txt') + or die "Could not open file $!"; + +opendir my $odh, $basedir or die "can't open directory: $!"; + +while ( readdir $odh ) { + chomp; + next if $_ eq '.' or $_ eq '..'; + + my $jobID1 = $_; + print "Open $jobID1\n"; + + opendir my $idh, "$basedir/$jobID1" or die "can't open directory: $!"; + + while ( readdir $idh ) { + chomp; + next if $_ eq '.' or $_ eq '..'; + my $jobID2 = $_; + + print $fh "$jobID1$jobID2.eadm $jobID1 $jobID2\n"; + } + + closedir $idh or die "can't close directory: $!"; +} +closedir $odh or die "can't close directory: $!"; +close $fh; diff --git a/syncDB.pl b/syncDB.pl index e2ca660..1226c57 100755 --- a/syncDB.pl +++ b/syncDB.pl @@ -47,7 +47,9 @@ my $dbh = DBI->connect( my $sth_select_job = $dbh->prepare(qq{ SELECT id, user_id, job_id, cluster_id, - start_time, stop_time, duration, num_nodes + start_time, stop_time, duration, num_nodes, + has_profile, mem_used_max, flops_any_avg, mem_bw_avg, + ib_bw_avg, file_bw_avg FROM job WHERE job_id=? }); @@ -55,25 +57,27 @@ my $sth_select_job = $dbh->prepare(qq{ my $sth_update_job = $dbh->prepare(qq{ UPDATE job SET has_profile = ?, - mem_used = ?, - flops_any = ?, - mem_bw = ? + mem_used_max = ?, + flops_any_avg = ?, + mem_bw_avg = ? WHERE id=?; }); my $jobcount = 0; my $wrongjobcount = 0; -my ($user_id, $num_nodes, $start_time, $stop_time, $queue, $duration, $db_id); +my ($TS, $TE); +my $counter = 0; -opendir my $dh, $basedir or die "can't open directory: $!"; -while ( readdir $dh ) { - chomp; - next if $_ eq '.' or $_ eq '..'; +open(my $fh, '<:encoding(UTF-8)', './jobIds.txt') + or die "Could not open file $!"; +$TS = time(); - my $jobID = $_; - my $needsUpdate = 0; +while ( <$fh> ) { - my $jobmeta_json = read_file("$basedir/$jobID/meta.json"); + my ($jobID, $path1, $path2) = split ' ', $_; + $counter++; + + my $jobmeta_json = read_file("$basedir/$path1/$path2/meta.json"); my $job = decode_json $jobmeta_json; my @row = $dbh->selectrow_array($sth_select_job, undef, $jobID); my ($db_id, $db_user_id, $db_job_id, $db_cluster_id, $db_start_time, $db_stop_time, $db_duration, $db_num_nodes); @@ -90,30 +94,48 @@ while ( readdir $dh ) { $db_duration, $db_num_nodes) = @row; - my $footprint = $job->{footprint}; + my $stats = $job->{statistics}; if ( $job->{user_id} ne $db_user_id ) { - print "$jobID $job->{user_id} $db_user_id\n"; + print "jobID $jobID $job->{user_id} $db_user_id\n"; $job->{user_id} = $db_user_id; } + # if ( $job->{start_time} != $db_start_time ) { + # print "start $jobID $job->{start_time} $db_start_time\n"; + # } + # if ( $job->{stop_time} != $db_stop_time ) { + # print "stop $jobID $job->{stop_time} $db_stop_time\n"; + # } + if ( $job->{duration} != $db_duration ) { + my $difference = $job->{duration} - $db_duration; + if ( abs($difference) > 120 ) { + print "duration $jobID $job->{duration} $db_duration $difference\n"; + } + } - # print "$footprint->{mem_used}->{avg}, $footprint->{flops_any}->{avg}, $footprint->{mem_bw}->{avg}\n"; - - # $sth_update_job->execute( - # 1, - # $footprint->{mem_used}->{avg}, - # $footprint->{flops_any}->{avg}, - # $footprint->{mem_bw}->{avg}, - # $db_id - # ); + $sth_update_job->execute( + 1, + $stats->{mem_used}->{max}, + $stats->{flops_any}->{avg}, + $stats->{mem_bw}->{avg}, + $db_id + ); $jobcount++; } else { print "$jobID NOT in DB!\n"; } + + if ( $counter == 50 ) { + $TE = time(); + my $rate = $counter/($TE-$TS); + $counter = 0; + print "Processing $rate jobs per second\n"; + $TS = $TE; + } } -closedir $dh or die "can't close directory: $!"; $dbh->disconnect; +close $fh; print "$wrongjobcount of $jobcount need update\n";