ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2024-12-26 05:19:05 +01:00
Code Issues Packages Releases Activity

Update sync script.

Browse Source 
Add performance metrics to query script
This commit is contained in:
Jan Eitzinger 2019-05-06 08:33:05 +02:00
parent 5c3c2950bf
commit 7e88dac94c
3 changed files with 152 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
acQuery.pl
View File

@ -47,22 +47,30 @@ DateTime::Format::Strptime->new(
my $help = 0;
my $man = 0;
my $hasprofile = '';
my $mode = 'count';
my $user = '';
my $project = '';
my @numnodes;
my @starttime;
my @duration;
my @mem_used;
my @mem_bandwidth;
my @flops_any;
GetOptions (
'help' => \$help,
'man' => \$man,
'hasprofile=s' => \$hasprofile,
'mode=s' => \$mode,
'user=s' => \$user,
'project=s' => \$project,
'numnodes=i{2}' => \@numnodes,
'starttime=s{2}' => \@starttime,
'duration=s{2}' => \@duration
'duration=s{2}' => \@duration,
'mem_used=i{2}' => \@mem_used,
'mem_bandwidth=i{2}' => \@mem_bandwidth,
'flops_any=i{2}' => \@flops_any
) or pod2usage(2);
my %attr = (
@ -227,6 +235,34 @@ if ( @duration ) {
buildCondition('duration');
}
if ( @mem_used ) {
$hasprofile = 'true';
($add, $from, $to) = processRange($mem_used[0], $mem_used[1]);
buildCondition('mem_used');
}
if ( @mem_bandwidth ) {
$hasprofile = 'true';
($add, $from, $to) = processRange($mem_bandwidth[0], $mem_bandwidth[1]);
buildCondition('mem_bw');
}
if ( @flops_any ) {
$hasprofile = 'true';
($add, $from, $to) = processRange($flops_any[0], $flops_any[1]);
buildCondition('flops_any');
}
if ( $hasprofile ) {
if ( $hasprofile eq 'true' ) {
push @conditions, "has_profile=1";
} elsif ( $hasprofile eq 'false' ) {
push @conditions, "has_profile=0";
} else {
print "Unknown value for option has_profile: $hasprofile. Can be true or false.\n";
}
}
my $query;
my $conditionstring;

4
acSync.pl
View File

@ -33,8 +33,8 @@ use Data::Dumper;
use JSON::MaybeXS qw(encode_json decode_json);
use DBI;
my $database = 'jobDB';
my $basedir = './data';
my $database = $ARGV[0];
my $basedir = $ARGV[1];
my %attr = (
PrintError => 1,

113
syncDB.pl Executable file
View File

@ -0,0 +1,113 @@
#!/usr/bin/env perl
# =======================================================================================
#
# Author: Jan Eitzinger (je), jan.eitzinger@fau.de
# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# =======================================================================================
use strict;
use warnings;
use utf8;
use File::Slurp;
use Data::Dumper;
use JSON::MaybeXS qw(encode_json decode_json);
use DBI;
my $database = $ARGV[0];
my $basedir = $ARGV[1];
my %attr = (
PrintError => 1,
RaiseError => 1
);
my $dbh = DBI->connect(
"DBI:SQLite:dbname=$database", "", "", \%attr)
or die "Could not connect to database: $DBI::errstr";
my $sth_select_job = $dbh->prepare(qq{
SELECT id, user_id, job_id, cluster_id,
start_time, stop_time, duration, num_nodes
FROM job
WHERE job_id=?
});
my $sth_update_job = $dbh->prepare(qq{
UPDATE job
SET has_profile = ?,
mem_used = ?,
flops_any = ?,
mem_bw = ?
WHERE id=?;
});
my $jobcount = 0;
my $wrongjobcount = 0;
my ($user_id, $num_nodes, $start_time, $stop_time, $queue, $duration, $db_id);
opendir my $dh, $basedir or die "can't open directory: $!";
while ( readdir $dh ) {
chomp;
next if $_ eq '.' or $_ eq '..';
my $jobID = $_;
my $needsUpdate = 0;
my $jobmeta_json = read_file("$basedir/$jobID/meta.json");
my $job = decode_json $jobmeta_json;
my @row = $dbh->selectrow_array($sth_select_job, undef, $jobID);
my ($db_id, $db_user_id, $db_job_id, $db_cluster_id, $db_start_time, $db_stop_time, $db_duration, $db_num_nodes);
# print Dumper($job);
if ( @row ) {
($db_id,
$db_user_id,
$db_job_id,
$db_cluster_id,
$db_start_time,
$db_stop_time,
$db_duration,
$db_num_nodes) = @row;
my $footprint = $job->{footprint};
# print "$footprint->{mem_used}->{avg}, $footprint->{flops_any}->{avg}, $footprint->{mem_bw}->{avg}\n";
$sth_update_job->execute(
1,
$footprint->{mem_used}->{avg},
$footprint->{flops_any}->{avg},
$footprint->{mem_bw}->{avg},
$db_id
);
$jobcount++;
} else {
print "$jobID NOT in DB!\n";
}
}
closedir $dh or die "can't close directory: $!";
$dbh->disconnect;
print "$wrongjobcount of $jobcount need update\n";