mirror of
https://github.com/ClusterCockpit/cc-backend
synced 2024-11-10 08:57:25 +01:00
Initial checkin.
This commit is contained in:
Jan Eitzinger
parent
dee638f4d7
commit
c92ebc4149
2
LICENSE
2
LICENSE
@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2019
|
||||
Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
189
acImportPBS.pl
Executable file
189
acImportPBS.pl
Executable file
@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env perl
|
||||
# =======================================================================================
|
||||
#
|
||||
# Author: Jan Eitzinger (je), jan.eitzinger@fau.de
|
||||
# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
#
|
||||
# =======================================================================================
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use utf8;
|
||||
|
||||
use Data::Dumper;
|
||||
use DateTime::Format::Strptime;
|
||||
use DBI;
|
||||
|
||||
if ( $#ARGV < 1 ){
|
||||
die "Usage: $0 <DBFile> <importDIR>\n";
|
||||
}
|
||||
|
||||
my $database = $ARGV[0];
|
||||
my $basedir = $ARGV[1];
|
||||
|
||||
my %attr = (
|
||||
PrintError => 1,
|
||||
RaiseError => 1
|
||||
);
|
||||
|
||||
my $dbh = DBI->connect(
|
||||
"DBI:SQLite:dbname=$database", "", "", \%attr);
|
||||
|
||||
my $dateParser =
|
||||
DateTime::Format::Strptime->new(
|
||||
pattern => '%m/%d/%Y %H:%M:%S',
|
||||
time_zone => 'Europe/Berlin',
|
||||
on_error => 'undef'
|
||||
);
|
||||
|
||||
my $sth_insert_job = $dbh->prepare(qq{
|
||||
INSERT INTO job
|
||||
(job_id, user_id, project_id, cluster_id,
|
||||
start_time, stop_time, duration, walltime,
|
||||
job_state, num_nodes, node_list, has_profile)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
});
|
||||
|
||||
my $sth_select_job = $dbh->prepare(qq{
|
||||
SELECT id, user_id, job_id, cluster_id,
|
||||
start_time, stop_time, duration, num_nodes
|
||||
FROM job
|
||||
WHERE job_id=?
|
||||
});
|
||||
|
||||
my %JOBCACHE;
|
||||
|
||||
while( defined( my $file = glob($basedir . '/*' ) ) ) {
|
||||
|
||||
print "Processing $file ...";
|
||||
open(my $fh, "<","$file");
|
||||
|
||||
while ( my $record = <$fh> ) {
|
||||
if ( $record =~ /(.*);([A-Z]);(.*?);(.*)/ ) {
|
||||
my $dt = $dateParser->parse_datetime($1);
|
||||
my $timestamp = $dt->epoch;
|
||||
my $job_state = $2;
|
||||
my $job_id = $3;
|
||||
my $jobinfo = $4;
|
||||
my @data = split(/ /, $jobinfo);
|
||||
my $queue;
|
||||
my $user_id;
|
||||
my $project_id;
|
||||
my $start_time;
|
||||
my $stop_time;
|
||||
my $walltime;
|
||||
my @nodes;
|
||||
my $num_nodes;
|
||||
my $node_list;
|
||||
|
||||
foreach my $prop ( @data ) {
|
||||
if ( $prop =~ /user=(.*)/ ) {
|
||||
$user_id = $1;
|
||||
} elsif ( $prop =~ /group=(.*)/ ) {
|
||||
$project_id = $1;
|
||||
} elsif ( $prop =~ /start=(.*)/ ) {
|
||||
$start_time = $1;
|
||||
} elsif ( $prop =~ /end=(.*)/ ) {
|
||||
$stop_time = $1;
|
||||
} elsif ( $prop =~ /queue=(.*)/ ) {
|
||||
$queue = $1;
|
||||
} elsif ( $prop =~ /Resource_List\.walltime=([0-9]+):([0-9]+):([0-9]+)/ ) {
|
||||
$walltime = $1 * 3600 + $2 * 60 + $3;
|
||||
} elsif ( $prop =~ /exec_host=(.*)/ ) {
|
||||
my $hostlist = $1;
|
||||
my @hosts = split(/\+/, $hostlist);
|
||||
|
||||
foreach my $host ( @hosts ) {
|
||||
if ( $host =~ /(.*?)\/0/) {
|
||||
push @nodes, $1;
|
||||
}
|
||||
}
|
||||
|
||||
$num_nodes = @nodes;
|
||||
$node_list = join(',', @nodes);
|
||||
}
|
||||
}
|
||||
|
||||
if ( $job_state eq 'S' ) {
|
||||
$JOBCACHE{$job_id} = {
|
||||
'user_id' => $user_id,
|
||||
'project_id' => $project_id,
|
||||
'start_time' => $start_time,
|
||||
'walltime' => $walltime,
|
||||
'num_nodes' => $num_nodes,
|
||||
'node_list' => $node_list
|
||||
};
|
||||
} elsif ( $job_state eq 'E' ) {
|
||||
delete $JOBCACHE{$job_id};
|
||||
} elsif ( $job_state eq 'D' or $job_state eq 'A' ) {
|
||||
my $job;
|
||||
|
||||
if (exists $JOBCACHE{$job_id}){
|
||||
$job = $JOBCACHE{$job_id};
|
||||
} else {
|
||||
next;
|
||||
}
|
||||
# print Dumper($job);
|
||||
$user_id = $job->{'user_id'};
|
||||
$project_id = $job->{'project_id'};
|
||||
$start_time = $job->{'start_time'};
|
||||
$stop_time = $timestamp;
|
||||
$walltime = $job->{'walltime'};
|
||||
$num_nodes = $job->{'num_nodes'};
|
||||
$node_list = $job->{'node_list'};
|
||||
delete $JOBCACHE{$job_id};
|
||||
}
|
||||
|
||||
if ( $job_state eq 'E' or
|
||||
$job_state eq 'D' or
|
||||
$job_state eq 'A' )
|
||||
{
|
||||
my $duration = $stop_time - $start_time;
|
||||
|
||||
# check if job already exists
|
||||
my @row = $dbh->selectrow_array($sth_select_job, undef, $job_id);
|
||||
|
||||
if ( @row ) {
|
||||
print "Job $job_id already exists!\n";
|
||||
} else {
|
||||
$sth_insert_job->execute(
|
||||
$job_id,
|
||||
$user_id,
|
||||
$project_id,
|
||||
"emmy",
|
||||
$start_time,
|
||||
$stop_time,
|
||||
$duration,
|
||||
$walltime,
|
||||
$job_state,
|
||||
$num_nodes,
|
||||
$node_list,
|
||||
0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
close $fh or die "can't close file $!";
|
||||
print " done\n";
|
||||
}
|
||||
|
||||
$dbh->disconnect;
|
||||
158
acImportSlurm.pl
Executable file
158
acImportSlurm.pl
Executable file
@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env perl
|
||||
# =======================================================================================
|
||||
#
|
||||
# Author: Jan Eitzinger (je), jan.eitzinger@fau.de
|
||||
# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
#
|
||||
# =======================================================================================
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use utf8;
|
||||
|
||||
use Data::Dumper;
|
||||
use DateTime::Format::Strptime;
|
||||
use DBI;
|
||||
|
||||
if ( $#ARGV < 1 ){
|
||||
die "Usage: $0 <DBFile> <importDIR>\n";
|
||||
}
|
||||
|
||||
my $database = $ARGV[0];
|
||||
my $basedir = $ARGV[1];
|
||||
|
||||
my %attr = (
|
||||
PrintError => 1,
|
||||
RaiseError => 1
|
||||
);
|
||||
|
||||
my $dbh = DBI->connect(
|
||||
"DBI:SQLite:dbname=$database", "", "", \%attr);
|
||||
|
||||
my $dateParser =
|
||||
DateTime::Format::Strptime->new(
|
||||
pattern => '%Y-%m-%dT%H:%M:%S',
|
||||
time_zone => 'Europe/Berlin',
|
||||
on_error => 'undef'
|
||||
);
|
||||
|
||||
sub parse_nodelist {
|
||||
my $nodestr = shift;
|
||||
my @nodes;
|
||||
|
||||
if ( $nodestr =~ /([a-z]+)\[(.*)\]/) {
|
||||
my $prefix = $1;
|
||||
my $list = $2;
|
||||
my @listitems = split(',', $list);
|
||||
|
||||
foreach my $item ( @listitems ){
|
||||
if ( $item =~ /([0-9]+)-([0-9]+)/ ){
|
||||
foreach my $nodeId ( $1 ... $2 ){
|
||||
push @nodes, $prefix.$nodeId;
|
||||
}
|
||||
} else {
|
||||
push @nodes, $prefix.$item;
|
||||
}
|
||||
}
|
||||
|
||||
return join(',', @nodes);
|
||||
} else {
|
||||
return $nodestr;
|
||||
}
|
||||
}
|
||||
|
||||
my $sth_insert_job = $dbh->prepare(qq{
|
||||
INSERT INTO job
|
||||
(job_id, user_id, project_id, cluster_id,
|
||||
start_time, stop_time, duration, walltime,
|
||||
job_state, num_nodes, node_list, has_profile)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
});
|
||||
|
||||
my $sth_select_job = $dbh->prepare(qq{
|
||||
SELECT id, user_id, job_id, cluster_id,
|
||||
start_time, stop_time, duration, num_nodes
|
||||
FROM job
|
||||
WHERE job_id=?
|
||||
});
|
||||
|
||||
my %JOBCACHE;
|
||||
my $dt;
|
||||
|
||||
while( defined( my $file = glob($basedir . '/*' ) ) ) {
|
||||
|
||||
print "Processing $file ...";
|
||||
open(my $fh, "<","$file");
|
||||
my $columns = <$fh>;
|
||||
|
||||
while ( my $record = <$fh> ) {
|
||||
|
||||
my @fields = split(/\|/, $record);
|
||||
|
||||
if ( $fields[1] =~ /^[0-9]+$/) {
|
||||
|
||||
my $cluster_id = $fields[0];
|
||||
my $job_id = $fields[1];
|
||||
my $user_id = $fields[2];
|
||||
my $project_id = $fields[3];
|
||||
$dt = $dateParser->parse_datetime($fields[5]);
|
||||
my $start_time = $dt->epoch;
|
||||
$dt = $dateParser->parse_datetime($fields[6]);
|
||||
my $stop_time = $dt->epoch;
|
||||
my $num_nodes = $fields[11];
|
||||
my $node_list = parse_nodelist($fields[13]);
|
||||
my $job_state = $fields[10];
|
||||
$job_state =~ s/ by [0-9]+//;
|
||||
my $walltime = 0;
|
||||
|
||||
my $duration = $stop_time - $start_time;
|
||||
|
||||
# check if job already exists
|
||||
my @row = $dbh->selectrow_array($sth_select_job, undef, $job_id);
|
||||
|
||||
if ( @row ) {
|
||||
print "Job $job_id already exists!\n";
|
||||
} else {
|
||||
$sth_insert_job->execute(
|
||||
$job_id,
|
||||
$user_id,
|
||||
$project_id,
|
||||
$cluster_id,
|
||||
$start_time,
|
||||
$stop_time,
|
||||
$duration,
|
||||
$walltime,
|
||||
$job_state,
|
||||
$num_nodes,
|
||||
$node_list,
|
||||
0);
|
||||
}
|
||||
} else {
|
||||
# print "$fields[1] \n";
|
||||
next;
|
||||
}
|
||||
}
|
||||
|
||||
close $fh or die "can't close file $!";
|
||||
print " done\n";
|
||||
}
|
||||
|
||||
$dbh->disconnect;
|
||||
361
acQuery.pl
Executable file
361
acQuery.pl
Executable file
@ -0,0 +1,361 @@
|
||||
#!/usr/bin/env perl
|
||||
# =======================================================================================
|
||||
#
|
||||
# Author: Jan Eitzinger (je), jan.eitzinger@fau.de
|
||||
# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
#
|
||||
# =======================================================================================
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use utf8;
|
||||
|
||||
use Data::Dumper;
|
||||
use Getopt::Long;
|
||||
use Pod::Usage;
|
||||
use DateTime::Format::Strptime;
|
||||
use DBI;
|
||||
|
||||
my $database = 'jobDB';
|
||||
my @conditions;
|
||||
my ($add, $from, $to);
|
||||
|
||||
my $dateParser =
|
||||
DateTime::Format::Strptime->new(
|
||||
pattern => '%d.%m.%Y',
|
||||
time_zone => 'Europe/Berlin',
|
||||
on_error => 'undef'
|
||||
);
|
||||
|
||||
my $help = 0;
|
||||
my $man = 0;
|
||||
my $mode = 'count';
|
||||
my $user = '';
|
||||
my $project = '';
|
||||
my @numnodes;
|
||||
my @starttime;
|
||||
my @duration;
|
||||
|
||||
GetOptions (
|
||||
'help' => \$help,
|
||||
'man' => \$man,
|
||||
'mode=s' => \$mode,
|
||||
'user=s' => \$user,
|
||||
'project=s' => \$project,
|
||||
'numnodes=i{2}' => \@numnodes,
|
||||
'starttime=s{2}' => \@starttime,
|
||||
'duration=s{2}' => \@duration
|
||||
) or pod2usage(2);
|
||||
|
||||
my %attr = (
|
||||
PrintError => 1,
|
||||
RaiseError => 1
|
||||
);
|
||||
|
||||
if ( $#ARGV == 0 ) {
|
||||
$database = $ARGV[0];
|
||||
}
|
||||
|
||||
my $dbh = DBI->connect(
|
||||
"DBI:SQLite:dbname=$database", "", "", \%attr)
|
||||
or die("Cannot connect to database $database\n");
|
||||
|
||||
sub parseDate {
|
||||
my $str = shift;
|
||||
my $dt;
|
||||
|
||||
if ( $str ){
|
||||
$dt = $dateParser->parse_datetime($str);
|
||||
|
||||
if ( $dt ) {
|
||||
return $dt->epoch;
|
||||
} else {
|
||||
print "Cannot parse datetime string $str: Ignoring!\n";
|
||||
return 0;
|
||||
}
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
sub parseDuration {
|
||||
my $str = shift;
|
||||
|
||||
if ( $str =~ /([0-9]+)h/ ) {
|
||||
return $1 * 3600;
|
||||
|
||||
} elsif ( $str =~ /([0-9]+)m/ ) {
|
||||
return $1 * 60;
|
||||
|
||||
} elsif ( $str =~ /([0-9]+)s/ ) {
|
||||
return $1;
|
||||
|
||||
} elsif ( $str =~ /([0-9]+)/ ) {
|
||||
return $1;
|
||||
|
||||
} else {
|
||||
print "Cannot parse duration string $str: Ignoring!\n";
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
sub formatDuration {
|
||||
my $ts = shift;
|
||||
|
||||
}
|
||||
|
||||
sub processRange {
|
||||
my $lower = shift;
|
||||
my $upper = shift;
|
||||
|
||||
if ( $lower && $upper ){
|
||||
return (3, $lower, $upper);
|
||||
} elsif ( $lower && !$upper ){
|
||||
return (1, $lower, 0);
|
||||
} elsif ( !$lower && $upper ){
|
||||
return (2, 0, $upper);
|
||||
}
|
||||
}
|
||||
|
||||
sub buildCondition {
|
||||
my $name = shift;
|
||||
|
||||
if ( $add ) {
|
||||
if ( $add == 1 ) {
|
||||
push @conditions, "$name < $from";
|
||||
} elsif ( $add == 2 ) {
|
||||
push @conditions, "$name > $to";
|
||||
} elsif ( $add == 3 ) {
|
||||
push @conditions, "$name BETWEEN $from AND $to";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub printJobStat {
|
||||
my $conditionstring = shift;
|
||||
|
||||
my $query = 'SELECT COUNT(id), SUM(duration)/3600 FROM job '.$conditionstring;
|
||||
my ($count, $coreHours) = $dbh->selectrow_array($query);
|
||||
|
||||
print "=================================\n";
|
||||
print "Job count: $count\n";
|
||||
print "Core hours: $coreHours \n";
|
||||
|
||||
$query = 'SELECT num_nodes, COUNT(*) FROM job '.$conditionstring.' GROUP BY 1';
|
||||
my @histo_num_nodes = $dbh->selectall_array($query);
|
||||
print "\nHistogram: Number of nodes\n";
|
||||
print "nodes\tcount\n";
|
||||
|
||||
foreach my $bin ( @histo_num_nodes ) {
|
||||
print "$bin->[0]\t$bin->[1]\n";
|
||||
}
|
||||
|
||||
$query = 'SELECT duration/3600, COUNT(*) FROM job '.$conditionstring.' GROUP BY 1';
|
||||
my @histo_runtime = $dbh->selectall_array($query);
|
||||
print "\nHistogram: Runtime\n";
|
||||
print "hours\tcount\n";
|
||||
|
||||
foreach my $bin ( @histo_runtime ) {
|
||||
print "$bin->[0]\t$bin->[1]\n";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub printJob {
|
||||
my $job = shift;
|
||||
|
||||
my $jobString = <<"END_JOB";
|
||||
=================================
|
||||
JobId: $job->{job_id}
|
||||
UserId: $job->{user_id}
|
||||
Number of nodes: $job->{num_nodes}
|
||||
From $job->{start_time} to $job->{stop_time}
|
||||
Duration $job->{duration}
|
||||
END_JOB
|
||||
|
||||
print $jobString;
|
||||
}
|
||||
|
||||
pod2usage(1) if $help;
|
||||
pod2usage(-verbose => 2) if $man;
|
||||
|
||||
# build query conditions
|
||||
if ( $user ) {
|
||||
push @conditions, "user_id=\'$user\'";
|
||||
}
|
||||
|
||||
if ( $project ) {
|
||||
push @conditions, "project_id=\'$project\'";
|
||||
}
|
||||
|
||||
|
||||
if ( @numnodes ) {
|
||||
($add, $from, $to) = processRange($numnodes[0], $numnodes[1]);
|
||||
buildCondition('num_nodes');
|
||||
}
|
||||
|
||||
if ( @starttime ) {
|
||||
($add, $from, $to) = processRange( parseDate($starttime[0]), parseDate($starttime[1]));
|
||||
buildCondition('start_time');
|
||||
}
|
||||
|
||||
if ( @duration ) {
|
||||
($add, $from, $to) = processRange( parseDuration($duration[0]), parseDuration($duration[1]));
|
||||
buildCondition('duration');
|
||||
}
|
||||
|
||||
my $query;
|
||||
my $conditionstring;
|
||||
|
||||
if ( @conditions ){
|
||||
$conditionstring = ' WHERE ';
|
||||
$conditionstring .= join(' AND ',@conditions);
|
||||
}
|
||||
|
||||
# handle mode
|
||||
if ( $mode eq 'query' ) {
|
||||
$query = 'SELECT * FROM job'.$conditionstring;
|
||||
print "$query\n";
|
||||
exit;
|
||||
}
|
||||
|
||||
if ( $mode eq 'count' ) {
|
||||
$query = 'SELECT COUNT(*) FROM job'.$conditionstring;
|
||||
my ($count) = $dbh->selectrow_array($query);
|
||||
print "COUNT $count\n";
|
||||
exit;
|
||||
}
|
||||
|
||||
if ( $mode eq 'stat' ) {
|
||||
printJobStat($conditionstring);
|
||||
exit;
|
||||
}
|
||||
|
||||
$query = 'SELECT * FROM job'.$conditionstring;
|
||||
my $sth = $dbh->prepare($query);
|
||||
$sth->execute;
|
||||
my %row;
|
||||
$sth->bind_columns( \( @row{ @{$sth->{NAME_lc} } } ));
|
||||
|
||||
if ( $mode eq 'list' ) {
|
||||
while ($sth->fetch) {
|
||||
printJob(\%row);
|
||||
}
|
||||
} elsif ( $mode eq 'ids' ) {
|
||||
while ($sth->fetch) {
|
||||
print "$row{job_id}\n";
|
||||
}
|
||||
} else {
|
||||
die "ERROR Unknown mode $mode!\n";
|
||||
}
|
||||
|
||||
__END__
|
||||
|
||||
=head1 NAME
|
||||
|
||||
acQuery.pl - Wrapper script to access sqlite job database.
|
||||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
acQuery.pl [options] -- <DB file>
|
||||
|
||||
Help Options:
|
||||
--help Show help text
|
||||
--man Show man page
|
||||
--mode <mode> Set the operation mode
|
||||
--user <user_id> Search for jobs of specific user
|
||||
--project <project_id> Search for jobs of specific project
|
||||
--duration <from> <to> Specify duration range of jobs
|
||||
--numnodes <from> <to> Specify range for number of nodes of job
|
||||
--starttime <from> <to> Specify range for start time of jobs
|
||||
|
||||
=head1 OPTIONS
|
||||
|
||||
=over 8
|
||||
|
||||
=item B<--help>
|
||||
Show a brief help information.
|
||||
|
||||
=item B<--man>
|
||||
Read the manual, with examples
|
||||
|
||||
=item B<--mode [012]>
|
||||
Specify output mode. Mode can be one of:
|
||||
|
||||
=over 4
|
||||
|
||||
=item B<ids>
|
||||
Print list of job ids matching conditions. One job id per line. (default mode)
|
||||
|
||||
=item B<query>
|
||||
Print the query string and then exit.
|
||||
|
||||
=item B<count>
|
||||
Only output the number of jobs matching the conditions.
|
||||
|
||||
=item B<list>
|
||||
Output a record of every job matching the conditions.
|
||||
|
||||
=item B<stat>
|
||||
Output job statistic for all jobs matching the conditions.
|
||||
|
||||
=back
|
||||
|
||||
=item B<--user>
|
||||
Search job for a specific user id.
|
||||
|
||||
=item B<--project>
|
||||
Search job for a specific project.
|
||||
|
||||
=item B<--duration>
|
||||
Specify condition for job duration. This option takes two arguments: If both
|
||||
arguments are positive integers the condition is duration between first
|
||||
argument and second argument. If the second argument is zero condition is duration
|
||||
smaller than first argument. If first argument is zero condition is duration
|
||||
larger than second argument. Duration can be in seconds, minutes (append m) or
|
||||
hours (append h).
|
||||
|
||||
=item B<--numnodes>
|
||||
Specify condition for number of node range of job. This option takes two
|
||||
arguments: If both arguments are positive integers the condition is number of
|
||||
nodes between first argument and second argument. If the second argument is
|
||||
zero condition is number of nodes smaller than first argument. If first
|
||||
argument is zero condition is number of nodes larger than second argument.
|
||||
|
||||
=item B<--starttime>
|
||||
Specify condition for the starttime of job. This option takes two
|
||||
arguments: If both arguments are positive integers the condition is start time
|
||||
between first argument and second argument. If the second argument is
|
||||
zero condition is start time smaller than first argument. If first
|
||||
argument is zero condition is start time larger than second argument.
|
||||
Start time must be given as date in the following format: %d.%m.%Y
|
||||
|
||||
=back
|
||||
|
||||
=head1 DESCRIPTION
|
||||
|
||||
=head1 EXAMPLES
|
||||
|
||||
=head1 AUTHOR
|
||||
|
||||
Jan Eitzinger - L<https://hpc.fau.de/person/jan-eitzinger/>
|
||||
|
||||
=cut
|
||||
98
acSync.pl
Executable file
98
acSync.pl
Executable file
@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env perl
|
||||
# =======================================================================================
|
||||
#
|
||||
# Author: Jan Eitzinger (je), jan.eitzinger@fau.de
|
||||
# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
#
|
||||
# =======================================================================================
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use utf8;
|
||||
|
||||
use File::Slurp;
|
||||
use Data::Dumper;
|
||||
use JSON::MaybeXS qw(encode_json decode_json);
|
||||
use DBI;
|
||||
|
||||
my $database = 'jobDB';
|
||||
my $basedir = './data';
|
||||
|
||||
my %attr = (
|
||||
PrintError => 1,
|
||||
RaiseError => 1
|
||||
);
|
||||
|
||||
my $dbh = DBI->connect(
|
||||
"DBI:SQLite:dbname=$database", "", "", \%attr);
|
||||
|
||||
my $sth_select_job = $dbh->prepare(qq{
|
||||
SELECT id, user_id, job_id, cluster_id,
|
||||
start_time, stop_time, duration, num_nodes
|
||||
FROM job
|
||||
WHERE job_id=?
|
||||
});
|
||||
|
||||
my $jobcount = 0;
|
||||
my $wrongjobcount = 0;
|
||||
|
||||
opendir my $dh, $basedir or die "can't open directory: $!";
|
||||
while ( readdir $dh ) {
|
||||
chomp;
|
||||
next if $_ eq '.' or $_ eq '..';
|
||||
|
||||
my $jobID = $_;
|
||||
my $needsUpdate = 0;
|
||||
|
||||
my $jobmeta_json = read_file("$basedir/$jobID/meta.json");
|
||||
my $job = decode_json $jobmeta_json;
|
||||
my @row = $dbh->selectrow_array($sth_select_job, undef, $jobID);
|
||||
|
||||
if ( @row ) {
|
||||
|
||||
$jobcount++;
|
||||
# print Dumper(@row);
|
||||
my $duration_diff = abs($job->{duration} - $row[6]);
|
||||
|
||||
if ( $duration_diff > 120 ) {
|
||||
$needsUpdate = 1;
|
||||
# print "$jobID DIFF DURATION $duration_diff\n";
|
||||
# print "CC $row[4] - $row[5]\n";
|
||||
# print "DB $job->{start_time} - $job->{stop_time}\n"
|
||||
}
|
||||
|
||||
if ( $row[7] != $job->{num_nodes} ){
|
||||
$needsUpdate = 1;
|
||||
# print "$jobID DIFF NODES $row[7] $job->{num_nodes}\n";
|
||||
}
|
||||
} else {
|
||||
print "$jobID NOT in DB!\n";
|
||||
}
|
||||
|
||||
if ( $needsUpdate ){
|
||||
$wrongjobcount++;
|
||||
print "$jobID\n";
|
||||
}
|
||||
}
|
||||
closedir $dh or die "can't close directory: $!";
|
||||
$dbh->disconnect;
|
||||
|
||||
print "$wrongjobcount of $jobcount need update\n";
|
||||
10
initDB.sql
Normal file
10
initDB.sql
Normal file
@ -0,0 +1,10 @@
|
||||
CREATE TABLE job ( id INTEGER PRIMARY KEY,
|
||||
job_id TEXT, user_id TEXT, project_id TEXT, cluster_id TEXT,
|
||||
start_time INTEGER, stop_time INTEGER, duration INTEGER,
|
||||
walltime INTEGER, job_state TEXT,
|
||||
num_nodes INTEGER, node_list TEXT, has_profile INTEGER,
|
||||
mem_used REAL, flops_any REAL, mem_bw REAL, ib_bw REAL, file_bw REAL);
|
||||
CREATE TABLE tag ( id INTEGER PRIMARY KEY, tag_type TEXT, tag_name TEXT);
|
||||
CREATE TABLE jobtag ( job_id INTEGER, tag_id INTEGER, PRIMARY KEY (job_id, tag_id),
|
||||
FOREIGN KEY (job_id) REFERENCES job (id) ON DELETE CASCADE ON UPDATE NO ACTION,
|
||||
FOREIGN KEY (tag_id) REFERENCES tag (id) ON DELETE CASCADE ON UPDATE NO ACTION );
|
||||
Loading…
Reference in New Issue
Block a user