From 72386b78e237f68c754297a6e36936336dfc682b Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 2 Feb 2022 16:31:22 +0100 Subject: [PATCH] Cleanup obsolete perl skripts --- utils/acImportPBS.pl | 189 --------------- utils/acImportSlurm.pl | 158 ------------- utils/acQuery.pl | 522 ----------------------------------------- utils/initDB.sql | 10 - 4 files changed, 879 deletions(-) delete mode 100755 utils/acImportPBS.pl delete mode 100755 utils/acImportSlurm.pl delete mode 100755 utils/acQuery.pl delete mode 100644 utils/initDB.sql diff --git a/utils/acImportPBS.pl b/utils/acImportPBS.pl deleted file mode 100755 index 570a1a6..0000000 --- a/utils/acImportPBS.pl +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env perl -# ======================================================================================= -# -# Author: Jan Eitzinger (je), jan.eitzinger@fau.de -# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# ======================================================================================= - -use strict; -use warnings; -use utf8; - -use Data::Dumper; -use DateTime::Format::Strptime; -use DBI; - -if ( $#ARGV < 1 ){ - die "Usage: $0 \n"; -} - -my $database = $ARGV[0]; -my $basedir = $ARGV[1]; - -my %attr = ( - PrintError => 1, - RaiseError => 1 -); - -my $dbh = DBI->connect( - "DBI:SQLite:dbname=$database", "", "", \%attr); - -my $dateParser = -DateTime::Format::Strptime->new( - pattern => '%m/%d/%Y %H:%M:%S', - time_zone => 'Europe/Berlin', - on_error => 'undef' -); - -my $sth_insert_job = $dbh->prepare(qq{ - INSERT INTO job - (job_id, user_id, project_id, cluster_id, - start_time, stop_time, duration, walltime, - job_state, num_nodes, node_list, has_profile) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?) - }); - -my $sth_select_job = $dbh->prepare(qq{ - SELECT id, user_id, job_id, cluster_id, - start_time, stop_time, duration, num_nodes - FROM job - WHERE job_id=? - }); - -my %JOBCACHE; - -while( defined( my $file = glob($basedir . '/*' ) ) ) { - - print "Processing $file ..."; - open(my $fh, "<","$file"); - - while ( my $record = <$fh> ) { - if ( $record =~ /(.*);([A-Z]);(.*?);(.*)/ ) { - my $dt = $dateParser->parse_datetime($1); - my $timestamp = $dt->epoch; - my $job_state = $2; - my $job_id = $3; - my $jobinfo = $4; - my @data = split(/ /, $jobinfo); - my $queue; - my $user_id; - my $project_id; - my $start_time; - my $stop_time; - my $walltime; - my @nodes; - my $num_nodes; - my $node_list; - - foreach my $prop ( @data ) { - if ( $prop =~ /user=(.*)/ ) { - $user_id = $1; - } elsif ( $prop =~ /group=(.*)/ ) { - $project_id = $1; - } elsif ( $prop =~ /start=(.*)/ ) { - $start_time = $1; - } elsif ( $prop =~ /end=(.*)/ ) { - $stop_time = $1; - } elsif ( $prop =~ /queue=(.*)/ ) { - $queue = $1; - } elsif ( $prop =~ /Resource_List\.walltime=([0-9]+):([0-9]+):([0-9]+)/ ) { - $walltime = $1 * 3600 + $2 * 60 + $3; - } elsif ( $prop =~ /exec_host=(.*)/ ) { - my $hostlist = $1; - my @hosts = split(/\+/, $hostlist); - - foreach my $host ( @hosts ) { - if ( $host =~ /(.*?)\/0/) { - push @nodes, $1; - } - } - - $num_nodes = @nodes; - $node_list = join(',', @nodes); - } - } - - if ( $job_state eq 'S' ) { - $JOBCACHE{$job_id} = { - 'user_id' => $user_id, - 'project_id' => $project_id, - 'start_time' => $start_time, - 'walltime' => $walltime, - 'num_nodes' => $num_nodes, - 'node_list' => $node_list - }; - } elsif ( $job_state eq 'E' ) { - delete $JOBCACHE{$job_id}; - } elsif ( $job_state eq 'D' or $job_state eq 'A' ) { - my $job; - - if (exists $JOBCACHE{$job_id}){ - $job = $JOBCACHE{$job_id}; - } else { - next; - } - # print Dumper($job); - $user_id = $job->{'user_id'}; - $project_id = $job->{'project_id'}; - $start_time = $job->{'start_time'}; - $stop_time = $timestamp; - $walltime = $job->{'walltime'}; - $num_nodes = $job->{'num_nodes'}; - $node_list = $job->{'node_list'}; - delete $JOBCACHE{$job_id}; - } - - if ( $job_state eq 'E' or - $job_state eq 'D' or - $job_state eq 'A' ) - { - my $duration = $stop_time - $start_time; - - # check if job already exists - my @row = $dbh->selectrow_array($sth_select_job, undef, $job_id); - - if ( @row ) { - print "Job $job_id already exists!\n"; - } else { - $sth_insert_job->execute( - $job_id, - $user_id, - $project_id, - "emmy", - $start_time, - $stop_time, - $duration, - $walltime, - $job_state, - $num_nodes, - $node_list, - 0); - } - } - } - } - - close $fh or die "can't close file $!"; - print " done\n"; -} - -$dbh->disconnect; diff --git a/utils/acImportSlurm.pl b/utils/acImportSlurm.pl deleted file mode 100755 index 7aaddad..0000000 --- a/utils/acImportSlurm.pl +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env perl -# ======================================================================================= -# -# Author: Jan Eitzinger (je), jan.eitzinger@fau.de -# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# ======================================================================================= - -use strict; -use warnings; -use utf8; - -use Data::Dumper; -use DateTime::Format::Strptime; -use DBI; - -if ( $#ARGV < 1 ){ - die "Usage: $0 \n"; -} - -my $database = $ARGV[0]; -my $basedir = $ARGV[1]; - -my %attr = ( - PrintError => 1, - RaiseError => 1 -); - -my $dbh = DBI->connect( - "DBI:SQLite:dbname=$database", "", "", \%attr); - -my $dateParser = -DateTime::Format::Strptime->new( - pattern => '%Y-%m-%dT%H:%M:%S', - time_zone => 'Europe/Berlin', - on_error => 'undef' -); - -sub parse_nodelist { - my $nodestr = shift; - my @nodes; - - if ( $nodestr =~ /([a-z]+)\[(.*)\]/) { - my $prefix = $1; - my $list = $2; - my @listitems = split(',', $list); - - foreach my $item ( @listitems ){ - if ( $item =~ /([0-9]+)-([0-9]+)/ ){ - foreach my $nodeId ( $1 ... $2 ){ - push @nodes, $prefix.$nodeId; - } - } else { - push @nodes, $prefix.$item; - } - } - - return join(',', @nodes); - } else { - return $nodestr; - } -} - -my $sth_insert_job = $dbh->prepare(qq{ - INSERT INTO job - (job_id, user_id, project_id, cluster_id, - start_time, stop_time, duration, walltime, - job_state, num_nodes, node_list, has_profile) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?) - }); - -my $sth_select_job = $dbh->prepare(qq{ - SELECT id, user_id, job_id, cluster_id, - start_time, stop_time, duration, num_nodes - FROM job - WHERE job_id=? - }); - -my %JOBCACHE; -my $dt; - -while( defined( my $file = glob($basedir . '/*' ) ) ) { - - print "Processing $file ..."; - open(my $fh, "<","$file"); - my $columns = <$fh>; - - while ( my $record = <$fh> ) { - - my @fields = split(/\|/, $record); - - if ( $fields[1] =~ /^[0-9]+$/) { - - my $cluster_id = $fields[0]; - my $job_id = $fields[1]; - my $user_id = $fields[2]; - my $project_id = $fields[3]; - $dt = $dateParser->parse_datetime($fields[5]); - my $start_time = $dt->epoch; - $dt = $dateParser->parse_datetime($fields[6]); - my $stop_time = $dt->epoch; - my $num_nodes = $fields[11]; - my $node_list = parse_nodelist($fields[13]); - my $job_state = $fields[10]; - $job_state =~ s/ by [0-9]+//; - my $walltime = 0; - - my $duration = $stop_time - $start_time; - - # check if job already exists - my @row = $dbh->selectrow_array($sth_select_job, undef, $job_id); - - if ( @row ) { - print "Job $job_id already exists!\n"; - } else { - $sth_insert_job->execute( - $job_id, - $user_id, - $project_id, - $cluster_id, - $start_time, - $stop_time, - $duration, - $walltime, - $job_state, - $num_nodes, - $node_list, - 0); - } - } else { - # print "$fields[1] \n"; - next; - } - } - - close $fh or die "can't close file $!"; - print " done\n"; -} - -$dbh->disconnect; diff --git a/utils/acQuery.pl b/utils/acQuery.pl deleted file mode 100755 index 7a96eed..0000000 --- a/utils/acQuery.pl +++ /dev/null @@ -1,522 +0,0 @@ -#!/usr/bin/env perl -# ======================================================================================= -# -# Author: Jan Eitzinger (je), jan.eitzinger@fau.de -# Copyright (c) 2020 RRZE, University Erlangen-Nuremberg -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# ======================================================================================= - -use strict; -use warnings; -use utf8; - -use Data::Dumper; -use Getopt::Long; -use Pod::Usage; -use DateTime; -use DateTime::Format::Strptime; -use DBI; - -my $database = 'jobDB'; -my @conditions; -my ($add, $from, $to); - -my $dateParser = -DateTime::Format::Strptime->new( - pattern => '%d.%m.%Y/%H:%M', - time_zone => 'Europe/Berlin', - on_error => 'undef' -); - -my $help = 0; -my $man = 0; -my $hasprofile = ''; -my $mode = 'count'; -my $user = ''; -my $jobID = ''; -my $project = ''; -my @numnodes; -my @starttime; -my @duration; -my @mem_used; -my @mem_bandwidth; -my @flops_any; - -GetOptions ( - 'help' => \$help, - 'man' => \$man, - 'hasprofile=s' => \$hasprofile, - 'mode=s' => \$mode, - 'user=s' => \$user, - 'job=s' => \$jobID, - 'project=s' => \$project, - 'numnodes=i{2}' => \@numnodes, - 'starttime=s{2}' => \@starttime, - 'duration=s{2}' => \@duration, - 'mem_used=i{2}' => \@mem_used, - 'mem_bandwidth=i{2}' => \@mem_bandwidth, - 'flops_any=i{2}' => \@flops_any -) or pod2usage(2); - -my %attr = ( - PrintError => 1, - RaiseError => 1 -); - -if ( $#ARGV == 0 ) { - $database = $ARGV[0]; -} - -my $dbh = DBI->connect( - "DBI:SQLite:dbname=$database", "", "", \%attr) - or die("Cannot connect to database $database\n"); - -sub parseDate { - my $str = shift; - my $dt; - - if ( $str ){ - $dt = $dateParser->parse_datetime($str); - - if ( $dt ) { - return $dt->epoch; - } else { - print "Cannot parse datetime string $str: Ignoring!\n"; - return 0; - } - } else { - return 0; - } -} - -sub parseDuration { - my $str = shift; - - if ( $str =~ /([0-9]+)h/ ) { - return $1 * 3600; - - } elsif ( $str =~ /([0-9]+)m/ ) { - return $1 * 60; - - } elsif ( $str =~ /([0-9]+)s/ ) { - return $1; - - } elsif ( $str =~ /([0-9]+)/ ) { - return $1; - - } else { - print "Cannot parse duration string $str: Ignoring!\n"; - return 0; - } -} - -sub formatDuration { - my $ts = shift; - -} - -sub processRange { - my $lower = shift; - my $upper = shift; - - if ( $lower && $upper ){ - return (3, $lower, $upper); - } elsif ( $lower && !$upper ){ - return (1, $lower, 0); - } elsif ( !$lower && $upper ){ - return (2, 0, $upper); - } -} - -sub buildCondition { - my $name = shift; - - if ( $add ) { - if ( $add == 1 ) { - push @conditions, "$name < $from"; - } elsif ( $add == 2 ) { - push @conditions, "$name > $to"; - } elsif ( $add == 3 ) { - push @conditions, "$name BETWEEN $from AND $to"; - } - } -} - -sub printPerfStat { - my $conditionstring = shift; - - my $query = 'SELECT COUNT(*), SUM(duration)/3600, SUM(duration*num_nodes)/3600 FROM job '.$conditionstring; - my ($count, $walltime, $nodeHours) = $dbh->selectrow_array($query); - - if ( $count > 0 ) { - $query = 'SELECT COUNT(*) FROM job '.$conditionstring.' AND has_profile=1'; - my ($perfcount) = $dbh->selectrow_array($query); - print "=================================\n"; - print "Job count: $count\n"; - print "Jobs with performance profile: $perfcount\n"; - print "Total walltime [h]: $walltime \n"; - print "Total node hours [h]: $nodeHours \n"; - - $query = 'SELECT ROUND(mem_used_max), COUNT(*) FROM job '.$conditionstring.' AND has_profile=1 GROUP BY 1'; - my @histo_mem_used = $dbh->selectall_array($query); - print "\nHistogram: Mem used\n"; - print "Mem\tcount\n"; - - foreach my $bin ( @histo_mem_used ) { - my $bar = log $bin->[1]; my $str = ''; - while (length($str)<$bar) { $str = $str.'*'; } - print "$bin->[0]\t$bin->[1]\t$str\n"; - } - - $query = 'SELECT ROUND(mem_bw_avg), COUNT(*) FROM job '.$conditionstring.' AND has_profile=1 GROUP BY 1'; - my @histo_mem_bandwidth = $dbh->selectall_array($query); - print "\nHistogram: Memory bandwidth\n"; - print "BW\tcount\n"; - - foreach my $bin ( @histo_mem_bandwidth ) { - my $bar = log $bin->[1]; my $str = ''; - while (length($str)<$bar) { $str = $str.'*'; } - print "$bin->[0]\t$bin->[1]\t$str\n"; - } - - $query = 'SELECT ROUND(flops_any_avg), COUNT(*) FROM job '.$conditionstring.' AND has_profile=1 GROUP BY 1'; - my @histo_flops_any = $dbh->selectall_array($query); - print "\nHistogram: Flops any\n"; - print "flops\tcount\n"; - - foreach my $bin ( @histo_flops_any ) { - my $bar = log $bin->[1]; my $str = ''; - while (length($str)<$bar) { $str = $str.'*'; } - print "$bin->[0]\t$bin->[1]\t$str\n"; - } - } else { - print "No jobs\n"; - } -} - -sub printJobStat { - my $conditionstring = shift; - - my $query = 'SELECT COUNT(id), SUM(duration)/3600, SUM(duration*num_nodes)/3600 FROM job '.$conditionstring; - my ($count, $walltime, $nodeHours) = $dbh->selectrow_array($query); - - if ( $count > 0 ) { - print "=================================\n"; - print "Job count: $count\n"; - print "Total walltime [h]: $walltime \n"; - print "Total node hours [h]: $nodeHours \n"; - - $query = 'SELECT num_nodes, COUNT(*) FROM job '.$conditionstring.' GROUP BY 1'; - my @histo_num_nodes = $dbh->selectall_array($query); - print "\nHistogram: Number of nodes\n"; - print "nodes\tcount\n"; - - foreach my $bin ( @histo_num_nodes ) { - my $bar = log $bin->[1]; my $str = ''; - while (length($str)<$bar) { $str = $str.'*'; } - print "$bin->[0]\t$bin->[1]\t$str\n"; - } - - $query = 'SELECT duration/3600, COUNT(*) FROM job '.$conditionstring.' GROUP BY 1'; - my @histo_runtime = $dbh->selectall_array($query); - print "\nHistogram: Walltime\n"; - print "hours\tcount\n"; - - foreach my $bin ( @histo_runtime ) { - my $bar = log $bin->[1]; my $str = ''; - while (length($str)<$bar) { $str = $str.'*'; } - print "$bin->[0]\t$bin->[1]\t$str\n"; - } - } else { - print "No jobs\n"; - } -} - -sub printJob { - my $job = shift; - my $durationHours = sprintf("%.2f", $job->{duration}/3600); - my $startDatetime = DateTime->from_epoch(epoch=>$job->{start_time}, time_zone => 'Europe/Berlin',); - my $stopDatetime = DateTime->from_epoch(epoch=>$job->{stop_time}, time_zone => 'Europe/Berlin',); - - my $jobString = <<"END_JOB"; -================================= -JobId: $job->{job_id} -UserId: $job->{user_id} -Number of nodes: $job->{num_nodes} -From $startDatetime to $stopDatetime -Duration $durationHours hours -END_JOB - - print $jobString; -} - -pod2usage(1) if $help; -pod2usage(-verbose => 2) if $man; - -if ( $jobID ) { - my $sth = $dbh->prepare("SELECT * FROM job WHERE job_id=\'$jobID\'"); - $sth->execute; - my %row; - $sth->bind_columns( \( @row{ @{$sth->{NAME_lc} } } )); - while ($sth->fetch) { - printJob(\%row); - } - exit; -} - -# build query conditions -if ( $user ) { - push @conditions, "user_id=\'$user\'"; -} - -if ( $project ) { - push @conditions, "project_id=\'$project\'"; -} - -if ( @numnodes ) { - ($add, $from, $to) = processRange($numnodes[0], $numnodes[1]); - buildCondition('num_nodes'); -} - -if ( @starttime ) { - ($add, $from, $to) = processRange( parseDate($starttime[0]), parseDate($starttime[1])); - buildCondition('start_time'); -} - -if ( @duration ) { - ($add, $from, $to) = processRange( parseDuration($duration[0]), parseDuration($duration[1])); - buildCondition('duration'); -} - -if ( @mem_used ) { - $hasprofile = 'true'; - ($add, $from, $to) = processRange($mem_used[0], $mem_used[1]); - buildCondition('mem_used_max'); -} - -if ( @mem_bandwidth ) { - $hasprofile = 'true'; - ($add, $from, $to) = processRange($mem_bandwidth[0], $mem_bandwidth[1]); - buildCondition('mem_bw_avg'); -} - -if ( @flops_any ) { - $hasprofile = 'true'; - ($add, $from, $to) = processRange($flops_any[0], $flops_any[1]); - buildCondition('flops_any_avg'); -} - -if ( $hasprofile ) { - if ( $hasprofile eq 'true' ) { - push @conditions, "has_profile=1"; - } elsif ( $hasprofile eq 'false' ) { - push @conditions, "has_profile=0"; - } else { - print "Unknown value for option has_profile: $hasprofile. Can be true or false.\n"; - } -} - -my $query; -my $conditionstring = ''; - -if ( @conditions ){ - $conditionstring = ' WHERE '; - $conditionstring .= join(' AND ',@conditions); -} - -# handle mode -if ( $mode eq 'query' ) { - $query = 'SELECT * FROM job'.$conditionstring; - print "$query\n"; - exit; -} - -if ( $mode eq 'count' ) { - $query = 'SELECT COUNT(*) FROM job'.$conditionstring; - my ($count) = $dbh->selectrow_array($query); - print "COUNT $count\n"; - exit; -} - -if ( $mode eq 'stat' ) { - printJobStat($conditionstring); - exit; -} - -if ( $mode eq 'perf' ) { - printPerfStat($conditionstring); - exit; -} - - -$query = 'SELECT * FROM job'.$conditionstring; -my $sth = $dbh->prepare($query); -$sth->execute; -my %row; -$sth->bind_columns( \( @row{ @{$sth->{NAME_lc} } } )); - -if ( $mode eq 'list' ) { - while ($sth->fetch) { - printJob(\%row); - } -} elsif ( $mode eq 'ids' ) { - while ($sth->fetch) { - print "$row{job_id}\n"; - } -} else { - die "ERROR Unknown mode $mode!\n"; -} - -__END__ - -=head1 NAME - -acQuery.pl - Wrapper script to access sqlite job database. - -=head1 SYNOPSIS - - acQuery.pl [options] -- - - Help Options: - --help Show help text - --man Show man page - --hasprofile Only show jobs with timerseries metric data - --mode Set the operation mode - --job Search for a specific job - --user Search for jobs of specific user - --project Search for jobs of specific project - --numnodes Specify range for number of nodes of job - --starttime Specify range for start time of jobs - --duration Specify duration range of jobs - --mem_used Specify range for average main memory capacity of job - --mem_bandwidth Specify range for average main memory bandwidth of job - --flops_any Specify range for average flop any rate of job - -=head1 OPTIONS - -=over 8 - -=item B<--help> -Show a brief help information. - -=item B<--man> -Read the manual, with examples - -=item B<--hasprofile [true|false]> -Only show jobs with or without timerseries metric data - -=item B<--mode [ids|query|count|list|stat|perf]> -Specify output mode. Mode can be one of: - -=over 4 - -=item B -Print list of job ids matching conditions. One job id per line. - -=item B -Print the query string and then exit. - -=item B -Only output the number of jobs matching the conditions. (Default mode) - -=item B -Output a record of every job matching the conditions. - -=item B -Output job statistic for all jobs matching the conditions. - -=item B -Output job performance footprint statistic for all jobs matching the conditions. - -=back - -=item B<--user> -Search job for a specific user id. - -=item B<--project> -Search job for a specific project. - -=item B<--duration> -Specify condition for job duration. This option takes two arguments: If both -arguments are positive integers the condition is duration between first -argument and second argument. If the second argument is zero condition is duration -smaller than first argument. If first argument is zero condition is duration -larger than second argument. Duration can be in seconds, minutes (append m) or -hours (append h). - -=item B<--numnodes> -Specify condition for number of node range of job. This option takes two -arguments: If both arguments are positive integers the condition is number of -nodes between first argument and second argument. If the second argument is -zero condition is number of nodes smaller than first argument. If first -argument is zero condition is number of nodes larger than second argument. - -=item B<--starttime> -Specify condition for the starttime of job. This option takes two -arguments: If both arguments are positive integers the condition is start time -between first argument and second argument. If the second argument is -zero condition is start time smaller than first argument. If first -argument is zero condition is start time larger than second argument. -Start time must be given as date in the following format: %d.%m.%Y/%H:%M. - -=item B<--mem_used> -Specify condition for average main memory capacity used by job. This option takes two -arguments: If both arguments are positive integers the condition is memory used is -between first argument and second argument. If the second argument is -zero condition is memory used is smaller than first argument. If first -argument is zero condition is memory used is larger than second argument. - -=item B<--mem_bandwidth> -Specify condition for average main memory bandwidth used by job. This option takes two -arguments: If both arguments are positive integers the condition is memory bandwidth is -between first argument and second argument. If the second argument is -zero condition is memory bandwidth is smaller than first argument. If first -argument is zero condition is memory bandwidth is larger than second argument. - -=item B<--flops_any> -Specify condition for average flops any of job. This option takes two -arguments: If both arguments are positive integers the condition is flops any is -between first argument and second argument. If the second argument is -zero condition is flops any is smaller than first argument. If first -argument is zero condition is flops any is larger than second argument. - -=back - -=head1 DESCRIPTION - -This script allows to create queries for the sqlite job database. Output format can -be a list of job ids (ids), a verbose output of job infos (list), just the job query -without executing it (query), a statistic analysis for jobs mathing the query conditions (stat) -and just the job count (count). Job count is the default. The script expects the sqlite -database in a file jobDB in the same directory. Optionally one can specify another database -file name as command line argument. - -=head1 EXAMPLES - -C<./acQuery.pl --duration 5m 0 --numnodes 0 50> - -C<./acQuery.pl --project exzi --starttime 01.01.2019/00:00 31.03.2019/23:59 --mode stat> - -=head1 AUTHOR - -Jan Eitzinger - L - -=cut diff --git a/utils/initDB.sql b/utils/initDB.sql deleted file mode 100644 index c9727c8..0000000 --- a/utils/initDB.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE TABLE job ( id INTEGER PRIMARY KEY, - job_id TEXT, user_id TEXT, project_id TEXT, cluster_id TEXT, - start_time INTEGER, stop_time INTEGER, duration INTEGER, - walltime INTEGER, job_state TEXT, - num_nodes INTEGER, node_list TEXT, has_profile INTEGER, - mem_used_max REAL, flops_any_avg REAL, mem_bw_avg REAL, ib_bw_avg REAL, file_bw_avg REAL); -CREATE TABLE tag ( id INTEGER PRIMARY KEY, tag_type TEXT, tag_name TEXT); -CREATE TABLE jobtag ( job_id INTEGER, tag_id INTEGER, PRIMARY KEY (job_id, tag_id), - FOREIGN KEY (job_id) REFERENCES job (id) ON DELETE CASCADE ON UPDATE NO ACTION, - FOREIGN KEY (tag_id) REFERENCES tag (id) ON DELETE CASCADE ON UPDATE NO ACTION );