237 lines
11 KiB
Python
237 lines
11 KiB
Python
import click
|
|
from click.core import ParameterSource
|
|
|
|
import os
|
|
import numpy as np
|
|
|
|
from auto_grader.utils import clean_root, build_root, run_root
|
|
from auto_grader.utils.display import ERROR_BOX, WARNING_BOX, indent_text
|
|
from auto_grader.utils.config_parser import grdr_config
|
|
from auto_grader.types import StudentVarType, run_log
|
|
|
|
|
|
@click.command(help='Builds and Runs full student root. Then compares file if in numpy txt matrix format.\n' +
|
|
'Command run is: \n\n' +
|
|
'[COMMAND] [PROCOPT] [NUMPROC] [EXECUTABLE-PREFIX]<exe + flags autodetermined by config>\n\n' +
|
|
'Example with defaults:\n\n' +
|
|
'mpirun -n 4 <exe + flags>')
|
|
@click.argument('number', required=True, type=click.INT)
|
|
@click.option('-p', '--path', default='./roots',
|
|
type=click.Path(exists=True, file_okay=False,
|
|
dir_okay=True, resolve_path=True),
|
|
help='Path to directory conaining student roots in format (student)/assignment_[NUMBER]/root_(...).')
|
|
@click.option('--tol', default=1e-1,
|
|
type=click.FLOAT,
|
|
help='Tollerance passed to numpy.allclose for result comparison')
|
|
@click.option('-c', '--run-command', default='mpirun', type=click.STRING,
|
|
help='Overrides command that is used to run roots. The default command is `mpirun`.')
|
|
@click.option('--procopt', default='-n ', type=click.STRING,
|
|
help='Indicates the option flag to be used to specify process number from [COMMAND]. The default value is `-n `.')
|
|
@click.option('-n', '--numproc', default=4, type=click.INT,
|
|
help='Selects the number of mpi processes used to run the root. The default value is `4`.')
|
|
@click.option('-t', '--run-timeout', default=10, type=click.INT,
|
|
help='Sets timeout for the run, defaults to 10s.')
|
|
@click.option('--solutions-path', default='./solutions',
|
|
type=click.Path(exists=True, file_okay=False,
|
|
dir_okay=True, resolve_path=True),
|
|
help='Path to directory conaining solution root in format assignment_[NUMBER]/. Solution must contain configuration grdr_config.toml')
|
|
@click.option('-s', '--student', default='', type=StudentVarType(),
|
|
help='Specify wich student to run the test for. If omitted test is run for all found students.')
|
|
@click.option('--executable-prefix', default='', type=click.STRING,
|
|
help='This prefix is appended to the executable name when the run command is launched (some launchers require `./`. The default value is ``.')
|
|
@click.option('--compile-command', default='make', type=click.STRING,
|
|
help='Overrides command that is run to build roots. The default command is `make`.')
|
|
@click.option('--clean-command', default='make distclean', type=click.STRING,
|
|
help='Overrides command that is run to clean roots. The default command is `make distclean`.')
|
|
@click.option('--compile-timeout', default=10, type=click.INT,
|
|
help='Sets timeout of both clean and build commands, defaults to 10s.')
|
|
@click.option('--verbose', is_flag=True, default=False,
|
|
help='Display the results of each attempted test as a detailed summary')
|
|
@click.pass_context
|
|
def correctness(
|
|
context,
|
|
number,
|
|
path,
|
|
tol,
|
|
run_command,
|
|
procopt,
|
|
numproc,
|
|
run_timeout,
|
|
student,
|
|
solutions_path,
|
|
executable_prefix,
|
|
compile_command,
|
|
clean_command,
|
|
compile_timeout,
|
|
verbose
|
|
):
|
|
|
|
current_wd = os.getcwd()
|
|
students = os.listdir(path)
|
|
config_path = solutions_path+f'/assignment_{number}/.grdr_config.toml'
|
|
config = grdr_config(config_path)
|
|
|
|
# -- Parse run options ---------------------------------------------------#
|
|
if context.get_parameter_source('run-command') != ParameterSource.COMMANDLINE and config.run_config['command'] != None:
|
|
run_command = config.run_config['command']
|
|
|
|
if context.get_parameter_source('procopt') != ParameterSource.COMMANDLINE and config.run_config['procopt'] != None:
|
|
procopt = config.run_config['procopt']
|
|
|
|
if context.get_parameter_source('numproc') != ParameterSource.COMMANDLINE and config.run_config['numproc'] != None:
|
|
numproc = config.run_config['numproc']
|
|
|
|
if context.get_parameter_source('run-timeout') != ParameterSource.COMMANDLINE and config.run_config['timeout'] != None:
|
|
run_timeout = config.run_config['timeout']
|
|
|
|
if len(config.run_config['_signatures']) == 0:
|
|
click.echo(
|
|
f"{ERROR_BOX}: No signature detected in configuration file. A signature list, even if empty (signature = []) us required)")
|
|
|
|
# -- Parse compile options ----------------------------------------------#
|
|
if context.get_parameter_source('compile-command') != ParameterSource.COMMANDLINE and config.compile_config['command'] != None:
|
|
compile_command = config.compile_config['command']
|
|
|
|
if context.get_parameter_source('clean-command') != ParameterSource.COMMANDLINE and config.compile_config['clean-command'] != None:
|
|
clean_command = config.compile_config['clean-command']
|
|
|
|
if context.get_parameter_source('compile-timeout') != ParameterSource.COMMANDLINE and config.compile_config['timeout'] != None:
|
|
compile_timeout = config.compile_config['timeout']
|
|
|
|
# -------------------------------------------------------------------------#
|
|
if config.correctness_config["ref-solution"] == None:
|
|
click.echo(
|
|
f"{ERROR_BOX}: No ref-solution detected in configuration file.")
|
|
return
|
|
reference_solution = np.empty((0))
|
|
try:
|
|
reference_solution = np.loadtxt(
|
|
config.correctness_config["ref-solution"])
|
|
except Exception as e:
|
|
click.echo(
|
|
f"{ERROR_BOX}: Failed to load reference solution. Except:{e}")
|
|
return
|
|
|
|
if config.correctness_config["local-solution"] == None:
|
|
click.echo(
|
|
f"{ERROR_BOX}: No local-solution detected in configuration file.")
|
|
return
|
|
local_solution_path: str = config.correctness_config["local-solution"]
|
|
if config.correctness_config["signature"] == None:
|
|
click.echo(
|
|
f"{ERROR_BOX}: No signature detected in configuration file.")
|
|
return
|
|
|
|
if student and student not in students:
|
|
click.echo(
|
|
f"{ERROR_BOX}: No student {student} in {os.path.relpath(path)}.")
|
|
return
|
|
|
|
if student:
|
|
students = [student]
|
|
asg_dirs = [os.path.join(
|
|
path, student, f"assignment_{number}") for student in students]
|
|
|
|
for asg_dir, student in zip(asg_dirs, students):
|
|
if not os.path.exists(asg_dir) or not os.path.isdir(asg_dir):
|
|
click.echo(
|
|
f"{WARNING_BOX}: No {student}/assignment_{number} was found, skipping...")
|
|
continue
|
|
local_roots = [os.path.join(asg_dir, dir)
|
|
for dir in os.listdir(asg_dir) if 'root_' in dir and os.path.isdir(os.path.join(asg_dir, dir))]
|
|
if len(local_roots) == 0:
|
|
click.echo(
|
|
f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}")
|
|
continue
|
|
click.echo(f"[{student}]:")
|
|
|
|
command_prefix = f'{run_command} {procopt}{numproc} {executable_prefix}'
|
|
|
|
for root in local_roots:
|
|
os.chdir(root)
|
|
executables = [file for file in os.listdir(
|
|
os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)]
|
|
for executable in executables:
|
|
os.remove(executable)
|
|
|
|
if os.path.exists(local_solution_path):
|
|
os.remove(local_solution_path)
|
|
|
|
clean_log = clean_root(root, clean_command)
|
|
if not clean_log.run_success or not clean_log.cmd_return_code == 0:
|
|
click.echo(
|
|
f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}")
|
|
continue
|
|
|
|
build_log = build_root(root, compile_command,
|
|
timeout=compile_timeout)
|
|
if not build_log.run_success or not build_log.cmd_return_code == 0:
|
|
click.echo(
|
|
f"{indent_text('[BUILD]:' + build_log.oneline(type_as_prefix=False), 4)}")
|
|
continue
|
|
executables = [file for file in os.listdir(
|
|
os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)]
|
|
|
|
local_command = command_prefix + executables[0] + ' '
|
|
rn_log = run_root(root, local_command +
|
|
config.correctness_config['signature'], timeout=run_timeout)
|
|
if not rn_log.run_success or not rn_log.cmd_return_code == 0:
|
|
click.echo(
|
|
f"{indent_text('[RUN]:' + rn_log.oneline(type_as_prefix=False), 4)}")
|
|
continue
|
|
|
|
run_success = True
|
|
summary = ''
|
|
cmd_return_code = 0
|
|
cmd_stderr = ''
|
|
|
|
if run_success and not os.path.exists(local_solution_path):
|
|
run_success = False
|
|
summary = f'Output file {local_solution_path} was not produced by run'
|
|
cmd_return_code = 1
|
|
|
|
local_solution = np.empty((0))
|
|
if run_success:
|
|
try:
|
|
local_solution = np.loadtxt(local_solution_path)
|
|
except Exception as e:
|
|
run_success = False
|
|
summary = f'Failed to load output file {local_solution_path}. Except: {e}'
|
|
cmd_return_code = 1
|
|
|
|
if run_success:
|
|
if cmd_return_code == 0 and reference_solution.shape != local_solution.shape:
|
|
cmd_return_code = 1
|
|
cmd_stderr = f'Correctness comparison failed due to missmatching data shapes. ref: {reference_solution.shape}, student: {local_solution.shape}'
|
|
if cmd_return_code == 0 and not np.allclose(local_solution, reference_solution, atol=tol):
|
|
cmd_return_code = 1
|
|
cmd_stderr = f'Correctness comparison failed due to missmatching data. Diff max|ref - student| = { np.max(np.abs(reference_solution- local_solution))}'
|
|
|
|
log = run_log(
|
|
'CORRECTNESS',
|
|
f"PHONEY: compare {config.correctness_config['ref-solution']}{os.path.abspath(local_solution_path)}",
|
|
os.getcwd(),
|
|
run_success,
|
|
summary,
|
|
cmd_return_code,
|
|
'',
|
|
cmd_stderr)
|
|
|
|
if not verbose:
|
|
click.echo(
|
|
f"{indent_text(log.oneline(type_as_prefix=False), 4)}")
|
|
else:
|
|
click.echo(f"{indent_text(log.as_str(),4)}")
|
|
|
|
if os.path.exists(local_solution_path):
|
|
os.remove(local_solution_path)
|
|
|
|
for executable in executables:
|
|
os.remove(executable)
|
|
clean_log = clean_root(root, clean_command)
|
|
if not clean_log.run_success or not clean_log.cmd_return_code == 0:
|
|
click.echo(
|
|
f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}")
|
|
continue
|
|
os.chdir(current_wd)
|