ft: new config and interface for correctness/compile/run, temporary disabled dissect, cleaned dev
This commit is contained in:
Erik Fabrizzi
@@ -1,32 +1,141 @@
|
||||
import click
|
||||
from click.core import ParameterSource
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import shlex
|
||||
import tomllib
|
||||
import numpy as np
|
||||
from auto_grader.utils.display import ERROR_BOX, WARNING_BOX
|
||||
|
||||
from auto_grader.utils import clean_root, build_root, run_root
|
||||
from auto_grader.utils.display import ERROR_BOX, WARNING_BOX, indent_text
|
||||
from auto_grader.utils.config_parser import grdr_config
|
||||
from auto_grader.types import StudentVarType, run_log
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.command(help='Builds and Runs full student root. Then compares file if in numpy txt matrix format.\n' +
|
||||
'Command run is: \n\n' +
|
||||
'[COMMAND] [PROCOPT] [NUMPROC] [EXECUTABLE-PREFIX]<exe + flags autodetermined by config>\n\n' +
|
||||
'Example with defaults:\n\n' +
|
||||
'mpirun -n 4 <exe + flags>')
|
||||
@click.argument('number', required=True, type=click.INT)
|
||||
@click.option('-p', '--path', default='./roots', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True))
|
||||
@click.option('-c', '--compile-command', default='make', type=click.STRING)
|
||||
@click.option('-r', '--run-command', default='mpirun', type=click.STRING)
|
||||
@click.option('-n', '--nproc-flag', default="-n 5", type=click.STRING)
|
||||
@click.option('-v', '--verbose', is_flag=True, default=False)
|
||||
@click.option('-s', '--solution-path', default='./solutions', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True))
|
||||
def correctness(number, solution_path, path, compile_command, run_command, nproc_flag, verbose):
|
||||
@click.option('-p', '--path', default='./roots',
|
||||
type=click.Path(exists=True, file_okay=False,
|
||||
dir_okay=True, resolve_path=True),
|
||||
help='Path to directory conaining student roots in format (student)/assignment_[NUMBER]/root_(...).')
|
||||
@click.option('--tol', default=1e-1,
|
||||
type=click.FLOAT,
|
||||
help='Tollerance passed to numpy.allclose for result comparison')
|
||||
@click.option('-c', '--run-command', default='mpirun', type=click.STRING,
|
||||
help='Overrides command that is used to run roots. The default command is `mpirun`.')
|
||||
@click.option('--procopt', default='-n ', type=click.STRING,
|
||||
help='Indicates the option flag to be used to specify process number from [COMMAND]. The default value is `-n `.')
|
||||
@click.option('-n', '--numproc', default=4, type=click.INT,
|
||||
help='Selects the number of mpi processes used to run the root. The default value is `4`.')
|
||||
@click.option('-t', '--run-timeout', default=10, type=click.INT,
|
||||
help='Sets timeout for the run, defaults to 10s.')
|
||||
@click.option('--solutions-path', default='./solutions',
|
||||
type=click.Path(exists=True, file_okay=False,
|
||||
dir_okay=True, resolve_path=True),
|
||||
help='Path to directory conaining solution root in format assignment_[NUMBER]/. Solution must contain configuration grdr_config.toml')
|
||||
@click.option('-s', '--student', default='', type=StudentVarType(),
|
||||
help='Specify wich student to run the test for. If omitted test is run for all found students.')
|
||||
@click.option('--executable-prefix', default='', type=click.STRING,
|
||||
help='This prefix is appended to the executable name when the run command is launched (some launchers require `./`. The default value is ``.')
|
||||
@click.option('--compile-command', default='make', type=click.STRING,
|
||||
help='Overrides command that is run to build roots. The default command is `make`.')
|
||||
@click.option('--clean-command', default='make distclean', type=click.STRING,
|
||||
help='Overrides command that is run to clean roots. The default command is `make distclean`.')
|
||||
@click.option('--compile-timeout', default=10, type=click.INT,
|
||||
help='Sets timeout of both clean and build commands, defaults to 10s.')
|
||||
@click.option('--verbose', is_flag=True, default=False,
|
||||
help='Display the results of each attempted test as a detailed summary')
|
||||
@click.pass_context
|
||||
def correctness(
|
||||
context,
|
||||
number,
|
||||
path,
|
||||
tol,
|
||||
run_command,
|
||||
procopt,
|
||||
numproc,
|
||||
run_timeout,
|
||||
student,
|
||||
solutions_path,
|
||||
executable_prefix,
|
||||
compile_command,
|
||||
clean_command,
|
||||
compile_timeout,
|
||||
verbose
|
||||
):
|
||||
|
||||
current_wd = os.getcwd()
|
||||
students = os.listdir(path)
|
||||
config_path = solutions_path+f'/assignment_{number}/.grdr_config.toml'
|
||||
config = grdr_config(config_path)
|
||||
|
||||
# -- Parse run options ---------------------------------------------------#
|
||||
if context.get_parameter_source('run-command') != ParameterSource.COMMANDLINE and config.run_config['command'] != None:
|
||||
run_command = config.run_config['command']
|
||||
|
||||
if context.get_parameter_source('procopt') != ParameterSource.COMMANDLINE and config.run_config['procopt'] != None:
|
||||
procopt = config.run_config['procopt']
|
||||
|
||||
if context.get_parameter_source('numproc') != ParameterSource.COMMANDLINE and config.run_config['numproc'] != None:
|
||||
numproc = config.run_config['numproc']
|
||||
|
||||
if context.get_parameter_source('run-timeout') != ParameterSource.COMMANDLINE and config.run_config['timeout'] != None:
|
||||
run_timeout = config.run_config['timeout']
|
||||
|
||||
if len(config.run_config['_signatures']) == 0:
|
||||
click.echo(
|
||||
f"{ERROR_BOX}: No signature detected in configuration file. A signature list, even if empty (signature = []) us required)")
|
||||
|
||||
# -- Parse compile options ----------------------------------------------#
|
||||
if context.get_parameter_source('compile-command') != ParameterSource.COMMANDLINE and config.compile_config['command'] != None:
|
||||
compile_command = config.compile_config['command']
|
||||
|
||||
if context.get_parameter_source('clean-command') != ParameterSource.COMMANDLINE and config.compile_config['clean-command'] != None:
|
||||
clean_command = config.compile_config['clean-command']
|
||||
|
||||
if context.get_parameter_source('compile-timeout') != ParameterSource.COMMANDLINE and config.compile_config['timeout'] != None:
|
||||
compile_timeout = config.compile_config['timeout']
|
||||
|
||||
# -------------------------------------------------------------------------#
|
||||
if config.correctness_config["ref-solution"] == None:
|
||||
click.echo(
|
||||
f"{ERROR_BOX}: No ref-solution detected in configuration file.")
|
||||
return
|
||||
reference_solution = np.empty((0))
|
||||
try:
|
||||
reference_solution = np.loadtxt(
|
||||
config.correctness_config["ref-solution"])
|
||||
except Exception as e:
|
||||
click.echo(
|
||||
f"{ERROR_BOX}: Failed to load reference solution. Except:{e}")
|
||||
return
|
||||
|
||||
if config.correctness_config["local-solution"] == None:
|
||||
click.echo(
|
||||
f"{ERROR_BOX}: No local-solution detected in configuration file.")
|
||||
return
|
||||
local_solution_path: str = config.correctness_config["local-solution"]
|
||||
if config.correctness_config["signature"] == None:
|
||||
click.echo(
|
||||
f"{ERROR_BOX}: No signature detected in configuration file.")
|
||||
return
|
||||
|
||||
if student and student not in students:
|
||||
click.echo(
|
||||
f"{ERROR_BOX}: No student {student} in {os.path.relpath(path)}.")
|
||||
return
|
||||
|
||||
if student:
|
||||
students = [student]
|
||||
asg_dirs = [os.path.join(
|
||||
path, student, f"assignment_{number}") for student in students]
|
||||
|
||||
for asg_dir, student in zip(asg_dirs, students):
|
||||
|
||||
current_wd = os.getcwd()
|
||||
|
||||
if not os.path.exists(asg_dir) or not os.path.isdir(asg_dir):
|
||||
click.echo(
|
||||
f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}")
|
||||
f"{WARNING_BOX}: No {student}/assignment_{number} was found, skipping...")
|
||||
continue
|
||||
local_roots = [os.path.join(asg_dir, dir)
|
||||
for dir in os.listdir(asg_dir) if 'root_' in dir and os.path.isdir(os.path.join(asg_dir, dir))]
|
||||
@@ -34,123 +143,94 @@ def correctness(number, solution_path, path, compile_command, run_command, nproc
|
||||
click.echo(
|
||||
f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}")
|
||||
continue
|
||||
click.echo(f"[{student}]:")
|
||||
|
||||
if verbose:
|
||||
click.echo(f"Testing compile for {student}:assignment_{number}")
|
||||
else:
|
||||
click.echo(f"({student}:assignment_{number}):")
|
||||
command_prefix = f'{run_command} {procopt}{numproc} {executable_prefix}'
|
||||
|
||||
for root in local_roots:
|
||||
os.chdir(current_wd)
|
||||
if verbose:
|
||||
click.echo(f"\t[{os.path.basename(root)}]: ", nl=False)
|
||||
click.echo(f"entering {os.path.relpath(root)}")
|
||||
os.chdir(root)
|
||||
result = subprocess.run(
|
||||
shlex.split("make distclean"), capture_output=True, text=True)
|
||||
|
||||
result = subprocess.run(
|
||||
shlex.split(compile_command), capture_output=True, text=True)
|
||||
|
||||
if verbose:
|
||||
if (result.returncode != 0):
|
||||
click.echo(
|
||||
f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False)
|
||||
else:
|
||||
click.echo(
|
||||
f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: ", nl=False)
|
||||
click.echo(f"stdout: {result.stdout}")
|
||||
click.echo(f"stderr: {result.stderr}")
|
||||
else:
|
||||
if (result.returncode != 0):
|
||||
click.echo(
|
||||
f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False)
|
||||
click.echo(f"stderr: {result.stderr[:69]:69s}...")
|
||||
|
||||
else:
|
||||
click.echo(
|
||||
f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: SUCCESS")
|
||||
|
||||
executables = [os.path.join(root, file)
|
||||
for file in os.listdir(root) if 'exe-' in file and os.path.isfile(os.path.join(root, file))]
|
||||
|
||||
sol_as_path = os.path.join(solution_path, f"assignment_{number}")
|
||||
config_path = os.path.join(sol_as_path, "test_config.toml")
|
||||
if not os.path.isdir(sol_as_path):
|
||||
click.echo(
|
||||
f"{ERROR_BOX}: No reference solution for assignment_{number} found.", err=True)
|
||||
return
|
||||
if not os.path.exists(config_path) or not os.path.isfile(config_path):
|
||||
click.echo(
|
||||
f"{ERROR_BOX}: No test configuration for assignment_{number} found.", err=True)
|
||||
return
|
||||
|
||||
config = dict()
|
||||
with open(config_path, 'rb') as handle:
|
||||
config = tomllib.load(handle)
|
||||
|
||||
if not 'ref_solution' in config["correctness"]:
|
||||
click.echo(
|
||||
f"{ERROR_BOX}: No function dissect config found for assignment_{number}.", err=True)
|
||||
return
|
||||
|
||||
ref_solution = config['correctness']['ref_solution']
|
||||
ref_solution_path = os.path.join(sol_as_path, ref_solution)
|
||||
|
||||
if not 'ref_param' in config["correctness"]:
|
||||
click.echo(
|
||||
f"{ERROR_BOX}: No function dissect config found for assignment_{number}.", err=True)
|
||||
return
|
||||
ref_param = config['correctness']['ref_param']
|
||||
ref_param_path = os.path.join(sol_as_path, ref_param)
|
||||
local_run_command = " ".join([run_command, nproc_flag,
|
||||
executables[0], ref_param_path])
|
||||
local_solution_path = os.path.join(root, 'p.dat')
|
||||
executables = [file for file in os.listdir(
|
||||
os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)]
|
||||
for executable in executables:
|
||||
os.remove(executable)
|
||||
|
||||
if os.path.exists(local_solution_path):
|
||||
os.remove(local_solution_path)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
shlex.split(local_run_command), capture_output=True, text=True, timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
click.echo("Run Timeout")
|
||||
result.returncode = 1
|
||||
clean_log = clean_root(root, clean_command)
|
||||
if not clean_log.run_success or not clean_log.cmd_return_code == 0:
|
||||
click.echo(
|
||||
f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}")
|
||||
continue
|
||||
|
||||
if verbose:
|
||||
if (result.returncode != 0):
|
||||
click.echo(
|
||||
f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False)
|
||||
continue
|
||||
else:
|
||||
click.echo(
|
||||
f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: ", nl=False)
|
||||
click.echo(f"stdout: {result.stdout}")
|
||||
click.echo(f"stderr: {result.stderr}")
|
||||
build_log = build_root(root, compile_command,
|
||||
timeout=compile_timeout)
|
||||
if not build_log.run_success or not build_log.cmd_return_code == 0:
|
||||
click.echo(
|
||||
f"{indent_text('[BUILD]:' + build_log.oneline(type_as_prefix=False), 4)}")
|
||||
continue
|
||||
executables = [file for file in os.listdir(
|
||||
os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)]
|
||||
|
||||
local_command = command_prefix + executables[0] + ' '
|
||||
rn_log = run_root(root, local_command +
|
||||
config.correctness_config['signature'], timeout=run_timeout)
|
||||
if not rn_log.run_success or not rn_log.cmd_return_code == 0:
|
||||
click.echo(
|
||||
f"{indent_text('[RUN]:' + rn_log.oneline(type_as_prefix=False), 4)}")
|
||||
continue
|
||||
|
||||
run_success = True
|
||||
summary = ''
|
||||
cmd_return_code = 0
|
||||
cmd_stderr = ''
|
||||
|
||||
if run_success and not os.path.exists(local_solution_path):
|
||||
run_success = False
|
||||
summary = f'Output file {local_solution_path} was not produced by run'
|
||||
cmd_return_code = 1
|
||||
|
||||
local_solution = np.empty((0))
|
||||
if run_success:
|
||||
try:
|
||||
local_solution = np.loadtxt(local_solution_path)
|
||||
except Exception as e:
|
||||
run_success = False
|
||||
summary = f'Failed to load output file {local_solution_path}. Except: {e}'
|
||||
cmd_return_code = 1
|
||||
|
||||
if run_success:
|
||||
if cmd_return_code == 0 and reference_solution.shape != local_solution.shape:
|
||||
cmd_return_code = 1
|
||||
cmd_stderr = f'Correctness comparison failed due to missmatching data shapes. ref: {reference_solution.shape}, student: {local_solution.shape}'
|
||||
if cmd_return_code == 0 and not np.allclose(local_solution, reference_solution, atol=tol):
|
||||
cmd_return_code = 1
|
||||
cmd_stderr = f'Correctness comparison failed due to missmatching data. Diff max|ref - student| = { np.max(np.abs(reference_solution- local_solution))}'
|
||||
|
||||
log = run_log(
|
||||
'CORRECTNESS',
|
||||
f"PHONEY: compare {config.correctness_config['ref-solution']}{os.path.abspath(local_solution_path)}",
|
||||
os.getcwd(),
|
||||
run_success,
|
||||
summary,
|
||||
cmd_return_code,
|
||||
'',
|
||||
cmd_stderr)
|
||||
|
||||
if not verbose:
|
||||
click.echo(
|
||||
f"{indent_text(log.oneline(type_as_prefix=False), 4)}")
|
||||
else:
|
||||
if (result.returncode != 0):
|
||||
click.echo(
|
||||
f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False)
|
||||
click.echo(f"stderr: {result.stderr[:69]:69s}...")
|
||||
continue
|
||||
else:
|
||||
click.echo(
|
||||
f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: SUCCESS")
|
||||
click.echo(f"{indent_text(log.as_str(),4)}")
|
||||
|
||||
if not os.path.exists(local_solution_path):
|
||||
data_files = [file for file in os.listdir(
|
||||
root) if '.dat' in file]
|
||||
print(data_files)
|
||||
continue
|
||||
if os.path.exists(local_solution_path):
|
||||
os.remove(local_solution_path)
|
||||
|
||||
local_solution_data = np.loadtxt(local_solution_path, usecols=True)
|
||||
ref_solution_data = np.loadtxt(ref_solution_path, usecols=True)
|
||||
|
||||
if local_solution_data.shape != ref_solution_data.shape:
|
||||
click.echo("Correctness Test Failed, Dimenstions Not Matching")
|
||||
for executable in executables:
|
||||
os.remove(executable)
|
||||
clean_log = clean_root(root, clean_command)
|
||||
if not clean_log.run_success or not clean_log.cmd_return_code == 0:
|
||||
click.echo(
|
||||
f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}")
|
||||
continue
|
||||
elif not np.allclose(local_solution_data, ref_solution_data, rtol=1e-1):
|
||||
click.echo("Correctness Test Failed: Data Not Matching")
|
||||
continue
|
||||
else:
|
||||
click.echo("Correctness Test Passed")
|
||||
os.chdir(current_wd)
|
||||
|
||||
Reference in New Issue
Block a user