ErikFabrizzi/auto-grader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ft: run command prototype and heavy refactor

Browse Source
This commit is contained in:
Erik Fabrizzi
2025-10-29 14:21:56 +01:00
parent 04a261aa41
commit 265bcb9192
11 changed files with 897 additions and 755 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
src/auto_grader/commands/test/correctness.py Normal file
View File

@@ -0,0 +1,156 @@
import click
import os
import subprocess
import shlex
import tomllib
import numpy as np
from auto_grader.utils.display import ERROR_BOX, WARNING_BOX
@click.command()
@click.argument('number', required=True, type=click.INT)
@click.option('-p', '--path', default='./roots', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True))
@click.option('-c', '--compile-command', default='make', type=click.STRING)
@click.option('-r', '--run-command', default='mpirun', type=click.STRING)
@click.option('-n', '--nproc-flag', default="-n 5", type=click.STRING)
@click.option('-v', '--verbose', is_flag=True, default=False)
@click.option('-s', '--solution-path', default='./solutions', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True))
def correctness(number, solution_path, path, compile_command, run_command, nproc_flag, verbose):
students = os.listdir(path)
asg_dirs = [os.path.join(
path, student, f"assignment_{number}") for student in students]
for asg_dir, student in zip(asg_dirs, students):
current_wd = os.getcwd()
if not os.path.exists(asg_dir) or not os.path.isdir(asg_dir):
click.echo(
f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}")
continue
local_roots = [os.path.join(asg_dir, dir)
for dir in os.listdir(asg_dir) if 'root_' in dir and os.path.isdir(os.path.join(asg_dir, dir))]
if len(local_roots) == 0:
click.echo(
f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}")
continue
if verbose:
click.echo(f"Testing compile for {student}:assignment_{number}")
else:
click.echo(f"({student}:assignment_{number}):")
for root in local_roots:
os.chdir(current_wd)
if verbose:
click.echo(f"\t[{os.path.basename(root)}]: ", nl=False)
click.echo(f"entering {os.path.relpath(root)}")
os.chdir(root)
result = subprocess.run(
shlex.split("make distclean"), capture_output=True, text=True)
result = subprocess.run(
shlex.split(compile_command), capture_output=True, text=True)
if verbose:
if (result.returncode != 0):
click.echo(
f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False)
else:
click.echo(
f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: ", nl=False)
click.echo(f"stdout: {result.stdout}")
click.echo(f"stderr: {result.stderr}")
else:
if (result.returncode != 0):
click.echo(
f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False)
click.echo(f"stderr: {result.stderr[:69]:69s}...")
else:
click.echo(
f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: SUCCESS")
executables = [os.path.join(root, file)
for file in os.listdir(root) if 'exe-' in file and os.path.isfile(os.path.join(root, file))]
sol_as_path = os.path.join(solution_path, f"assignment_{number}")
config_path = os.path.join(sol_as_path, "test_config.toml")
if not os.path.isdir(sol_as_path):
click.echo(
f"{ERROR_BOX}: No reference solution for assignment_{number} found.", err=True)
return
if not os.path.exists(config_path) or not os.path.isfile(config_path):
click.echo(
f"{ERROR_BOX}: No test configuration for assignment_{number} found.", err=True)
return
config = dict()
with open(config_path, 'rb') as handle:
config = tomllib.load(handle)
if not 'ref_solution' in config["correctness"]:
click.echo(
f"{ERROR_BOX}: No function dissect config found for assignment_{number}.", err=True)
return
ref_solution = config['correctness']['ref_solution']
ref_solution_path = os.path.join(sol_as_path, ref_solution)
if not 'ref_param' in config["correctness"]:
click.echo(
f"{ERROR_BOX}: No function dissect config found for assignment_{number}.", err=True)
return
ref_param = config['correctness']['ref_param']
ref_param_path = os.path.join(sol_as_path, ref_param)
local_run_command = " ".join([run_command, nproc_flag,
executables[0], ref_param_path])
local_solution_path = os.path.join(root, 'p.dat')
if os.path.exists(local_solution_path):
os.remove(local_solution_path)
try:
result = subprocess.run(
shlex.split(local_run_command), capture_output=True, text=True, timeout=10)
except subprocess.TimeoutExpired:
click.echo("Run Timeout")
result.returncode = 1
if verbose:
if (result.returncode != 0):
click.echo(
f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False)
continue
else:
click.echo(
f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: ", nl=False)
click.echo(f"stdout: {result.stdout}")
click.echo(f"stderr: {result.stderr}")
else:
if (result.returncode != 0):
click.echo(
f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False)
click.echo(f"stderr: {result.stderr[:69]:69s}...")
continue
else:
click.echo(
f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: SUCCESS")
if not os.path.exists(local_solution_path):
data_files = [file for file in os.listdir(
root) if '.dat' in file]
print(data_files)
continue
local_solution_data = np.loadtxt(local_solution_path, usecols=True)
ref_solution_data = np.loadtxt(ref_solution_path, usecols=True)
if local_solution_data.shape != ref_solution_data.shape:
click.echo("Correctness Test Failed, Dimenstions Not Matching")
continue
elif not np.allclose(local_solution_data, ref_solution_data, rtol=1e-1):
click.echo("Correctness Test Failed: Data Not Matching")
continue
else:
click.echo("Correctness Test Passed")