added run and correctness in dissect

src/auto_grader/cli.py

@@ -8,6 +8,7 @@ import shlex
 import tomllib
 import re
 import difflib
+import numpy as np
 from unidecode import unidecode
 
 SUCCESS_BOX = "[\033[0;92mSUCCESS\033[0m]"
@@ -123,7 +124,7 @@ def _extract_prototypes_to_header(filename, functions, header_filename):
 
     matches = func_pattern.findall(code)
     if not matches:
-        print(f"No functions found in {filename}")
+        click.echo(f"{WARNING_BOX}:No definitions found for {functions}")
         return
 
     prototypes = []
@@ -407,6 +408,7 @@ def compile(number, path, command, verbose):
 @click.option('-s', '--solution-path', default='./solutions', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True))
 @click.option('-p', '--path', default='./roots', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True))
 def dissect(number, solution_path, path):
+
     sol_as_path = os.path.join(solution_path, f"assignment_{number}")
     config_path = os.path.join(sol_as_path, "test_config.toml")
 
@@ -422,40 +424,53 @@ def dissect(number, solution_path, path):
 
     with open(config_path, 'rb') as handle:
         config = tomllib.load(handle)
+
     if not 'test_functions' in config["dissect"]:
         click.echo(
             f"{ERROR_BOX}: No function dissect config found for assignment_{number}.", err=True)
         return
     test_functions = config['dissect']['test_functions']
-    if not 'all_functions' in config["dissect"]:
+
-        click.echo(
-            f"{ERROR_BOX}: No function dissect config found for assignment_{number}.", err=True)
-        return
-    all_functions = config['dissect']['all_functions']
     if not 'build_cmd' in config["dissect"]:
         click.echo(
             f"{ERROR_BOX}: No build config found for assignment_{number}.", err=True)
         return
     static_functions = config['dissect']['static_functions']
+
     if not 'build_cmd' in config["dissect"]:
         click.echo(
             f"{ERROR_BOX}: No build config found for assignment_{number}.", err=True)
         return
     build_cmd = config['dissect']['build_cmd']
+
     if not 'run_cmd' in config["dissect"]:
         click.echo(
             f"{ERROR_BOX}: No run config found for assignment_{number}.", err=True)
         return
     run_cmd = config['dissect']['run_cmd']
+
     if not 'solution_files' in config["dissect"]:
         click.echo(
             f"{ERROR_BOX}: No solution_files config found for assignment_{number}.", err=True)
         return
     solution_files = config['dissect']['solution_files']
 
+    if not 'ref_solution' in config["correctness"]:
+        click.echo(
+            f"{ERROR_BOX}: No ref_solution config found for assignment_{number}.", err=True)
+        return
+    ref_solution = config['correctness']['ref_solution']
+
+    if not 'user_solution' in config["correctness"]:
+        click.echo(
+            f"{ERROR_BOX}: No user_solution config found for assignment_{number}.", err=True)
+        return
+    user_solution = config['correctness']['user_solution']
+
     students = os.listdir(path)
     asg_dirs = [os.path.join(
         path, student, f"assignment_{number}") for student in students]
+
     for asg_dir, student in zip(asg_dirs, students):
         if not os.path.exists(asg_dir) or not os.path.isdir(asg_dir):
             click.echo(
@@ -469,7 +484,7 @@ def dissect(number, solution_path, path):
             continue
         current_wd = os.getcwd()
         click.echo(f"({student}:assignment_{number}):")
-        failed = False
+
         flags = [f"-DGRDR_TEST_{func.upper()}" for func in test_functions]
         for root in local_roots:
             root_src_paths = [os.path.join(root, 'src', file)
@@ -485,8 +500,8 @@ def dissect(number, solution_path, path):
                 _extract_prototypes_to_header(
                     dest, static_functions, os.path.join(sol_as_path, 'src', 'aid.h'))
 
-                _unstaticize_file(dest, all_functions)
+                _unstaticize_file(dest, test_functions)
-                _weaken_file(dest, all_functions)
+                _weaken_file(dest, test_functions)
                 _staticize_file(dest, static_functions)
 
                 result = 0
@@ -497,8 +512,7 @@ def dissect(number, solution_path, path):
                     result = subprocess.run(
                         ['make', 'distclean'],  capture_output=True, text=True)
                     result = subprocess.run(
-                        shlex.split(command),  capture_output=True, text=True)
+                        shlex.split(command),  capture_output=True, text=True, timeout=10)
-
                     if result.returncode == 0:
                         click.echo(
                             f"\t[{os.path.basename(root)}]:[BUILD]:[{function}]:{SUCCESS_BOX}")
@@ -506,6 +520,42 @@ def dissect(number, solution_path, path):
                         click.echo(
                             f"\t[{os.path.basename(root)}]:[BUILD]:[{function}]:{ERROR_BOX}")
                         click.echo(f"\t\tstderr: {result.stderr[:61]:61s}...")
+                        continue
+
+                    ref_sol_path = os.path.join(sol_as_path, ref_solution)
+                    usr_sol_path = os.path.join(sol_as_path, user_solution)
+
+                    if os.path.exists(usr_sol_path):
+                        os.remove(usr_sol_path)
+
+                    try:
+                        result = subprocess.run(
+                            shlex.split(run_cmd),  capture_output=True, text=True, timeout=10)
+                    except subprocess.TimeoutExpired:
+                        result.returncode = 1
+                        click.echo(
+                            f"\t[{os.path.basename(root)}]:[RUN]:[{function}]:{ERROR_BOX} -- Timeout")
+
+                    if result.returncode == 0:
+                        click.echo(
+                            f"\t[{os.path.basename(root)}]:[RUN]:[{function}]:{SUCCESS_BOX}")
+                    else:
+                        click.echo(
+                            f"\t[{os.path.basename(root)}]:[RUN]:[{function}]:{ERROR_BOX}")
+                        click.echo(f"\t\tstderr: {result.stderr[:61]:61s}...")
+                        continue
+
+                    ref_sol_data = np.loadtxt(ref_sol_path)
+                    usr_sol_data = np.loadtxt(usr_sol_path)
+
+                    if np.allclose(ref_sol_data, usr_sol_data) == 0:
+                        click.echo(
+                            f"\t[{os.path.basename(root)}]:[CORRECT]:[{function}]:{SUCCESS_BOX}")
+                    else:
+                        click.echo(
+                            f"\t[{os.path.basename(root)}]:[CORRECT]:[{function}]:{ERROR_BOX}")
+                        click.echo(f"\t\tstderr: {result.stderr[:61]:61s}...")
+                        continue
                 os.chdir(current_wd)