feat: checks name the solution files they read

src/Check/DatabaseCheck.php

@@ -11,6 +11,7 @@
 use PhpSchool\PhpWorkshop\Event\Event;
 use PhpSchool\PhpWorkshop\Event\EventDispatcher;
 use PhpSchool\PhpWorkshop\Event\ExerciseRunnerEvent;
+use PhpSchool\PhpWorkshop\Exercise\ExerciseInterface;
 use PhpSchool\PhpWorkshop\ExerciseCheck\DatabaseExerciseCheck;
 use PhpSchool\PhpWorkshop\Result\Failure;
 use PhpSchool\PhpWorkshop\Result\Success;
@@ -24,7 +25,7 @@
  * student's solution has been executed, allowing you to first seed the database and then verify the contents of the
  * database.
  */
-class DatabaseCheck implements ListenableCheckInterface
+class DatabaseCheck implements ListenableCheckInterface, ReadsSolutionFiles
 {
     private Filesystem $filesystem;
     private ?string $dbContent = null;
@@ -113,6 +114,11 @@ function (CliExerciseRunnerEvent|CgiExerciseRunnerEvent $e) {
         });
     }
 
+    public function getSolutionFiles(ExerciseInterface $exercise): array
+    {
+        return ['db.sqlite'];
+    }
+
     private function getPDO(string $path): PDO
     {
         $db = new PDO('sqlite:' . $path);

src/Check/FileComparisonCheck.php

@@ -6,6 +6,7 @@
 
 use InvalidArgumentException;
 use PhpSchool\PhpWorkshop\Exception\SolutionFileDoesNotExistException;
+use PhpSchool\PhpWorkshop\Exercise\ExerciseInterface;
 use PhpSchool\PhpWorkshop\Exercise\ExerciseType;
 use PhpSchool\PhpWorkshop\ExerciseCheck\FileComparisonExerciseCheck;
 use PhpSchool\PhpWorkshop\ExerciseRunner\Context\ExecutionContext;
@@ -20,7 +21,7 @@
  * This check verifies that any additional files which should be created by a student, match the ones
  * created by the reference solution.
  */
-class FileComparisonCheck implements SimpleCheckInterface
+class FileComparisonCheck implements SimpleCheckInterface, ReadsSolutionFiles
 {
     /**
      * Return the check's name.
@@ -76,6 +77,20 @@ public function check(ExecutionContext $context): ResultInterface
         return Success::fromCheck($this);
     }
 
+    public function getSolutionFiles(ExerciseInterface $exercise): array
+    {
+        if (!$exercise instanceof FileComparisonExerciseCheck) {
+            return [];
+        }
+
+        $files = [];
+        foreach ($exercise->getFilesToCompare() as $key => $file) {
+            $files[] = $this->getOptionsAndFile($key, $file)[1];
+        }
+
+        return $files;
+    }
+
     /**
      * @param int|string $key
      * @param string|array<string, string> $file

src/Check/ReadsSolutionFiles.php

@@ -0,0 +1,19 @@
+<?php
+
+declare(strict_types=1);
+
+namespace PhpSchool\PhpWorkshop\Check;
+
+use PhpSchool\PhpWorkshop\Exercise\ExerciseInterface;
+
+/**
+ * A check that reads files the solutions wrote when they ran, eg. a database. The sandbox makes sure they are in the
+ * solution's directory afterwards.
+ */
+interface ReadsSolutionFiles
+{
+    /**
+     * @return list<string> relative to the solution's directory, eg. ['db.sqlite']
+     */
+    public function getSolutionFiles(ExerciseInterface $exercise): array;
+}

src/ExerciseDispatcher.php

@@ -6,6 +6,7 @@
 
 use PhpSchool\PhpWorkshop\Check\CheckRepository;
 use PhpSchool\PhpWorkshop\Check\ListenableCheckInterface;
+use PhpSchool\PhpWorkshop\Check\ReadsSolutionFiles;
 use PhpSchool\PhpWorkshop\Check\PhpLintCheck;
 use PhpSchool\PhpWorkshop\Check\SimpleCheckInterface;
 use PhpSchool\PhpWorkshop\Event\EventDispatcher;
@@ -106,16 +107,17 @@ public function requireCheck(string $requiredCheck): void
      */
     public function verify(ExerciseInterface $exercise, Input $input): ResultAggregator
     {
-        $context = ExecutionContext::fromInputAndExercise($input, $exercise);
-
         $runner = $this->runnerManager->getRunner($exercise);
 
         $exercise->defineListeners($this->eventDispatcher);
 
-        foreach ([...$runner->getRequiredChecks(), ...$exercise->getRequiredChecks()] as $requiredCheck) {
+        $requiredChecks = [...$runner->getRequiredChecks(), ...$exercise->getRequiredChecks()];
+        foreach ($requiredChecks as $requiredCheck) {
             $this->requireCheck($requiredCheck);
         }
 
+        $context = ExecutionContext::fromInputAndExercise($input, $exercise, $this->getSolutionFiles($requiredChecks, $exercise));
+
         $this->eventDispatcher->dispatch(new ExerciseRunnerEvent('verify.start', $context));
 
         $this->validateChecks($this->checksToRunBefore, $exercise);
@@ -235,4 +237,23 @@ public function getChecksToRunAfter(): array
     {
         return $this->checksToRunAfter;
     }
+
+    /**
+     * The files the required checks read from the solutions after they ran.
+     *
+     * @param array<class-string> $requiredChecks
+     * @return list<string> eg. ['db.sqlite']
+     */
+    private function getSolutionFiles(array $requiredChecks, ExerciseInterface $exercise): array
+    {
+        $files = [];
+        foreach ($requiredChecks as $requiredCheck) {
+            $check = $this->checkRepository->getByClass($requiredCheck);
+            if ($check instanceof ReadsSolutionFiles) {
+                $files = [...$files, ...$check->getSolutionFiles($exercise)];
+            }
+        }
+
+        return $files;
+    }
 }

src/ExerciseRunner/CgiRunner.php

@@ -122,8 +122,8 @@ function (RequestInterface $request) use ($context, $scenario, $reference, $stud
             ),
         );
 
-        $reference->finish();
-        $student->finish();
+        $reference->finish($context);
+        $student->finish($context);
 
         $this->eventDispatcher->dispatch(new CgiExerciseRunnerEvent('cgi.verify.finish', $context, $scenario));
         return $result;
@@ -340,7 +340,7 @@ public function run(ExecutionContext $context, OutputInterface $output): bool
             );
         }
 
-        $student->finish();
+        $student->finish($context);
 
         $this->eventDispatcher->dispatch(new CgiExerciseRunnerEvent('cgi.run.finish', $context, $scenario));
         return $success;

src/ExerciseRunner/CliRunner.php

@@ -121,8 +121,8 @@ function (Collection $args) use ($context, $scenario, $reference, $student) {
             ),
         );
 
-        $reference->finish();
-        $student->finish();
+        $reference->finish($context);
+        $student->finish($context);
 
         $this->eventDispatcher->dispatch(new CliExerciseRunnerEvent('cli.verify.finish', $context, $scenario));
 
@@ -252,7 +252,7 @@ public function run(ExecutionContext $context, OutputInterface $output): bool
             );
         }
 
-        $student->finish();
+        $student->finish($context);
 
         $this->eventDispatcher->dispatch(new CliExerciseRunnerEvent('cli.run.finish', $context, $scenario));
 

src/ExerciseRunner/Context/ExecutionContext.php

@@ -9,14 +9,21 @@
 
 class ExecutionContext
 {
+    /**
+     * @param list<string> $solutionFiles see getSolutionFiles()
+     */
     public function __construct(
         private string $studentExecutionDirectory,
         private string $referenceExecutionDirectory,
         private ExerciseInterface $exercise,
         private Input $input,
+        private array $solutionFiles = [],
     ) {}
 
-    public static function fromInputAndExercise(Input $input, ExerciseInterface $exercise): ExecutionContext
+    /**
+     * @param list<string> $solutionFiles see getSolutionFiles()
+     */
+    public static function fromInputAndExercise(Input $input, ExerciseInterface $exercise, array $solutionFiles = []): ExecutionContext
     {
         $program = $input->hasArgument('program') ? dirname($input->getRequiredArgument('program')) : (string) getcwd();
 
@@ -25,6 +32,7 @@ public static function fromInputAndExercise(Input $input, ExerciseInterface $exe
             System::randomTempDir(),
             $exercise,
             $input,
+            $solutionFiles,
         );
     }
 
@@ -64,4 +72,14 @@ public function getReferenceExecutionDirectory(): string
     {
         return $this->referenceExecutionDirectory;
     }
+
+    /**
+     * The files the checks read from the solutions' directories after they ran, see ReadsSolutionFiles.
+     *
+     * @return list<string> eg. ['db.sqlite']
+     */
+    public function getSolutionFiles(): array
+    {
+        return $this->solutionFiles;
+    }
 }

src/Process/Sandbox/Docker/DockerSandbox.php

@@ -4,6 +4,7 @@
 
 namespace PhpSchool\PhpWorkshop\Process\Sandbox\Docker;
 
+use PhpSchool\PhpWorkshop\ExerciseRunner\Context\ExecutionContext;
 use PhpSchool\PhpWorkshop\Process\Port\PortPicker;
 use PhpSchool\PhpWorkshop\Process\Sandbox\ProcessInput;
 use PhpSchool\PhpWorkshop\Process\Sandbox\Sandbox;
@@ -99,7 +100,10 @@ public function getPort(): int
         return $this->port ?? throw new \RuntimeException('The scenario does not expose a port');
     }
 
-    public function finish(): void
+    /**
+     * The directory is mounted, so the solution files are there already.
+     */
+    public function finish(ExecutionContext $context): void
     {
         if ($this->port !== null) {
             $this->portPicker->release($this->port);

src/Process/Sandbox/Host/HostSandbox.php

@@ -4,6 +4,7 @@
 
 namespace PhpSchool\PhpWorkshop\Process\Sandbox\Host;
 
+use PhpSchool\PhpWorkshop\ExerciseRunner\Context\ExecutionContext;
 use PhpSchool\PhpWorkshop\Process\ProcessNotFoundException;
 use PhpSchool\PhpWorkshop\Process\Sandbox\ProcessInput;
 use PhpSchool\PhpWorkshop\Process\Sandbox\Sandbox;
@@ -45,7 +46,10 @@ public function getPort(): int
         return $this->port ?? throw new \RuntimeException('The scenario does not expose a port');
     }
 
-    public function finish(): void {}
+    /**
+     * The code ran in the directory, so the solution files are there already.
+     */
+    public function finish(ExecutionContext $context): void {}
 
     /**
      * Symfony Process inherits the parent env, so we unset every inherited var by setting it to false.

src/Process/Sandbox/Sandbox.php

@@ -4,6 +4,8 @@
 
 namespace PhpSchool\PhpWorkshop\Process\Sandbox;
 
+use PhpSchool\PhpWorkshop\ExerciseRunner\Context\ExecutionContext;
+
 /**
  * Where one solution's code runs. Every execution of it happens here and listens on the same port.
  */
@@ -17,7 +19,7 @@ public function execute(ProcessInput $processInput): SandboxExecution;
     public function getPort(): int;
 
     /**
-     * Once every execution is done, frees the port.
+     * Once every execution is done: the context's solution files are in the directory and the port is free.
      */
-    public function finish(): void;
+    public function finish(ExecutionContext $context): void;
 }

test/Check/DatabaseCheckTest.php

@@ -51,6 +51,7 @@ public function testCheckMeta(): void
     {
         $this->assertEquals('Database Verification Check', $this->check->getName());
         $this->assertEquals(DatabaseExerciseCheck::class, $this->check->getExerciseInterface());
+        $this->assertSame(['db.sqlite'], $this->check->getSolutionFiles($this->exercise));
     }
 
     private function getRunnerManager(ExerciseInterface $exercise, EventDispatcher $eventDispatcher): MockObject

test/Check/FileComparisonCheckTest.php

@@ -74,6 +74,13 @@ public function testFailureIsReturnedIfStudentFileDosNotMatchReferenceFile(): vo
         $this->assertEquals($failure->getActualValue(), "somegibberish");
     }
 
+    public function testSolutionFilesAreTheFilesToCompare(): void
+    {
+        $exercise = new FileComparisonExercise(['some-file.txt', 'other-file.txt' => ['strip' => '/\d{2}:\d{2}/']]);
+
+        $this->assertSame(['some-file.txt', 'other-file.txt'], $this->check->getSolutionFiles($exercise));
+    }
+
     public function testSuccessIsReturnedIfFilesMatch(): void
     {
         $exercise = new FileComparisonExercise(['some-file.txt']);

test/ExerciseDispatcherTest.php

@@ -6,6 +6,7 @@
 use PhpSchool\PhpWorkshop\Check\CheckRepository;
 use PhpSchool\PhpWorkshop\Check\ListenableCheckInterface;
 use PhpSchool\PhpWorkshop\Check\PhpLintCheck;
+use PhpSchool\PhpWorkshop\Check\ReadsSolutionFiles;
 use PhpSchool\PhpWorkshop\Check\SimpleCheckInterface;
 use PhpSchool\PhpWorkshop\Event\EventDispatcher;
 use PhpSchool\PhpWorkshop\Event\EventInterface;
@@ -240,6 +241,37 @@ public function testVerify(): void
         $this->assertTrue($result->isSuccessful());
     }
 
+    public function testVerifyGivesTheRunnerTheFilesTheChecksRead(): void
+    {
+        $exercise = new CliExerciseImpl('Some Exercise');
+
+        $check = $this->createMockForIntersectionOfInterfaces([SimpleCheckInterface::class, ReadsSolutionFiles::class]);
+        $check->method('canRun')->willReturn(true);
+        $check->method('getPosition')->willReturn(SimpleCheckInterface::CHECK_AFTER);
+        $check->method('getExerciseInterface')->willReturn(ExerciseInterface::class);
+        $check->method('check')->willReturn(new Success('Success!'));
+        $check->method('getSolutionFiles')->with($exercise)->willReturn(['db.sqlite']);
+
+        $runner = $this->createMock(ExerciseRunnerInterface::class);
+        $runner->method('getRequiredChecks')->willReturn([get_class($check)]);
+        $runner->expects($this->once())
+            ->method('verify')
+            ->with($this->callback(fn(ExecutionContext $context) => $context->getSolutionFiles() === ['db.sqlite']))
+            ->willReturn(new Success('Success!'));
+
+        $runnerManager = $this->createMock(RunnerManager::class);
+        $runnerManager->method('getRunner')->with($exercise)->willReturn($runner);
+
+        $exerciseDispatcher = new ExerciseDispatcher(
+            $runnerManager,
+            new ResultAggregator(),
+            new EventDispatcher(new ResultAggregator()),
+            new CheckRepository([$check]),
+        );
+
+        $exerciseDispatcher->verify($exercise, new Input('app', ['program' => $this->file]));
+    }
+
     public function testVerifyOnlyRunsRequiredChecks(): void
     {
         $input = new Input('app', ['program' => $this->file]);

test/ExerciseRunner/CliRunnerTest.php

@@ -6,6 +6,7 @@
 use PhpSchool\PhpWorkshop\Check\CodeExistsCheck;
 use PhpSchool\PhpWorkshop\Exercise\Scenario\CliScenario;
 use PhpSchool\PhpWorkshop\Exercise\Scenario\ExerciseScenario;
+use PhpSchool\PhpWorkshop\ExerciseRunner\Context\ExecutionContext;
 use PhpSchool\PhpWorkshop\ExerciseRunner\Context\TestContext;
 use PhpSchool\PhpWorkshop\ExerciseRunner\EnvironmentManager;
 use PhpSchool\PhpWorkshop\Listener\OutputRunInfoListener;
@@ -236,7 +237,7 @@ public function getPort(): int
                 return 54321;
             }
 
-            public function finish(): void {}
+            public function finish(ExecutionContext $context): void {}
         };
         $factory = $this->createMock(SandboxFactory::class);
         $factory->method('create')->willReturnCallback(function (string $directory, ExerciseScenario $scenario) use ($sandbox) {

test/Process/Sandbox/Docker/DockerSandboxTest.php

@@ -2,6 +2,7 @@
 
 namespace PhpSchool\PhpWorkshopTest\Process\Sandbox\Docker;
 
+use PhpSchool\PhpWorkshop\ExerciseRunner\Context\TestContext;
 use PhpSchool\PhpWorkshop\Process\Sandbox\Docker\DockerSandboxFactory;
 use PhpSchool\PhpWorkshop\Process\Port\PortPicker;
 use PhpSchool\PhpWorkshop\Exercise\Scenario\CliScenario;
@@ -141,7 +142,7 @@ public function testAddressIsTheSamePortOnTheGivenHost(): void
 
     public function testFinishReleasesThePort(): void
     {
-        $this->factory()->create(__DIR__, (new CliScenario())->exposePort())->finish();
+        $this->factory()->create(__DIR__, (new CliScenario())->exposePort())->finish(new TestContext());
 
         static::assertSame([47213], $this->released);
     }