From 534a3246978f155362035e8c2047f4be3081798b Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Tue, 12 Aug 2025 13:46:17 +0100 Subject: [PATCH 1/3] Added file IO to allow reading and writing of JSON files to work with Shimmy. Also added print statements for the three types of evaluation that this function supports. --- evaluation_function.wl | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/evaluation_function.wl b/evaluation_function.wl index 7862ce6..0e32d55 100644 --- a/evaluation_function.wl +++ b/evaluation_function.wl @@ -3,6 +3,7 @@ (* The basic evaluation function code*) equalQNumeric[answer_, response_, params_] := Module[{tolerance}, + Print["Evaluating Equal Numeric"]; tolerance = If[Lookup[params, "tolerance_is_absolute", False], Lookup[params, "tolerance", 0], Lookup[params, "tolerance", 0] * answer @@ -15,6 +16,7 @@ equalQNumeric[answer_, response_, params_] := Module[{tolerance}, ] equalQOther[answer_, response_, params_] := Module[{correctQ}, + Print["Evaluating Equal Other"]; <| "error" -> Null, "is_correct" -> TrueQ[answer == response] @@ -111,11 +113,13 @@ StructureMatchQ[response_, answerTemplate_, namedVariables_, Atomic -> OptionValue[Atomic]]]] equalQStructure[answer_, response_, params_] := Module[{namedVariables,correctQ}, + Print["Evaluating Structure"]; namedVariables = ToExpression[Lookup[params,"named_variables",{}],TraditionalForm]; correctQ = StructureMatchQ[ ToExpression[ToString[response],TraditionalForm], ToExpression[ToString[answer],TraditionalForm], namedVariables]; + <| "error" -> Null, "is_correct" -> correctQ @@ -137,6 +141,11 @@ evalQ[answer_, response_, params_] := Module[{}, EvaluationFunction[answer_, response_, params_] := Module[{tolerance, correctQ, error}, result = evalQ[answer, response, params]; + Print["EvalFn"]; + Print[answer]; + Print[response]; + Print[params]; + Print[result]; <| "is_correct" -> result["is_correct"], "feedback" -> If[result["is_correct"], @@ -146,3 +155,22 @@ EvaluationFunction[answer_, response_, params_] := Module[{tolerance, correctQ, "error" -> result["error"] |> ]; + +evalQuestionIO = Function[ + Module[{jsonData, result}, + jsonData = Import[#1, "JSON"] //. List :> Association; + answer = jsonData["answerTemplate"]; + response = jsonData["response"]; + params = jsonData["params"]; + Print["evalQuestionIO"]; + Print[answer]; + Print[response]; + Print[params]; + result = EvaluationFunction[answer, response, params]; + Export[#2, result, "JSON", "Compact" -> True] + ] +]; + +argv = Rest[$ScriptCommandLine]; +evalQuestionIO[argv[[1]], argv[[2]]] + From 8cd37723fca118c50242476ca8266aee7aeeef2a Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Tue, 12 Aug 2025 14:31:49 +0100 Subject: [PATCH 2/3] Implemented change to allow comparisonType to be passed to the EvalFn --- evaluation_function.wl | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/evaluation_function.wl b/evaluation_function.wl index 0e32d55..6d59168 100644 --- a/evaluation_function.wl +++ b/evaluation_function.wl @@ -128,9 +128,9 @@ equalQStructure[answer_, response_, params_] := Module[{namedVariables,correctQ} (* The evaluation function itself *) -evalQ[answer_, response_, params_] := Module[{}, +evalQ[type_, answer_, response_, params_] := Module[{}, Which[ - Lookup[params,"equality_test","None"] == "structure", + type == "structure", equalQStructure[answer,response,params], NumericQ[answer], equalQNumeric[answer, response, params], @@ -139,13 +139,8 @@ evalQ[answer_, response_, params_] := Module[{}, ] ]; -EvaluationFunction[answer_, response_, params_] := Module[{tolerance, correctQ, error}, - result = evalQ[answer, response, params]; - Print["EvalFn"]; - Print[answer]; - Print[response]; - Print[params]; - Print[result]; +EvaluationFunction[type_, answer_, response_, params_] := Module[{tolerance, correctQ, error}, + result = evalQ[type, answer, response, params]; <| "is_correct" -> result["is_correct"], "feedback" -> If[result["is_correct"], @@ -159,14 +154,11 @@ EvaluationFunction[answer_, response_, params_] := Module[{tolerance, correctQ, evalQuestionIO = Function[ Module[{jsonData, result}, jsonData = Import[#1, "JSON"] //. List :> Association; + type = jsonData["comparisonType"]; answer = jsonData["answerTemplate"]; response = jsonData["response"]; params = jsonData["params"]; - Print["evalQuestionIO"]; - Print[answer]; - Print[response]; - Print[params]; - result = EvaluationFunction[answer, response, params]; + result = EvaluationFunction[type, answer, response, params]; Export[#2, result, "JSON", "Compact" -> True] ] ]; From 719ab52e44745d64e553afbeaae582897fbcb787 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Wed, 13 Aug 2025 16:43:04 +0100 Subject: [PATCH 3/3] Fixed IO and stucture --- Dockerfile | 28 ++++++++++------------------ config.json | 2 +- evaluation_function.wl | 29 ++++++++++++++++++++--------- structure_match_example1.json | 9 ++++++--- 4 files changed, 37 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3c6458a..97093ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,4 @@ -FROM ghcr.io/lambda-feedback/evaluation-function-base/wolfram:latest AS final - -# Use the WSTP server as a kernel backend -ENV WSTP_SERVER="true" +FROM ghcr.io/lambda-feedback/evaluation-function-base/wolfram:latest as base # Command to start the evaluation function with ENV FUNCTION_COMMAND="wolframscript" @@ -12,22 +9,17 @@ ENV FUNCTION_ARGS="-f,/app/evaluation_function.wl" # Interface to use for the evaluation function ENV FUNCTION_INTERFACE="file" +ENV LOG_LEVEL="DEBUG" + # Copy the evaluation function to the app directory COPY ./evaluation_function.wl /app/evaluation_function.wl -# Final layer for private images, which contains the wolfram licence key, -# and is started with the shimmy handle command. -FROM final AS final-private - -# Copy the mathpass secret to the Wolfram Engine licensing directory. -# See https://hub.docker.com/r/wolframresearch/wolframengine for more information. -RUN --mount=type=secret,id=mathpass \ - mkdir -p /tmp/home/.WolframEngine/Licensing && \ - cp /run/secrets/mathpass /tmp/home/.WolframEngine/Licensing/mathpass - +FROM base AS with-license +COPY ./dist/LICENCE.txt /home/wolframengine/.WolframEngine/Licensing/mathpass -RUN apt-get update && apt-get install -y \ - netcat \ - && rm -rf /var/lib/apt/lists/* +FROM base AS without-license +# no COPY, no error -COPY ./entrypoint.sh /entrypoint.sh \ No newline at end of file +# Choose final stage with build arg +ARG WITH_LICENSE=false +FROM ${WITH_LICENSE:+with-license}${WITH_LICENSE:-without-license} \ No newline at end of file diff --git a/config.json b/config.json index d13fa81..32d7442 100644 --- a/config.json +++ b/config.json @@ -1,3 +1,3 @@ { - "EvaluationFunctionName": "" + "EvaluationFunctionName": "wolframEvaluationFunction" } diff --git a/evaluation_function.wl b/evaluation_function.wl index 6d59168..d0d573d 100644 --- a/evaluation_function.wl +++ b/evaluation_function.wl @@ -140,25 +140,36 @@ evalQ[type_, answer_, response_, params_] := Module[{}, ]; EvaluationFunction[type_, answer_, response_, params_] := Module[{tolerance, correctQ, error}, + Print["Running Evaluation Function"]; result = evalQ[type, answer, response, params]; - <| - "is_correct" -> result["is_correct"], - "feedback" -> If[result["is_correct"], + Print["Results"]; + Print[result]; + feedback = If[result["is_correct"], Lookup[params, "correct_response_feedback", "Correct!"], Lookup[params, "incorrect_response_feedback", "Incorrect!"] - ], - "error" -> result["error"] + ]; + <| + "command" -> "eval", + "result" -> <| + "is_correct" -> result["is_correct"], + "feedback" -> feedback, + "error" -> result["error"] + |> |> ]; evalQuestionIO = Function[ Module[{jsonData, result}, jsonData = Import[#1, "JSON"] //. List :> Association; - type = jsonData["comparisonType"]; - answer = jsonData["answerTemplate"]; - response = jsonData["response"]; - params = jsonData["params"]; + requestData = jsonData["params"]; + answer = requestData["answer"]; + response = requestData["response"]; + params = requestData["params"]; + type = params["comparisonType"]; + Print["Evaluating Response Against Answer"]; result = EvaluationFunction[type, answer, response, params]; + Print["Response"]; + Print[result]; Export[#2, result, "JSON", "Compact" -> True] ] ]; diff --git a/structure_match_example1.json b/structure_match_example1.json index e826b45..1a11877 100644 --- a/structure_match_example1.json +++ b/structure_match_example1.json @@ -1,10 +1,13 @@ { - "comparisonType":"structure", - "answerTemplate":"Sin[p x + q]", + "method": "eval", + "params": { + "answer":"Sin[p x + q]", "response":"Sin[a x + b]", "params":{ + "comparisonType":"structure", "named_variables":"{x}", "correct_response_feedback":"Your answer is correct!", "incorrect_response_feedback":"Your answer is incorrect!" } -} \ No newline at end of file + } +}